ant-notifications mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From hi...@apache.org
Subject svn commit: r1373534 - in /ant/ivy/core/trunk: ./ doc/use/ src/java/org/apache/ivy/ src/java/org/apache/ivy/ant/ src/java/org/apache/ivy/core/retrieve/ src/java/org/apache/ivy/util/ test/java/org/apache/ivy/core/retrieve/
Date Wed, 15 Aug 2012 17:32:47 GMT
Author: hibou
Date: Wed Aug 15 17:32:47 2012
New Revision: 1373534

URL: http://svn.apache.org/viewvc?rev=1373534&view=rev
Log:
IVY-1252 : symlinkmass feature based on symlink feature of ivy:retrieve (Thanks to Gene Smith)

Modified:
    ant/ivy/core/trunk/CHANGES.txt
    ant/ivy/core/trunk/doc/use/retrieve.html
    ant/ivy/core/trunk/src/java/org/apache/ivy/Main.java
    ant/ivy/core/trunk/src/java/org/apache/ivy/ant/IvyRetrieve.java
    ant/ivy/core/trunk/src/java/org/apache/ivy/core/retrieve/RetrieveEngine.java
    ant/ivy/core/trunk/src/java/org/apache/ivy/core/retrieve/RetrieveOptions.java
    ant/ivy/core/trunk/src/java/org/apache/ivy/util/FileUtil.java
    ant/ivy/core/trunk/test/java/org/apache/ivy/core/retrieve/RetrieveTest.java

Modified: ant/ivy/core/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/ant/ivy/core/trunk/CHANGES.txt?rev=1373534&r1=1373533&r2=1373534&view=diff
==============================================================================
--- ant/ivy/core/trunk/CHANGES.txt (original)
+++ ant/ivy/core/trunk/CHANGES.txt Wed Aug 15 17:32:47 2012
@@ -111,6 +111,7 @@ for detailed view of each issue, please 
 	Ruslan Shevchenko
 	John Shields
 	Nihal Sinha
+	Gene Smith
 	Simon Steiner
 	Johan Stuyts
 	John Tinetti
@@ -141,6 +142,7 @@ for detailed view of each issue, please 
 
 - NEW: Support Conditional Setting of a Property (IVY-1367)
 - NEW: Exposing some parent metadata (organisation, module, revision, branch) as properties
(IVY-1288) (Thanks to Jean-Louis Boudart)
+- NEW: symlinkmass feature based on symlink feature of ivy:retrieve (IVY-1252) (Thanks to
Gene Smith)
 
 - IMPROVEMENT: add support for source bundles from p2 repositories
 - IMPROVEMENT: add support for source URI from OBR repositories

Modified: ant/ivy/core/trunk/doc/use/retrieve.html
URL: http://svn.apache.org/viewvc/ant/ivy/core/trunk/doc/use/retrieve.html?rev=1373534&r1=1373533&r2=1373534&view=diff
==============================================================================
--- ant/ivy/core/trunk/doc/use/retrieve.html (original)
+++ ant/ivy/core/trunk/doc/use/retrieve.html Wed Aug 15 17:32:47 2012
@@ -62,7 +62,17 @@ Possible values are:
 <li><b>always</b></li> always overwrite the destination file
 <li><b>never</b></li> never overwrite the destination file
 </ul></td><td>No. Defaults to 'newer'.</td></tr>
-    <tr><td>symlink</td><td>true to create symbolic links, false
to copy the artifacts. The destination of the symbolic links depends on the value of the useOrigin
attribute <span class="since">(since 2.0)</span></td><td>No. Defaults
to false</td></tr>
+    <tr><td>symlink</td><td>true to create symbolic links, false
to copy the artifacts.
+    The destination of the symbolic links depends on the value of the useOrigin attribute.
+    (requires "ln" to be a valid command, and to support the options -s and -f (works on
UNIX/Linux, on other systems you may need to script "ln")
+    <span class="since">(since 2.0)</span></td><td>No. Defaults to
false</td></tr>
+    <tr><td>symlinkmass</td><td>true to create symbolic links in
mass, false to copy the artifacts.
+    "symlinkmass" overrides "symlink" if both are set to "true".
+    "symlinkmass" will create the same symbolic links "symlink" does, but with a single process
call to "sh" with batched "ln" commands passed in as standard input (works on UNIX/Linux,
on other systems you may need to script it)
+    Far large lists of resolved jars, this can be dramatically faster.
+    The destination of the symbolic links depends on the value of the useOrigin attribute.
+    The events "StartRetrieveArtifactEvent" and EndRetrieveEvent are NOT fired by this activity,
because it is not clear when they should be called.
+    <span class="since">(since 2.4)</span></td><td>No. Defaults to
false</td></tr>
     <tr><td>settingsRef</td><td>A reference to the ivy settings that
must be used by this task <span class="since">(since 2.0)</span></td><td>No,
'ivy.instance' is taken by default.</td></tr></tbody>
     <tr><td>log</td><td>the log setting to use during the resolve
and retrieve process. <span class="since">(since 2.0)</span><br/>
 Available options are the same as for [[ant:resolve]] when used to trigger resolve automatically
(see [[ant:postresolvetask]]), or the following for the retrieve process only:

Modified: ant/ivy/core/trunk/src/java/org/apache/ivy/Main.java
URL: http://svn.apache.org/viewvc/ant/ivy/core/trunk/src/java/org/apache/ivy/Main.java?rev=1373534&r1=1373533&r2=1373534&view=diff
==============================================================================
--- ant/ivy/core/trunk/src/java/org/apache/ivy/Main.java (original)
+++ ant/ivy/core/trunk/src/java/org/apache/ivy/Main.java Wed Aug 15 17:32:47 2012
@@ -287,7 +287,8 @@ public final class Main {
                         .setUseOrigin(line.hasOption("useOrigin"))
                         .setDestIvyPattern(ivyPattern)
                         .setArtifactFilter(FilterHelper.getArtifactTypeFilter(line.getOptionValues("types")))
-                        .setMakeSymlinks(line.hasOption("symlink")));
+                        .setMakeSymlinks(line.hasOption("symlink"))
+                        .setMakeSymlinksInMass(line.hasOption("symlinkmass")));
             }
             if (line.hasOption("cachepath")) {
                 outputCachePath(ivy, cache, md, confs, line.getOptionValue("cachepath",

Modified: ant/ivy/core/trunk/src/java/org/apache/ivy/ant/IvyRetrieve.java
URL: http://svn.apache.org/viewvc/ant/ivy/core/trunk/src/java/org/apache/ivy/ant/IvyRetrieve.java?rev=1373534&r1=1373533&r2=1373534&view=diff
==============================================================================
--- ant/ivy/core/trunk/src/java/org/apache/ivy/ant/IvyRetrieve.java (original)
+++ ant/ivy/core/trunk/src/java/org/apache/ivy/ant/IvyRetrieve.java Wed Aug 15 17:32:47 2012
@@ -50,6 +50,8 @@ public class IvyRetrieve extends IvyPost
     private boolean sync = false;
 
     private boolean symlink = false;
+
+    private boolean symlinkmass = false;
     
     private String overwriteMode = RetrieveOptions.OVERWRITEMODE_NEWER;
 
@@ -106,6 +108,7 @@ public class IvyRetrieve extends IvyPost
                             .setOverwriteMode(getOverwriteMode())
                             .setUseOrigin(isUseOrigin())
                             .setMakeSymlinks(symlink)
+                            .setMakeSymlinksInMass(symlinkmass)
                             .setResolveId(getResolveId())
                             .setMapper(mapper == null ? null : new MapperAdapter(mapper)));
 
@@ -168,6 +171,13 @@ public class IvyRetrieve extends IvyPost
         this.symlink = symlink;
     }
 
+    /**
+     * Option to create symlinks in one mass action, instead of separately.
+     */
+    public void setSymlinkmass(boolean symlinkmass) {
+        this.symlinkmass = symlinkmass;
+    }
+
     public void setOverwriteMode(String overwriteMode) {
         if (!OVERWRITEMODE_VALUES.contains(overwriteMode)) {
             throw new IllegalArgumentException("invalid overwriteMode value '" + overwriteMode
+ "'. "

Modified: ant/ivy/core/trunk/src/java/org/apache/ivy/core/retrieve/RetrieveEngine.java
URL: http://svn.apache.org/viewvc/ant/ivy/core/trunk/src/java/org/apache/ivy/core/retrieve/RetrieveEngine.java?rev=1373534&r1=1373533&r2=1373534&view=diff
==============================================================================
--- ant/ivy/core/trunk/src/java/org/apache/ivy/core/retrieve/RetrieveEngine.java (original)
+++ ant/ivy/core/trunk/src/java/org/apache/ivy/core/retrieve/RetrieveEngine.java Wed Aug 15
17:32:47 2012
@@ -110,6 +110,7 @@ public class RetrieveEngine {
         }
 
         try {
+            Map/*<File, File>*/ destToSrcMap = null;
             Map artifactsToCopy = determineArtifactsToCopy(mrid, destFilePattern, options);
             File fileRetrieveRoot = settings.resolveFile(
                 IvyPatternHelper.getTokenRoot(destFilePattern));
@@ -122,6 +123,12 @@ public class RetrieveEngine {
             // for sync)
             Collection targetIvysStructure = new HashSet(); // same for ivy files
 
+            if (options.isMakeSymlinksInMass()) {
+                // The HashMap is of "destToSrc" because src could go two places, but dest
can only
+                // come from one
+                destToSrcMap = new HashMap();
+            }
+
             // do retrieve
             long totalCopiedSize = 0;
             for (Iterator iter = artifactsToCopy.keySet().iterator(); iter.hasNext();) {
@@ -139,17 +146,29 @@ public class RetrieveEngine {
                     if (!settings.isCheckUpToDate() || !upToDate(archive, destFile, options))
{
                         Message.verbose("\t\tto " + destFile);
                         if (this.eventManager != null) {
-                            this.eventManager.fireIvyEvent(
-                                new StartRetrieveArtifactEvent(artifact, destFile));
+                            // There is no unitary event for the mass sym linking.
+                            // skip the event declaration.
+                            if (!options.isMakeSymlinksInMass()) {
+                                this.eventManager.fireIvyEvent(new StartRetrieveArtifactEvent(
+                                        artifact, destFile));
+                            }
                         }
-                        if (options.isMakeSymlinks()) {
+                        if (options.isMakeSymlinksInMass()) {
+                            if (FileUtil.prepareCopy(archive, destFile, true)) {
+                                destToSrcMap.put(destFile, archive);
+                            }
+                        } else if (options.isMakeSymlinks()) {
                             FileUtil.symlink(archive, destFile, null, true);
                         } else {
                             FileUtil.copy(archive, destFile, null, true);
                         }
                         if (this.eventManager != null) {
-                            this.eventManager.fireIvyEvent(
-                                new EndRetrieveArtifactEvent(artifact, destFile));
+                            // There is no unitary event for the mass sym linking.
+                            // skip the event declaration.
+                            if (!options.isMakeSymlinksInMass()) {
+                                this.eventManager.fireIvyEvent(new EndRetrieveArtifactEvent(
+                                        artifact, destFile));
+                            }
                         }
                         totalCopiedSize += destFile.length();
                         report.addCopiedFile(destFile, artifact);
@@ -168,6 +187,11 @@ public class RetrieveEngine {
                 }
             }
 
+            if (options.isMakeSymlinksInMass()) {
+                Message.verbose("\tMass symlinking " + destToSrcMap.size() + " files");
+                FileUtil.symlinkInMass(destToSrcMap, true);
+            }
+
             if (options.isSync()) {
                 Message.verbose("\tsyncing...");
                 

Modified: ant/ivy/core/trunk/src/java/org/apache/ivy/core/retrieve/RetrieveOptions.java
URL: http://svn.apache.org/viewvc/ant/ivy/core/trunk/src/java/org/apache/ivy/core/retrieve/RetrieveOptions.java?rev=1373534&r1=1373533&r2=1373534&view=diff
==============================================================================
--- ant/ivy/core/trunk/src/java/org/apache/ivy/core/retrieve/RetrieveOptions.java (original)
+++ ant/ivy/core/trunk/src/java/org/apache/ivy/core/retrieve/RetrieveOptions.java Wed Aug
15 17:32:47 2012
@@ -75,6 +75,12 @@ public class RetrieveOptions extends Log
     private boolean makeSymlinks = false;
 
     /**
+     * True if symbolic links should be created all at once, instead of one at a time. Works
only on
+     * OS supporting with both "sh" (a shell) and "ln" (the link command).
+     */
+    private boolean makeSymlinksInMass = false;
+
+    /**
      * The id used to store the resolve information.
      */
     private String resolveId;
@@ -147,11 +153,20 @@ public class RetrieveOptions extends Log
         return makeSymlinks;
     }
 
+    public boolean isMakeSymlinksInMass() {
+        return makeSymlinksInMass;
+    }
+
     public RetrieveOptions setMakeSymlinks(boolean makeSymlinks) {
         this.makeSymlinks = makeSymlinks;
         return this;
     }
 
+    public RetrieveOptions setMakeSymlinksInMass(boolean makeSymlinksInMass) {
+        this.makeSymlinksInMass = makeSymlinksInMass;
+        return this;
+    }
+
     public boolean isSync() {
         return sync;
     }

Modified: ant/ivy/core/trunk/src/java/org/apache/ivy/util/FileUtil.java
URL: http://svn.apache.org/viewvc/ant/ivy/core/trunk/src/java/org/apache/ivy/util/FileUtil.java?rev=1373534&r1=1373533&r2=1373534&view=diff
==============================================================================
--- ant/ivy/core/trunk/src/java/org/apache/ivy/util/FileUtil.java (original)
+++ ant/ivy/core/trunk/src/java/org/apache/ivy/util/FileUtil.java Wed Aug 15 17:32:47 2012
@@ -31,9 +31,13 @@ import java.net.URL;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
+import java.util.Iterator;
 import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
 import java.util.Stack;
 import java.util.StringTokenizer;
+import java.util.regex.Pattern;
 
 import org.apache.ivy.util.url.URLHandlerRegistry;
 
@@ -52,22 +56,75 @@ public final class FileUtil {
 
     private static final byte[] EMPTY_BUFFER = new byte[0];
 
-    public static void symlink(File src, File dest, CopyProgressListener l, boolean overwrite)
-            throws IOException {
+    private static final Pattern ALLOWED_PATH_PATTERN = Pattern.compile("[\\w-./\\\\:~ ]+");
+
+    public static void symlinkInMass(Map/* <File, File> */destToSrcMap, boolean overwrite)
throws IOException {
+
+        // This pattern could be more forgiving if somebody wanted it to be...
+        // ...but this should satisfy 99+% of all needs, without letting unsafe operations
be done.
+        // If you paths is not supported, you then skip this mass option.
+        // NOTE: A space inside the path is allowed (I can't control other programmers who
like them
+        // in their working directory names)...
+        // but trailing spaces on file names will be checked otherwise and refused.
         try {
-            if (dest.exists()) {
-                if (!dest.isFile()) {
-                    throw new IOException("impossible to copy: destination is not a file:
" + dest);
+            StringBuffer sb = new StringBuffer();
+
+            Iterator keyItr = destToSrcMap.entrySet().iterator();
+            while (keyItr.hasNext()) {
+                Entry/*<File, File>*/ entry = (Entry) keyItr.next();
+                File destFile = (File) entry.getKey();
+                File srcFile = (File) entry.getValue();
+                if (!ALLOWED_PATH_PATTERN.matcher(srcFile.getAbsolutePath()).matches()) {
+                    throw new IOException("Unsafe file to 'mass' symlink: '"
+                            + srcFile.getAbsolutePath() + "'");
                 }
-                if (!overwrite) {
-                    Message.verbose(dest + " already exists, nothing done");
-                    return;
+                if (!ALLOWED_PATH_PATTERN.matcher(destFile.getAbsolutePath()).matches())
{
+                    throw new IOException("Unsafe file to 'mass' symlink to: '"
+                            + destFile.getAbsolutePath() + "'");
+                }
+
+                // Add to our buffer of commands
+                sb.append("ln -s -f \"" + srcFile.getAbsolutePath() + "\"  \"" + destFile.getAbsolutePath()
+ "\";");
+                if (keyItr.hasNext()) {
+                    sb.append("\n");
                 }
             }
-            if (dest.getParentFile() != null) {
-                dest.getParentFile().mkdirs();
+
+            String commands = sb.toString();
+            // Run the buffer of commands we have built.
+            Runtime runtime = Runtime.getRuntime();
+            Message.verbose("executing \"sh\" of:\n\t" + commands.replace("\n", "\n\t"));
+            Process process = runtime.exec("sh");
+            OutputStream os = process.getOutputStream();
+            os.write(commands.getBytes("UTF-8"));
+            os.flush();
+            os.close();
+
+            if (process.waitFor() != 0) {
+                InputStream errorStream = process.getErrorStream();
+                InputStreamReader isr = new InputStreamReader(errorStream);
+                BufferedReader br = new BufferedReader(isr);
+
+                StringBuffer error = new StringBuffer();
+                String line;
+                while ((line = br.readLine()) != null) {
+                    error.append(line);
+                    error.append('\n');
+                }
+
+                throw new IOException("error running ln commands with 'sh':\n" + error);
             }
+        } catch (InterruptedException x) {
+            Thread.currentThread().interrupt();
+        }
+    }
 
+    public static void symlink(File src, File dest, CopyProgressListener l, boolean overwrite)
+            throws IOException {
+        if (!prepareCopy(src, dest, overwrite)) {
+            return;
+        }
+        try {
             Runtime runtime = Runtime.getRuntime();
             Message.verbose("executing 'ln -s -f " + src.getAbsolutePath() + " " + dest.getPath()
                     + "'");
@@ -114,8 +171,7 @@ public final class FileUtil {
         return copy(src, dest, l, false);
     }
 
-    public static boolean copy(File src, File dest, CopyProgressListener l, boolean overwrite)
-            throws IOException {
+    public static boolean prepareCopy(File src, File dest, boolean overwrite) throws IOException
{
         if (dest.exists()) {
             if (!dest.isFile()) {
                 throw new IOException("impossible to copy: destination is not a file: " +
dest);
@@ -129,6 +185,17 @@ public final class FileUtil {
                 return false;
             }
         }
+        if (dest.getParentFile() != null) {
+            dest.getParentFile().mkdirs();
+        }
+        return true;
+    }
+
+    public static boolean copy(File src, File dest, CopyProgressListener l, boolean overwrite)
+            throws IOException {
+        if (!prepareCopy(src, dest, overwrite)) {
+            return false;
+        }
         copy(new FileInputStream(src), dest, l);
         long srcLen = src.length();
         long destLen = dest.length();

Modified: ant/ivy/core/trunk/test/java/org/apache/ivy/core/retrieve/RetrieveTest.java
URL: http://svn.apache.org/viewvc/ant/ivy/core/trunk/test/java/org/apache/ivy/core/retrieve/RetrieveTest.java?rev=1373534&r1=1373533&r2=1373534&view=diff
==============================================================================
--- ant/ivy/core/trunk/test/java/org/apache/ivy/core/retrieve/RetrieveTest.java (original)
+++ ant/ivy/core/trunk/test/java/org/apache/ivy/core/retrieve/RetrieveTest.java Wed Aug 15
17:32:47 2012
@@ -195,17 +195,37 @@ public class RetrieveTest extends TestCa
         ModuleDescriptor md = report.getModuleDescriptor();
         assertNotNull(md);
 
+        getRetrieveOptions().setMakeSymlinks(true);
+
         String pattern = "build/test/retrieve/[module]/[conf]/[artifact]-[revision].[ext]";
-        ivy
-                .retrieve(md.getModuleRevisionId(), pattern, getRetrieveOptions().setMakeSymlinks(
-                    true));
+        ivy.retrieve(md.getModuleRevisionId(), pattern, getRetrieveOptions());
         assertLink(IvyPatternHelper.substitute(pattern, "org1", "mod1.2", "2.0", "mod1.2",
"jar",
             "jar", "default"));
 
         pattern = "build/test/retrieve/[module]/[conf]/[type]s/[artifact]-[revision].[ext]";
-        ivy
-                .retrieve(md.getModuleRevisionId(), pattern, getRetrieveOptions().setMakeSymlinks(
-                    true));
+        ivy.retrieve(md.getModuleRevisionId(), pattern, getRetrieveOptions());
+        assertLink(IvyPatternHelper.substitute(pattern, "org1", "mod1.2", "2.0", "mod1.2",
"jar",
+            "jar", "default"));
+    }
+
+    public void testRetrieveWithSymlinksMass() throws Exception {
+        // mod1.1 depends on mod1.2
+        ResolveReport report = ivy.resolve(new File(
+                "test/repositories/1/org1/mod1.1/ivys/ivy-1.0.xml").toURL(),
+            getResolveOptions(new String[] {"*"}));
+        assertNotNull(report);
+        ModuleDescriptor md = report.getModuleDescriptor();
+        assertNotNull(md);
+
+        getRetrieveOptions().setMakeSymlinksInMass(true);
+
+        String pattern = "build/test/retrieve/[module]/[conf]/[artifact]-[revision].[ext]";
+        ivy.retrieve(md.getModuleRevisionId(), pattern, getRetrieveOptions());
+        assertLink(IvyPatternHelper.substitute(pattern, "org1", "mod1.2", "2.0", "mod1.2",
"jar",
+            "jar", "default"));
+
+        pattern = "build/test/retrieve/[module]/[conf]/[type]s/[artifact]-[revision].[ext]";
+        ivy.retrieve(md.getModuleRevisionId(), pattern, getRetrieveOptions());
         assertLink(IvyPatternHelper.substitute(pattern, "org1", "mod1.2", "2.0", "mod1.2",
"jar",
             "jar", "default"));
     }



Mime
View raw message