pig-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From aniket...@apache.org
Subject svn commit: r1579006 - in /pig/trunk: CHANGES.txt src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/JobControlCompiler.java
Date Tue, 18 Mar 2014 18:36:10 GMT
Author: aniket486
Date: Tue Mar 18 18:36:10 2014
New Revision: 1579006

URL: http://svn.apache.org/r1579006
Log:
PIG-3815: Hadoop bug causes to pig to fail silently with jar cache (aniket486)

Modified:
    pig/trunk/CHANGES.txt
    pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/JobControlCompiler.java

Modified: pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1579006&r1=1579005&r2=1579006&view=diff
==============================================================================
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Tue Mar 18 18:36:10 2014
@@ -99,6 +99,8 @@ OPTIMIZATIONS
  
 BUG FIXES
 
+PIG-3815: Hadoop bug causes to pig to fail silently with jar cache (aniket486)
+
 PIG-3816: Incorrect Javadoc for launchPlan() method (kyungho via prkommireddi)
 
 PIG-3673: Divide by zero error in runpigmix.pl script (suhassatish via daijy)

Modified: pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/JobControlCompiler.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/JobControlCompiler.java?rev=1579006&r1=1579005&r2=1579006&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/JobControlCompiler.java
(original)
+++ pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/JobControlCompiler.java
Tue Mar 18 18:36:10 2014
@@ -20,6 +20,7 @@ package org.apache.pig.backend.hadoop.ex
 import java.io.File;
 import java.io.FileOutputStream;
 import java.io.IOException;
+import java.io.InputStream;
 import java.io.OutputStream;
 import java.lang.reflect.Method;
 import java.net.URI;
@@ -1639,7 +1640,10 @@ public class JobControlCompiler{
                         log.info("Found " + url + " in jar cache at "+ stagingDir);
                         long curTime = System.currentTimeMillis();
                         fs.setTimes(jarPath, -1, curTime);
-                        return jarPath;
+                        // PIG-3815 In hadoop 1.0, addFileToClassPath uses : as separator
+                        // jarPath has full uri at this point, we need to remove hdfs://nn:port
+                        // part to avoid parsing errors on backend
+                        return new Path(jarPath.toUri().getPath());
                     }
                 }
             }
@@ -1647,11 +1651,15 @@ public class JobControlCompiler{
             // attempt to copy to cache else return null
             fs.mkdirs(cacheDir, FileLocalizer.OWNER_ONLY_PERMS);
             Path cacheFile = new Path(cacheDir, filename);
-            OutputStream os = FileSystem.create(fs, cacheFile, FileLocalizer.OWNER_ONLY_PERMS);
+            OutputStream os = null;
+            InputStream is = null;
             try {
-                IOUtils.copyBytes(url.openStream(), os, 4096, true);
+                os = FileSystem.create(fs, cacheFile, FileLocalizer.OWNER_ONLY_PERMS);
+                is = url.openStream();
+                IOUtils.copyBytes(is, os, 4096, true);
             } finally {
-                os.close();
+                org.apache.commons.io.IOUtils.closeQuietly(is);
+                org.apache.commons.io.IOUtils.closeQuietly(os);
             }
             return cacheFile;
 
@@ -1686,13 +1694,12 @@ public class JobControlCompiler{
 
         Path dst = new Path(FileLocalizer.getTemporaryPath(pigContext).toUri().getPath(),
suffix);
         FileSystem fs = dst.getFileSystem(conf);
-        OutputStream os = fs.create(dst);
+        OutputStream os = null;
         try {
+            os = fs.create(dst);
             IOUtils.copyBytes(url.openStream(), os, 4096, true);
         } finally {
-            // IOUtils can not close both the input and the output properly in a finally
-            // as we can get an exception in between opening the stream and calling the method
-            os.close();
+            org.apache.commons.io.IOUtils.closeQuietly(os);
         }
         return dst;
     }



Mime
View raw message