Mailing-List: contact commits-help@pig.apache.org; run by ezmlm
Precedence: bulk
Reply-To: dev@pig.apache.org
Content-Type: text/plain; charset="utf-8"
MIME-Version: 1.0
Content-Transfer-Encoding: 7bit
Subject: svn commit: r1579006 - in /pig/trunk: CHANGES.txt
 src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/JobControlCompiler.java
Date: Tue, 18 Mar 2014 18:36:10 -0000
To: commits@pig.apache.org
From: aniket486@apache.org
Message-Id: <20140318183610.BD0312388993@eris.apache.org>

Author: aniket486
Date: Tue Mar 18 18:36:10 2014
New Revision: 1579006

URL: http://svn.apache.org/r1579006
Log:
PIG-3815: Hadoop bug causes to pig to fail silently with jar cache (aniket486)

Modified:
    pig/trunk/CHANGES.txt
    pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/JobControlCompiler.java

Modified: pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1579006&r1=1579005&r2=1579006&view=diff
==============================================================================
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Tue Mar 18 18:36:10 2014
@@ -99,6 +99,8 @@ OPTIMIZATIONS
  
 BUG FIXES
 
+PIG-3815: Hadoop bug causes to pig to fail silently with jar cache (aniket486)
+
 PIG-3816: Incorrect Javadoc for launchPlan() method (kyungho via prkommireddi)
 
 PIG-3673: Divide by zero error in runpigmix.pl script (suhassatish via daijy)

Modified: pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/JobControlCompiler.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/JobControlCompiler.java?rev=1579006&r1=1579005&r2=1579006&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/JobControlCompiler.java (original)
+++ pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/JobControlCompiler.java Tue Mar 18 18:36:10 2014
@@ -20,6 +20,7 @@ package org.apache.pig.backend.hadoop.ex
 import java.io.File;
 import java.io.FileOutputStream;
 import java.io.IOException;
+import java.io.InputStream;
 import java.io.OutputStream;
 import java.lang.reflect.Method;
 import java.net.URI;
@@ -1639,7 +1640,10 @@ public class JobControlCompiler{
                         log.info("Found " + url + " in jar cache at "+ stagingDir);
                         long curTime = System.currentTimeMillis();
                         fs.setTimes(jarPath, -1, curTime);
-                        return jarPath;
+                        // PIG-3815 In hadoop 1.0, addFileToClassPath uses : as separator
+                        // jarPath has full uri at this point, we need to remove hdfs://nn:port
+                        // part to avoid parsing errors on backend
+                        return new Path(jarPath.toUri().getPath());
                     }
                 }
             }
@@ -1647,11 +1651,15 @@ public class JobControlCompiler{
             // attempt to copy to cache else return null
             fs.mkdirs(cacheDir, FileLocalizer.OWNER_ONLY_PERMS);
             Path cacheFile = new Path(cacheDir, filename);
-            OutputStream os = FileSystem.create(fs, cacheFile, FileLocalizer.OWNER_ONLY_PERMS);
+            OutputStream os = null;
+            InputStream is = null;
             try {
-                IOUtils.copyBytes(url.openStream(), os, 4096, true);
+                os = FileSystem.create(fs, cacheFile, FileLocalizer.OWNER_ONLY_PERMS);
+                is = url.openStream();
+                IOUtils.copyBytes(is, os, 4096, true);
             } finally {
-                os.close();
+                org.apache.commons.io.IOUtils.closeQuietly(is);
+                org.apache.commons.io.IOUtils.closeQuietly(os);
             }
             return cacheFile;
 
@@ -1686,13 +1694,12 @@ public class JobControlCompiler{
 
         Path dst = new Path(FileLocalizer.getTemporaryPath(pigContext).toUri().getPath(), suffix);
         FileSystem fs = dst.getFileSystem(conf);
-        OutputStream os = fs.create(dst);
+        OutputStream os = null;
         try {
+            os = fs.create(dst);
             IOUtils.copyBytes(url.openStream(), os, 4096, true);
         } finally {
-            // IOUtils can not close both the input and the output properly in a finally
-            // as we can get an exception in between opening the stream and calling the method
-            os.close();
+            org.apache.commons.io.IOUtils.closeQuietly(os);
         }
         return dst;
     }