pig-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From aniket...@apache.org
Subject svn commit: r1579354 - /pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/JobControlCompiler.java
Date Wed, 19 Mar 2014 19:06:38 GMT
Author: aniket486
Date: Wed Mar 19 19:06:38 2014
New Revision: 1579354

URL: http://svn.apache.org/r1579354
Log:
PIG-3815: Hadoop bug causes to pig to fail silently with jar cache- minor refactor

Modified:
    pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/JobControlCompiler.java

Modified: pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/JobControlCompiler.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/JobControlCompiler.java?rev=1579354&r1=1579353&r2=1579354&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/JobControlCompiler.java
(original)
+++ pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/JobControlCompiler.java
Wed Mar 19 19:06:38 2014
@@ -1632,25 +1632,16 @@ public class JobControlCompiler{
             String checksum = DigestUtils.shaHex(url.openStream());
             FileSystem fs = FileSystem.get(conf);
             Path cacheDir = new Path(stagingDir, checksum);
-            FileStatus [] statuses = fs.listStatus(cacheDir);
-            if (statuses != null) {
-                for (FileStatus stat : statuses) {
-                    Path jarPath = stat.getPath();
-                    if(jarPath.getName().equals(filename)) {
-                        log.info("Found " + url + " in jar cache at "+ stagingDir);
-                        long curTime = System.currentTimeMillis();
-                        fs.setTimes(jarPath, -1, curTime);
-                        // PIG-3815 In hadoop 0.20, addFileToClassPath uses : as separator
-                        // jarPath has full uri at this point, we need to remove hdfs://nn:port
-                        // part to avoid parsing errors on backend
-                        return new Path(jarPath.toUri().getPath());
-                    }
-                }
+            Path cacheFile = new Path(cacheDir, filename);
+            if (fs.exists(cacheFile)) {
+               log.info("Found " + url + " in jar cache at "+ stagingDir);
+               long curTime = System.currentTimeMillis();
+               fs.setTimes(cacheFile, -1, curTime);
+               return cacheFile;
             }
             log.info("Url "+ url + " was not found in jarcache at "+ stagingDir);
             // attempt to copy to cache else return null
             fs.mkdirs(cacheDir, FileLocalizer.OWNER_ONLY_PERMS);
-            Path cacheFile = new Path(cacheDir, filename);
             OutputStream os = null;
             InputStream is = null;
             try {



Mime
View raw message