pig-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From roh...@apache.org
Subject svn commit: r1729685 - in /pig/trunk: ./ src/org/apache/pig/backend/hadoop/executionengine/tez/ src/org/apache/pig/backend/hadoop/executionengine/tez/plan/
Date Wed, 10 Feb 2016 18:17:48 GMT
Author: rohini
Date: Wed Feb 10 18:17:48 2016
New Revision: 1729685

URL: http://svn.apache.org/viewvc?rev=1729685&view=rev
Log:
PIG-4800: EvalFunc.getCacheFiles() fails for different namenode (rohini)

Modified:
    pig/trunk/CHANGES.txt
    pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/TezJobCompiler.java
    pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/TezResourceManager.java
    pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/plan/TezOperPlan.java

Modified: pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1729685&r1=1729684&r2=1729685&view=diff
==============================================================================
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Wed Feb 10 18:17:48 2016
@@ -89,6 +89,8 @@ PIG-4639: Add better parser for Apache H
 
 BUG FIXES
 
+PIG-4800: EvalFunc.getCacheFiles() fails for different namenode (rohini)
+
 PIG-4790: Join after union fail due to UnionOptimizer (rohini)
 
 PIG-4686: Backend code should not call AvroStorageUtils.getPaths (mitdesai via rohini)

Modified: pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/TezJobCompiler.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/TezJobCompiler.java?rev=1729685&r1=1729684&r2=1729685&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/TezJobCompiler.java (original)
+++ pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/TezJobCompiler.java Wed
Feb 10 18:17:48 2016
@@ -100,10 +100,7 @@ public class TezJobCompiler {
             }
             String cacheFiles = pigContext.getProperties().getProperty("pig.streaming.cache.files");
             if (cacheFiles != null) {
-                for (String file : cacheFiles.split(",")) {
-                    // Do new URI() before passing to Path constructor else it encodes #
when there is symlink
-                    TezResourceManager.getInstance().addTezResource(new Path(new URI(file.trim())).toUri());
-                }
+                addCacheResources(cacheFiles.split(","));
             }
             for (Map.Entry<String, LocalResource> entry : localResources.entrySet())
{
                 log.info("Local resource: " + entry.getKey());
@@ -119,6 +116,20 @@ public class TezJobCompiler {
         }
     }
 
+    private void addCacheResources(String[] fileNames) throws Exception {
+        for (String fileName : fileNames) {
+            fileName = fileName.trim();
+            if (fileName.length() > 0) {
+                URI resourceURI = new URI(fileName);
+                String fragment = resourceURI.getFragment();
+
+                Path remoteFsPath = new Path(resourceURI);
+                String resourceName = (fragment != null && fragment.length() >
0) ? fragment : remoteFsPath.getName();
+                TezResourceManager.getInstance().addTezResource(resourceName, remoteFsPath);
+            }
+        }
+    }
+
     private String createDagInfo(String script) throws IOException {
         String dagInfo;
         try {

Modified: pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/TezResourceManager.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/TezResourceManager.java?rev=1729685&r1=1729684&r2=1729685&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/TezResourceManager.java
(original)
+++ pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/TezResourceManager.java
Wed Feb 10 18:17:48 2016
@@ -130,7 +130,8 @@ public class TezResourceManager {
             // The resource name will be symlinked to the resource path in the
             // container's working directory.
             Path resourcePath = resources.get(resourceName);
-            FileStatus fstat = remoteFs.getFileStatus(resourcePath);
+            FileSystem fileSystem = resourcePath.getFileSystem(conf);
+            FileStatus fstat = fileSystem.getFileStatus(resourcePath);
 
             LocalResource tezResource = LocalResource.newInstance(
                     ConverterUtils.getYarnUrlFromPath(fstat.getPath()),

Modified: pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/plan/TezOperPlan.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/plan/TezOperPlan.java?rev=1729685&r1=1729684&r2=1729685&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/plan/TezOperPlan.java
(original)
+++ pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/plan/TezOperPlan.java
Wed Feb 10 18:17:48 2016
@@ -154,7 +154,7 @@ public class TezOperPlan extends Operato
                 URI resourceURI = new URI(fileName);
                 String fragment = resourceURI.getFragment();
 
-                Path remoteFsPath = new Path(resourceURI.getPath());
+                Path remoteFsPath = new Path(resourceURI);
                 String resourceName = (fragment != null && fragment.length() >
0) ? fragment : remoteFsPath.getName();
 
                 addExtraResource(resourceName, remoteFsPath);



Mime
View raw message