pig-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From da...@apache.org
Subject svn commit: r1370393 - in /pig/branches/branch-0.9: CHANGES.txt src/org/apache/pig/PigServer.java src/org/apache/pig/impl/PigContext.java src/org/apache/pig/impl/util/JarManager.java src/org/apache/pig/scripting/ScriptEngine.java
Date Tue, 07 Aug 2012 17:47:09 GMT
Author: daijy
Date: Tue Aug  7 17:47:08 2012
New Revision: 1370393

URL: http://svn.apache.org/viewvc?rev=1370393&view=rev
Log:
PIG-2761: With hadoop23 importing modules inside python script does not work

Modified:
    pig/branches/branch-0.9/CHANGES.txt
    pig/branches/branch-0.9/src/org/apache/pig/PigServer.java
    pig/branches/branch-0.9/src/org/apache/pig/impl/PigContext.java
    pig/branches/branch-0.9/src/org/apache/pig/impl/util/JarManager.java
    pig/branches/branch-0.9/src/org/apache/pig/scripting/ScriptEngine.java

Modified: pig/branches/branch-0.9/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.9/CHANGES.txt?rev=1370393&r1=1370392&r2=1370393&view=diff
==============================================================================
--- pig/branches/branch-0.9/CHANGES.txt (original)
+++ pig/branches/branch-0.9/CHANGES.txt Tue Aug  7 17:47:08 2012
@@ -30,6 +30,8 @@ PIG-2619: HBaseStorage constructs a Scan
 
 BUG FIXES
 
+PIG-2761: With hadoop23 importing modules inside python script does not work (rohini via
daijy)
+
 PIG-2775: Register jar does not goes to classpath in some cases (daijy)
 
 PIG-2666: LoadFunc.setLocation() is not called when pig script only has Order By (daijy)

Modified: pig/branches/branch-0.9/src/org/apache/pig/PigServer.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.9/src/org/apache/pig/PigServer.java?rev=1370393&r1=1370392&r2=1370393&view=diff
==============================================================================
--- pig/branches/branch-0.9/src/org/apache/pig/PigServer.java (original)
+++ pig/branches/branch-0.9/src/org/apache/pig/PigServer.java Tue Aug  7 17:47:08 2012
@@ -563,11 +563,16 @@ public class PigServer {
             throw new FrontendException(msg, errCode,
                     PigException.USER_ENVIRONMENT);
         }
+        String cwd = new File(".").getCanonicalPath();
+        String filePath = f.getCanonicalPath();
+        //Use the relative path in the jar, if the path specified is relative
+        String nameInJar = filePath.equals(cwd + File.separator + path) ? 
+                filePath.substring(cwd.length() + 1) : filePath;
+        pigContext.addScriptFile(nameInJar, filePath);
         if(scriptingLang != null) {
             ScriptEngine se = ScriptEngine.getInstance(scriptingLang);    
-            se.registerFunctions(path, namespace, pigContext);
+            se.registerFunctions(nameInJar, namespace, pigContext);
         }
-        pigContext.addScriptFile(path);
     }
 
     /**

Modified: pig/branches/branch-0.9/src/org/apache/pig/impl/PigContext.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.9/src/org/apache/pig/impl/PigContext.java?rev=1370393&r1=1370392&r2=1370393&view=diff
==============================================================================
--- pig/branches/branch-0.9/src/org/apache/pig/impl/PigContext.java (original)
+++ pig/branches/branch-0.9/src/org/apache/pig/impl/PigContext.java Tue Aug  7 17:47:08 2012
@@ -28,6 +28,7 @@ import java.net.URLClassLoader;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.HashMap;
+import java.util.LinkedHashMap;
 import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
@@ -91,7 +92,9 @@ public class PigContext implements Seria
     private Properties properties;
     
     //  script files that are needed to run a job
+    @Deprecated
     public List<String> scriptFiles = new ArrayList<String>();
+    private Map<String,File> aliasedScriptFiles = new LinkedHashMap<String,File>();
     
     //  script jars that are needed to run a script - jython.jar etc
     public List<String> scriptJars = new ArrayList<String>(2);
@@ -212,13 +215,31 @@ public class PigContext implements Seria
             log.error("Failed to set tracker at: " + newLocation);
         }
     }
-    
+  
+    /**
+     * calls: addScriptFile(path, new File(path)), ensuring that a given path is
+     * added to the jar at most once.
+     * @param path
+     * @throws MalformedURLException
+     */
     public void addScriptFile(String path) throws MalformedURLException {
         if (path != null) {
-            scriptFiles.add(path);
+            aliasedScriptFiles.put(path.replaceFirst("^/", ""), new File(path));
         }
     }
-    
+
+    /**
+     * this method adds script files that must be added to the shipped jar
+     * named differently from their local fs path.
+     * @param name  name in the jar
+     * @param path  path on the local fs
+     */
+    public void addScriptFile(String name, String path) {
+        if (path != null) {
+            aliasedScriptFiles.put(name.replaceFirst("^/", ""), new File(path));
+        }
+    }
+   
     public void addJar(String path) throws MalformedURLException {
         if (path != null) {
             URL resource = (new File(path)).toURI().toURL();
@@ -234,6 +255,14 @@ public class PigContext implements Seria
         }
     }
 
+    /**
+     * script files as name/file pairs to be added to the job jar
+     * @return name/file pairs
+     */
+    public Map<String,File> getScriptFiles() {
+        return aliasedScriptFiles;
+    }
+
     public void rename(String oldName, String newName) throws IOException {
         if (oldName.equals(newName)) {
             return;

Modified: pig/branches/branch-0.9/src/org/apache/pig/impl/util/JarManager.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.9/src/org/apache/pig/impl/util/JarManager.java?rev=1370393&r1=1370392&r2=1370393&view=diff
==============================================================================
--- pig/branches/branch-0.9/src/org/apache/pig/impl/util/JarManager.java (original)
+++ pig/branches/branch-0.9/src/org/apache/pig/impl/util/JarManager.java Tue Aug  7 17:47:08
2012
@@ -128,6 +128,18 @@ public class JarManager {
         	String path = pigContext.scriptFiles.get(i);
         	addStream(jarFile, path, new FileInputStream(new File(path)),contents);
         }
+        for (Map.Entry<String, File> entry : pigContext.getScriptFiles().entrySet())
{
+            InputStream stream = null;
+            if (entry.getValue().exists()) {
+                stream = new FileInputStream(entry.getValue());
+            } else {
+                stream = PigContext.getClassLoader().getResourceAsStream(entry.getValue().getPath());
+            }
+            if (stream==null) {
+                throw new IOException("Cannot find " + entry.getValue().getPath());
+            }
+            addStream(jarFile, entry.getKey(), stream, contents);
+        }
         
         jarFile.putNextEntry(new ZipEntry("pigContext"));
         new ObjectOutputStream(jarFile).writeObject(pigContext);

Modified: pig/branches/branch-0.9/src/org/apache/pig/scripting/ScriptEngine.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.9/src/org/apache/pig/scripting/ScriptEngine.java?rev=1370393&r1=1370392&r2=1370393&view=diff
==============================================================================
--- pig/branches/branch-0.9/src/org/apache/pig/scripting/ScriptEngine.java (original)
+++ pig/branches/branch-0.9/src/org/apache/pig/scripting/ScriptEngine.java Tue Aug  7 17:47:08
2012
@@ -132,10 +132,23 @@ public abstract class ScriptEngine {
                 throw new IllegalStateException("could not find existing file "+scriptPath,
e);
             }
         } else {
-            if (file.isAbsolute()) {
-                is = ScriptEngine.class.getResourceAsStream(scriptPath);
-            } else {
-                is = ScriptEngine.class.getResourceAsStream("/" + scriptPath);
+            // Try system, current and context classloader.
+            is = ScriptEngine.class.getResourceAsStream(scriptPath);
+            if (is == null) {
+                is = getResourceUsingClassLoader(scriptPath, ScriptEngine.class.getClassLoader());
+            }
+            if (is == null) {
+                is = getResourceUsingClassLoader(scriptPath, Thread.currentThread().getContextClassLoader());
+            }
+            if (is == null && !file.isAbsolute()) {
+                String path = "/" + scriptPath;
+                is = ScriptEngine.class.getResourceAsStream(path);
+                if (is == null) {
+                    is = getResourceUsingClassLoader(path, ScriptEngine.class.getClassLoader());
+                }
+                if (is == null) {
+                    is = getResourceUsingClassLoader(path, Thread.currentThread().getContextClassLoader());
+                }
             }
         }
         
@@ -148,6 +161,13 @@ public abstract class ScriptEngine {
         return is;
     }
     
+    private static InputStream getResourceUsingClassLoader(String fullFilename, ClassLoader
loader) {
+        if (loader != null) {
+            return loader.getResourceAsStream(fullFilename);
+        }
+        return null;
+    } 
+    
     public static final String NAMESPACE_SEPARATOR = ".";
        
     /**



Mime
View raw message