pig-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From o...@apache.org
Subject svn commit: r598736 - in /incubator/pig/trunk: CHANGES.txt src/org/apache/pig/PigServer.java src/org/apache/pig/impl/PigContext.java src/org/apache/pig/impl/util/JarManager.java test/org/apache/pig/test/TestPigServer.java
Date Tue, 27 Nov 2007 19:15:05 GMT
Author: olga
Date: Tue Nov 27 11:15:04 2007
New Revision: 598736

URL: http://svn.apache.org/viewvc?rev=598736&view=rev
Log:
Fix for PIG-11

Added:
    incubator/pig/trunk/test/org/apache/pig/test/TestPigServer.java
Modified:
    incubator/pig/trunk/CHANGES.txt
    incubator/pig/trunk/src/org/apache/pig/PigServer.java
    incubator/pig/trunk/src/org/apache/pig/impl/PigContext.java
    incubator/pig/trunk/src/org/apache/pig/impl/util/JarManager.java

Modified: incubator/pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/incubator/pig/trunk/CHANGES.txt?rev=598736&r1=598735&r2=598736&view=diff
==============================================================================
--- incubator/pig/trunk/CHANGES.txt (original)
+++ incubator/pig/trunk/CHANGES.txt Tue Nov 27 11:15:04 2007
@@ -10,6 +10,7 @@
   OPTIMIZATIONS
 
   BUG FIXES
+        PIG-11 Add capability to search for jar file to register.
 
   	PIG-24 Files that were incorrectly placed under test/reports have been
 	removed.  ant clean now cleans test/reports.  (milindb via gates)

Modified: incubator/pig/trunk/src/org/apache/pig/PigServer.java
URL: http://svn.apache.org/viewvc/incubator/pig/trunk/src/org/apache/pig/PigServer.java?rev=598736&r1=598735&r2=598736&view=diff
==============================================================================
--- incubator/pig/trunk/src/org/apache/pig/PigServer.java (original)
+++ incubator/pig/trunk/src/org/apache/pig/PigServer.java Tue Nov 27 11:15:04 2007
@@ -23,6 +23,11 @@
 import java.util.HashMap;
 import java.util.Iterator;
 import java.util.Map;
+import java.util.Enumeration;
+import java.util.Vector;
+import java.net.URL;
+import java.net.URI;
+import java.net.URISyntaxException;
 
 import org.apache.hadoop.dfs.DistributedFileSystem;
 import org.apache.hadoop.fs.FileSystem;
@@ -146,11 +151,59 @@
     	pigContext.registerFunction(function, functionSpec);
     }
     
-    public void registerJar(String path) throws IOException{
-    	File f = new File(path);
-    	if (!f.canRead())
-    		throw new IOException("Can't read " + path);
-        pigContext.addJar(path);
+    private URL locateJarFromResources(String jarName) throws IOException {
+        Enumeration<URL> urls = ClassLoader.getSystemResources(jarName);
+        URL resourceLocation = null;
+        
+        if (urls.hasMoreElements()) {
+        	resourceLocation = urls.nextElement();
+        }
+        
+        if (pigContext.debug && urls.hasMoreElements()) {
+            String logMessage = "Found multiple resources that match " 
+                + jarName + ": " + resourceLocation;
+            
+            while (urls.hasMoreElements()) {
+            	logMessage += (logMessage + urls.nextElement() + "; ");
+            }
+            
+            pigContext.getLogger().debug(logMessage);
+        }
+    
+        return resourceLocation;
+    }
+    
+    /**
+     * Registers a jar file. Name of the jar file can be an absolute or 
+     * relative path.
+     * 
+     * If multiple resources are found with the specified name, the
+     * first one is registered as returned by getSystemResources.
+     * A warning is issued to inform the user.
+     * 
+     * @param name of the jar file to register
+     * @throws IOException
+     */
+    public void registerJar(String name) throws IOException {
+        // first try to locate jar via system resources
+        // if this fails, try by using "name" as File (this preserves 
+        // compatibility with case when user passes absolute path or path 
+    	// relative to current working directory.)    	
+        if (name != null) {
+            URL resource = locateJarFromResources(name);
+
+            if (resource == null) {
+                File f = new File(name);
+                
+                if (!f.canRead()) {
+                    throw new IOException("Can't read jar file: " + name);
+                }
+                
+                resource = f.toURI().toURL();
+            }
+
+            pigContext.addJar(resource);    	
+        }
     }
     
     /**

Modified: incubator/pig/trunk/src/org/apache/pig/impl/PigContext.java
URL: http://svn.apache.org/viewvc/incubator/pig/trunk/src/org/apache/pig/impl/PigContext.java?rev=598736&r1=598735&r2=598736&view=diff
==============================================================================
--- incubator/pig/trunk/src/org/apache/pig/impl/PigContext.java (original)
+++ incubator/pig/trunk/src/org/apache/pig/impl/PigContext.java Tue Nov 27 11:15:04 2007
@@ -19,6 +19,7 @@
 
 import java.io.FileInputStream;
 import java.io.IOException;
+import java.io.File;
 import java.io.InputStream;
 import java.io.Serializable;
 import java.lang.reflect.Constructor;
@@ -75,7 +76,7 @@
     transient private JobConf conf = null;        
     
     //  extra jar files that are needed to run a job
-    transient public List<String> extraJars = new LinkedList<String>();     
        
+    transient public List<URL> extraJars = new LinkedList<URL>();           
  
     
     //  The jars that should not be merged in. (Some functions may come from pig.jar and
we don't want the whole jar file.)
     transient public Vector<String> skipJars = new Vector<String>(2);    
@@ -369,9 +370,18 @@
 		}
     }
     
-    public void addJar(String path) throws MalformedURLException{
-        extraJars.add(path);
-        LogicalPlanBuilder.classloader = createCl(null);
+    public void addJar(String path) throws MalformedURLException {
+        if (path != null) {
+            URL resource = (new File(path)).toURI().toURL();
+            addJar(resource);
+        }
+    }
+    
+    public void addJar(URL resource) throws MalformedURLException{
+        if (resource != null) {
+            extraJars.add(resource);
+            LogicalPlanBuilder.classloader = createCl(null);
+        }
     }
 
     public void rename(String oldName, String newName) throws IOException {
@@ -466,7 +476,7 @@
             urls[0] = new URL("file:" + jarFile);
         }
         for (int i = 0; i < extraJars.size(); i++) {
-            urls[i + passedJar] = new URL("file:" + extraJars.get(i));
+            urls[i + passedJar] = extraJars.get(i);
         }
         return new URLClassLoader(urls, PigMapReduce.class.getClassLoader());
     }

Modified: incubator/pig/trunk/src/org/apache/pig/impl/util/JarManager.java
URL: http://svn.apache.org/viewvc/incubator/pig/trunk/src/org/apache/pig/impl/util/JarManager.java?rev=598736&r1=598735&r2=598736&view=diff
==============================================================================
--- incubator/pig/trunk/src/org/apache/pig/impl/util/JarManager.java (original)
+++ incubator/pig/trunk/src/org/apache/pig/impl/util/JarManager.java Tue Nov 27 11:15:04 2007
@@ -168,17 +168,29 @@
      */
     private static void mergeJar(JarOutputStream jarFile, String jar, String prefix, Map<String,
String> contents)
             throws FileNotFoundException, IOException {
-        JarInputStream jis = new JarInputStream(new FileInputStream(jar));
+        JarInputStream jarInput = new JarInputStream(new FileInputStream(jar));
+        
+        mergeJar(jarFile, jarInput, prefix, contents);
+    }
+    
+    private static void mergeJar(JarOutputStream jarFile, URL jar, String prefix, Map<String,
String> contents)
+    throws FileNotFoundException, IOException {
+        JarInputStream jarInput = new JarInputStream(jar.openStream());
+
+        mergeJar(jarFile, jarInput, prefix, contents);
+    }
+
+    private static void mergeJar(JarOutputStream jarFile, JarInputStream jarInput, String
prefix, Map<String, String> contents)
+    throws FileNotFoundException, IOException {
         JarEntry entry;
-        while ((entry = jis.getNextJarEntry()) != null) {
+        while ((entry = jarInput.getNextJarEntry()) != null) {
             if (prefix != null && !entry.getName().startsWith(prefix)) {
                 continue;
             }
-            addStream(jarFile, entry.getName(), jis, contents);
+            addStream(jarFile, entry.getName(), jarInput, contents);
         }
     }
-    
-    /**
+        /**
      * Adds a stream to a Jar file.
      * 
      * @param os

Added: incubator/pig/trunk/test/org/apache/pig/test/TestPigServer.java
URL: http://svn.apache.org/viewvc/incubator/pig/trunk/test/org/apache/pig/test/TestPigServer.java?rev=598736&view=auto
==============================================================================
--- incubator/pig/trunk/test/org/apache/pig/test/TestPigServer.java (added)
+++ incubator/pig/trunk/test/org/apache/pig/test/TestPigServer.java Tue Nov 27 11:15:04 2007
@@ -0,0 +1,235 @@
+
+package org.apache.pig.test;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.FileOutputStream;
+import java.io.OutputStreamWriter;
+import java.util.List;
+import java.util.Iterator;
+import java.net.URL;
+import java.net.URLClassLoader;
+import java.lang.reflect.Method;
+
+import org.apache.pig.PigServer;
+
+import org.junit.Test;
+import junit.framework.TestCase;
+
+
+public class TestPigServer extends TestCase {
+    private PigServer pig = new PigServer();
+
+    private final static String FILE_SEPARATOR = System.getProperty("file.separator");
+    
+    // make sure that name is included or not (depending on flag "included") 
+    // in the given list of stings
+    private static void verifyStringContained(List<URL> list, String name, boolean
included) {
+        Iterator<URL> iter = list.iterator();
+        boolean nameIsSubstring = false;
+        int count = 0;
+        
+        while (iter.hasNext()) {
+            if (iter.next().toString().contains(name)) {
+                nameIsSubstring = true;
+                ++count;
+            }
+        }
+        
+        if (included) {
+            assertTrue(nameIsSubstring);
+            assertTrue(count == 1);
+        }
+        else {
+            assertFalse(nameIsSubstring);
+        }
+    }
+    
+    // creates an empty jar file
+    private static void createFakeJarFile(String location, String name) 
+            throws IOException {
+        assertFalse((new File(name)).canRead());
+        
+        assertTrue((new File(location)).mkdirs());
+        
+        assertTrue((new File(location + FILE_SEPARATOR + name)).
+                    createNewFile());
+    }
+    
+    // dynamically add more resources to the system class loader
+    private static void registerNewResource(String file) throws Exception {
+        URL urlToAdd = new File(file).toURI().toURL();
+        URLClassLoader sysLoader = (URLClassLoader) ClassLoader.getSystemClassLoader();
+        Method addMethod = URLClassLoader.class.
+                            getDeclaredMethod("addURL",
+                                              new Class[]{URL.class});
+        addMethod.setAccessible(true);
+        addMethod.invoke(sysLoader, new Object[]{urlToAdd});
+    }
+    
+    private static void executeShellCommand(String cmd) throws Exception {
+        Process cmdProc = Runtime.getRuntime().exec(cmd);
+        
+        cmdProc.waitFor();
+        
+        assertTrue(cmdProc.exitValue() == 0);
+    }
+    
+    /**
+     * The jar file to register is not present
+     */
+    @Test
+    public void testRegisterJarFileNotPresent() throws Exception {
+        // resister a jar file that does not exist
+        
+        String jarName = "BadFileNameTestJarNotPresent.jar";
+        
+        // jar name is not present to start with
+        verifyStringContained(pig.getPigContext().extraJars, jarName, false);
+
+        boolean exceptionRaised = false;
+        try {
+            pig.registerJar(jarName);
+        }
+        catch (IOException e) {
+            exceptionRaised = true;
+        }        
+        assertTrue(exceptionRaised);
+        verifyStringContained(pig.getPigContext().extraJars, jarName, false);
+    }
+
+    /**
+     * Jar file to register is not present in the system resources
+     * in this case name of jar file is relative to current working dir
+     */
+    @Test
+    public void testRegisterJarLocalDir() throws Exception {
+        String dir1 = "test1_register_jar_local";
+        String dir2 = "test2_register_jar_local";
+        String jarLocation = dir1 + FILE_SEPARATOR +
+                              dir2 + FILE_SEPARATOR;
+        String jarName = "TestRegisterJarLocal.jar";
+        
+        createFakeJarFile(jarLocation, jarName);
+        
+        verifyStringContained(pig.getPigContext().extraJars, jarName, false);
+        
+        boolean exceptionRaised = false;
+        try {
+            pig.registerJar(jarLocation + jarName);
+        }
+        catch (IOException e) {
+            exceptionRaised = true;
+        }        
+        assertFalse(exceptionRaised);
+        verifyStringContained(pig.getPigContext().extraJars, jarName, true);
+
+        // clean-up
+        assertTrue((new File(jarLocation + jarName)).delete());
+        (new File(dir1 + FILE_SEPARATOR + dir2)).delete();
+        (new File(dir1)).delete();
+    }
+
+    /**
+     * Jar file is located via system resources
+     * Test verifies that even with multiple resources matching,
+     * only one of them is registered.
+     */
+    @Test
+    public void testRegisterJarFromResources () throws Exception {
+        String dir = "test_register_jar_res_dir";
+        String subDir1 = "test_register_jar_res_sub_dir1";
+        String subDir2 = "test_register_jar_res_sub_dir2";
+        String jarName = "TestRegisterJarFromRes.jar";
+        String jarLocation1 = dir + FILE_SEPARATOR + subDir1 + FILE_SEPARATOR;
+        String jarLocation2 = dir + FILE_SEPARATOR + subDir2 + FILE_SEPARATOR;
+        
+        createFakeJarFile(jarLocation1, jarName);
+        createFakeJarFile(jarLocation2, jarName);
+        
+        verifyStringContained(pig.getPigContext().extraJars, jarName, false);
+        
+        registerNewResource(jarLocation1);
+        registerNewResource(jarLocation2);
+        
+        boolean exceptionRaised = false;
+        try {
+            pig.registerJar(jarName);
+        }
+        catch (IOException e) {
+            exceptionRaised = true;
+        }
+        assertFalse(exceptionRaised);
+        verifyStringContained(pig.getPigContext().extraJars, jarName, true);
+
+        // clean-up
+        assertTrue((new File(jarLocation1 + jarName)).delete());
+        assertTrue((new File(jarLocation2 + jarName)).delete());
+        (new File(jarLocation1)).delete();
+        (new File(jarLocation2)).delete();
+        (new File(dir)).delete();
+    }
+
+    /**
+     * Use a resource inside a jar file.
+     * Verify that the containing jar file is registered correctly.
+     * @throws Exception
+     */
+    @Test
+    public void testRegisterJarResourceInJar() throws Exception {
+        String dir = "test_register_jar_res_in_jar";
+        String subDir = "sub_dir";
+        String jarName = "TestRegisterJarNonEmpty.jar";
+        String className = "TestRegisterJar";
+        String javaSrc = "package " + subDir + "; class " + className + " { }";
+
+        // create dirs
+        (new File(dir + FILE_SEPARATOR + subDir)).mkdirs();
+
+        // generate java file
+        FileOutputStream outStream = 
+            new FileOutputStream(new File(dir + FILE_SEPARATOR + subDir +
+                                    FILE_SEPARATOR + className + ".java"));
+        
+        OutputStreamWriter outWriter = new OutputStreamWriter(outStream);
+        outWriter.write(javaSrc);
+        outWriter.close();
+        
+        // compile
+        executeShellCommand("javac " + dir + FILE_SEPARATOR + subDir +
+                               FILE_SEPARATOR + className + ".java");
+
+        // remove src file
+        (new File(dir + FILE_SEPARATOR + subDir +
+                  FILE_SEPARATOR + className + ".java")).delete();
+
+        // generate jar file
+        executeShellCommand("jar -cf " + dir + FILE_SEPARATOR + jarName + " " +
+                              "-C " + dir + " " + subDir);
+        
+        // remove class file and sub_dir
+        (new File(dir + FILE_SEPARATOR + subDir +
+                  FILE_SEPARATOR + className + ".class")).delete();
+        (new File(dir + FILE_SEPARATOR + subDir)).delete();
+        
+        // register resource
+        registerNewResource(dir + FILE_SEPARATOR + jarName);
+        
+        // load the specific resource
+        boolean exceptionRaised = false;
+        try {
+            pig.registerJar("sub_dir/TestRegisterJar.class");
+        }
+        catch (IOException e) {
+            exceptionRaised = true;
+        }
+        
+        // verify proper jar file is located
+        assertFalse(exceptionRaised);
+        verifyStringContained(pig.getPigContext().extraJars, jarName, true);
+
+        // clean up Jar file and test dir
+        (new File(dir + FILE_SEPARATOR + jarName)).delete();
+        (new File(dir)).delete();
+    }
+}
\ No newline at end of file



Mime
View raw message