pig-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jul...@apache.org
Subject svn commit: r1293967 - in /pig/trunk: ./ src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/ src/org/apache/pig/impl/util/ test/org/apache/pig/test/ test/resources/ test/resources/org/ test/resources/org/apache/ test/resources/org/apache/...
Date Sun, 26 Feb 2012 23:48:35 GMT
Author: julien
Date: Sun Feb 26 23:48:35 2012
New Revision: 1293967

URL: http://svn.apache.org/viewvc?rev=1293967&view=rev
Log:
PIG-2532: Registered classes fail deserialization in frontend (traviscrawford via julien)

Added:
    pig/trunk/test/org/apache/pig/test/TestRegisteredJarVisibility.java
    pig/trunk/test/resources/
    pig/trunk/test/resources/org/
    pig/trunk/test/resources/org/apache/
    pig/trunk/test/resources/org/apache/pig/
    pig/trunk/test/resources/org/apache/pig/test/
    pig/trunk/test/resources/org/apache/pig/test/RegisteredJarVisibilityLoader.java
    pig/trunk/test/resources/org/apache/pig/test/RegisteredJarVisibilitySchema.java
Modified:
    pig/trunk/CHANGES.txt
    pig/trunk/build.xml
    pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MapReduceLauncher.java
    pig/trunk/src/org/apache/pig/impl/util/UDFContext.java

Modified: pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1293967&r1=1293966&r2=1293967&view=diff
==============================================================================
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Sun Feb 26 23:48:35 2012
@@ -100,6 +100,8 @@ OPTIMIZATIONS
 
 BUG FIXES
 
+PIG-2532: Registered classes fail deserialization in frontend (traviscrawford via julien)
+
 PIG-2549: org.apache.pig.piggybank.storage.avro - Broken documentation link for AvroStorage
(chrisas via daijy)
 
 PIG-2322: varargs functions do not get passed the arguments in Python embedding (julien)

Modified: pig/trunk/build.xml
URL: http://svn.apache.org/viewvc/pig/trunk/build.xml?rev=1293967&r1=1293966&r2=1293967&view=diff
==============================================================================
--- pig/trunk/build.xml (original)
+++ pig/trunk/build.xml Sun Feb 26 23:48:35 2012
@@ -266,7 +266,7 @@
           <source path="${src.shims.dir}"/>
           <source path="${src.shims.test.dir}"/>
           <source path="tutorial/src"/>
-          <source path="${test.src.dir}" excluding="e2e/pig/udfs/java/"/>
+          <source path="${test.src.dir}" excluding="e2e/pig/udfs/java/|resources/"/>
           <output path="${build.dir.eclipse-main-classes}" />
           <library pathref="eclipse.classpath" exported="true" />
           <!--library pathref="classpath" exported="false"/-->
@@ -464,7 +464,7 @@
             <param name="dist" value="${test.build.classes}" />
             <param name="cp" value="test.classpath" />
             <!-- don't compile PigTestLoader -->
-            <param name="excludes" value="**/PigTestLoader.java" />
+            <param name="excludes" value="**/PigTestLoader.java **/resources/**" />
         </antcall>
 
         <antcall target="compile-sources-all-warnings" inheritRefs="true" inheritall="true">
@@ -472,7 +472,7 @@
             <param name="dist" value="${test.build.classes}" />
             <param name="cp" value="test.classpath" />
             <!-- don't compile PigTestLoader -->
-            <param name="excludes" value="**/PigTestLoader.java" />            
+            <param name="excludes" value="**/PigTestLoader.java **/resources/**" />
         </antcall>
     	
     	<copy file="${basedir}/test/hbase-site.xml" tofile="${test.build.classes}/hbase-site.xml"/>

Modified: pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MapReduceLauncher.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MapReduceLauncher.java?rev=1293967&r1=1293966&r2=1293967&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MapReduceLauncher.java
(original)
+++ pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MapReduceLauncher.java
Sun Feb 26 23:48:35 2012
@@ -64,6 +64,7 @@ import org.apache.pig.impl.plan.VisitorE
 import org.apache.pig.impl.plan.CompilationMessageCollector.MessageType;
 import org.apache.pig.impl.util.ConfigurationValidator;
 import org.apache.pig.impl.util.LogUtils;
+import org.apache.pig.impl.util.UDFContext;
 import org.apache.pig.impl.util.Utils;
 import org.apache.pig.tools.pigstats.PigStats;
 import org.apache.pig.tools.pigstats.PigStatsUtil;
@@ -249,8 +250,17 @@ public class MapReduceLauncher extends L
             }
             
             completeFailedJobsInThisRun.clear();
-            
-            Thread jcThread = new Thread(jc);
+
+            // Set the thread UDFContext so registered classes are available.
+            final UDFContext udfContext = UDFContext.getUDFContext();
+            Thread jcThread = new Thread(jc) {
+                @Override
+                public void run() {
+                    UDFContext.setUdfContext(udfContext);
+                    super.run();
+                }
+            };
+
             jcThread.setUncaughtExceptionHandler(jctExceptionHandler);
             
             jcThread.setContextClassLoader(PigContext.getClassLoader());

Modified: pig/trunk/src/org/apache/pig/impl/util/UDFContext.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/impl/util/UDFContext.java?rev=1293967&r1=1293966&r2=1293967&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/impl/util/UDFContext.java (original)
+++ pig/trunk/src/org/apache/pig/impl/util/UDFContext.java Sun Feb 26 23:48:35 2012
@@ -54,6 +54,13 @@ public class UDFContext {
     /*
      *  internal pig use only - should NOT be called from user code
      */
+    public static void setUdfContext(UDFContext udfContext) {
+        tss.set(udfContext);
+    }
+
+    /*
+     *  internal pig use only - should NOT be called from user code
+     */
     public void setClientSystemProps(Properties properties) {
         clientSysProps = properties;
     }

Added: pig/trunk/test/org/apache/pig/test/TestRegisteredJarVisibility.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestRegisteredJarVisibility.java?rev=1293967&view=auto
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestRegisteredJarVisibility.java (added)
+++ pig/trunk/test/org/apache/pig/test/TestRegisteredJarVisibility.java Sun Feb 26 23:48:35
2012
@@ -0,0 +1,163 @@
+package org.apache.pig.test;
+
+import java.io.ByteArrayInputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+import java.util.jar.JarOutputStream;
+import java.util.zip.ZipEntry;
+
+import javax.tools.JavaCompiler;
+import javax.tools.JavaFileObject;
+import javax.tools.StandardJavaFileManager;
+import javax.tools.ToolProvider;
+
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+import com.google.common.io.ByteStreams;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.fs.Path;
+import org.apache.pig.ExecType;
+import org.apache.pig.PigServer;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+/**
+ * Ensure classes from a registered jar are available in the UDFContext.
+ * Please see PIG-2532 for additional details.
+ */
+public class TestRegisteredJarVisibility {
+    private static final Log LOG = LogFactory.getLog(TestRegisteredJarVisibility.class);
+    private static final String JAR_FILE_NAME = "test-foo-loader.jar";
+    private static final String PACKAGE_NAME = "org.apache.pig.test";
+    // Actual data is not important. Reusing an existing input file.
+    private static final File INPUT_FILE = new File("test/data/pigunit/top_queries_input_data.txt");
+
+    private MiniCluster cluster;
+    private File jarFile;
+
+    @Before()
+    public void setUp() throws IOException {
+
+        String testResourcesDir =  "test/resources/" + PACKAGE_NAME.replace(".", "/");
+
+        String testBuildDataDir = "build/test/data";
+        // Create the test data directory if needed
+        File testDataDir = new File(testBuildDataDir,
+                TestRegisteredJarVisibility.class.getCanonicalName());
+        testDataDir.mkdirs();
+
+        jarFile = new File(testDataDir, JAR_FILE_NAME);
+
+        File[] javaFiles = new File[]{
+                new File(testResourcesDir, "RegisteredJarVisibilityLoader.java"),
+                new File(testResourcesDir, "RegisteredJarVisibilitySchema.java")};
+
+        List<File> classFiles = compile(javaFiles);
+
+        // Canonical class name to class file
+        Map<String, File> filesToJar = Maps.newHashMap();
+        for (File classFile : classFiles) {
+            filesToJar.put(
+                    PACKAGE_NAME + "." + classFile.getName().replace(".class", ""),
+                    classFile);
+        }
+
+        jar(filesToJar);
+
+        cluster = MiniCluster.buildCluster();
+    }
+
+    @After()
+    public void tearDown() {
+        cluster.shutDown();
+    }
+
+    @Test()
+    public void testRegisteredJarVisibilitySchemaNotOnClasspath() {
+        boolean exceptionThrown = false;
+        try {
+            Class.forName("org.apache.pig.test.FooSchema");
+        } catch (ClassNotFoundException e) {
+            exceptionThrown = true;
+        }
+        Assert.assertTrue(exceptionThrown);
+    }
+
+    @Test()
+    public void testRegisteredJarVisibility() throws IOException {
+        cluster.getFileSystem().copyFromLocalFile(
+                new Path("file://" + INPUT_FILE.getAbsolutePath()), new Path(INPUT_FILE.getName()));
+        PigServer pigServer = new PigServer(ExecType.MAPREDUCE, cluster.getProperties());
+
+        String query = "register " + jarFile.getAbsolutePath() + ";\n"
+                + "a = load '" + INPUT_FILE.getName()
+                + "' using org.apache.pig.test.RegisteredJarVisibilityLoader();";
+        LOG.info("Running pig script:\n" + query);
+        pigServer.registerScript(new ByteArrayInputStream(query.getBytes()));
+
+        pigServer.openIterator("a");
+        pigServer.shutdown();
+    }
+
+    private List<File> compile(File[] javaFiles) {
+        LOG.info("Compiling: " + Arrays.asList(javaFiles));
+
+        JavaCompiler compiler = ToolProvider.getSystemJavaCompiler();
+        StandardJavaFileManager fileManager = compiler.getStandardFileManager(null, null,
null);
+        Iterable<? extends JavaFileObject> compilationUnits1 =
+                fileManager.getJavaFileObjects(javaFiles);
+        JavaCompiler.CompilationTask task =
+                compiler.getTask(null, fileManager, null, null, null, compilationUnits1);
+        task.call();
+
+        List<File> classFiles = Lists.newArrayList();
+        for (File javaFile : javaFiles) {
+            File classFile = new File(javaFile.getAbsolutePath().replace(".java", ".class"));
+            classFile.deleteOnExit();
+            Assert.assertTrue(classFile.exists());
+            classFiles.add(classFile);
+            LOG.info("Created " + classFile.getAbsolutePath());
+        }
+
+        return classFiles;
+    }
+
+    /**
+     * Create a jar file containing the generated classes.
+     *
+     * @param filesToJar map of canonical class name to class file
+     * @throws IOException on error
+     */
+    private void jar(Map<String, File> filesToJar) throws IOException {
+        LOG.info("Creating jar file containing: " + filesToJar);
+
+        JarOutputStream jos = new JarOutputStream(new FileOutputStream(jarFile.getAbsolutePath()));
+        try {
+            for (Map.Entry<String, File> entry : filesToJar.entrySet()) {
+                String zipEntryName = entry.getKey().replace(".", "/") + ".class";
+                LOG.info("Adding " + zipEntryName + " to " + jarFile.getAbsolutePath());
+                jos.putNextEntry(new ZipEntry(zipEntryName));
+                InputStream classInputStream = new FileInputStream(entry.getValue().getAbsolutePath());
+                try {
+                    ByteStreams.copy(classInputStream, jos);
+                } finally {
+                    classInputStream.close();
+                }
+            }
+        } finally {
+            jos.close();
+        }
+        Assert.assertTrue(jarFile.exists());
+        LOG.info("Created " + jarFile.getAbsolutePath());
+    }
+}

Added: pig/trunk/test/resources/org/apache/pig/test/RegisteredJarVisibilityLoader.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/resources/org/apache/pig/test/RegisteredJarVisibilityLoader.java?rev=1293967&view=auto
==============================================================================
--- pig/trunk/test/resources/org/apache/pig/test/RegisteredJarVisibilityLoader.java (added)
+++ pig/trunk/test/resources/org/apache/pig/test/RegisteredJarVisibilityLoader.java Sun Feb
26 23:48:35 2012
@@ -0,0 +1,37 @@
+package org.apache.pig.test;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.pig.builtin.PigStorage;
+import org.apache.pig.impl.util.UDFContext;
+
+import java.io.IOException;
+import java.util.Properties;
+
+/**
+ * Please see {@link TestRegisteredJarVisibility} for information about this class.
+ */
+public class RegisteredJarVisibilityLoader extends PigStorage {
+    private static final Log LOG = LogFactory.getLog(RegisteredJarVisibilityLoader.class);
+    private static final String REGISTERED_JAR_VISIBILITY_SCHEMA = "registered.jar.visibility.schema";
+
+    @Override
+    public void setLocation(String location, Job job) throws IOException {
+        UDFContext udfContext = UDFContext.getUDFContext();
+        Properties properties = udfContext.getUDFProperties(RegisteredJarVisibilityLoader.class);
+
+        if (!properties.containsKey(REGISTERED_JAR_VISIBILITY_SCHEMA)) {
+            LOG.info("Storing " + RegisteredJarVisibilitySchema.class.getName() + " in UDFContext.");
+            properties.put(REGISTERED_JAR_VISIBILITY_SCHEMA, new RegisteredJarVisibilitySchema());
+            LOG.info("Stored " + RegisteredJarVisibilitySchema.class.getName() + " in UDFContext.");
+        } else {
+            LOG.info("Retrieving " + REGISTERED_JAR_VISIBILITY_SCHEMA + " from UDFContext.");
+            RegisteredJarVisibilitySchema registeredJarVisibilitySchema =
+                    (RegisteredJarVisibilitySchema) properties.get(REGISTERED_JAR_VISIBILITY_SCHEMA);
+            LOG.info("Retrieved " + REGISTERED_JAR_VISIBILITY_SCHEMA + " from UDFContext.");
+        }
+
+        super.setLocation(location, job);
+    }
+}

Added: pig/trunk/test/resources/org/apache/pig/test/RegisteredJarVisibilitySchema.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/resources/org/apache/pig/test/RegisteredJarVisibilitySchema.java?rev=1293967&view=auto
==============================================================================
--- pig/trunk/test/resources/org/apache/pig/test/RegisteredJarVisibilitySchema.java (added)
+++ pig/trunk/test/resources/org/apache/pig/test/RegisteredJarVisibilitySchema.java Sun Feb
26 23:48:35 2012
@@ -0,0 +1,9 @@
+package org.apache.pig.test;
+
+import java.io.Serializable;
+
+/**
+ * Please see {@link TestRegisteredJarVisibility} for information about this class.
+ */
+public class RegisteredJarVisibilitySchema implements Serializable {
+}



Mime
View raw message