incubator-oozie-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ange...@apache.org
Subject svn commit: r1228429 - in /incubator/oozie/branches/3.1/core/src: main/java/org/apache/oozie/service/WorkflowAppService.java test/java/org/apache/oozie/action/hadoop/PigTestCase.java test/java/org/apache/oozie/service/TestLiteWorkflowAppService.java
Date Fri, 06 Jan 2012 21:57:23 GMT
Author: angeloh
Date: Fri Jan  6 21:57:22 2012
New Revision: 1228429

URL: http://svn.apache.org/viewvc?rev=1228429&view=rev
Log:
Closes OOZIE-564 Embedded pig within python fails when run using oozie -- Adding new file

Added:
    incubator/oozie/branches/3.1/core/src/test/java/org/apache/oozie/action/hadoop/PigTestCase.java
Modified:
    incubator/oozie/branches/3.1/core/src/main/java/org/apache/oozie/service/WorkflowAppService.java
    incubator/oozie/branches/3.1/core/src/test/java/org/apache/oozie/service/TestLiteWorkflowAppService.java

Modified: incubator/oozie/branches/3.1/core/src/main/java/org/apache/oozie/service/WorkflowAppService.java
URL: http://svn.apache.org/viewvc/incubator/oozie/branches/3.1/core/src/main/java/org/apache/oozie/service/WorkflowAppService.java?rev=1228429&r1=1228428&r2=1228429&view=diff
==============================================================================
--- incubator/oozie/branches/3.1/core/src/main/java/org/apache/oozie/service/WorkflowAppService.java
(original)
+++ incubator/oozie/branches/3.1/core/src/main/java/org/apache/oozie/service/WorkflowAppService.java
Fri Jan  6 21:57:22 2012
@@ -201,6 +201,11 @@ public abstract class WorkflowAppService
                 if (entry.getKey().startsWith("oozie.")) {
                     String name = entry.getKey();
                     String value = entry.getValue();
+                    // Append application lib jars of both parent and child in
+                    // subworkflow to APP_LIB_PATH_LIST
+                    if ((conf.get(name) != null) && name.equals(APP_LIB_PATH_LIST))
{
+                        value = value + "," + conf.get(name);
+                    }
                     conf.set(name, value);
                 }
             }

Added: incubator/oozie/branches/3.1/core/src/test/java/org/apache/oozie/action/hadoop/PigTestCase.java
URL: http://svn.apache.org/viewvc/incubator/oozie/branches/3.1/core/src/test/java/org/apache/oozie/action/hadoop/PigTestCase.java?rev=1228429&view=auto
==============================================================================
--- incubator/oozie/branches/3.1/core/src/test/java/org/apache/oozie/action/hadoop/PigTestCase.java
(added)
+++ incubator/oozie/branches/3.1/core/src/test/java/org/apache/oozie/action/hadoop/PigTestCase.java
Fri Jan  6 21:57:22 2012
@@ -0,0 +1,84 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.oozie.action.hadoop;
+
+import java.io.FileInputStream;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.util.concurrent.Callable;
+
+import org.antlr.runtime.ANTLRReaderStream;
+import org.apache.hadoop.filecache.DistributedCache;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.oozie.test.XFsTestCase;
+import org.apache.oozie.util.ClassUtils;
+import org.apache.oozie.util.IOUtils;
+import org.python.util.jython;
+
+import com.google.common.primitives.Booleans;
+
+public abstract class PigTestCase extends XFsTestCase implements Callable<Void> {
+    protected static String pigScript;
+    private static String commonPigScript = "set job.name 'test'\n" +
+             "set debug on\n" +
+             "A = load '$IN' using PigStorage(':');\n" +
+             "B = foreach A generate $0 as id;\n" +
+             "store B into '$OUT' USING PigStorage();";
+
+    public void testPigScript() throws Exception {
+        pigScript = commonPigScript;
+        DoAs.call(getTestUser(), this);
+
+    }
+
+    // testing embedded Pig feature of Pig 0.9
+    public void testEmbeddedPigWithinPython() throws Exception {
+        FileSystem fs = getFileSystem();
+        Path jythonJar = new Path(getFsTestCaseDir(), "jython.jar");
+        InputStream is = new FileInputStream(ClassUtils.findContainingJar(jython.class));
+        OutputStream os = fs.create(jythonJar);
+        IOUtils.copyStream(is, os);
+
+        Path antlrRuntimeJar = new Path(getFsTestCaseDir(), "antlr_runtime.jar");
+        is = new FileInputStream(ClassUtils.findContainingJar(ANTLRReaderStream.class));
+        os = fs.create(antlrRuntimeJar);
+        IOUtils.copyStream(is, os);
+
+        Path guavaJar = new Path(getFsTestCaseDir(), "guava.jar");
+        is = new FileInputStream(ClassUtils.findContainingJar(Booleans.class));
+        os = fs.create(guavaJar);
+        IOUtils.copyStream(is, os);
+
+        DistributedCache.addFileToClassPath(new Path(jythonJar.toUri().getPath()), getFileSystem().getConf());
+        DistributedCache.addFileToClassPath(new Path(antlrRuntimeJar.toUri().getPath()),
getFileSystem().getConf());
+        DistributedCache.addFileToClassPath(new Path(guavaJar.toUri().getPath()), getFileSystem().getConf());
+
+        Path inputDir = new Path(getFsTestCaseDir(), "input");
+        Path outputDir = new Path(getFsTestCaseDir(), "output");
+        pigScript = "#!/usr/bin/python"
+                + "\nfrom org.apache.pig.scripting import Pig"
+                + "\nP = Pig.compile(\"\"\"" + "\n" + commonPigScript + "\n"+ "\"\"\")"
+                + "\ninput = " + "'"+inputDir.toUri().getPath()+"'"
+                + "\noutput = " + "'"+outputDir.toUri().getPath()+"'"
+                + "\nQ = P.bind({'IN':input, 'OUT':output})"
+                + "\nQ.runSingle()";
+        DoAs.call(getTestUser(), this);
+    }
+
+}

Modified: incubator/oozie/branches/3.1/core/src/test/java/org/apache/oozie/service/TestLiteWorkflowAppService.java
URL: http://svn.apache.org/viewvc/incubator/oozie/branches/3.1/core/src/test/java/org/apache/oozie/service/TestLiteWorkflowAppService.java?rev=1228429&r1=1228428&r2=1228429&view=diff
==============================================================================
--- incubator/oozie/branches/3.1/core/src/test/java/org/apache/oozie/service/TestLiteWorkflowAppService.java
(original)
+++ incubator/oozie/branches/3.1/core/src/test/java/org/apache/oozie/service/TestLiteWorkflowAppService.java
Fri Jan  6 21:57:22 2012
@@ -551,4 +551,89 @@ public class TestLiteWorkflowAppService 
         }
     }
 
+    public void testCreateprotoConfWithSubWorkflow_Case1_ParentWorkflowContainingLibs() throws
Exception {
+        // When parent workflow has an non-empty lib directory,
+        // APP_LIB_PATH_LIST should contain libraries from both parent and
+        // subworkflow (child)
+        Services services = new Services();
+        try {
+            services.init();
+            Reader reader = IOUtils.getResourceAsReader("wf-schema-valid.xml", -1);
+            Writer writer = new FileWriter(getTestCaseDir() + "/workflow.xml");
+            IOUtils.copyCharStream(reader, writer);
+
+            createTestCaseSubDir("lib");
+            writer = new FileWriter(getTestCaseDir() + "/lib/childdependency1.jar");
+            writer.write("bla bla");
+            writer.close();
+            writer = new FileWriter(getTestCaseDir() + "/lib/childdependency2.so");
+            writer.write("bla bla");
+            writer.close();
+            WorkflowAppService wps = Services.get().get(WorkflowAppService.class);
+            Configuration jobConf = new XConfiguration();
+            jobConf.set(OozieClient.APP_PATH, "file://" + getTestCaseDir() + File.separator
+ "workflow.xml");
+            jobConf.set(OozieClient.USER_NAME, getTestUser());
+            jobConf.set(OozieClient.GROUP_NAME, getTestGroup());
+            jobConf.set(WorkflowAppService.APP_LIB_PATH_LIST, "parentdependency1.jar");
+            injectKerberosInfo(jobConf);
+            Configuration protoConf = wps.createProtoActionConf(jobConf, "authToken", true);
+            assertEquals(getTestUser(), protoConf.get(OozieClient.USER_NAME));
+            assertEquals(getTestGroup(), protoConf.get(OozieClient.GROUP_NAME));
+
+            assertEquals(3, protoConf.getStrings(WorkflowAppService.APP_LIB_PATH_LIST).length);
+            String f1 = protoConf.getStrings(WorkflowAppService.APP_LIB_PATH_LIST)[0];
+            String f2 = protoConf.getStrings(WorkflowAppService.APP_LIB_PATH_LIST)[1];
+            String f3 = protoConf.getStrings(WorkflowAppService.APP_LIB_PATH_LIST)[2];
+            String ref1 = "parentdependency1.jar";
+            String ref2 = getTestCaseDir() + "/lib/childdependency1.jar";
+            String ref3 = getTestCaseDir() + "/lib/childdependency2.so";
+            Assert.assertTrue(f1.equals(ref1));
+            Assert.assertTrue(f2.equals(ref2));
+            Assert.assertTrue(f3.equals(ref3));
+        }
+        finally {
+            services.destroy();
+        }
+    }
+
+    public void testCreateprotoConfWithSubWorkflow_Case2_ParentWorkflowWithoutLibs() throws
Exception {
+        // When parent workflow has an empty (or missing) lib directory,
+        // APP_LIB_PATH_LIST should contain libraries from only the subworkflow
+        // (child)
+        Services services = new Services();
+        try {
+            services.init();
+            Reader reader = IOUtils.getResourceAsReader("wf-schema-valid.xml", -1);
+            Writer writer = new FileWriter(getTestCaseDir() + "/workflow.xml");
+            IOUtils.copyCharStream(reader, writer);
+
+            createTestCaseSubDir("lib");
+            writer = new FileWriter(getTestCaseDir() + "/lib/childdependency1.jar");
+            writer.write("bla bla");
+            writer.close();
+            writer = new FileWriter(getTestCaseDir() + "/lib/childdependency2.so");
+            writer.write("bla bla");
+            writer.close();
+            WorkflowAppService wps = Services.get().get(WorkflowAppService.class);
+            Configuration jobConf = new XConfiguration();
+            jobConf.set(OozieClient.APP_PATH, "file://" + getTestCaseDir() + File.separator
+ "workflow.xml");
+            jobConf.set(OozieClient.USER_NAME, getTestUser());
+            jobConf.set(OozieClient.GROUP_NAME, getTestGroup());
+            injectKerberosInfo(jobConf);
+            Configuration protoConf = wps.createProtoActionConf(jobConf, "authToken", true);
+            assertEquals(getTestUser(), protoConf.get(OozieClient.USER_NAME));
+            assertEquals(getTestGroup(), protoConf.get(OozieClient.GROUP_NAME));
+
+            assertEquals(2, protoConf.getStrings(WorkflowAppService.APP_LIB_PATH_LIST).length);
+            String f1 = protoConf.getStrings(WorkflowAppService.APP_LIB_PATH_LIST)[0];
+            String f2 = protoConf.getStrings(WorkflowAppService.APP_LIB_PATH_LIST)[1];
+            String ref1 = getTestCaseDir() + "/lib/childdependency1.jar";
+            String ref2 = getTestCaseDir() + "/lib/childdependency2.so";
+            Assert.assertTrue(f1.equals(ref1));
+            Assert.assertTrue(f2.equals(ref2));
+        }
+        finally {
+            services.destroy();
+        }
+    }
 }



Mime
View raw message