falcon-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From rag...@apache.org
Subject incubator-falcon git commit: FALCON-908 Remove jars that are not needed. Contributed by Paul Isaychuk
Date Wed, 10 Dec 2014 21:24:52 GMT
Repository: incubator-falcon
Updated Branches:
  refs/heads/master 73c9b9f54 -> 4133c9633


FALCON-908 Remove jars that are not needed. Contributed by Paul Isaychuk


Project: http://git-wip-us.apache.org/repos/asf/incubator-falcon/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-falcon/commit/4133c963
Tree: http://git-wip-us.apache.org/repos/asf/incubator-falcon/tree/4133c963
Diff: http://git-wip-us.apache.org/repos/asf/incubator-falcon/diff/4133c963

Branch: refs/heads/master
Commit: 4133c9633baf709a234c493cb973ac30d3bcb14c
Parents: 73c9b9f
Author: Raghav Kumar Gautam <raghav@apache.org>
Authored: Wed Dec 10 13:24:45 2014 -0800
Committer: Raghav Kumar Gautam <raghav@apache.org>
Committed: Wed Dec 10 13:24:45 2014 -0800

----------------------------------------------------------------------
 falcon-regression/CHANGES.txt                   |   2 +
 .../falcon/regression/core/bundle/Bundle.java   |  11 ++--
 .../helpers/entity/AbstractEntityHelper.java    |  32 +++++-----
 .../falcon/regression/ProcessLibPathTest.java   |  10 +--
 .../resources/ELbundle/workflow/workflow.xml    |  61 -------------------
 .../src/test/resources/IvoryClient/IvoryCLI.jar | Bin 4386324 -> 0 bytes
 .../src/test/resources/ivory-oozie-lib-0.1.jar  | Bin 8530 -> 0 bytes
 .../oozie/lib/oozie-examples-3.1.5.jar          | Bin 15826 -> 0 bytes
 .../src/test/resources/oozie/workflow.xml       |   4 +-
 .../lib/oozie-examples-3.1.5.jar                | Bin 15826 -> 0 bytes
 10 files changed, 28 insertions(+), 92 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/4133c963/falcon-regression/CHANGES.txt
----------------------------------------------------------------------
diff --git a/falcon-regression/CHANGES.txt b/falcon-regression/CHANGES.txt
index 8ae1c7a..f7b9781 100644
--- a/falcon-regression/CHANGES.txt
+++ b/falcon-regression/CHANGES.txt
@@ -36,6 +36,8 @@ Trunk (Unreleased)
    via Samarth Gupta)
 
   IMPROVEMENTS
+   FALCON-908 Remove jars that are not needed (Paul Isaychuk via Raghav Kumar Gautam)
+
    FALCON-939 Fixing few typos and removing unused stuff (Paul Isaychuk via Ruslan Ostafiychuk)
 
    FALCON-930 Delete old project name (ivory) from code and xmls (Ruslan Ostafiychuk via
Raghav

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/4133c963/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/bundle/Bundle.java
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/bundle/Bundle.java
b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/bundle/Bundle.java
index 4c913f5..91abe89 100644
--- a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/bundle/Bundle.java
+++ b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/bundle/Bundle.java
@@ -572,10 +572,11 @@ public class Bundle {
     }
 
 
-    public void verifyDependencyListing(ColoHelper coloHelper) {
+    public void verifyDependencyListing(ColoHelper coloHelper)
+        throws InterruptedException, IOException, AuthenticationException, URISyntaxException
{
         //display dependencies of process:
         String dependencies = coloHelper.getProcessHelper().getDependencies(
-            Util.readEntityName(getProcessData()));
+            Util.readEntityName(getProcessData())).getEntityList().toString();
 
         //verify presence
         for (String cluster : clusters) {
@@ -585,15 +586,13 @@ public class Bundle {
             Assert.assertTrue(dependencies.contains("(feed) " + Util.readEntityName(feed)));
             for (String cluster : clusters) {
                 Assert.assertTrue(coloHelper.getFeedHelper().getDependencies(
-                    Util.readEntityName(feed))
+                    Util.readEntityName(feed)).getEntityList().toString()
                     .contains("(cluster) " + Util.readEntityName(cluster)));
             }
             Assert.assertFalse(coloHelper.getFeedHelper().getDependencies(
-                Util.readEntityName(feed))
+                Util.readEntityName(feed)).getEntityList().toString()
                 .contains("(process)" + Util.readEntityName(getProcessData())));
         }
-
-
     }
 
     public void addProcessInput(String feed, String feedName) {

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/4133c963/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/helpers/entity/AbstractEntityHelper.java
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/helpers/entity/AbstractEntityHelper.java
b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/helpers/entity/AbstractEntityHelper.java
index 797554b..b365a4c 100644
--- a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/helpers/entity/AbstractEntityHelper.java
+++ b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/helpers/entity/AbstractEntityHelper.java
@@ -29,7 +29,6 @@ import org.apache.falcon.regression.core.util.ExecUtil;
 import org.apache.falcon.regression.core.util.FileUtil;
 import org.apache.falcon.regression.core.util.HCatUtil;
 import org.apache.falcon.regression.core.util.InstanceUtil;
-import org.apache.falcon.regression.core.util.OSUtil;
 import org.apache.falcon.regression.core.util.OozieUtil;
 import org.apache.falcon.regression.core.util.Util;
 import org.apache.falcon.regression.core.util.Util.URLS;
@@ -56,10 +55,6 @@ public abstract class AbstractEntityHelper {
 
     private static final Logger LOGGER = Logger.getLogger(AbstractEntityHelper.class);
 
-    protected static final String CLIENT_LOCATION = OSUtil.RESOURCES
-        + OSUtil.getPath("IvoryClient", "IvoryCLI.jar");
-    protected static final String BASE_COMMAND = "java -jar " + CLIENT_LOCATION;
-
     private static boolean setAuthenticate() {
         String value = Config.getProperty("isAuthenticationSet");
         value = (null == value) ? "true" : value;
@@ -495,17 +490,6 @@ public abstract class AbstractEntityHelper {
             .createAndSendRequestProcessInstance(url, params, allColo, null);
     }
 
-    public String list() {
-        return ExecUtil.executeCommandGetOutput(
-            BASE_COMMAND + " entity -list -url " + this.hostname + " -type " + getEntityType());
-    }
-
-    public String getDependencies(String entityName) {
-        return ExecUtil.executeCommandGetOutput(
-            BASE_COMMAND + " entity -dependency -url " + this.hostname + " -type "
-                + getEntityType() + " -name " + entityName);
-    }
-
     public List<String> getArchiveInfo() throws IOException, JSchException {
         return Util.getStoreInfo(this, "/archive/" + getEntityType().toUpperCase());
     }
@@ -597,4 +581,20 @@ public abstract class AbstractEntityHelper {
         }
         return (InstancesResult) InstanceUtil.sendRequestProcessInstance(url, user);
     }
+
+    /**
+     * Get list of all dependencies of a given entity.
+     * @param entityName entity name
+     * @return response
+     * @throws URISyntaxException
+     * @throws AuthenticationException
+     * @throws InterruptedException
+     * @throws IOException
+     */
+    public ServiceResponse getDependencies(String entityName)
+        throws URISyntaxException, AuthenticationException, InterruptedException, IOException
{
+        String url = createUrl(this.hostname + URLS.DEPENDENCIES.getValue(), getEntityType(),
+            entityName + colo);
+        return Util.sendRequest(url, "get", null, null);
+    }
 }

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/4133c963/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ProcessLibPathTest.java
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ProcessLibPathTest.java
b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ProcessLibPathTest.java
index 6b1dec3..f335f04 100644
--- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ProcessLibPathTest.java
+++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ProcessLibPathTest.java
@@ -60,9 +60,6 @@ public class ProcessLibPathTest extends BaseTestClass {
     @BeforeClass(alwaysRun = true)
     public void createTestData() throws Exception {
         LOGGER.info("in @BeforeClass");
-
-        //common lib for both test cases
-        HadoopUtil.uploadDir(clusterFS, testLibDir, OSUtil.RESOURCES_OOZIE + "lib");
         Bundle b = BundleUtil.readELBundle();
         b.generateUniqueBundle();
         b = new Bundle(b, cluster);
@@ -98,7 +95,7 @@ public class ProcessLibPathTest extends BaseTestClass {
     }
 
     /**
-     * Test which test a process with no lib folder in workflow location.
+     * Tests a process with no lib folder in workflow location.
      *
      * @throws Exception
      */
@@ -106,7 +103,6 @@ public class ProcessLibPathTest extends BaseTestClass {
     public void setDifferentLibPathWithNoLibFolderInWorkflowfLocaltion() throws Exception
{
         String workflowDir = testLibDir + "/aggregatorLib1/";
         HadoopUtil.uploadDir(clusterFS, workflowDir, OSUtil.RESOURCES_OOZIE);
-        HadoopUtil.deleteDirIfExists(workflowDir + "/lib", clusterFS);
         bundles[0].setProcessWorkflow(workflowDir);
         LOGGER.info("processData: " + Util.prettyPrintXml(process));
         bundles[0].submitFeedsScheduleProcess(prism);
@@ -125,8 +121,8 @@ public class ProcessLibPathTest extends BaseTestClass {
         String workflowDir = testLibDir + "/aggregatorLib2/";
         HadoopUtil.uploadDir(clusterFS, workflowDir, OSUtil.RESOURCES_OOZIE);
         HadoopUtil.recreateDir(clusterFS, workflowDir + "/lib");
-        HadoopUtil.copyDataToFolder(clusterFS, workflowDir + "/lib",
-            OSUtil.RESOURCES + "ivory-oozie-lib-0.1.jar");
+        HadoopUtil.copyDataToFolder(clusterFS, workflowDir + "/lib/invalid.jar",
+            OSUtil.RESOURCES + "feed-s4Replication.xml");
         bundles[0].setProcessWorkflow(workflowDir);
         LOGGER.info("processData: " + Util.prettyPrintXml(process));
         bundles[0].submitFeedsScheduleProcess(prism);

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/4133c963/falcon-regression/merlin/src/test/resources/ELbundle/workflow/workflow.xml
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin/src/test/resources/ELbundle/workflow/workflow.xml b/falcon-regression/merlin/src/test/resources/ELbundle/workflow/workflow.xml
deleted file mode 100644
index 4a80109..0000000
--- a/falcon-regression/merlin/src/test/resources/ELbundle/workflow/workflow.xml
+++ /dev/null
@@ -1,61 +0,0 @@
-<!--
-  Licensed to the Apache Software Foundation (ASF) under one
-  or more contributor license agreements.  See the NOTICE file
-  distributed with this work for additional information
-  regarding copyright ownership.  The ASF licenses this file
-  to you under the Apache License, Version 2.0 (the
-  "License"); you may not use this file except in compliance
-  with the License.  You may obtain a copy of the License at
-  
-       http://www.apache.org/licenses/LICENSE-2.0
-  
-  Unless required by applicable law or agreed to in writing, software
-  distributed under the License is distributed on an "AS IS" BASIS,
-  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  See the License for the specific language governing permissions and
-  limitations under the License.
--->
-<workflow-app xmlns="uri:oozie:workflow:0.2" name="aggregator-wf">
-    <start to="aggregator"/>
-    <action name="aggregator">
-        <map-reduce>
-            <job-tracker>${jobTracker}</job-tracker>
-            <name-node>${nameNode}</name-node>
-            <prepare>
-                <delete path="${outputData}"/>
-            </prepare>
-            <configuration>
-                <property>
-                    <name>mapred.job.queue.name</name>
-                    <value>${queueName}</value>
-                </property>
-                <property>
-                    <name>mapred.mapper.class</name>
-                    <value>org.apache.oozie.example.SampleMapper</value>
-                </property>
-                <property>
-                    <name>mapred.reducer.class</name>
-                    <value>org.apache.oozie.example.SampleReducer</value>
-                </property>
-                <property>
-                    <name>mapred.map.tasks</name>
-                    <value>1</value>
-                </property>
-                <property>
-                    <name>mapred.input.dir</name>
-                    <value>${inputData}</value>
-                </property>
-                <property>
-                    <name>mapred.output.dir</name>
-                    <value>${outputData}</value>
-                </property>
-            </configuration>
-        </map-reduce>
-        <ok to="end"/>
-        <error to="fail"/>
-    </action>
-    <kill name="fail">
-        <message>Map/Reduce failed, error message[${wf:errorMessage(wf:lastErrorNode())}]</message>
-    </kill>
-    <end name="end"/>
-</workflow-app>

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/4133c963/falcon-regression/merlin/src/test/resources/IvoryClient/IvoryCLI.jar
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin/src/test/resources/IvoryClient/IvoryCLI.jar b/falcon-regression/merlin/src/test/resources/IvoryClient/IvoryCLI.jar
deleted file mode 100644
index 319bf5a..0000000
Binary files a/falcon-regression/merlin/src/test/resources/IvoryClient/IvoryCLI.jar and /dev/null
differ

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/4133c963/falcon-regression/merlin/src/test/resources/ivory-oozie-lib-0.1.jar
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin/src/test/resources/ivory-oozie-lib-0.1.jar b/falcon-regression/merlin/src/test/resources/ivory-oozie-lib-0.1.jar
deleted file mode 100755
index 992feab..0000000
Binary files a/falcon-regression/merlin/src/test/resources/ivory-oozie-lib-0.1.jar and /dev/null
differ

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/4133c963/falcon-regression/merlin/src/test/resources/oozie/lib/oozie-examples-3.1.5.jar
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin/src/test/resources/oozie/lib/oozie-examples-3.1.5.jar
b/falcon-regression/merlin/src/test/resources/oozie/lib/oozie-examples-3.1.5.jar
deleted file mode 100755
index 380c6f3..0000000
Binary files a/falcon-regression/merlin/src/test/resources/oozie/lib/oozie-examples-3.1.5.jar
and /dev/null differ

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/4133c963/falcon-regression/merlin/src/test/resources/oozie/workflow.xml
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin/src/test/resources/oozie/workflow.xml b/falcon-regression/merlin/src/test/resources/oozie/workflow.xml
index 4a80109..8c9cdaf 100644
--- a/falcon-regression/merlin/src/test/resources/oozie/workflow.xml
+++ b/falcon-regression/merlin/src/test/resources/oozie/workflow.xml
@@ -31,11 +31,11 @@
                 </property>
                 <property>
                     <name>mapred.mapper.class</name>
-                    <value>org.apache.oozie.example.SampleMapper</value>
+                    <value>org.apache.hadoop.mapred.lib.IdentityMapper</value>
                 </property>
                 <property>
                     <name>mapred.reducer.class</name>
-                    <value>org.apache.oozie.example.SampleReducer</value>
+                    <value>org.apache.hadoop.mapred.lib.IdentityReducer</value>
                 </property>
                 <property>
                     <name>mapred.map.tasks</name>

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/4133c963/falcon-regression/merlin/src/test/resources/workflows/aggregatorNoOutput/lib/oozie-examples-3.1.5.jar
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin/src/test/resources/workflows/aggregatorNoOutput/lib/oozie-examples-3.1.5.jar
b/falcon-regression/merlin/src/test/resources/workflows/aggregatorNoOutput/lib/oozie-examples-3.1.5.jar
deleted file mode 100644
index 380c6f3..0000000
Binary files a/falcon-regression/merlin/src/test/resources/workflows/aggregatorNoOutput/lib/oozie-examples-3.1.5.jar
and /dev/null differ


Mime
View raw message