falcon-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ajayyad...@apache.org
Subject falcon git commit: FALCON-1449 Move getEntityProperties method to EntityUtil. Contributed by Ajay Yadava.
Date Tue, 15 Sep 2015 11:06:10 GMT
Repository: falcon
Updated Branches:
  refs/heads/master bb659d40b -> 3a8659611


FALCON-1449 Move getEntityProperties method to EntityUtil. Contributed by Ajay Yadava.


Project: http://git-wip-us.apache.org/repos/asf/falcon/repo
Commit: http://git-wip-us.apache.org/repos/asf/falcon/commit/3a865961
Tree: http://git-wip-us.apache.org/repos/asf/falcon/tree/3a865961
Diff: http://git-wip-us.apache.org/repos/asf/falcon/diff/3a865961

Branch: refs/heads/master
Commit: 3a8659611c82b928ebb2abc7b5ec4629d4156402
Parents: bb659d4
Author: Ajay Yadava <ajaynsit@gmail.com>
Authored: Tue Sep 15 16:35:34 2015 +0530
Committer: Ajay Yadava <ajaynsit@gmail.com>
Committed: Tue Sep 15 16:35:34 2015 +0530

----------------------------------------------------------------------
 CHANGES.txt                                     |  2 +
 .../org/apache/falcon/entity/EntityUtil.java    | 40 +++++++++++++++
 .../apache/falcon/entity/EntityUtilTest.java    | 51 ++++++++++++++++++++
 .../apache/falcon/oozie/OozieEntityBuilder.java | 44 ++---------------
 .../OozieOrchestrationWorkflowBuilder.java      |  6 +--
 .../feed/FeedReplicationCoordinatorBuilder.java |  4 +-
 .../feed/FeedRetentionCoordinatorBuilder.java   |  3 +-
 .../ProcessExecutionCoordinatorBuilder.java     |  2 +-
 .../ProcessExecutionWorkflowBuilder.java        |  2 +-
 9 files changed, 106 insertions(+), 48 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/falcon/blob/3a865961/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 3a00a1e..bd24a86 100755
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -8,6 +8,8 @@ Trunk (Unreleased)
     FALCON-1027 Falcon proxy user support(Sowmya Ramesh)
 
   IMPROVEMENTS
+    FALCON-1449 Move getEntityProperties method to EntityUtil.(Ajay Yadava)
+
     FALCON-1357 Update CHANGES.txt to change 0.7 branch to release.(Ajay Yadava)
 
     FALCON-1414 Add all fields in filterBy to the entity list output.(Narayan Periwal via
Ajay Yadava)

http://git-wip-us.apache.org/repos/asf/falcon/blob/3a865961/common/src/main/java/org/apache/falcon/entity/EntityUtil.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/falcon/entity/EntityUtil.java b/common/src/main/java/org/apache/falcon/entity/EntityUtil.java
index 25d9008..2f05b1f 100644
--- a/common/src/main/java/org/apache/falcon/entity/EntityUtil.java
+++ b/common/src/main/java/org/apache/falcon/entity/EntityUtil.java
@@ -34,6 +34,7 @@ import org.apache.falcon.entity.v0.Frequency;
 import org.apache.falcon.entity.v0.SchemaHelper;
 import org.apache.falcon.entity.v0.cluster.Cluster;
 import org.apache.falcon.entity.v0.cluster.ClusterLocationType;
+import org.apache.falcon.entity.v0.cluster.Property;
 import org.apache.falcon.entity.v0.feed.ClusterType;
 import org.apache.falcon.entity.v0.feed.Feed;
 import org.apache.falcon.entity.v0.process.LateInput;
@@ -66,6 +67,7 @@ import java.util.Iterator;
 import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
+import java.util.Properties;
 import java.util.Set;
 import java.util.TimeZone;
 
@@ -311,6 +313,44 @@ public final class EntityUtil {
         return startCal.getTime();
     }
 
+
+    public static Properties getEntityProperties(Entity myEntity) {
+        Properties properties = new Properties();
+        switch (myEntity.getEntityType()) {
+        case CLUSTER:
+            org.apache.falcon.entity.v0.cluster.Properties clusterProps = ((Cluster) myEntity).getProperties();
+            if (clusterProps != null) {
+                for (Property prop : clusterProps.getProperties()) {
+                    properties.put(prop.getName(), prop.getValue());
+                }
+            }
+            break;
+
+        case FEED:
+            org.apache.falcon.entity.v0.feed.Properties feedProps = ((Feed) myEntity).getProperties();
+            if (feedProps != null) {
+                for (org.apache.falcon.entity.v0.feed.Property prop : feedProps.getProperties())
{
+                    properties.put(prop.getName(), prop.getValue());
+                }
+            }
+            break;
+
+        case PROCESS:
+            org.apache.falcon.entity.v0.process.Properties processProps = ((Process) myEntity).getProperties();
+            if (processProps != null) {
+                for (org.apache.falcon.entity.v0.process.Property prop : processProps.getProperties())
{
+                    properties.put(prop.getName(), prop.getValue());
+                }
+            }
+            break;
+
+        default:
+            throw new IllegalArgumentException("Unhandled entity type " + myEntity.getEntityType());
+        }
+        return properties;
+    }
+
+
     public static int getInstanceSequence(Date startTime, Frequency frequency, TimeZone tz,
Date instanceTime) {
         if (startTime.after(instanceTime)) {
             return -1;

http://git-wip-us.apache.org/repos/asf/falcon/blob/3a865961/common/src/test/java/org/apache/falcon/entity/EntityUtilTest.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/falcon/entity/EntityUtilTest.java b/common/src/test/java/org/apache/falcon/entity/EntityUtilTest.java
index cfdc84d..f6a4679 100644
--- a/common/src/test/java/org/apache/falcon/entity/EntityUtilTest.java
+++ b/common/src/test/java/org/apache/falcon/entity/EntityUtilTest.java
@@ -28,6 +28,7 @@ import org.apache.falcon.entity.v0.Frequency;
 import org.apache.falcon.entity.v0.SchemaHelper;
 import org.apache.falcon.entity.v0.feed.Feed;
 import org.apache.falcon.entity.v0.feed.LateArrival;
+import org.apache.falcon.entity.v0.feed.Property;
 import org.apache.falcon.entity.v0.process.Cluster;
 import org.apache.falcon.entity.v0.process.Process;
 import org.apache.falcon.hadoop.HadoopClientFactory;
@@ -42,6 +43,7 @@ import java.text.DateFormat;
 import java.text.ParseException;
 import java.text.SimpleDateFormat;
 import java.util.Date;
+import java.util.Properties;
 import java.util.TimeZone;
 
 /**
@@ -254,6 +256,55 @@ public class EntityUtilTest extends AbstractTestBase {
     }
 
     @Test
+    public void testGetFeedProperties() {
+        Feed feed = new Feed();
+        org.apache.falcon.entity.v0.feed.Properties props = new org.apache.falcon.entity.v0.feed.Properties();
+        Property queue = new Property();
+        String name = "Q";
+        String value = "head of Q division!";
+        queue.setName(name);
+        queue.setValue(value);
+        props.getProperties().add(queue);
+        feed.setProperties(props);
+        Properties actual = EntityUtil.getEntityProperties(feed);
+        Assert.assertEquals(actual.size(), 1);
+        Assert.assertEquals(actual.getProperty(name), value);
+    }
+
+    @Test
+    public void testGetProcessProperties() {
+        org.apache.falcon.entity.v0.cluster.Cluster cluster = new org.apache.falcon.entity.v0.cluster.Cluster();
+        org.apache.falcon.entity.v0.cluster.Properties props = new org.apache.falcon.entity.v0.cluster.Properties();
+        org.apache.falcon.entity.v0.cluster.Property priority = new org.apache.falcon.entity.v0.cluster.Property();
+        String name = "priority";
+        String value = "Sister of Moriarity!";
+        priority.setName(name);
+        priority.setValue(value);
+        props.getProperties().add(priority);
+        cluster.setProperties(props);
+        Properties actual = EntityUtil.getEntityProperties(cluster);
+        Assert.assertEquals(actual.size(), 1);
+        Assert.assertEquals(actual.getProperty(name), value);
+    }
+
+    @Test
+    public void testGetClusterProperties() {
+        Process process = new Process();
+        org.apache.falcon.entity.v0.process.Properties props = new org.apache.falcon.entity.v0.process.Properties();
+        org.apache.falcon.entity.v0.process.Property priority = new org.apache.falcon.entity.v0.process.Property();
+        String name = "M";
+        String value = "Minions!";
+        priority.setName(name);
+        priority.setValue(value);
+        props.getProperties().add(priority);
+        process.setProperties(props);
+        Properties actual = EntityUtil.getEntityProperties(process);
+        Assert.assertEquals(actual.size(), 1);
+        Assert.assertEquals(actual.getProperty(name), value);
+
+    }
+
+    @Test
     public void testGetLateProcessFeed() throws FalconException {
         Feed feed = new Feed();
 

http://git-wip-us.apache.org/repos/asf/falcon/blob/3a865961/oozie/src/main/java/org/apache/falcon/oozie/OozieEntityBuilder.java
----------------------------------------------------------------------
diff --git a/oozie/src/main/java/org/apache/falcon/oozie/OozieEntityBuilder.java b/oozie/src/main/java/org/apache/falcon/oozie/OozieEntityBuilder.java
index 9a6b14c..f9230da 100644
--- a/oozie/src/main/java/org/apache/falcon/oozie/OozieEntityBuilder.java
+++ b/oozie/src/main/java/org/apache/falcon/oozie/OozieEntityBuilder.java
@@ -23,11 +23,11 @@ import org.apache.commons.lang3.StringUtils;
 import org.apache.falcon.FalconException;
 import org.apache.falcon.entity.CatalogStorage;
 import org.apache.falcon.entity.ClusterHelper;
+import org.apache.falcon.entity.EntityUtil;
 import org.apache.falcon.entity.v0.Entity;
 import org.apache.falcon.entity.v0.cluster.Cluster;
 import org.apache.falcon.entity.v0.cluster.ClusterLocationType;
 import org.apache.falcon.entity.v0.cluster.Interfacetype;
-import org.apache.falcon.entity.v0.cluster.Property;
 import org.apache.falcon.entity.v0.feed.Feed;
 import org.apache.falcon.entity.v0.process.Output;
 import org.apache.falcon.entity.v0.process.Process;
@@ -165,8 +165,8 @@ public abstract class OozieEntityBuilder<T extends Entity> {
         }
     }
 
-    protected Properties createAppProperties(Cluster cluster, String wfName) throws FalconException
{
-        Properties properties = getEntityProperties(cluster);
+    protected Properties createAppProperties(Cluster cluster) throws FalconException {
+        Properties properties = EntityUtil.getEntityProperties(cluster);
         properties.setProperty(AbstractWorkflowEngine.NAME_NODE, ClusterHelper.getStorageUrl(cluster));
         properties.setProperty(AbstractWorkflowEngine.JOB_TRACKER, ClusterHelper.getMREndPoint(cluster));
         properties.setProperty("colo.name", cluster.getColo());
@@ -175,7 +175,7 @@ public abstract class OozieEntityBuilder<T extends Entity> {
             properties.setProperty(OozieClient.USE_SYSTEM_LIBPATH, "true");
         }
         properties.setProperty("falcon.libpath",
-                ClusterHelper.getLocation(cluster, ClusterLocationType.WORKING).getPath()
 + "/lib");
+                ClusterHelper.getLocation(cluster, ClusterLocationType.WORKING).getPath()
+ "/lib");
 
         return properties;
     }
@@ -215,42 +215,6 @@ public abstract class OozieEntityBuilder<T extends Entity> {
         return hiveConf;
     }
 
-    protected Properties getEntityProperties(Entity myEntity) {
-        Properties properties = new Properties();
-        switch (myEntity.getEntityType()) {
-        case CLUSTER:
-            org.apache.falcon.entity.v0.cluster.Properties clusterProps = ((Cluster) myEntity).getProperties();
-            if (clusterProps != null) {
-                for (Property prop : clusterProps.getProperties()) {
-                    properties.put(prop.getName(), prop.getValue());
-                }
-            }
-            break;
-
-        case FEED:
-            org.apache.falcon.entity.v0.feed.Properties feedProps = ((Feed) myEntity).getProperties();
-            if (feedProps != null) {
-                for (org.apache.falcon.entity.v0.feed.Property prop : feedProps.getProperties())
{
-                    properties.put(prop.getName(), prop.getValue());
-                }
-            }
-            break;
-
-        case PROCESS:
-            org.apache.falcon.entity.v0.process.Properties processProps = ((Process) myEntity).getProperties();
-            if (processProps != null) {
-                for (org.apache.falcon.entity.v0.process.Property prop : processProps.getProperties())
{
-                    properties.put(prop.getName(), prop.getValue());
-                }
-            }
-            break;
-
-        default:
-            throw new IllegalArgumentException("Unhandled entity type " + myEntity.getEntityType());
-        }
-        return properties;
-    }
-
     protected void propagateCatalogTableProperties(Output output, CatalogStorage tableStorage,
Properties props) {
         String prefix = "falcon_" + output.getName();
 

http://git-wip-us.apache.org/repos/asf/falcon/blob/3a865961/oozie/src/main/java/org/apache/falcon/oozie/OozieOrchestrationWorkflowBuilder.java
----------------------------------------------------------------------
diff --git a/oozie/src/main/java/org/apache/falcon/oozie/OozieOrchestrationWorkflowBuilder.java
b/oozie/src/main/java/org/apache/falcon/oozie/OozieOrchestrationWorkflowBuilder.java
index f7193a3..671ae6b 100644
--- a/oozie/src/main/java/org/apache/falcon/oozie/OozieOrchestrationWorkflowBuilder.java
+++ b/oozie/src/main/java/org/apache/falcon/oozie/OozieOrchestrationWorkflowBuilder.java
@@ -164,7 +164,7 @@ public abstract class OozieOrchestrationWorkflowBuilder<T extends Entity>
extend
 
     protected void addAdditionalReplicationProperties(ACTION replicationAction) {
         List<String> args = replicationAction.getJava().getArg();
-        Properties props = getEntityProperties(entity);
+        Properties props = EntityUtil.getEntityProperties(entity);
 
         for (ReplicationDistCpOption distcpOption : ReplicationDistCpOption.values()) {
             String propertyValue = props.getProperty(distcpOption.getName());
@@ -427,8 +427,8 @@ public abstract class OozieOrchestrationWorkflowBuilder<T extends Entity>
extend
         props.put(MR_JOB_PRIORITY, "NORMAL");
 
         //props in entity override the set props.
-        props.putAll(getEntityProperties(entity));
-        props.putAll(createAppProperties(cluster, entity.getName()));
+        props.putAll(EntityUtil.getEntityProperties(entity));
+        props.putAll(createAppProperties(cluster));
         return props;
     }
 

http://git-wip-us.apache.org/repos/asf/falcon/blob/3a865961/oozie/src/main/java/org/apache/falcon/oozie/feed/FeedReplicationCoordinatorBuilder.java
----------------------------------------------------------------------
diff --git a/oozie/src/main/java/org/apache/falcon/oozie/feed/FeedReplicationCoordinatorBuilder.java
b/oozie/src/main/java/org/apache/falcon/oozie/feed/FeedReplicationCoordinatorBuilder.java
index db08d60..b0e46f0 100644
--- a/oozie/src/main/java/org/apache/falcon/oozie/feed/FeedReplicationCoordinatorBuilder.java
+++ b/oozie/src/main/java/org/apache/falcon/oozie/feed/FeedReplicationCoordinatorBuilder.java
@@ -182,7 +182,7 @@ public class FeedReplicationCoordinatorBuilder extends OozieCoordinatorBuilder<F
 
         propagateLateDataProperties(instancePaths, sourceStorage.getType().name(), props);
         // Add the custom properties set in feed. Else, dryrun won't catch any missing props.
-        props.putAll(getEntityProperties(entity));
+        props.putAll(EntityUtil.getEntityProperties(entity));
         workflow.setConfiguration(getConfig(props));
         action.setWorkflow(workflow);
 
@@ -336,7 +336,7 @@ public class FeedReplicationCoordinatorBuilder extends OozieCoordinatorBuilder<F
             timeoutInMillis = THIRTY_MINUTES;
         }
 
-        Properties props = getEntityProperties(entity);
+        Properties props = EntityUtil.getEntityProperties(entity);
         String timeout = props.getProperty(TIMEOUT);
         if (timeout!=null) {
             try{

http://git-wip-us.apache.org/repos/asf/falcon/blob/3a865961/oozie/src/main/java/org/apache/falcon/oozie/feed/FeedRetentionCoordinatorBuilder.java
----------------------------------------------------------------------
diff --git a/oozie/src/main/java/org/apache/falcon/oozie/feed/FeedRetentionCoordinatorBuilder.java
b/oozie/src/main/java/org/apache/falcon/oozie/feed/FeedRetentionCoordinatorBuilder.java
index ce9ef9a..a4b5f35 100644
--- a/oozie/src/main/java/org/apache/falcon/oozie/feed/FeedRetentionCoordinatorBuilder.java
+++ b/oozie/src/main/java/org/apache/falcon/oozie/feed/FeedRetentionCoordinatorBuilder.java
@@ -21,6 +21,7 @@ package org.apache.falcon.oozie.feed;
 import org.apache.falcon.FalconException;
 import org.apache.falcon.LifeCycle;
 import org.apache.falcon.Tag;
+import org.apache.falcon.entity.EntityUtil;
 import org.apache.falcon.entity.FeedHelper;
 import org.apache.falcon.entity.v0.Frequency.TimeUnit;
 import org.apache.falcon.entity.v0.SchemaHelper;
@@ -78,7 +79,7 @@ public class FeedRetentionCoordinatorBuilder extends OozieCoordinatorBuilder<Fee
         workflow.setAppPath(getStoragePath(wfProps.getProperty(OozieEntityBuilder.ENTITY_PATH)));
         props.putAll(getProperties(coordPath, coordName));
         // Add the custom properties set in feed. Else, dryrun won't catch any missing props.
-        props.putAll(getEntityProperties(entity));
+        props.putAll(EntityUtil.getEntityProperties(entity));
         workflow.setConfiguration(getConfig(props));
         ACTION action = new ACTION();
         action.setWorkflow(workflow);

http://git-wip-us.apache.org/repos/asf/falcon/blob/3a865961/oozie/src/main/java/org/apache/falcon/oozie/process/ProcessExecutionCoordinatorBuilder.java
----------------------------------------------------------------------
diff --git a/oozie/src/main/java/org/apache/falcon/oozie/process/ProcessExecutionCoordinatorBuilder.java
b/oozie/src/main/java/org/apache/falcon/oozie/process/ProcessExecutionCoordinatorBuilder.java
index d6d42e1..7154e40 100644
--- a/oozie/src/main/java/org/apache/falcon/oozie/process/ProcessExecutionCoordinatorBuilder.java
+++ b/oozie/src/main/java/org/apache/falcon/oozie/process/ProcessExecutionCoordinatorBuilder.java
@@ -92,7 +92,7 @@ public class ProcessExecutionCoordinatorBuilder extends OozieCoordinatorBuilder<
         WORKFLOW wf = new WORKFLOW();
         wf.setAppPath(getStoragePath(wfProps.getProperty(OozieEntityBuilder.ENTITY_PATH)));
         // Add the custom properties set in feed. Else, dryrun won't catch any missing props.
-        props.putAll(getEntityProperties(entity));
+        props.putAll(EntityUtil.getEntityProperties(entity));
         wf.setConfiguration(getConfig(props));
 
         // set coord action to parent wf

http://git-wip-us.apache.org/repos/asf/falcon/blob/3a865961/oozie/src/main/java/org/apache/falcon/oozie/process/ProcessExecutionWorkflowBuilder.java
----------------------------------------------------------------------
diff --git a/oozie/src/main/java/org/apache/falcon/oozie/process/ProcessExecutionWorkflowBuilder.java
b/oozie/src/main/java/org/apache/falcon/oozie/process/ProcessExecutionWorkflowBuilder.java
index ac436ca..3f49adb 100644
--- a/oozie/src/main/java/org/apache/falcon/oozie/process/ProcessExecutionWorkflowBuilder.java
+++ b/oozie/src/main/java/org/apache/falcon/oozie/process/ProcessExecutionWorkflowBuilder.java
@@ -197,7 +197,7 @@ public abstract class ProcessExecutionWorkflowBuilder extends OozieOrchestration
     }
 
     protected void propagateEntityProperties(CONFIGURATION conf, List<String> paramList)
{
-        Properties entityProperties = getEntityProperties(entity);
+        Properties entityProperties = EntityUtil.getEntityProperties(entity);
 
         // Propagate user defined properties to job configuration
         final List<org.apache.falcon.oozie.workflow.CONFIGURATION.Property> configuration
= conf.getProperty();


Mime
View raw message