falcon-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From pall...@apache.org
Subject falcon git commit: FALCON-1752 Revert FALCON-1573 from 0.9 branch
Date Mon, 18 Jan 2016 07:34:52 GMT
Repository: falcon
Updated Branches:
  refs/heads/0.9 6fb5cc89a -> 195b3c1dc


FALCON-1752 Revert FALCON-1573 from 0.9 branch


Project: http://git-wip-us.apache.org/repos/asf/falcon/repo
Commit: http://git-wip-us.apache.org/repos/asf/falcon/commit/195b3c1d
Tree: http://git-wip-us.apache.org/repos/asf/falcon/tree/195b3c1d
Diff: http://git-wip-us.apache.org/repos/asf/falcon/diff/195b3c1d

Branch: refs/heads/0.9
Commit: 195b3c1dccd36bcf23b8e20e94f1146698eed8ef
Parents: 6fb5cc8
Author: Pallavi Rao <pallavi.rao@inmobi.com>
Authored: Mon Jan 18 13:04:40 2016 +0530
Committer: Pallavi Rao <pallavi.rao@inmobi.com>
Committed: Mon Jan 18 13:04:40 2016 +0530

----------------------------------------------------------------------
 CHANGES.txt                                     |  2 -
 docs/src/site/twiki/FalconCLI.twiki             |  5 +-
 .../src/site/twiki/restapi/EntitySchedule.twiki | 70 +-------------------
 .../apache/falcon/oozie/OozieEntityBuilder.java | 23 -------
 .../workflow/engine/OozieWorkflowEngine.java    |  3 +-
 .../feed/OozieFeedWorkflowBuilderTest.java      | 16 -----
 .../OozieProcessWorkflowBuilderTest.java        | 18 -----
 7 files changed, 3 insertions(+), 134 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/falcon/blob/195b3c1d/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 78cc96b..775252d 100755
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -49,8 +49,6 @@ Proposed Release Version: 0.9
 
     FALCON-1588 Add ability to provide the path for recipe files in command line(Peeyush
Bishnoi via Ajay Yadava)
 
-    FALCON-1573 Supply user-defined properties to Oozie workflows during schedule(Daniel
Del Castillo via Ajay Yadava)
-
     FALCON-1559 Config changes required for native scheduler (Pallavi Rao)
 
     FALCON-1459 Ability to import from database(Venkat Ramachandran via Sowmya Ramesh)

http://git-wip-us.apache.org/repos/asf/falcon/blob/195b3c1d/docs/src/site/twiki/FalconCLI.twiki
----------------------------------------------------------------------
diff --git a/docs/src/site/twiki/FalconCLI.twiki b/docs/src/site/twiki/FalconCLI.twiki
index bee3342..9f91143 100644
--- a/docs/src/site/twiki/FalconCLI.twiki
+++ b/docs/src/site/twiki/FalconCLI.twiki
@@ -44,10 +44,7 @@ Once submitted, an entity can be scheduled using schedule option. Process
and fe
 Usage:
 $FALCON_HOME/bin/falcon entity  -type [process|feed] -name <<name>> -schedule
 
-Optional Arg : -skipDryRun -doAs <username>
--properties <<key1:val1,...,keyN:valN>>
-
-<a href="./Restapi/EntitySchedule.html">Optional params described here.</a>
+Optional Arg : -skipDryRun. When this argument is specified, Falcon skips oozie dryrun.
 
 Example:
 $FALCON_HOME/bin/falcon entity  -type process -name sampleProcess -schedule

http://git-wip-us.apache.org/repos/asf/falcon/blob/195b3c1d/docs/src/site/twiki/restapi/EntitySchedule.twiki
----------------------------------------------------------------------
diff --git a/docs/src/site/twiki/restapi/EntitySchedule.twiki b/docs/src/site/twiki/restapi/EntitySchedule.twiki
index 0dede9b..263d152 100644
--- a/docs/src/site/twiki/restapi/EntitySchedule.twiki
+++ b/docs/src/site/twiki/restapi/EntitySchedule.twiki
@@ -12,81 +12,15 @@ Schedule an entity.
    * :entity-name is name of the entity.
    * skipDryRun : Optional query param, Falcon skips oozie dryrun when value is set to true.
    * doAs <optional query param> allows the current user to impersonate the user passed
in doAs when interacting with the Falcon system.
-   * properties <key1:val1,...,keyN:valN> : Optional query param, supplies a set of
key-value pairs that will be available to the entity in the coordinator configuration. These
values will not override properties with the same name predefined in the entity specification.
For example, to change the scheduler used for scheduling the entity you would set the property
_falcon.scheduler_ in the properties parameter to _native_ to use the Falcon Scheduler or
to _oozie_ to use the Oozie Scheduler.
 
 
 ---++ Results
 Result of the schedule command.
 
 ---++ Examples
----+++ Oozie Workflow
-<verbatim>
-<workflow-app xmlns="uri:oozie:workflow:0.4" name="aggregator-wf">
-  <start to="aggregator" />
-  <action name="aggregator">
-    <java>
-      <job-tracker>${jobTracker}</job-tracker>
-      <name-node>${nameNode}</name-node>
-      <configuration>
-        <property>
-          <name>mapred.job.queue.name</name>
-          <value>${queueName}</value>
-        </property>
-      </configuration>
-      <main-class>com.company.hadoop.AggregatorJob</main-class>
-      <java-opts>-Dframework.instrumentation.host=${instrumentationServer}</java-opts>
-      <arg>--input.path=${inputBasePath}</arg>
-      <arg>--output.path=${outputBasePath}</arg>
-    </java>
-    <ok to="end" />
-    <error to="fail" />
-  </action>
-  <kill name="fail">
-    <message>Action failed, error message[${wf:errorMessage(wf:lastErrorNode())}]</message>
-  </kill>
-</workflow-app>
-</verbatim>
----+++ Submitted Process
-<verbatim>
-<?xml version="1.0" encoding="UTF-8"?>
-<!-- Daily sample process. Runs at 6th hour every day. Input - last day's hourly data.
Generates output for yesterday -->
-<process xmlns="uri:falcon:process:0.1" name="SampleProcess" >
-    <clusters>
-      <cluster name="primary-cluster">
-        <validity start="2012-04-03T06:00Z" end="2022-12-30T00:00Z" />
-      </cluster>
-    </clusters>
-
-    <parallel>1</parallel>
-    <order>FIFO</order>
-    <frequency>hours(1)</frequency>
-
-    <inputs>
-        <input name="input" feed="SampleInput" start="yesterday(0,0)" end="today(-1,0)"
/>
-    </inputs>
-
-    <outputs>
-        <output name="output" feed="SampleOutput" instance="yesterday(0,0)" />
-    </outputs>
-
-    <properties>
-        <property name="queueName" value="default" />
-        <property name="ssh.host" value="localhost" />
-        <property name="fileTimestamp" value="${coord:formatTime(coord:nominalTime(),
'yyyy-MM-dd')}" />
-        <property name="instrumentationServer" value="${coord:conf('instrumentation.host')}"
/>
-    </properties>
-
-    <workflow engine="oozie" path="/examples/apps/aggregator" />
-    <retry policy="exp-backoff" delay="minutes(5)" attempts="3" />
-    
-    <late-process policy="exp-backoff" delay="hours(1)">
-        <late-input input="input" workflow-path="/projects/bootcamp/workflow/lateinput"
/>
-    </late-process>
-</process>
-</verbatim>
 ---+++ Rest Call
 <verbatim>
-POST http://localhost:15000/api/entities/schedule/process/SampleProcess?skipDryRun=false&doAs=joe&properties=instrumentation.host:intrumentation.localdomain
+POST http://localhost:15000/api/entities/schedule/process/SampleProcess?skipDryRun=false&doAs=joe
 </verbatim>
 ---+++ Result
 <verbatim>
@@ -96,5 +30,3 @@ POST http://localhost:15000/api/entities/schedule/process/SampleProcess?skipDryR
     "status": "SUCCEEDED"
 }
 </verbatim>
----+++ Notes
-In this example, the value of _framework.instrumentation.host_ in the Oozie workflow will
be _intrumentation.localdomain_ which is the property passed when the process is scheduled.

http://git-wip-us.apache.org/repos/asf/falcon/blob/195b3c1d/oozie/src/main/java/org/apache/falcon/oozie/OozieEntityBuilder.java
----------------------------------------------------------------------
diff --git a/oozie/src/main/java/org/apache/falcon/oozie/OozieEntityBuilder.java b/oozie/src/main/java/org/apache/falcon/oozie/OozieEntityBuilder.java
index a36ee79..e0af30d 100644
--- a/oozie/src/main/java/org/apache/falcon/oozie/OozieEntityBuilder.java
+++ b/oozie/src/main/java/org/apache/falcon/oozie/OozieEntityBuilder.java
@@ -60,7 +60,6 @@ import java.io.OutputStream;
 import java.io.StringWriter;
 import java.util.ArrayList;
 import java.util.List;
-import java.util.Map;
 import java.util.Map.Entry;
 import java.util.Properties;
 
@@ -108,28 +107,6 @@ public abstract class OozieEntityBuilder<T extends Entity> {
 
     public abstract Properties build(Cluster cluster, Path buildPath) throws FalconException;
 
-    public Properties build(Cluster cluster, Path buildPath, Map<String, String> properties)
throws FalconException {
-        Properties builderProperties = build(cluster, buildPath);
-        if (properties == null || properties.isEmpty()) {
-            return builderProperties;
-        }
-
-        Properties propertiesCopy = new Properties();
-        propertiesCopy.putAll(properties);
-
-        // Builder properties shadow any user-defined property
-        for(String propertyName : builderProperties.stringPropertyNames()) {
-            String propertyValue = builderProperties.getProperty(propertyName);
-            if (propertiesCopy.contains(propertyName)) {
-                LOG.warn("User provided property {} is already declared in the entity and
will be ignored.",
-                    propertyName);
-            }
-            propertiesCopy.put(propertyName, propertyValue);
-        }
-
-        return propertiesCopy;
-    }
-
     protected String getStoragePath(Path path) {
         if (path != null) {
             return getStoragePath(path.toString());

http://git-wip-us.apache.org/repos/asf/falcon/blob/195b3c1d/oozie/src/main/java/org/apache/falcon/workflow/engine/OozieWorkflowEngine.java
----------------------------------------------------------------------
diff --git a/oozie/src/main/java/org/apache/falcon/workflow/engine/OozieWorkflowEngine.java
b/oozie/src/main/java/org/apache/falcon/workflow/engine/OozieWorkflowEngine.java
index d4b45a6..403dee9 100644
--- a/oozie/src/main/java/org/apache/falcon/workflow/engine/OozieWorkflowEngine.java
+++ b/oozie/src/main/java/org/apache/falcon/workflow/engine/OozieWorkflowEngine.java
@@ -170,12 +170,11 @@ public class OozieWorkflowEngine extends AbstractWorkflowEngine {
                 Cluster cluster = STORE.get(EntityType.CLUSTER, clusterName);
                 prepareEntityBuildPath(entity, cluster);
                 Path buildPath = EntityUtil.getNewStagingPath(cluster, entity);
-                Properties properties = builder.build(cluster, buildPath, suppliedProps);
+                Properties properties = builder.build(cluster, buildPath);
                 if (properties == null) {
                     LOG.info("Entity {} is not scheduled on cluster {}", entity.getName(),
cluster);
                     continue;
                 }
-
                 //Do dryRun of coords before schedule as schedule is asynchronous
                 dryRunInternal(cluster, new Path(properties.getProperty(OozieEntityBuilder.ENTITY_PATH)),
skipDryRun);
                 scheduleEntity(clusterName, properties, entity);

http://git-wip-us.apache.org/repos/asf/falcon/blob/195b3c1d/oozie/src/test/java/org/apache/falcon/oozie/feed/OozieFeedWorkflowBuilderTest.java
----------------------------------------------------------------------
diff --git a/oozie/src/test/java/org/apache/falcon/oozie/feed/OozieFeedWorkflowBuilderTest.java
b/oozie/src/test/java/org/apache/falcon/oozie/feed/OozieFeedWorkflowBuilderTest.java
index a553ea7..8f927e5 100644
--- a/oozie/src/test/java/org/apache/falcon/oozie/feed/OozieFeedWorkflowBuilderTest.java
+++ b/oozie/src/test/java/org/apache/falcon/oozie/feed/OozieFeedWorkflowBuilderTest.java
@@ -911,20 +911,4 @@ public class OozieFeedWorkflowBuilderTest extends AbstractTestBase {
         property.setValue(umask);
         cluster.getProperties().getProperties().add(property);
     }
-
-    @Test
-    public void testUserDefinedProperties() throws Exception {
-        Map<String, String> suppliedProps = new HashMap<>();
-        suppliedProps.put("custom.property", "custom value");
-        suppliedProps.put("ENTITY_NAME", "MyEntity");
-
-        OozieEntityBuilder builder = OozieEntityBuilder.get(lifecycleRetentionFeed);
-        Path bundlePath = new Path("/projects/falcon/");
-        Properties props = builder.build(trgCluster, bundlePath, suppliedProps);
-
-        Assert.assertNotNull(props);
-        Assert.assertEquals(props.get("ENTITY_NAME"), lifecycleRetentionFeed.getName());
-        Assert.assertEquals(props.get("custom.property"), "custom value");
-    }
-
 }

http://git-wip-us.apache.org/repos/asf/falcon/blob/195b3c1d/oozie/src/test/java/org/apache/falcon/oozie/process/OozieProcessWorkflowBuilderTest.java
----------------------------------------------------------------------
diff --git a/oozie/src/test/java/org/apache/falcon/oozie/process/OozieProcessWorkflowBuilderTest.java
b/oozie/src/test/java/org/apache/falcon/oozie/process/OozieProcessWorkflowBuilderTest.java
index 8d824ba..9f492d7 100644
--- a/oozie/src/test/java/org/apache/falcon/oozie/process/OozieProcessWorkflowBuilderTest.java
+++ b/oozie/src/test/java/org/apache/falcon/oozie/process/OozieProcessWorkflowBuilderTest.java
@@ -75,7 +75,6 @@ import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
-import java.util.Properties;
 
 import static org.testng.Assert.assertEquals;
 import static org.testng.Assert.assertTrue;
@@ -768,21 +767,4 @@ public class OozieProcessWorkflowBuilderTest extends AbstractTestBase
{
         Assert.assertEquals(props.get(WorkflowExecutionArgs.OUTPUT_FEED_NAMES.getName()),
"impressions");
         Assert.assertEquals(props.get(WorkflowExecutionArgs.INPUT_FEED_NAMES.getName()),
"NONE");
     }
-
-    @Test
-    public void testUserDefinedProperties() throws Exception {
-        Map<String, String> suppliedProps = new HashMap<>();
-        suppliedProps.put("custom.property", "custom value");
-        suppliedProps.put("ENTITY_NAME", "MyEntity");
-
-        Process process = ConfigurationStore.get().get(EntityType.PROCESS, "clicksummary");
-        Path bundlePath = new Path("/projects/falcon/");
-        OozieEntityBuilder builder = OozieEntityBuilder.get(process);
-        Properties props = builder.build(cluster, bundlePath, suppliedProps);
-
-        Assert.assertNotNull(props);
-        Assert.assertEquals(props.get("ENTITY_NAME"), process.getName());
-        Assert.assertEquals(props.get("custom.property"), "custom value");
-    }
-
 }


Mime
View raw message