falcon-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From rostafiyc...@apache.org
Subject [2/2] falcon git commit: FALCON-1113 Clean up data files in merlin resource directory. Create better names for them. Contributed by Paul Isaychuk
Date Tue, 24 Mar 2015 15:33:20 GMT
FALCON-1113 Clean up data files in merlin resource directory. Create better names for them. Contributed by Paul Isaychuk


Project: http://git-wip-us.apache.org/repos/asf/falcon/repo
Commit: http://git-wip-us.apache.org/repos/asf/falcon/commit/1aac3a50
Tree: http://git-wip-us.apache.org/repos/asf/falcon/tree/1aac3a50
Diff: http://git-wip-us.apache.org/repos/asf/falcon/diff/1aac3a50

Branch: refs/heads/master
Commit: 1aac3a5027f1f43209cc4f9bcf8844822d6f7b9c
Parents: 969d10e
Author: Ruslan Ostafiychuk <rostafiychuk@apache.org>
Authored: Tue Mar 24 17:32:00 2015 +0200
Committer: Ruslan Ostafiychuk <rostafiychuk@apache.org>
Committed: Tue Mar 24 17:32:00 2015 +0200

----------------------------------------------------------------------
 falcon-regression/CHANGES.txt                   |   3 +
 .../falcon/regression/core/util/OSUtil.java     |   2 -
 .../falcon/regression/FeedLateRerunTest.java    |   4 +-
 .../falcon/regression/FeedReplicationTest.java  |  17 +-
 .../falcon/regression/ProcessLateRerunTest.java |   4 +-
 .../falcon/regression/ProcessLibPathTest.java   |   2 +-
 .../regression/hcat/HCatRetentionTest.java      |   2 +-
 .../PrismFeedReplicationPartitionExpTest.java   |  11 +-
 .../falcon/regression/prism/RetentionTest.java  |   3 +-
 .../2ndLateData/Configuration.java.ignore       |  73 ----
 .../2ndLateData/DateValidator.java.ignore       |  87 ----
 .../2ndLateData/TimeUnit.java.ignore            |  36 --
 .../2ndLateData/dataFile1.txt                   |  73 ++++
 .../2ndLateData/dataFile2.txt                   |  87 ++++
 .../2ndLateData/dataFile3.txt                   |  36 ++
 .../OozieExampleInputData/lateData/log_15.txt   |  19 -
 .../EntityInstanceMessage.java.ignore           | 412 -------------------
 .../EntityInstanceMessageCreator.java.ignore    |  61 ---
 .../normalInput/MessageProducer.java.ignore     | 139 -------
 .../normalInput/dataFile.properties             |  30 ++
 .../normalInput/dataFile.xml                    |  63 +++
 .../normalInput/dataFile1.txt                   |  61 +++
 .../normalInput/dataFile2.txt                   | 412 +++++++++++++++++++
 .../normalInput/dataFile3.txt                   | 139 +++++++
 .../normalInput/dataFile4.txt                   | 174 ++++++++
 .../normalInput/jms-config.properties           |  31 --
 .../OozieExampleInputData/normalInput/log4j.xml |  63 ---
 .../normalInput/log_01.txt                      | 174 --------
 .../ReplicationResources/cluster-0.1.xml        |  41 --
 .../ReplicationResources/feed-s4Replication.xml |  38 --
 .../test/resources/ReplicationResources/id.pig  |  20 -
 .../ReplicationResources/log4testng.properties  |  29 --
 .../ReplicationResources/process-agg.xml        |  52 ---
 .../src/test/resources/feed-s4Replication.xml   |  42 --
 .../merlin/src/test/resources/log_01.txt        |  18 -
 35 files changed, 1096 insertions(+), 1362 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/falcon/blob/1aac3a50/falcon-regression/CHANGES.txt
----------------------------------------------------------------------
diff --git a/falcon-regression/CHANGES.txt b/falcon-regression/CHANGES.txt
index 5a897fc..2024016 100644
--- a/falcon-regression/CHANGES.txt
+++ b/falcon-regression/CHANGES.txt
@@ -63,6 +63,9 @@ Trunk (Unreleased)
    via Samarth Gupta)
 
   IMPROVEMENTS
+   FALCON-1113 Clean up data files in merlin resource directory. Create better names for them
+   (Paul Isaychuk via Ruslan Ostafiychuk)
+
    FALCON-1103 RetentionTest stabilization - remove check of all retention job actions
    (Paul Isaychuk via Ruslan Ostafiychuk)
 

http://git-wip-us.apache.org/repos/asf/falcon/blob/1aac3a50/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/OSUtil.java
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/OSUtil.java b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/OSUtil.java
index cc9e7ea..ab27ccf 100644
--- a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/OSUtil.java
+++ b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/OSUtil.java
@@ -38,8 +38,6 @@ public final class OSUtil {
     public static final String RESOURCES_OOZIE = String.format(RESOURCES + "oozie%s", SEPARATOR);
     public static final String OOZIE_EXAMPLE_INPUT_DATA =
         String.format(RESOURCES + "OozieExampleInputData%s", SEPARATOR);
-    public static final String OOZIE_EXAMPLE_INPUT_LATE_INPUT =
-        OSUtil.OOZIE_EXAMPLE_INPUT_DATA + "lateData";
     public static final String NORMAL_INPUT =
         String.format(OOZIE_EXAMPLE_INPUT_DATA + "normalInput%s", SEPARATOR);
     public static final String SINGLE_FILE =

http://git-wip-us.apache.org/repos/asf/falcon/blob/1aac3a50/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/FeedLateRerunTest.java
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/FeedLateRerunTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/FeedLateRerunTest.java
index 9751003..2c8346d 100644
--- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/FeedLateRerunTest.java
+++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/FeedLateRerunTest.java
@@ -194,7 +194,7 @@ public class FeedLateRerunTest extends BaseTestClass {
             for (String location : missingDependencies) {
                 if (tempCount==1) {
                     LOGGER.info("Transferring data to : " + location);
-                    HadoopUtil.copyDataToFolder(clusterFS1, location, OSUtil.RESOURCES + "feed-s4Replication.xml");
+                    HadoopUtil.copyDataToFolder(clusterFS1, location, OSUtil.NORMAL_INPUT + "dataFile.xml");
                     tempCount++;
                 }
             }
@@ -207,7 +207,7 @@ public class FeedLateRerunTest extends BaseTestClass {
         for (String dependency : missingDependencies) {
             if (tempCounter==1) {
                 LOGGER.info("Transferring late data to : " + dependency);
-                HadoopUtil.copyDataToFolder(clusterFS1, dependency, OSUtil.RESOURCES + "log4j.properties");
+                HadoopUtil.copyDataToFolder(clusterFS1, dependency, OSUtil.NORMAL_INPUT + "dataFile.properties");
             }
             tempCounter++;
         }

http://git-wip-us.apache.org/repos/asf/falcon/blob/1aac3a50/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/FeedReplicationTest.java
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/FeedReplicationTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/FeedReplicationTest.java
index f3a8318..eb8c4fe 100644
--- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/FeedReplicationTest.java
+++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/FeedReplicationTest.java
@@ -147,9 +147,8 @@ public class FeedReplicationTest extends BaseTestClass {
         Path toSource = new Path(sourceLocation);
         Path toTarget = new Path(targetLocation);
         if (dataFlag) {
-            HadoopUtil.copyDataToFolder(cluster1FS, sourceLocation,
-                    OSUtil.RESOURCES + "feed-s4Replication.xml");
-            HadoopUtil.copyDataToFolder(cluster1FS, sourceLocation, OSUtil.RESOURCES + "log_01.txt");
+            HadoopUtil.copyDataToFolder(cluster1FS, sourceLocation, OSUtil.NORMAL_INPUT + "dataFile.xml");
+            HadoopUtil.copyDataToFolder(cluster1FS, sourceLocation, OSUtil.NORMAL_INPUT + "dataFile1.txt");
         }
 
         //check if coordinator exists
@@ -236,9 +235,8 @@ public class FeedReplicationTest extends BaseTestClass {
         Path toTarget = new Path(targetLocation);
 
         if (dataFlag) {
-            HadoopUtil.copyDataToFolder(cluster1FS, sourceLocation,
-                    OSUtil.RESOURCES + "feed-s4Replication.xml");
-            HadoopUtil.copyDataToFolder(cluster1FS, sourceLocation, OSUtil.RESOURCES + "log_01.txt");
+            HadoopUtil.copyDataToFolder(cluster1FS, sourceLocation, OSUtil.NORMAL_INPUT + "dataFile.xml");
+            HadoopUtil.copyDataToFolder(cluster1FS, sourceLocation, OSUtil.NORMAL_INPUT + "dataFile1.txt");
         }
 
         //check if all coordinators exist
@@ -336,9 +334,8 @@ public class FeedReplicationTest extends BaseTestClass {
         Path toSource = new Path(sourceLocation);
         Path toTarget = new Path(targetLocation);
         if (dataFlag) {
-            HadoopUtil.copyDataToFolder(cluster1FS, sourceLocation,
-                    OSUtil.RESOURCES + "feed-s4Replication.xml");
-            HadoopUtil.copyDataToFolder(cluster1FS, sourceLocation, OSUtil.RESOURCES + "log_01.txt");
+            HadoopUtil.copyDataToFolder(cluster1FS, sourceLocation, OSUtil.NORMAL_INPUT + "dataFile.xml");
+            HadoopUtil.copyDataToFolder(cluster1FS, sourceLocation, OSUtil.NORMAL_INPUT + "dataFile1.txt");
         }
 
         //check while instance is got created
@@ -357,7 +354,7 @@ public class FeedReplicationTest extends BaseTestClass {
 
         //create availability flag on source
         HadoopUtil.copyDataToFolder(cluster1FS, sourceLocation,
-                OSUtil.RESOURCES + availabilityFlagName);
+                OSUtil.OOZIE_EXAMPLE_INPUT_DATA + availabilityFlagName);
 
         //check if instance become running
         InstanceUtil.waitTillInstanceReachState(cluster2OC, Util.readEntityName(feed), 1,

http://git-wip-us.apache.org/repos/asf/falcon/blob/1aac3a50/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ProcessLateRerunTest.java
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ProcessLateRerunTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ProcessLateRerunTest.java
index 338d90a..dc032f1 100644
--- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ProcessLateRerunTest.java
+++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ProcessLateRerunTest.java
@@ -301,7 +301,7 @@ public class ProcessLateRerunTest extends BaseTestClass {
                 for (String location : missingDependencies) {
                     if (tempCount==1) {
                         LOGGER.info("Transferring data to : " + location);
-                        HadoopUtil.copyDataToFolder(clusterFS, location, OSUtil.RESOURCES + "feed-s4Replication.xml");
+                        HadoopUtil.copyDataToFolder(clusterFS, location, OSUtil.NORMAL_INPUT + "dataFile.xml");
                         tempCount++;
                     }
                 }
@@ -320,7 +320,7 @@ public class ProcessLateRerunTest extends BaseTestClass {
             for (String dependency : missingDependencies) {
                 if (tempCounter==dataFolder) {
                     LOGGER.info("Transferring late data to : " + dependency);
-                    HadoopUtil.copyDataToFolder(clusterFS, dependency, OSUtil.RESOURCES + "log4j.properties");
+                    HadoopUtil.copyDataToFolder(clusterFS, dependency, OSUtil.NORMAL_INPUT + "dataFile.properties");
                 }
                 tempCounter++;
             }

http://git-wip-us.apache.org/repos/asf/falcon/blob/1aac3a50/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ProcessLibPathTest.java
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ProcessLibPathTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ProcessLibPathTest.java
index 7cb98cf..bc2978f 100644
--- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ProcessLibPathTest.java
+++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ProcessLibPathTest.java
@@ -118,7 +118,7 @@ public class ProcessLibPathTest extends BaseTestClass {
         HadoopUtil.uploadDir(clusterFS, workflowDir, OSUtil.RESOURCES_OOZIE);
         HadoopUtil.recreateDir(clusterFS, workflowDir + "/lib");
         HadoopUtil.copyDataToFolder(clusterFS, workflowDir + "/lib/invalid.jar",
-            OSUtil.RESOURCES + "feed-s4Replication.xml");
+            OSUtil.NORMAL_INPUT + "dataFile.xml");
         bundles[0].setProcessWorkflow(workflowDir);
         LOGGER.info("processData: " + Util.prettyPrintXml(process));
         bundles[0].submitFeedsScheduleProcess(prism);

http://git-wip-us.apache.org/repos/asf/falcon/blob/1aac3a50/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/hcat/HCatRetentionTest.java
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/hcat/HCatRetentionTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/hcat/HCatRetentionTest.java
index 83554e2..2213c1d 100644
--- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/hcat/HCatRetentionTest.java
+++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/hcat/HCatRetentionTest.java
@@ -117,7 +117,7 @@ public class HCatRetentionTest extends BaseTestClass {
                 freqType.getFormatter());
             AssertUtil.checkForListSizes(dataDates, dataDateStrings);
             final List<String> dataFolders = HadoopUtil.flattenAndPutDataInFolder(clusterFS,
-                OSUtil.OOZIE_EXAMPLE_INPUT_LATE_INPUT, baseTestHDFSDir, dataDateStrings);
+                OSUtil.SINGLE_FILE, baseTestHDFSDir, dataDateStrings);
             addPartitionsToExternalTable(cli, dBName, tableName, freqType, dataDates, dataFolders);
             List<String> initialData =
                 getHadoopDataFromDir(clusterFS, baseTestHDFSDir, testDir, freqType);

http://git-wip-us.apache.org/repos/asf/falcon/blob/1aac3a50/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/PrismFeedReplicationPartitionExpTest.java
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/PrismFeedReplicationPartitionExpTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/PrismFeedReplicationPartitionExpTest.java
index b51c4a4..97d4e67 100755
--- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/PrismFeedReplicationPartitionExpTest.java
+++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/PrismFeedReplicationPartitionExpTest.java
@@ -72,13 +72,10 @@ public class PrismFeedReplicationPartitionExpTest extends BaseTestClass {
     private String testDirWithDate = testBaseDir1 + testDate;
     private String testDirWithDateSourceTarget = testBaseDir4 + testDate;
     private String testDirWithDateSource1 = testBaseDirServer1Source + testDate;
-    private String testFile1 = OSUtil.RESOURCES
-        + OSUtil.getPath("ReplicationResources", "feed-s4Replication.xml");
-    private String testFile2 = OSUtil.RESOURCES + OSUtil.getPath("ReplicationResources", "id.pig");
-    private String testFile3 = OSUtil.RESOURCES
-        + OSUtil.getPath("ReplicationResources", "cluster-0.1.xml");
-    private String testFile4 = OSUtil.RESOURCES
-        + OSUtil.getPath("ReplicationResources", "log4testng.properties");
+    private String testFile1 = OSUtil.NORMAL_INPUT + "dataFile.xml";
+    private String testFile2 = OSUtil.RESOURCES + OSUtil.getPath("pig", "id.pig");
+    private String testFile3 = OSUtil.RESOURCES + OSUtil.getPath("ELbundle", "cluster-0.1.xml");
+    private String testFile4 = OSUtil.NORMAL_INPUT + "dataFile.properties";
     private static final Logger LOGGER =
         Logger.getLogger(PrismFeedReplicationPartitionExpTest.class);
 

http://git-wip-us.apache.org/repos/asf/falcon/blob/1aac3a50/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/RetentionTest.java
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/RetentionTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/RetentionTest.java
index c049a7e..99e240d 100644
--- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/RetentionTest.java
+++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/RetentionTest.java
@@ -146,8 +146,7 @@ public class RetentionTest extends BaseTestClass {
         LOGGER.info("dataDates = " + dataDates);
         dataDates.add(HadoopUtil.SOMETHING_RANDOM);
         if (withData) {
-            HadoopUtil.flattenAndPutDataInFolder(clusterFS, OSUtil.RESOURCES + "log_01.txt",
-                testHDFSDir, dataDates);
+            HadoopUtil.flattenAndPutDataInFolder(clusterFS, OSUtil.SINGLE_FILE, testHDFSDir, dataDates);
         } else {
             HadoopUtil.createFolders(clusterFS, testHDFSDir, dataDates);
         }

http://git-wip-us.apache.org/repos/asf/falcon/blob/1aac3a50/falcon-regression/merlin/src/test/resources/OozieExampleInputData/2ndLateData/Configuration.java.ignore
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin/src/test/resources/OozieExampleInputData/2ndLateData/Configuration.java.ignore b/falcon-regression/merlin/src/test/resources/OozieExampleInputData/2ndLateData/Configuration.java.ignore
deleted file mode 100644
index 1ab0140..0000000
--- a/falcon-regression/merlin/src/test/resources/OozieExampleInputData/2ndLateData/Configuration.java.ignore
+++ /dev/null
@@ -1,73 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ivory.entity.common;
-
-import java.util.Iterator;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.concurrent.ConcurrentHashMap;
-
-public class Configuration implements Iterable<Map.Entry<String, String>>, Cloneable {
-
-  private final Map<String, String> properties;
-
-  public Configuration() {
-    properties = new ConcurrentHashMap<String, String>();
-  }
-
-  public Configuration(Map<String, String> properties) {
-    this.properties = properties;
-  }
-
-  public void addConfiguration(Configuration config) {
-    for (Entry<String, String> entry : config) {
-      properties.put(entry.getKey(), entry.getValue());
-    }
-  }
-
-  public Configuration addAndReturnNewConfiguration(Configuration config) {
-    Map<String, String> newProperties = new ConcurrentHashMap<String, String>(properties);
-    for (Entry<String, String> entry : config) {
-      newProperties.put(entry.getKey(), entry.getValue());
-    }
-    return new Configuration(newProperties);
-  }
-
-  public String getConf(String name) {
-    return properties.get(name);
-  }
-
-  public void setConf(String name, String value) {
-    properties.put(name, value);
-  }
-
-  public void setConf(String name, String value, String defaultValue) {
-    if (value == null) {
-      properties.put(name, defaultValue);
-    } else {
-      properties.put(name, value);
-    }
-  }
-
-  @Override
-  public Iterator<Entry<String, String>> iterator() {
-    return properties.entrySet().iterator();
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/falcon/blob/1aac3a50/falcon-regression/merlin/src/test/resources/OozieExampleInputData/2ndLateData/DateValidator.java.ignore
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin/src/test/resources/OozieExampleInputData/2ndLateData/DateValidator.java.ignore b/falcon-regression/merlin/src/test/resources/OozieExampleInputData/2ndLateData/DateValidator.java.ignore
deleted file mode 100644
index f02ed34..0000000
--- a/falcon-regression/merlin/src/test/resources/OozieExampleInputData/2ndLateData/DateValidator.java.ignore
+++ /dev/null
@@ -1,87 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.ivory.entity.common;
-
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
-public class DateValidator {
-
-	private Pattern pattern;
-	private Matcher matcher;
-
-	private static final String DATE_PATTERN = "(2\\d\\d\\d|19\\d\\d)-(0[1-9]|1[012])-(0[1-9]|1[0-9]|2[0-9]|3[01])T([0-1][0-9]|2[0-3]):([0-5][0-9])Z";
-
-	public DateValidator() {
-		pattern = Pattern.compile(DATE_PATTERN);
-	}
-
-	/**
-	 * Validate date format with regular expression
-	 * 
-	 * @param date
-	 *            date address for validation
-	 * @return true valid date fromat, false invalid date format
-	 */
-	public boolean validate(final String date) {
-
-		matcher = pattern.matcher(date);
-
-		if (matcher.matches()) {
-
-			matcher.reset();
-
-			if (matcher.find()) {
-
-				int year = Integer.parseInt(matcher.group(1));
-				String month = matcher.group(2);
-				String day = matcher.group(3);
-
-				if (day.equals("31")
-						&& (month.equals("4") || month.equals("6")
-								|| month.equals("9") || month.equals("11")
-								|| month.equals("04") || month.equals("06") || month
-									.equals("09"))) {
-					return false; // only 1,3,5,7,8,10,12 has 31 days
-				} else if (month.equals("2") || month.equals("02")) {
-					// leap year
-					if (year % 4 == 0) {
-						if (day.equals("30") || day.equals("31")) {
-							return false;
-						} else {
-							return true;
-						}
-					} else {
-						if (day.equals("29") || day.equals("30")
-								|| day.equals("31")) {
-							return false;
-						} else {
-							return true;
-						}
-					}
-				} else {
-					return true;
-				}
-			} else {
-				return false;
-			}
-		} else {
-			return false;
-		}
-	}
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/falcon/blob/1aac3a50/falcon-regression/merlin/src/test/resources/OozieExampleInputData/2ndLateData/TimeUnit.java.ignore
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin/src/test/resources/OozieExampleInputData/2ndLateData/TimeUnit.java.ignore b/falcon-regression/merlin/src/test/resources/OozieExampleInputData/2ndLateData/TimeUnit.java.ignore
deleted file mode 100644
index 67f9d4c..0000000
--- a/falcon-regression/merlin/src/test/resources/OozieExampleInputData/2ndLateData/TimeUnit.java.ignore
+++ /dev/null
@@ -1,36 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ivory.entity.common;
-
-import java.util.Calendar;
-
-public enum TimeUnit {
-    MINUTE(Calendar.MINUTE), HOUR(Calendar.HOUR), DAY(Calendar.DATE), MONTH(Calendar.MONTH), END_OF_DAY(Calendar.DATE), END_OF_MONTH(
-        Calendar.MONTH), NONE(-1);
-
-    private int calendarUnit;
-
-    private TimeUnit(int calendarUnit) {
-        this.calendarUnit = calendarUnit;
-    }
-
-    public int getCalendarUnit() {
-        return calendarUnit;
-    }
-}

http://git-wip-us.apache.org/repos/asf/falcon/blob/1aac3a50/falcon-regression/merlin/src/test/resources/OozieExampleInputData/2ndLateData/dataFile1.txt
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin/src/test/resources/OozieExampleInputData/2ndLateData/dataFile1.txt b/falcon-regression/merlin/src/test/resources/OozieExampleInputData/2ndLateData/dataFile1.txt
new file mode 100644
index 0000000..1ab0140
--- /dev/null
+++ b/falcon-regression/merlin/src/test/resources/OozieExampleInputData/2ndLateData/dataFile1.txt
@@ -0,0 +1,73 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ivory.entity.common;
+
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.concurrent.ConcurrentHashMap;
+
+public class Configuration implements Iterable<Map.Entry<String, String>>, Cloneable {
+
+  private final Map<String, String> properties;
+
+  public Configuration() {
+    properties = new ConcurrentHashMap<String, String>();
+  }
+
+  public Configuration(Map<String, String> properties) {
+    this.properties = properties;
+  }
+
+  public void addConfiguration(Configuration config) {
+    for (Entry<String, String> entry : config) {
+      properties.put(entry.getKey(), entry.getValue());
+    }
+  }
+
+  public Configuration addAndReturnNewConfiguration(Configuration config) {
+    Map<String, String> newProperties = new ConcurrentHashMap<String, String>(properties);
+    for (Entry<String, String> entry : config) {
+      newProperties.put(entry.getKey(), entry.getValue());
+    }
+    return new Configuration(newProperties);
+  }
+
+  public String getConf(String name) {
+    return properties.get(name);
+  }
+
+  public void setConf(String name, String value) {
+    properties.put(name, value);
+  }
+
+  public void setConf(String name, String value, String defaultValue) {
+    if (value == null) {
+      properties.put(name, defaultValue);
+    } else {
+      properties.put(name, value);
+    }
+  }
+
+  @Override
+  public Iterator<Entry<String, String>> iterator() {
+    return properties.entrySet().iterator();
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/falcon/blob/1aac3a50/falcon-regression/merlin/src/test/resources/OozieExampleInputData/2ndLateData/dataFile2.txt
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin/src/test/resources/OozieExampleInputData/2ndLateData/dataFile2.txt b/falcon-regression/merlin/src/test/resources/OozieExampleInputData/2ndLateData/dataFile2.txt
new file mode 100644
index 0000000..e59db94
--- /dev/null
+++ b/falcon-regression/merlin/src/test/resources/OozieExampleInputData/2ndLateData/dataFile2.txt
@@ -0,0 +1,87 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ivory.entity.common;
+
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+public class DateValidator {
+
+	private Pattern pattern;
+	private Matcher matcher;
+
+	private static final String DATE_PATTERN = "(2\\d\\d\\d|19\\d\\d)-(0[1-9]|1[012])-(0[1-9]|1[0-9]|2[0-9]|3[01])T([0-1][0-9]|2[0-3]):([0-5][0-9])Z";
+
+	public DateValidator() {
+		pattern = Pattern.compile(DATE_PATTERN);
+	}
+
+	/**
+	 * Validate date format with regular expression
+	 * 
+	 * @param date
+	 *            date address for validation
+	 * @return true valid date fromat, false invalid date format
+	 */
+	public boolean validate(final String date) {
+
+		matcher = pattern.matcher(date);
+
+		if (matcher.matches()) {
+
+			matcher.reset();
+
+			if (matcher.find()) {
+
+				int year = Integer.parseInt(matcher.group(1));
+				String month = matcher.group(2);
+				String day = matcher.group(3);
+
+				if (day.equals("31")
+						&& (month.equals("4") || month.equals("6")
+								|| month.equals("9") || month.equals("11")
+								|| month.equals("04") || month.equals("06") || month
+									.equals("09"))) {
+					return false; // only 1,3,5,7,8,10,12 has 31 days
+				} else if (month.equals("2") || month.equals("02")) {
+					// leap year
+					if (year % 4 == 0) {
+						if (day.equals("30") || day.equals("31")) {
+							return false;
+						} else {
+							return true;
+						}
+					} else {
+						if (day.equals("29") || day.equals("30")
+								|| day.equals("31")) {
+							return false;
+						} else {
+							return true;
+						}
+					}
+				} else {
+					return true;
+				}
+			} else {
+				return false;
+			}
+		} else {
+			return false;
+		}
+	}
+}

http://git-wip-us.apache.org/repos/asf/falcon/blob/1aac3a50/falcon-regression/merlin/src/test/resources/OozieExampleInputData/2ndLateData/dataFile3.txt
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin/src/test/resources/OozieExampleInputData/2ndLateData/dataFile3.txt b/falcon-regression/merlin/src/test/resources/OozieExampleInputData/2ndLateData/dataFile3.txt
new file mode 100644
index 0000000..67f9d4c
--- /dev/null
+++ b/falcon-regression/merlin/src/test/resources/OozieExampleInputData/2ndLateData/dataFile3.txt
@@ -0,0 +1,36 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ivory.entity.common;
+
+import java.util.Calendar;
+
+public enum TimeUnit {
+    MINUTE(Calendar.MINUTE), HOUR(Calendar.HOUR), DAY(Calendar.DATE), MONTH(Calendar.MONTH), END_OF_DAY(Calendar.DATE), END_OF_MONTH(
+        Calendar.MONTH), NONE(-1);
+
+    private int calendarUnit;
+
+    private TimeUnit(int calendarUnit) {
+        this.calendarUnit = calendarUnit;
+    }
+
+    public int getCalendarUnit() {
+        return calendarUnit;
+    }
+}

http://git-wip-us.apache.org/repos/asf/falcon/blob/1aac3a50/falcon-regression/merlin/src/test/resources/OozieExampleInputData/lateData/log_15.txt
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin/src/test/resources/OozieExampleInputData/lateData/log_15.txt b/falcon-regression/merlin/src/test/resources/OozieExampleInputData/lateData/log_15.txt
deleted file mode 100644
index f3f94c5..0000000
--- a/falcon-regression/merlin/src/test/resources/OozieExampleInputData/lateData/log_15.txt
+++ /dev/null
@@ -1,19 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-[15] LOG!!!!!!
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/falcon/blob/1aac3a50/falcon-regression/merlin/src/test/resources/OozieExampleInputData/normalInput/EntityInstanceMessage.java.ignore
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin/src/test/resources/OozieExampleInputData/normalInput/EntityInstanceMessage.java.ignore b/falcon-regression/merlin/src/test/resources/OozieExampleInputData/normalInput/EntityInstanceMessage.java.ignore
deleted file mode 100644
index f992f21..0000000
--- a/falcon-regression/merlin/src/test/resources/OozieExampleInputData/normalInput/EntityInstanceMessage.java.ignore
+++ /dev/null
@@ -1,412 +0,0 @@
-///**
-// * Licensed to the Apache Software Foundation (ASF) under one
-// * or more contributor license agreements.  See the NOTICE file
-// * distributed with this work for additional information
-// * regarding copyright ownership.  The ASF licenses this file
-// * to you under the Apache License, Version 2.0 (the
-// * "License"); you may not use this file except in compliance
-// * with the License.  You may obtain a copy of the License at
-// *
-// *     http://www.apache.org/licenses/LICENSE-2.0
-// *
-// * Unless required by applicable law or agreed to in writing, software
-// * distributed under the License is distributed on an "AS IS" BASIS,
-// * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// * See the License for the specific language governing permissions and
-// * limitations under the License.
-// */
-//
-//package org.apache.ivory.messaging;
-//
-//import org.apache.hadoop.conf.Configuration;
-//import org.apache.hadoop.fs.FileSystem;
-//import org.apache.hadoop.fs.Path;
-//import org.apache.hadoop.io.IOUtils;
-//import org.apache.log4j.Logger;
-//
-//import java.io.ByteArrayOutputStream;
-//import java.io.IOException;
-//import java.io.InputStream;
-//import java.util.HashMap;
-//import java.util.Map;
-//
-///**
-// * Value Object which is stored in JMS Topic as TextMessage
-// *
-// */
-//public class EntityInstanceMessage {
-//
-//	private final Map<ARG, String> msgs = new HashMap<ARG, String>();
-//	private static final String MSG_SEPERATOR = "$";
-//	private static final int ARG_LENGTH = EntityInstanceMessage.ARG.values().length;
-//	private static final Logger LOG = Logger
-//			.getLogger(EntityInstanceMessage.class);
-//	private static final String IVORY_PROCESS_TOPIC_NAME = "IVORY.PROCESS.TOPIC";
-//
-//	/**
-//	 * Enum for arguments that are used in coordinators to pass arguments to
-//	 * parent workflow
-//	 */
-//
-//	public enum entityOperation {
-//		GENERATE, DELETE, ARCHIVE, REPLICATE, CHMOD
-//	}
-//
-//	public enum ARG {
-//		PROCESS_NAME(0, "entityName"), FEED_NAME(1, "feedNames"), FEED_INSTANCE_PATH(
-//				2, "feedInstancePaths"), WORKFLOW_ID(3, "workflowId"), RUN_ID(
-//				4, "runId"), NOMINAL_TIME(5, "nominalTime"), TIME_STAMP(6,
-//				"timeStamp"), BROKER_URL(7, "brokerUrl"), BROKER_IMPL_CLASS(8,
-//				"brokerImplClass"), ENTITY_TYPE(9, "entityType"), OPERATION(10,
-//				"operation"), LOG_FILE(11, "logFile"), TOPIC_NAME(12,
-//				"topicName"), STATUS(13, "status"), BROKER_TTL(14, "broker.ttlInMins");
-//
-//		private int argOrder;
-//		private String argName;
-//
-//		private ARG(int argOrder, String argName) {
-//			this.argOrder = argOrder;
-//			this.argName = argName;
-//		}
-//
-//		public int ORDER() {
-//			return this.argOrder;
-//		}
-//
-//		public String NAME() {
-//			return this.argName;
-//		}
-//	}
-//
-//	public String getProcessName() {
-//		return this.msgs.get(ARG.PROCESS_NAME);
-//	}
-//
-//	public void setProcessName(String processName) {
-//		this.msgs.put(ARG.PROCESS_NAME, processName);
-//	}
-//
-//	public String getTopicName() {
-//		return this.msgs.get(ARG.TOPIC_NAME);
-//	}
-//
-//	public void setTopicName(String topicName) {
-//		this.msgs.put(ARG.TOPIC_NAME, topicName);
-//	}
-//
-//	public String getFeedName() {
-//		return this.msgs.get(ARG.FEED_NAME);
-//	}
-//
-//	public void setFeedName(String feedName) {
-//		this.msgs.put(ARG.FEED_NAME, feedName);
-//	}
-//
-//	public String getFeedInstancePath() {
-//		return this.msgs.get(ARG.FEED_INSTANCE_PATH);
-//	}
-//
-//	public void setFeedInstancePath(String feedInstancePath) {
-//		this.msgs.put(ARG.FEED_INSTANCE_PATH, feedInstancePath);
-//	}
-//
-//	public String getWorkflowId() {
-//		return this.msgs.get(ARG.WORKFLOW_ID);
-//	}
-//
-//	public void setWorkflowId(String workflowId) {
-//		this.msgs.put(ARG.WORKFLOW_ID, workflowId);
-//	}
-//
-//	public String getRunId() {
-//		return this.msgs.get(ARG.RUN_ID);
-//	}
-//
-//	public void setRunId(String runId) {
-//		this.msgs.put(ARG.RUN_ID, runId);
-//	}
-//
-//	public String getNominalTime() {
-//		return this.msgs.get(ARG.NOMINAL_TIME);
-//	}
-//
-//	public void setNominalTime(String nominalTime) {
-//		this.msgs.put(ARG.NOMINAL_TIME, nominalTime);
-//	}
-//
-//	public String getTimeStamp() {
-//		return this.msgs.get(ARG.TIME_STAMP);
-//	}
-//
-//	public void setTimeStamp(String timeStamp) {
-//		this.msgs.put(ARG.TIME_STAMP, timeStamp);
-//	}
-//
-//	public String getBrokerUrl() {
-//		return this.msgs.get(ARG.BROKER_URL);
-//	}
-//
-//	public void setBrokerUrl(String brokerUrl) {
-//		this.msgs.put(ARG.BROKER_URL, brokerUrl);
-//	}
-//
-//	public String getBrokerImplClass() {
-//		return this.msgs.get(ARG.BROKER_IMPL_CLASS);
-//	}
-//
-//	public void setBrokerImplClass(String brokerImplClass) {
-//		this.msgs.put(ARG.BROKER_IMPL_CLASS, brokerImplClass);
-//	}
-//
-//	public String getEntityType() {
-//		return this.msgs.get(ARG.ENTITY_TYPE);
-//	}
-//
-//	public void setEntityType(String entityType) {
-//		this.msgs.put(ARG.ENTITY_TYPE, entityType);
-//	}
-//
-//	public String getOperation() {
-//		return this.msgs.get(ARG.OPERATION);
-//	}
-//
-//	public void setOperation(String operation) {
-//		this.msgs.put(ARG.OPERATION, operation);
-//	}
-//
-//	public String getLogFile() {
-//		return this.msgs.get(ARG.LOG_FILE);
-//	}
-//
-//	public void setLogFile(String logFile) {
-//		this.msgs.put(ARG.LOG_FILE, logFile);
-//	}
-//
-//	public String getStatus() {
-//		return this.msgs.get(ARG.STATUS);
-//	}
-//
-//	public void setStatus(String status) {
-//		this.msgs.put(ARG.STATUS, status);
-//	}
-//
-//	public String getBrokerTTL() {
-//		return this.msgs.get(ARG.BROKER_TTL);
-//	}
-//
-//	public void setBrokerTTL(String brokerTTL) {
-//		this.msgs.put(ARG.BROKER_TTL, brokerTTL);
-//	}
-//
-//	@Override
-//	public String toString() {
-//		if (getEntityType().equalsIgnoreCase("PROCESS")
-//				&& getTopicName().equals(IVORY_PROCESS_TOPIC_NAME)) {
-//			return getIvoryMessage();
-//		}
-//		if (getEntityType().equalsIgnoreCase("FEED")) {
-//			return getFeedMessage();
-//		}
-//		return getProcessMessage();
-//
-//	}
-//
-//	private String getProcessMessage() {
-//		return getProcessName() + MSG_SEPERATOR + getFeedName() + MSG_SEPERATOR
-//				+ getFeedInstancePath() + MSG_SEPERATOR + getWorkflowId()
-//				+ MSG_SEPERATOR + getRunId() + MSG_SEPERATOR + getNominalTime()
-//				+ MSG_SEPERATOR + getTimeStamp();
-//	}
-//
-//	private String getIvoryMessage() {
-//		return getProcessName() + MSG_SEPERATOR + getFeedName() + MSG_SEPERATOR
-//				+ getFeedInstancePath() + MSG_SEPERATOR + getWorkflowId()
-//				+ MSG_SEPERATOR + getRunId() + MSG_SEPERATOR + getNominalTime()
-//				+ MSG_SEPERATOR + getTimeStamp() + MSG_SEPERATOR + getStatus();
-//	}
-//
-//	private String getFeedMessage() {
-//		return getFeedName() + MSG_SEPERATOR
-//				+ getFeedInstancePath() + MSG_SEPERATOR + getOperation()
-//				+ MSG_SEPERATOR + getWorkflowId() + MSG_SEPERATOR + getRunId()
-//				+ MSG_SEPERATOR + getNominalTime() + MSG_SEPERATOR
-//				+ getTimeStamp();
-//	}
-//
-//	/**
-//	 *
-//	 * @param args
-//	 *            - String array passed from oozie action for jms messaging
-//	 * @return ProcessMessage - Value object which is stored in JMS topic
-//	 */
-//	public static EntityInstanceMessage[] argsToMessage(String[] args) {
-//
-//		assert args.length == ARG_LENGTH : "Required number of arguments: "
-//				+ ARG_LENGTH;
-//
-//		String[] feedNames = getFeedNames(args);
-//
-//		String[] feedPaths;
-//		try {
-//			feedPaths = getFeedPaths(args);
-//		} catch (IOException e) {
-//			LOG.error("Error getting instance paths: ", e);
-//			throw new RuntimeException(e);
-//		}
-//
-//		EntityInstanceMessage[] processMessages = new EntityInstanceMessage[feedPaths.length];
-//		for (int i = 0; i < feedPaths.length; i++) {
-//			EntityInstanceMessage instanceMessage = new EntityInstanceMessage();
-//			instanceMessage
-//					.setProcessName(args[EntityInstanceMessage.ARG.PROCESS_NAME
-//							.ORDER()]);
-//			if (args[EntityInstanceMessage.ARG.ENTITY_TYPE.ORDER()]
-//					.equalsIgnoreCase("PROCESS")) {
-//				instanceMessage.setFeedName(feedNames[i]);
-//			} else {
-//				instanceMessage
-//						.setFeedName(args[EntityInstanceMessage.ARG.FEED_NAME
-//								.ORDER()]);
-//			}
-//			instanceMessage.setFeedInstancePath(feedPaths[i]);
-//			instanceMessage
-//					.setWorkflowId(args[EntityInstanceMessage.ARG.WORKFLOW_ID
-//							.ORDER()]);
-//			instanceMessage.setRunId(args[EntityInstanceMessage.ARG.RUN_ID
-//					.ORDER()]);
-//			instanceMessage
-//					.setNominalTime(args[EntityInstanceMessage.ARG.NOMINAL_TIME
-//							.ORDER()]);
-//			instanceMessage
-//					.setTimeStamp(args[EntityInstanceMessage.ARG.TIME_STAMP
-//							.ORDER()]);
-//			instanceMessage
-//					.setBrokerUrl(args[EntityInstanceMessage.ARG.BROKER_URL
-//							.ORDER()]);
-//			instanceMessage
-//					.setBrokerImplClass(args[EntityInstanceMessage.ARG.BROKER_IMPL_CLASS
-//							.ORDER()]);
-//			instanceMessage
-//					.setEntityType(args[EntityInstanceMessage.ARG.ENTITY_TYPE
-//							.ORDER()]);
-//			instanceMessage
-//					.setOperation(args[EntityInstanceMessage.ARG.OPERATION
-//							.ORDER()]);
-//			instanceMessage.setLogFile(args[EntityInstanceMessage.ARG.LOG_FILE
-//					.ORDER()]);
-//			instanceMessage
-//					.setTopicName(args[EntityInstanceMessage.ARG.TOPIC_NAME
-//							.ORDER()]);
-//			instanceMessage.setStatus(args[EntityInstanceMessage.ARG.STATUS
-//					.ORDER()]);
-//			instanceMessage.setBrokerTTL(args[EntityInstanceMessage.ARG.BROKER_TTL
-//			           					.ORDER()]);
-//
-//			processMessages[i] = instanceMessage;
-//		}
-//		return processMessages;
-//	}
-//
-//	private static String[] getFeedNames(String[] args) {
-//		String topicName = args[ARG.TOPIC_NAME.argOrder];
-//		if (topicName.equals(IVORY_PROCESS_TOPIC_NAME)) {
-//			return new String[] { args[EntityInstanceMessage.ARG.FEED_NAME
-//					.ORDER()] };
-//		}
-//		return args[EntityInstanceMessage.ARG.FEED_NAME.ORDER()].split(",");
-//	}
-//
-//	private static String[] getFeedPaths(String[] args) throws IOException {
-//		String entityType = args[EntityInstanceMessage.ARG.ENTITY_TYPE.ORDER()];
-//		String topicName = args[EntityInstanceMessage.ARG.TOPIC_NAME.ORDER()];
-//
-//		if (entityType.equalsIgnoreCase("PROCESS")
-//				&& topicName.equals(IVORY_PROCESS_TOPIC_NAME)) {
-//			LOG.debug("Returning instance paths for Ivory Topic: "
-//					+ args[EntityInstanceMessage.ARG.FEED_INSTANCE_PATH.ORDER()]);
-//			return new String[] { args[EntityInstanceMessage.ARG.FEED_INSTANCE_PATH
-//					.ORDER()] };
-//		}
-//
-//		if (entityType.equalsIgnoreCase("PROCESS")) {
-//			LOG.debug("Returning instance paths for process: "
-//					+ args[EntityInstanceMessage.ARG.FEED_INSTANCE_PATH.ORDER()]);
-//			return args[EntityInstanceMessage.ARG.FEED_INSTANCE_PATH.ORDER()]
-//					.split(",");
-//		}
-//		//
-//		Path logFile = new Path(
-//				args[EntityInstanceMessage.ARG.LOG_FILE.ORDER()]);
-//		FileSystem fs = FileSystem.get(logFile.toUri(), new Configuration());
-//		ByteArrayOutputStream writer = new ByteArrayOutputStream();
-//		InputStream instance = fs.open(logFile);
-//		IOUtils.copyBytes(instance, writer, 4096, true);
-//		String[] instancePaths = writer.toString().split("=");
-//		if (instancePaths.length == 1) {
-//			LOG.debug("Returning 0 instance paths for feed ");
-//			return new String[0];
-//		} else {
-//			LOG.debug("Returning instance paths for feed " + instancePaths[1]);
-//			return instancePaths[1].split(",");
-//		}
-//
-//	}
-//
-//	/**
-//	 *
-//	 * @param instanceMessages
-//	 *            - value object which is stored in JMS topic as TextMessage
-//	 * @return - String array.
-//	 */
-//	public static String[] messageToArgs(
-//			EntityInstanceMessage[] instanceMessages) {
-//		String[] args = new String[ARG_LENGTH];
-//
-//		args[EntityInstanceMessage.ARG.PROCESS_NAME.ORDER()] = instanceMessages[0]
-//				.getProcessName();
-//		StringBuilder feedNames = new StringBuilder();
-//		StringBuilder feedPaths = new StringBuilder();
-//
-//		for (EntityInstanceMessage instanceMessage : instanceMessages) {
-//			feedNames.append(instanceMessage.getFeedName()).append(",");
-//			feedPaths.append(instanceMessage.getFeedInstancePath()).append(",");
-//		}
-//		if (instanceMessages[0].getEntityType().equalsIgnoreCase("PROCESS")) {
-//			args[EntityInstanceMessage.ARG.FEED_NAME.ORDER()] = feedNames
-//					.toString();
-//
-//		} else {
-//			args[EntityInstanceMessage.ARG.FEED_NAME.ORDER()] = instanceMessages[0]
-//					.getFeedName();
-//		}
-//		args[EntityInstanceMessage.ARG.FEED_INSTANCE_PATH.ORDER()] = feedPaths
-//				.toString();
-//		args[EntityInstanceMessage.ARG.WORKFLOW_ID.ORDER()] = instanceMessages[0]
-//				.getWorkflowId();
-//		args[EntityInstanceMessage.ARG.RUN_ID.ORDER()] = instanceMessages[0]
-//				.getRunId();
-//		args[EntityInstanceMessage.ARG.NOMINAL_TIME.ORDER()] = instanceMessages[0]
-//				.getNominalTime();
-//		args[EntityInstanceMessage.ARG.TIME_STAMP.ORDER()] = instanceMessages[0]
-//				.getTimeStamp();
-//		args[EntityInstanceMessage.ARG.BROKER_URL.ORDER()] = instanceMessages[0]
-//				.getBrokerUrl();
-//		args[EntityInstanceMessage.ARG.BROKER_IMPL_CLASS.ORDER()] = instanceMessages[0]
-//				.getBrokerImplClass();
-//		args[EntityInstanceMessage.ARG.ENTITY_TYPE.ORDER()] = instanceMessages[0]
-//				.getEntityType();
-//		args[EntityInstanceMessage.ARG.OPERATION.ORDER()] = instanceMessages[0]
-//				.getOperation();
-//		args[EntityInstanceMessage.ARG.LOG_FILE.ORDER()] = instanceMessages[0]
-//				.getLogFile();
-//		args[EntityInstanceMessage.ARG.TOPIC_NAME.ORDER()] = instanceMessages[0]
-//				.getTopicName();
-//		args[EntityInstanceMessage.ARG.STATUS.ORDER()] = instanceMessages[0]
-//				.getStatus();
-//		args[EntityInstanceMessage.ARG.BROKER_TTL.ORDER()] = instanceMessages[0]
-//				.getBrokerTTL();
-//
-//		return args;
-//	}
-//
-//}

http://git-wip-us.apache.org/repos/asf/falcon/blob/1aac3a50/falcon-regression/merlin/src/test/resources/OozieExampleInputData/normalInput/EntityInstanceMessageCreator.java.ignore
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin/src/test/resources/OozieExampleInputData/normalInput/EntityInstanceMessageCreator.java.ignore b/falcon-regression/merlin/src/test/resources/OozieExampleInputData/normalInput/EntityInstanceMessageCreator.java.ignore
deleted file mode 100644
index f7e8bbb..0000000
--- a/falcon-regression/merlin/src/test/resources/OozieExampleInputData/normalInput/EntityInstanceMessageCreator.java.ignore
+++ /dev/null
@@ -1,61 +0,0 @@
-///**
-// * Licensed to the Apache Software Foundation (ASF) under one
-// * or more contributor license agreements.  See the NOTICE file
-// * distributed with this work for additional information
-// * regarding copyright ownership.  The ASF licenses this file
-// * to you under the Apache License, Version 2.0 (the
-// * "License"); you may not use this file except in compliance
-// * with the License.  You may obtain a copy of the License at
-// *
-// *     http://www.apache.org/licenses/LICENSE-2.0
-// *
-// * Unless required by applicable law or agreed to in writing, software
-// * distributed under the License is distributed on an "AS IS" BASIS,
-// * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// * See the License for the specific language governing permissions and
-// * limitations under the License.
-// */
-//
-//package org.apache.ivory.messaging;
-//
-//import javax.jms.JMSException;
-//import javax.jms.Session;
-//import javax.jms.TextMessage;
-//
-//import org.apache.log4j.Logger;
-//
-///**
-// * Ivory JMS message creator- creates JMS TextMessage
-// */
-//public class EntityInstanceMessageCreator  {
-//
-//	private static final Logger LOG = Logger
-//			.getLogger(EntityInstanceMessageCreator.class);
-//
-//	private TextMessage textMessage;
-//
-//	private final EntityInstanceMessage args;
-//
-//	public EntityInstanceMessageCreator(EntityInstanceMessage args) {
-//		this.args = args;
-//	}
-//
-//	public TextMessage createMessage(Session session) throws JMSException {
-//		this.textMessage = session.createTextMessage();
-//		this.textMessage.setText(this.args.toString());
-//		LOG.debug("Sending Message: " + this.textMessage.getText());
-//		// System.out.println("Sending Message: " + this.textMessage);
-//		return this.textMessage;
-//	}
-//
-//	@Override
-//	public String toString() {
-//		try {
-//			return this.textMessage.getText();
-//		} catch (JMSException e) {
-//			return e.getMessage();
-//		}
-//
-//	}
-//
-//}

http://git-wip-us.apache.org/repos/asf/falcon/blob/1aac3a50/falcon-regression/merlin/src/test/resources/OozieExampleInputData/normalInput/MessageProducer.java.ignore
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin/src/test/resources/OozieExampleInputData/normalInput/MessageProducer.java.ignore b/falcon-regression/merlin/src/test/resources/OozieExampleInputData/normalInput/MessageProducer.java.ignore
deleted file mode 100644
index 7234f52..0000000
--- a/falcon-regression/merlin/src/test/resources/OozieExampleInputData/normalInput/MessageProducer.java.ignore
+++ /dev/null
@@ -1,139 +0,0 @@
-///**
-// * Licensed to the Apache Software Foundation (ASF) under one
-// * or more contributor license agreements.  See the NOTICE file
-// * distributed with this work for additional information
-// * regarding copyright ownership.  The ASF licenses this file
-// * to you under the Apache License, Version 2.0 (the
-// * "License"); you may not use this file except in compliance
-// * with the License.  You may obtain a copy of the License at
-// *
-// *     http://www.apache.org/licenses/LICENSE-2.0
-// *
-// * Unless required by applicable law or agreed to in writing, software
-// * distributed under the License is distributed on an "AS IS" BASIS,
-// * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// * See the License for the specific language governing permissions and
-// * limitations under the License.
-// */
-//
-//package org.apache.ivory.messaging;
-//
-//import java.lang.reflect.InvocationTargetException;
-//
-//import javax.jms.Connection;
-//import javax.jms.ConnectionFactory;
-//import javax.jms.DeliveryMode;
-//import javax.jms.JMSException;
-//import javax.jms.Session;
-//import javax.jms.Topic;
-//
-//import org.apache.log4j.Logger;
-//
-///**
-// * Default Ivory Message Producer The configuration are loaded from
-// * jms-beans.xml
-// */
-//public class MessageProducer {
-//
-//	private Connection connection;
-//	private static final Logger LOG = Logger.getLogger(MessageProducer.class);
-//	private static final long DEFAULT_TTL = 3 * 24 * 60 * 60 * 1000;
-//
-//	/**
-//	 *
-//	 * @param arguments
-//	 *            - Accepts a Message to be send to JMS topic, creates a new
-//	 *            Topic based on topic name if it does not exist or else
-//	 *            existing topic with the same name is used to send the message.
-//	 * @throws JMSException
-//	 */
-//	protected void sendMessage(EntityInstanceMessage entityInstanceMessage)
-//			throws JMSException {
-//
-//		Session session = connection.createSession(false,
-//				Session.AUTO_ACKNOWLEDGE);
-//		Topic entityTopic = session.createTopic(entityInstanceMessage
-//				.getTopicName());
-//		javax.jms.MessageProducer producer = session
-//				.createProducer(entityTopic);
-//		producer.setDeliveryMode(DeliveryMode.PERSISTENT);
-//		long messageTTL = DEFAULT_TTL;
-//		try {
-//			long messageTTLinMins = Long.parseLong(entityInstanceMessage
-//					.getBrokerTTL());
-//			messageTTL = messageTTLinMins * 60 * 1000;
-//		} catch (NumberFormatException e) {
-//			LOG.error("Error in parsing broker.ttl, setting TTL to:"
-//					+ DEFAULT_TTL+ " milli-seconds");
-//		}
-//		producer.setTimeToLive(messageTTL);
-//		producer.send(new EntityInstanceMessageCreator(entityInstanceMessage)
-//				.createMessage(session));
-//
-//	}
-//
-//	/**
-//	 *
-//	 * @param args
-//	 *            - array of Strings, which will be used to create TextMessage
-//	 */
-//	public static void main(String[] args) {
-//		debug(args);
-//		EntityInstanceMessage[] entityInstanceMessage = EntityInstanceMessage
-//				.argsToMessage(args);
-//		if (entityInstanceMessage.length == 0) {
-//			LOG.warn("No operation on output feed");
-//			return;
-//		}
-//
-//		MessageProducer ivoryMessageProducer = new MessageProducer();
-//		try {
-//			ivoryMessageProducer.createAndStartConnection(
-//					args[EntityInstanceMessage.ARG.BROKER_IMPL_CLASS.ORDER()],
-//					"", "", entityInstanceMessage[0].getBrokerUrl());
-//			for (EntityInstanceMessage processMessage : entityInstanceMessage) {
-//				ivoryMessageProducer.sendMessage(processMessage);
-//			}
-//		} catch (JMSException e) {
-//			LOG.error(e);
-//			e.printStackTrace();
-//		} catch (Exception e) {
-//			LOG.error(e);
-//			e.printStackTrace();
-//		} finally {
-//			try {
-//				ivoryMessageProducer.connection.close();
-//			} catch (JMSException e) {
-//				e.printStackTrace();
-//			}
-//		}
-//
-//	}
-//
-//	private static void debug(String[] args) {
-//		if (LOG.isDebugEnabled()) {
-//			for (int i = 0; i < args.length; i++) {
-//				LOG.debug(args[i] + "::");
-//			}
-//		}
-//
-//	}
-//
-//	private void createAndStartConnection(String implementation,
-//			String userName, String password, String url) throws JMSException,
-//			ClassNotFoundException, IllegalArgumentException,
-//			SecurityException, InstantiationException, IllegalAccessException,
-//			InvocationTargetException, NoSuchMethodException {
-//
-//		Class<ConnectionFactory> clazz = (Class<ConnectionFactory>) MessageProducer.class
-//				.getClassLoader().loadClass(implementation);
-//
-//		ConnectionFactory connectionFactory = clazz.getConstructor(
-//				String.class, String.class, String.class).newInstance(userName,
-//				password, url);
-//
-//		connection = connectionFactory.createConnection();
-//		connection.start();
-//	}
-//
-//}

http://git-wip-us.apache.org/repos/asf/falcon/blob/1aac3a50/falcon-regression/merlin/src/test/resources/OozieExampleInputData/normalInput/dataFile.properties
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin/src/test/resources/OozieExampleInputData/normalInput/dataFile.properties b/falcon-regression/merlin/src/test/resources/OozieExampleInputData/normalInput/dataFile.properties
new file mode 100644
index 0000000..c9649d3
--- /dev/null
+++ b/falcon-regression/merlin/src/test/resources/OozieExampleInputData/normalInput/dataFile.properties
@@ -0,0 +1,30 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+
+
+#Global configuration for Ivory JMS, default to use embedded ActiveMQ.
+
+#Defualt Active MQ url
+#ivory.broker.url = tcp://localhost:61616?daemon=true
+
+#Embedded
+ivory.broker.url = vm://localhost?broker.useJmx=false&broker.persistent=true
+
+
+

http://git-wip-us.apache.org/repos/asf/falcon/blob/1aac3a50/falcon-regression/merlin/src/test/resources/OozieExampleInputData/normalInput/dataFile.xml
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin/src/test/resources/OozieExampleInputData/normalInput/dataFile.xml b/falcon-regression/merlin/src/test/resources/OozieExampleInputData/normalInput/dataFile.xml
new file mode 100644
index 0000000..e047aaf
--- /dev/null
+++ b/falcon-regression/merlin/src/test/resources/OozieExampleInputData/normalInput/dataFile.xml
@@ -0,0 +1,63 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<!--
+  Licensed to the Apache Software Foundation (ASF) under one
+  or more contributor license agreements.  See the NOTICE file
+  distributed with this work for additional information
+  regarding copyright ownership.  The ASF licenses this file
+  to you under the Apache License, Version 2.0 (the
+  "License"); you may not use this file except in compliance
+  with the License.  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+  -->
+
+<!DOCTYPE log4j:configuration SYSTEM "log4j.dtd">
+
+<log4j:configuration xmlns:log4j="http://jakarta.apache.org/log4j/">
+  <appender name="console" class="org.apache.log4j.ConsoleAppender">
+    <param name="Target" value="System.out"/>
+    <layout class="org.apache.log4j.PatternLayout">
+      <param name="ConversionPattern" value="%d %-5p - %m (%c{1}:%L)%n"/>
+    </layout>
+  </appender>
+
+  <appender name="FILE" class="org.apache.log4j.DailyRollingFileAppender">
+    <param name="File" value="/var/log/ivory/application.log"/>
+    <param name="Append" value="true"/>
+    <param name="Threshold" value="debug"/>
+    <layout class="org.apache.log4j.PatternLayout">
+      <param name="ConversionPattern" value="%d %-5p - %m (%c{1}:%L)%n"/>
+    </layout>
+  </appender>
+
+  <appender name="AUDIT" class="org.apache.log4j.DailyRollingFileAppender">
+    <param name="File" value="/var/log/ivory/audit.log"/>
+    <param name="Append" value="true"/>
+    <param name="Threshold" value="debug"/>
+    <layout class="org.apache.log4j.PatternLayout">
+      <param name="ConversionPattern" value="%d %-5p - %m%n"/>
+    </layout>
+  </appender>
+
+  <logger name="org.apache.ivory" additivity="false">
+    <level value="debug"/>
+    <appender-ref ref="console" />
+  </logger>
+
+  <logger name="AUDIT">
+    <level value="info"/>
+    <appender-ref ref="AUDIT" />
+  </logger>
+
+  <root>
+    <priority value ="info" />
+    <appender-ref ref="console" />
+  </root>
+
+</log4j:configuration>

http://git-wip-us.apache.org/repos/asf/falcon/blob/1aac3a50/falcon-regression/merlin/src/test/resources/OozieExampleInputData/normalInput/dataFile1.txt
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin/src/test/resources/OozieExampleInputData/normalInput/dataFile1.txt b/falcon-regression/merlin/src/test/resources/OozieExampleInputData/normalInput/dataFile1.txt
new file mode 100644
index 0000000..f7e8bbb
--- /dev/null
+++ b/falcon-regression/merlin/src/test/resources/OozieExampleInputData/normalInput/dataFile1.txt
@@ -0,0 +1,61 @@
+///**
+// * Licensed to the Apache Software Foundation (ASF) under one
+// * or more contributor license agreements.  See the NOTICE file
+// * distributed with this work for additional information
+// * regarding copyright ownership.  The ASF licenses this file
+// * to you under the Apache License, Version 2.0 (the
+// * "License"); you may not use this file except in compliance
+// * with the License.  You may obtain a copy of the License at
+// *
+// *     http://www.apache.org/licenses/LICENSE-2.0
+// *
+// * Unless required by applicable law or agreed to in writing, software
+// * distributed under the License is distributed on an "AS IS" BASIS,
+// * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// * See the License for the specific language governing permissions and
+// * limitations under the License.
+// */
+//
+//package org.apache.ivory.messaging;
+//
+//import javax.jms.JMSException;
+//import javax.jms.Session;
+//import javax.jms.TextMessage;
+//
+//import org.apache.log4j.Logger;
+//
+///**
+// * Ivory JMS message creator- creates JMS TextMessage
+// */
+//public class EntityInstanceMessageCreator  {
+//
+//	private static final Logger LOG = Logger
+//			.getLogger(EntityInstanceMessageCreator.class);
+//
+//	private TextMessage textMessage;
+//
+//	private final EntityInstanceMessage args;
+//
+//	public EntityInstanceMessageCreator(EntityInstanceMessage args) {
+//		this.args = args;
+//	}
+//
+//	public TextMessage createMessage(Session session) throws JMSException {
+//		this.textMessage = session.createTextMessage();
+//		this.textMessage.setText(this.args.toString());
+//		LOG.debug("Sending Message: " + this.textMessage.getText());
+//		// System.out.println("Sending Message: " + this.textMessage);
+//		return this.textMessage;
+//	}
+//
+//	@Override
+//	public String toString() {
+//		try {
+//			return this.textMessage.getText();
+//		} catch (JMSException e) {
+//			return e.getMessage();
+//		}
+//
+//	}
+//
+//}

http://git-wip-us.apache.org/repos/asf/falcon/blob/1aac3a50/falcon-regression/merlin/src/test/resources/OozieExampleInputData/normalInput/dataFile2.txt
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin/src/test/resources/OozieExampleInputData/normalInput/dataFile2.txt b/falcon-regression/merlin/src/test/resources/OozieExampleInputData/normalInput/dataFile2.txt
new file mode 100644
index 0000000..f992f21
--- /dev/null
+++ b/falcon-regression/merlin/src/test/resources/OozieExampleInputData/normalInput/dataFile2.txt
@@ -0,0 +1,412 @@
+///**
+// * Licensed to the Apache Software Foundation (ASF) under one
+// * or more contributor license agreements.  See the NOTICE file
+// * distributed with this work for additional information
+// * regarding copyright ownership.  The ASF licenses this file
+// * to you under the Apache License, Version 2.0 (the
+// * "License"); you may not use this file except in compliance
+// * with the License.  You may obtain a copy of the License at
+// *
+// *     http://www.apache.org/licenses/LICENSE-2.0
+// *
+// * Unless required by applicable law or agreed to in writing, software
+// * distributed under the License is distributed on an "AS IS" BASIS,
+// * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// * See the License for the specific language governing permissions and
+// * limitations under the License.
+// */
+//
+//package org.apache.ivory.messaging;
+//
+//import org.apache.hadoop.conf.Configuration;
+//import org.apache.hadoop.fs.FileSystem;
+//import org.apache.hadoop.fs.Path;
+//import org.apache.hadoop.io.IOUtils;
+//import org.apache.log4j.Logger;
+//
+//import java.io.ByteArrayOutputStream;
+//import java.io.IOException;
+//import java.io.InputStream;
+//import java.util.HashMap;
+//import java.util.Map;
+//
+///**
+// * Value Object which is stored in JMS Topic as TextMessage
+// *
+// */
+//public class EntityInstanceMessage {
+//
+//	private final Map<ARG, String> msgs = new HashMap<ARG, String>();
+//	private static final String MSG_SEPERATOR = "$";
+//	private static final int ARG_LENGTH = EntityInstanceMessage.ARG.values().length;
+//	private static final Logger LOG = Logger
+//			.getLogger(EntityInstanceMessage.class);
+//	private static final String IVORY_PROCESS_TOPIC_NAME = "IVORY.PROCESS.TOPIC";
+//
+//	/**
+//	 * Enum for arguments that are used in coordinators to pass arguments to
+//	 * parent workflow
+//	 */
+//
+//	public enum entityOperation {
+//		GENERATE, DELETE, ARCHIVE, REPLICATE, CHMOD
+//	}
+//
+//	public enum ARG {
+//		PROCESS_NAME(0, "entityName"), FEED_NAME(1, "feedNames"), FEED_INSTANCE_PATH(
+//				2, "feedInstancePaths"), WORKFLOW_ID(3, "workflowId"), RUN_ID(
+//				4, "runId"), NOMINAL_TIME(5, "nominalTime"), TIME_STAMP(6,
+//				"timeStamp"), BROKER_URL(7, "brokerUrl"), BROKER_IMPL_CLASS(8,
+//				"brokerImplClass"), ENTITY_TYPE(9, "entityType"), OPERATION(10,
+//				"operation"), LOG_FILE(11, "logFile"), TOPIC_NAME(12,
+//				"topicName"), STATUS(13, "status"), BROKER_TTL(14, "broker.ttlInMins");
+//
+//		private int argOrder;
+//		private String argName;
+//
+//		private ARG(int argOrder, String argName) {
+//			this.argOrder = argOrder;
+//			this.argName = argName;
+//		}
+//
+//		public int ORDER() {
+//			return this.argOrder;
+//		}
+//
+//		public String NAME() {
+//			return this.argName;
+//		}
+//	}
+//
+//	public String getProcessName() {
+//		return this.msgs.get(ARG.PROCESS_NAME);
+//	}
+//
+//	public void setProcessName(String processName) {
+//		this.msgs.put(ARG.PROCESS_NAME, processName);
+//	}
+//
+//	public String getTopicName() {
+//		return this.msgs.get(ARG.TOPIC_NAME);
+//	}
+//
+//	public void setTopicName(String topicName) {
+//		this.msgs.put(ARG.TOPIC_NAME, topicName);
+//	}
+//
+//	public String getFeedName() {
+//		return this.msgs.get(ARG.FEED_NAME);
+//	}
+//
+//	public void setFeedName(String feedName) {
+//		this.msgs.put(ARG.FEED_NAME, feedName);
+//	}
+//
+//	public String getFeedInstancePath() {
+//		return this.msgs.get(ARG.FEED_INSTANCE_PATH);
+//	}
+//
+//	public void setFeedInstancePath(String feedInstancePath) {
+//		this.msgs.put(ARG.FEED_INSTANCE_PATH, feedInstancePath);
+//	}
+//
+//	public String getWorkflowId() {
+//		return this.msgs.get(ARG.WORKFLOW_ID);
+//	}
+//
+//	public void setWorkflowId(String workflowId) {
+//		this.msgs.put(ARG.WORKFLOW_ID, workflowId);
+//	}
+//
+//	public String getRunId() {
+//		return this.msgs.get(ARG.RUN_ID);
+//	}
+//
+//	public void setRunId(String runId) {
+//		this.msgs.put(ARG.RUN_ID, runId);
+//	}
+//
+//	public String getNominalTime() {
+//		return this.msgs.get(ARG.NOMINAL_TIME);
+//	}
+//
+//	public void setNominalTime(String nominalTime) {
+//		this.msgs.put(ARG.NOMINAL_TIME, nominalTime);
+//	}
+//
+//	public String getTimeStamp() {
+//		return this.msgs.get(ARG.TIME_STAMP);
+//	}
+//
+//	public void setTimeStamp(String timeStamp) {
+//		this.msgs.put(ARG.TIME_STAMP, timeStamp);
+//	}
+//
+//	public String getBrokerUrl() {
+//		return this.msgs.get(ARG.BROKER_URL);
+//	}
+//
+//	public void setBrokerUrl(String brokerUrl) {
+//		this.msgs.put(ARG.BROKER_URL, brokerUrl);
+//	}
+//
+//	public String getBrokerImplClass() {
+//		return this.msgs.get(ARG.BROKER_IMPL_CLASS);
+//	}
+//
+//	public void setBrokerImplClass(String brokerImplClass) {
+//		this.msgs.put(ARG.BROKER_IMPL_CLASS, brokerImplClass);
+//	}
+//
+//	public String getEntityType() {
+//		return this.msgs.get(ARG.ENTITY_TYPE);
+//	}
+//
+//	public void setEntityType(String entityType) {
+//		this.msgs.put(ARG.ENTITY_TYPE, entityType);
+//	}
+//
+//	public String getOperation() {
+//		return this.msgs.get(ARG.OPERATION);
+//	}
+//
+//	public void setOperation(String operation) {
+//		this.msgs.put(ARG.OPERATION, operation);
+//	}
+//
+//	public String getLogFile() {
+//		return this.msgs.get(ARG.LOG_FILE);
+//	}
+//
+//	public void setLogFile(String logFile) {
+//		this.msgs.put(ARG.LOG_FILE, logFile);
+//	}
+//
+//	public String getStatus() {
+//		return this.msgs.get(ARG.STATUS);
+//	}
+//
+//	public void setStatus(String status) {
+//		this.msgs.put(ARG.STATUS, status);
+//	}
+//
+//	public String getBrokerTTL() {
+//		return this.msgs.get(ARG.BROKER_TTL);
+//	}
+//
+//	public void setBrokerTTL(String brokerTTL) {
+//		this.msgs.put(ARG.BROKER_TTL, brokerTTL);
+//	}
+//
+//	@Override
+//	public String toString() {
+//		if (getEntityType().equalsIgnoreCase("PROCESS")
+//				&& getTopicName().equals(IVORY_PROCESS_TOPIC_NAME)) {
+//			return getIvoryMessage();
+//		}
+//		if (getEntityType().equalsIgnoreCase("FEED")) {
+//			return getFeedMessage();
+//		}
+//		return getProcessMessage();
+//
+//	}
+//
+//	private String getProcessMessage() {
+//		return getProcessName() + MSG_SEPERATOR + getFeedName() + MSG_SEPERATOR
+//				+ getFeedInstancePath() + MSG_SEPERATOR + getWorkflowId()
+//				+ MSG_SEPERATOR + getRunId() + MSG_SEPERATOR + getNominalTime()
+//				+ MSG_SEPERATOR + getTimeStamp();
+//	}
+//
+//	private String getIvoryMessage() {
+//		return getProcessName() + MSG_SEPERATOR + getFeedName() + MSG_SEPERATOR
+//				+ getFeedInstancePath() + MSG_SEPERATOR + getWorkflowId()
+//				+ MSG_SEPERATOR + getRunId() + MSG_SEPERATOR + getNominalTime()
+//				+ MSG_SEPERATOR + getTimeStamp() + MSG_SEPERATOR + getStatus();
+//	}
+//
+//	private String getFeedMessage() {
+//		return getFeedName() + MSG_SEPERATOR
+//				+ getFeedInstancePath() + MSG_SEPERATOR + getOperation()
+//				+ MSG_SEPERATOR + getWorkflowId() + MSG_SEPERATOR + getRunId()
+//				+ MSG_SEPERATOR + getNominalTime() + MSG_SEPERATOR
+//				+ getTimeStamp();
+//	}
+//
+//	/**
+//	 *
+//	 * @param args
+//	 *            - String array passed from oozie action for jms messaging
+//	 * @return ProcessMessage - Value object which is stored in JMS topic
+//	 */
+//	public static EntityInstanceMessage[] argsToMessage(String[] args) {
+//
+//		assert args.length == ARG_LENGTH : "Required number of arguments: "
+//				+ ARG_LENGTH;
+//
+//		String[] feedNames = getFeedNames(args);
+//
+//		String[] feedPaths;
+//		try {
+//			feedPaths = getFeedPaths(args);
+//		} catch (IOException e) {
+//			LOG.error("Error getting instance paths: ", e);
+//			throw new RuntimeException(e);
+//		}
+//
+//		EntityInstanceMessage[] processMessages = new EntityInstanceMessage[feedPaths.length];
+//		for (int i = 0; i < feedPaths.length; i++) {
+//			EntityInstanceMessage instanceMessage = new EntityInstanceMessage();
+//			instanceMessage
+//					.setProcessName(args[EntityInstanceMessage.ARG.PROCESS_NAME
+//							.ORDER()]);
+//			if (args[EntityInstanceMessage.ARG.ENTITY_TYPE.ORDER()]
+//					.equalsIgnoreCase("PROCESS")) {
+//				instanceMessage.setFeedName(feedNames[i]);
+//			} else {
+//				instanceMessage
+//						.setFeedName(args[EntityInstanceMessage.ARG.FEED_NAME
+//								.ORDER()]);
+//			}
+//			instanceMessage.setFeedInstancePath(feedPaths[i]);
+//			instanceMessage
+//					.setWorkflowId(args[EntityInstanceMessage.ARG.WORKFLOW_ID
+//							.ORDER()]);
+//			instanceMessage.setRunId(args[EntityInstanceMessage.ARG.RUN_ID
+//					.ORDER()]);
+//			instanceMessage
+//					.setNominalTime(args[EntityInstanceMessage.ARG.NOMINAL_TIME
+//							.ORDER()]);
+//			instanceMessage
+//					.setTimeStamp(args[EntityInstanceMessage.ARG.TIME_STAMP
+//							.ORDER()]);
+//			instanceMessage
+//					.setBrokerUrl(args[EntityInstanceMessage.ARG.BROKER_URL
+//							.ORDER()]);
+//			instanceMessage
+//					.setBrokerImplClass(args[EntityInstanceMessage.ARG.BROKER_IMPL_CLASS
+//							.ORDER()]);
+//			instanceMessage
+//					.setEntityType(args[EntityInstanceMessage.ARG.ENTITY_TYPE
+//							.ORDER()]);
+//			instanceMessage
+//					.setOperation(args[EntityInstanceMessage.ARG.OPERATION
+//							.ORDER()]);
+//			instanceMessage.setLogFile(args[EntityInstanceMessage.ARG.LOG_FILE
+//					.ORDER()]);
+//			instanceMessage
+//					.setTopicName(args[EntityInstanceMessage.ARG.TOPIC_NAME
+//							.ORDER()]);
+//			instanceMessage.setStatus(args[EntityInstanceMessage.ARG.STATUS
+//					.ORDER()]);
+//			instanceMessage.setBrokerTTL(args[EntityInstanceMessage.ARG.BROKER_TTL
+//			           					.ORDER()]);
+//
+//			processMessages[i] = instanceMessage;
+//		}
+//		return processMessages;
+//	}
+//
+//	private static String[] getFeedNames(String[] args) {
+//		String topicName = args[ARG.TOPIC_NAME.argOrder];
+//		if (topicName.equals(IVORY_PROCESS_TOPIC_NAME)) {
+//			return new String[] { args[EntityInstanceMessage.ARG.FEED_NAME
+//					.ORDER()] };
+//		}
+//		return args[EntityInstanceMessage.ARG.FEED_NAME.ORDER()].split(",");
+//	}
+//
+//	private static String[] getFeedPaths(String[] args) throws IOException {
+//		String entityType = args[EntityInstanceMessage.ARG.ENTITY_TYPE.ORDER()];
+//		String topicName = args[EntityInstanceMessage.ARG.TOPIC_NAME.ORDER()];
+//
+//		if (entityType.equalsIgnoreCase("PROCESS")
+//				&& topicName.equals(IVORY_PROCESS_TOPIC_NAME)) {
+//			LOG.debug("Returning instance paths for Ivory Topic: "
+//					+ args[EntityInstanceMessage.ARG.FEED_INSTANCE_PATH.ORDER()]);
+//			return new String[] { args[EntityInstanceMessage.ARG.FEED_INSTANCE_PATH
+//					.ORDER()] };
+//		}
+//
+//		if (entityType.equalsIgnoreCase("PROCESS")) {
+//			LOG.debug("Returning instance paths for process: "
+//					+ args[EntityInstanceMessage.ARG.FEED_INSTANCE_PATH.ORDER()]);
+//			return args[EntityInstanceMessage.ARG.FEED_INSTANCE_PATH.ORDER()]
+//					.split(",");
+//		}
+//		//
+//		Path logFile = new Path(
+//				args[EntityInstanceMessage.ARG.LOG_FILE.ORDER()]);
+//		FileSystem fs = FileSystem.get(logFile.toUri(), new Configuration());
+//		ByteArrayOutputStream writer = new ByteArrayOutputStream();
+//		InputStream instance = fs.open(logFile);
+//		IOUtils.copyBytes(instance, writer, 4096, true);
+//		String[] instancePaths = writer.toString().split("=");
+//		if (instancePaths.length == 1) {
+//			LOG.debug("Returning 0 instance paths for feed ");
+//			return new String[0];
+//		} else {
+//			LOG.debug("Returning instance paths for feed " + instancePaths[1]);
+//			return instancePaths[1].split(",");
+//		}
+//
+//	}
+//
+//	/**
+//	 *
+//	 * @param instanceMessages
+//	 *            - value object which is stored in JMS topic as TextMessage
+//	 * @return - String array.
+//	 */
+//	public static String[] messageToArgs(
+//			EntityInstanceMessage[] instanceMessages) {
+//		String[] args = new String[ARG_LENGTH];
+//
+//		args[EntityInstanceMessage.ARG.PROCESS_NAME.ORDER()] = instanceMessages[0]
+//				.getProcessName();
+//		StringBuilder feedNames = new StringBuilder();
+//		StringBuilder feedPaths = new StringBuilder();
+//
+//		for (EntityInstanceMessage instanceMessage : instanceMessages) {
+//			feedNames.append(instanceMessage.getFeedName()).append(",");
+//			feedPaths.append(instanceMessage.getFeedInstancePath()).append(",");
+//		}
+//		if (instanceMessages[0].getEntityType().equalsIgnoreCase("PROCESS")) {
+//			args[EntityInstanceMessage.ARG.FEED_NAME.ORDER()] = feedNames
+//					.toString();
+//
+//		} else {
+//			args[EntityInstanceMessage.ARG.FEED_NAME.ORDER()] = instanceMessages[0]
+//					.getFeedName();
+//		}
+//		args[EntityInstanceMessage.ARG.FEED_INSTANCE_PATH.ORDER()] = feedPaths
+//				.toString();
+//		args[EntityInstanceMessage.ARG.WORKFLOW_ID.ORDER()] = instanceMessages[0]
+//				.getWorkflowId();
+//		args[EntityInstanceMessage.ARG.RUN_ID.ORDER()] = instanceMessages[0]
+//				.getRunId();
+//		args[EntityInstanceMessage.ARG.NOMINAL_TIME.ORDER()] = instanceMessages[0]
+//				.getNominalTime();
+//		args[EntityInstanceMessage.ARG.TIME_STAMP.ORDER()] = instanceMessages[0]
+//				.getTimeStamp();
+//		args[EntityInstanceMessage.ARG.BROKER_URL.ORDER()] = instanceMessages[0]
+//				.getBrokerUrl();
+//		args[EntityInstanceMessage.ARG.BROKER_IMPL_CLASS.ORDER()] = instanceMessages[0]
+//				.getBrokerImplClass();
+//		args[EntityInstanceMessage.ARG.ENTITY_TYPE.ORDER()] = instanceMessages[0]
+//				.getEntityType();
+//		args[EntityInstanceMessage.ARG.OPERATION.ORDER()] = instanceMessages[0]
+//				.getOperation();
+//		args[EntityInstanceMessage.ARG.LOG_FILE.ORDER()] = instanceMessages[0]
+//				.getLogFile();
+//		args[EntityInstanceMessage.ARG.TOPIC_NAME.ORDER()] = instanceMessages[0]
+//				.getTopicName();
+//		args[EntityInstanceMessage.ARG.STATUS.ORDER()] = instanceMessages[0]
+//				.getStatus();
+//		args[EntityInstanceMessage.ARG.BROKER_TTL.ORDER()] = instanceMessages[0]
+//				.getBrokerTTL();
+//
+//		return args;
+//	}
+//
+//}

http://git-wip-us.apache.org/repos/asf/falcon/blob/1aac3a50/falcon-regression/merlin/src/test/resources/OozieExampleInputData/normalInput/dataFile3.txt
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin/src/test/resources/OozieExampleInputData/normalInput/dataFile3.txt b/falcon-regression/merlin/src/test/resources/OozieExampleInputData/normalInput/dataFile3.txt
new file mode 100644
index 0000000..7234f52
--- /dev/null
+++ b/falcon-regression/merlin/src/test/resources/OozieExampleInputData/normalInput/dataFile3.txt
@@ -0,0 +1,139 @@
+///**
+// * Licensed to the Apache Software Foundation (ASF) under one
+// * or more contributor license agreements.  See the NOTICE file
+// * distributed with this work for additional information
+// * regarding copyright ownership.  The ASF licenses this file
+// * to you under the Apache License, Version 2.0 (the
+// * "License"); you may not use this file except in compliance
+// * with the License.  You may obtain a copy of the License at
+// *
+// *     http://www.apache.org/licenses/LICENSE-2.0
+// *
+// * Unless required by applicable law or agreed to in writing, software
+// * distributed under the License is distributed on an "AS IS" BASIS,
+// * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// * See the License for the specific language governing permissions and
+// * limitations under the License.
+// */
+//
+//package org.apache.ivory.messaging;
+//
+//import java.lang.reflect.InvocationTargetException;
+//
+//import javax.jms.Connection;
+//import javax.jms.ConnectionFactory;
+//import javax.jms.DeliveryMode;
+//import javax.jms.JMSException;
+//import javax.jms.Session;
+//import javax.jms.Topic;
+//
+//import org.apache.log4j.Logger;
+//
+///**
+// * Default Ivory Message Producer The configuration are loaded from
+// * jms-beans.xml
+// */
+//public class MessageProducer {
+//
+//	private Connection connection;
+//	private static final Logger LOG = Logger.getLogger(MessageProducer.class);
+//	private static final long DEFAULT_TTL = 3 * 24 * 60 * 60 * 1000;
+//
+//	/**
+//	 *
+//	 * @param arguments
+//	 *            - Accepts a Message to be send to JMS topic, creates a new
+//	 *            Topic based on topic name if it does not exist or else
+//	 *            existing topic with the same name is used to send the message.
+//	 * @throws JMSException
+//	 */
+//	protected void sendMessage(EntityInstanceMessage entityInstanceMessage)
+//			throws JMSException {
+//
+//		Session session = connection.createSession(false,
+//				Session.AUTO_ACKNOWLEDGE);
+//		Topic entityTopic = session.createTopic(entityInstanceMessage
+//				.getTopicName());
+//		javax.jms.MessageProducer producer = session
+//				.createProducer(entityTopic);
+//		producer.setDeliveryMode(DeliveryMode.PERSISTENT);
+//		long messageTTL = DEFAULT_TTL;
+//		try {
+//			long messageTTLinMins = Long.parseLong(entityInstanceMessage
+//					.getBrokerTTL());
+//			messageTTL = messageTTLinMins * 60 * 1000;
+//		} catch (NumberFormatException e) {
+//			LOG.error("Error in parsing broker.ttl, setting TTL to:"
+//					+ DEFAULT_TTL+ " milli-seconds");
+//		}
+//		producer.setTimeToLive(messageTTL);
+//		producer.send(new EntityInstanceMessageCreator(entityInstanceMessage)
+//				.createMessage(session));
+//
+//	}
+//
+//	/**
+//	 *
+//	 * @param args
+//	 *            - array of Strings, which will be used to create TextMessage
+//	 */
+//	public static void main(String[] args) {
+//		debug(args);
+//		EntityInstanceMessage[] entityInstanceMessage = EntityInstanceMessage
+//				.argsToMessage(args);
+//		if (entityInstanceMessage.length == 0) {
+//			LOG.warn("No operation on output feed");
+//			return;
+//		}
+//
+//		MessageProducer ivoryMessageProducer = new MessageProducer();
+//		try {
+//			ivoryMessageProducer.createAndStartConnection(
+//					args[EntityInstanceMessage.ARG.BROKER_IMPL_CLASS.ORDER()],
+//					"", "", entityInstanceMessage[0].getBrokerUrl());
+//			for (EntityInstanceMessage processMessage : entityInstanceMessage) {
+//				ivoryMessageProducer.sendMessage(processMessage);
+//			}
+//		} catch (JMSException e) {
+//			LOG.error(e);
+//			e.printStackTrace();
+//		} catch (Exception e) {
+//			LOG.error(e);
+//			e.printStackTrace();
+//		} finally {
+//			try {
+//				ivoryMessageProducer.connection.close();
+//			} catch (JMSException e) {
+//				e.printStackTrace();
+//			}
+//		}
+//
+//	}
+//
+//	private static void debug(String[] args) {
+//		if (LOG.isDebugEnabled()) {
+//			for (int i = 0; i < args.length; i++) {
+//				LOG.debug(args[i] + "::");
+//			}
+//		}
+//
+//	}
+//
+//	private void createAndStartConnection(String implementation,
+//			String userName, String password, String url) throws JMSException,
+//			ClassNotFoundException, IllegalArgumentException,
+//			SecurityException, InstantiationException, IllegalAccessException,
+//			InvocationTargetException, NoSuchMethodException {
+//
+//		Class<ConnectionFactory> clazz = (Class<ConnectionFactory>) MessageProducer.class
+//				.getClassLoader().loadClass(implementation);
+//
+//		ConnectionFactory connectionFactory = clazz.getConstructor(
+//				String.class, String.class, String.class).newInstance(userName,
+//				password, url);
+//
+//		connection = connectionFactory.createConnection();
+//		connection.start();
+//	}
+//
+//}


Mime
View raw message