falcon-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From samar...@apache.org
Subject [04/27] adding falcon-regression
Date Mon, 04 Aug 2014 10:04:03 GMT
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/bdcf001f/falcon-regression/merlin/src/test/resources/AvailabilityBundle/valid/bundle1/process-agg.xml
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin/src/test/resources/AvailabilityBundle/valid/bundle1/process-agg.xml b/falcon-regression/merlin/src/test/resources/AvailabilityBundle/valid/bundle1/process-agg.xml
new file mode 100644
index 0000000..6a0ee2d
--- /dev/null
+++ b/falcon-regression/merlin/src/test/resources/AvailabilityBundle/valid/bundle1/process-agg.xml
@@ -0,0 +1,51 @@
+<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+<!--
+  Licensed to the Apache Software Foundation (ASF) under one
+  or more contributor license agreements.  See the NOTICE file
+  distributed with this work for additional information
+  regarding copyright ownership.  The ASF licenses this file
+  to you under the Apache License, Version 2.0 (the
+  "License"); you may not use this file except in compliance
+  with the License.  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+  -->
+
+<process name="agregator-coord16" xmlns="uri:falcon:process:0.1" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
+
+      <clusters>
+        <cluster name="corp">
+           <validity end="2011-01-03T03:00Z" start="2010-01-02T01:00Z" />   
+        </cluster>
+      </clusters>
+	<parallel>2</parallel>
+	<order>FIFO</order>
+	<frequency>minutes(60)</frequency>    
+	<timezone>UTC</timezone>
+	
+	<inputs>
+		<input end="now(0,0)" start="now(0,-20)"
+			feed="raaw-logs16" name="inputData"/>
+	</inputs>
+	<outputs>
+		<output instance="now(0,0)" feed="agregated-logs16"
+			name="outputData" />
+	</outputs>
+	<properties>
+		<property name="queueName" value="default"/>
+	</properties>
+	<workflow path="/examples/apps/aggregator" />
+	<retry policy="backoff" delay="minutes(10)" attempts="3" />
+
+    <late-process policy="exp-backoff" delay="hours(1)">
+        <late-input feed="impression" workflow-path="hdfs://impression/late/workflow"/>
+        <late-input feed="clicks" workflow-path="hdfs://clicks/late/workflow" />
+    </late-process>
+</process>
+

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/bdcf001f/falcon-regression/merlin/src/test/resources/ELbundle/cluster-0.1.xml
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin/src/test/resources/ELbundle/cluster-0.1.xml b/falcon-regression/merlin/src/test/resources/ELbundle/cluster-0.1.xml
new file mode 100644
index 0000000..cdb14ae
--- /dev/null
+++ b/falcon-regression/merlin/src/test/resources/ELbundle/cluster-0.1.xml
@@ -0,0 +1,41 @@
+<?xml version="1.0"?>
+<!--
+  Licensed to the Apache Software Foundation (ASF) under one
+  or more contributor license agreements.  See the NOTICE file
+  distributed with this work for additional information
+  regarding copyright ownership.  The ASF licenses this file
+  to you under the Apache License, Version 2.0 (the
+  "License"); you may not use this file except in compliance
+  with the License.  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+  -->
+
+<cluster colo="ua1" description="" name="corp" xmlns="uri:falcon:cluster:0.1" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
+	<interfaces>
+		<interface type="readonly" endpoint="http://gs1001.grid.corp.inmobi.com:50070"
+			version="0.20.2" />
+		<interface type="write" endpoint="hdfs://gs1001.grid.corp.inmobi.com:54310"
+			version="0.20.2" />
+		<interface type="execute" endpoint="hdfs://gs1001.grid.corp.inmobi.com:54311" version="0.20.2" />
+		<interface type="workflow" endpoint="http://gs1001.grid.corp.inmobi.com:11000/oozie/"
+			version="3.1" />
+		<interface type="messaging" endpoint="tcp://gs1001.grid.corp.inmobi.com:61616?daemon=true"
+			version="5.1.6" />
+	</interfaces>
+	<locations>
+		<location name="staging" path="/projects/ivory/staging" />
+		<location name="temp" path="/tmp" />
+		<location name="working" path="/projectsTest/ivory/working" />
+	</locations>
+	<properties>
+		<property name="field1" value="value1" />
+		<property name="field2" value="value2" />
+	</properties>
+</cluster>

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/bdcf001f/falcon-regression/merlin/src/test/resources/ELbundle/feed-template1.xml
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin/src/test/resources/ELbundle/feed-template1.xml b/falcon-regression/merlin/src/test/resources/ELbundle/feed-template1.xml
new file mode 100644
index 0000000..2114bb9
--- /dev/null
+++ b/falcon-regression/merlin/src/test/resources/ELbundle/feed-template1.xml
@@ -0,0 +1,52 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  Licensed to the Apache Software Foundation (ASF) under one
+  or more contributor license agreements.  See the NOTICE file
+  distributed with this work for additional information
+  regarding copyright ownership.  The ASF licenses this file
+  to you under the Apache License, Version 2.0 (the
+  "License"); you may not use this file except in compliance
+  with the License.  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+  -->
+<feed description="clicks log" name="raaw-logs16" xmlns="uri:falcon:feed:0.1" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
+   <partitions>
+        <partition name="country" />
+        <partition name="colo" />
+    </partitions>
+<!--
+    <groups>online,bi</groups>-->
+
+    <frequency>minutes(20)</frequency>
+    <timezone>UTC</timezone>
+    <late-arrival cut-off="hours(6)" />
+
+    <clusters>
+        <cluster name="corp" type="source">
+            <validity start="2009-02-01T00:00Z" end="2099-05-01T00:00Z"
+     />
+            <retention limit="months(9000)" action="delete" /> <!-- Limit can be in Time or Instances 100, Action ENUM DELETE,ARCHIVE -->
+        </cluster>
+    </clusters>
+
+    <locations>
+        <location type="data" path="/samarth/input-data/rawLogs/${YEAR}/${MONTH}/${DAY}/${HOUR}/${MINUTE}" />
+        <location type="stats" path="/projects/ivory/clicksStats" />
+        <location type="meta" path="/projects/ivory/clicksMetaData" />
+    </locations>
+    
+    <ACL owner="testuser" group="group" permission="0x755" />
+    <schema location="/schema/clicks" provider="protobuf" />
+
+    <properties>
+        <property name="field1" value="value1" />
+        <property name="field2" value="value2" />
+    </properties>
+</feed>
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/bdcf001f/falcon-regression/merlin/src/test/resources/ELbundle/feed-template2.xml
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin/src/test/resources/ELbundle/feed-template2.xml b/falcon-regression/merlin/src/test/resources/ELbundle/feed-template2.xml
new file mode 100644
index 0000000..08c5f5d
--- /dev/null
+++ b/falcon-regression/merlin/src/test/resources/ELbundle/feed-template2.xml
@@ -0,0 +1,52 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  Licensed to the Apache Software Foundation (ASF) under one
+  or more contributor license agreements.  See the NOTICE file
+  distributed with this work for additional information
+  regarding copyright ownership.  The ASF licenses this file
+  to you under the Apache License, Version 2.0 (the
+  "License"); you may not use this file except in compliance
+  with the License.  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+  -->
+<feed description="clicks log" name="agregated-logs16" xmlns="uri:falcon:feed:0.1" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
+<!--   <partitions>
+        <partition name="fraud" />
+        <partition name="good" />
+    </partitions>
+
+    <groups>online,bi</groups>-->
+
+    <frequency>hours(1)</frequency>
+<timezone>UTC</timezone>
+    <late-arrival cut-off="hours(6)" />
+
+    <clusters>
+        <cluster name="corp" type="source">
+            <validity start="2009-02-01T01:00Z" end="2099-05-01T00:00Z"
+           />
+            <retention limit="hours(6)" action="delete" /> <!-- Limit can be in Time or Instances 100, Action ENUM DELETE,ARCHIVE -->
+        </cluster>
+    </clusters>
+
+    <locations>
+        <location type="data" path="/examples/output-data/aggregator/aggregatedLogs/${YEAR}/${MONTH}/${DAY}/${HOUR}/${MINUTE}" />
+        <location type="stats" path="/projects/ivory/clicksStats" />
+        <location type="meta" path="/projects/ivory/clicksMetaData" />
+    </locations>
+    
+    <ACL owner="testuser" group="group" permission="0x755" />
+    <schema location="/schema/clicks" provider="protobuf" />
+
+    <properties>
+        <property name="field1" value="value1" />
+        <property name="field2" value="value2" />
+    </properties>
+</feed>
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/bdcf001f/falcon-regression/merlin/src/test/resources/ELbundle/process-agg.xml
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin/src/test/resources/ELbundle/process-agg.xml b/falcon-regression/merlin/src/test/resources/ELbundle/process-agg.xml
new file mode 100644
index 0000000..3fb4c61
--- /dev/null
+++ b/falcon-regression/merlin/src/test/resources/ELbundle/process-agg.xml
@@ -0,0 +1,51 @@
+<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+<!--
+  Licensed to the Apache Software Foundation (ASF) under one
+  or more contributor license agreements.  See the NOTICE file
+  distributed with this work for additional information
+  regarding copyright ownership.  The ASF licenses this file
+  to you under the Apache License, Version 2.0 (the
+  "License"); you may not use this file except in compliance
+  with the License.  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+  -->
+
+<process name="agregator-coord16" xmlns="uri:falcon:process:0.1" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
+
+       <clusters>
+        <cluster name="corp">
+           <validity end="2011-01-03T03:00Z" start="2010-01-02T01:00Z" />  
+        </cluster>
+      </clusters>
+	<parallel>1</parallel>
+	<order>FIFO</order>
+       <frequency>minutes(5)</frequency>
+	<timezone>UTC</timezone>
+	
+	<inputs>
+		<input end="now(0,0)" start="now(0,-20)"
+			feed="raaw-logs16" name="inputData"/>
+	</inputs>
+	<outputs>
+		<output instance="now(0,0)" feed="agregated-logs16"
+			name="outputData" />
+	</outputs>
+	<properties>
+		<property name="queueName" value="default"/>
+		
+       <property name="fileTime" value="${formatTime(dateOffset(instanceTime(), 1, 'DAY'), 'yyyy-MMM-dd')}"/>
+       <property name="user" value="${user()}"/>
+   
+	</properties>
+	<workflow path="/examples/apps/aggregator"/>
+	<retry policy="periodic" delay="minutes(3)" attempts="3" />
+
+</process>
+

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/bdcf001f/falcon-regression/merlin/src/test/resources/ELbundle/workflow/workflow.xml
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin/src/test/resources/ELbundle/workflow/workflow.xml b/falcon-regression/merlin/src/test/resources/ELbundle/workflow/workflow.xml
new file mode 100644
index 0000000..4a80109
--- /dev/null
+++ b/falcon-regression/merlin/src/test/resources/ELbundle/workflow/workflow.xml
@@ -0,0 +1,61 @@
+<!--
+  Licensed to the Apache Software Foundation (ASF) under one
+  or more contributor license agreements.  See the NOTICE file
+  distributed with this work for additional information
+  regarding copyright ownership.  The ASF licenses this file
+  to you under the Apache License, Version 2.0 (the
+  "License"); you may not use this file except in compliance
+  with the License.  You may obtain a copy of the License at
+  
+       http://www.apache.org/licenses/LICENSE-2.0
+  
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+-->
+<workflow-app xmlns="uri:oozie:workflow:0.2" name="aggregator-wf">
+    <start to="aggregator"/>
+    <action name="aggregator">
+        <map-reduce>
+            <job-tracker>${jobTracker}</job-tracker>
+            <name-node>${nameNode}</name-node>
+            <prepare>
+                <delete path="${outputData}"/>
+            </prepare>
+            <configuration>
+                <property>
+                    <name>mapred.job.queue.name</name>
+                    <value>${queueName}</value>
+                </property>
+                <property>
+                    <name>mapred.mapper.class</name>
+                    <value>org.apache.oozie.example.SampleMapper</value>
+                </property>
+                <property>
+                    <name>mapred.reducer.class</name>
+                    <value>org.apache.oozie.example.SampleReducer</value>
+                </property>
+                <property>
+                    <name>mapred.map.tasks</name>
+                    <value>1</value>
+                </property>
+                <property>
+                    <name>mapred.input.dir</name>
+                    <value>${inputData}</value>
+                </property>
+                <property>
+                    <name>mapred.output.dir</name>
+                    <value>${outputData}</value>
+                </property>
+            </configuration>
+        </map-reduce>
+        <ok to="end"/>
+        <error to="fail"/>
+    </action>
+    <kill name="fail">
+        <message>Map/Reduce failed, error message[${wf:errorMessage(wf:lastErrorNode())}]</message>
+    </kill>
+    <end name="end"/>
+</workflow-app>

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/bdcf001f/falcon-regression/merlin/src/test/resources/EmptyInputTagProcess/cluster-0.1.xml
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin/src/test/resources/EmptyInputTagProcess/cluster-0.1.xml b/falcon-regression/merlin/src/test/resources/EmptyInputTagProcess/cluster-0.1.xml
new file mode 100644
index 0000000..cdb14ae
--- /dev/null
+++ b/falcon-regression/merlin/src/test/resources/EmptyInputTagProcess/cluster-0.1.xml
@@ -0,0 +1,41 @@
+<?xml version="1.0"?>
+<!--
+  Licensed to the Apache Software Foundation (ASF) under one
+  or more contributor license agreements.  See the NOTICE file
+  distributed with this work for additional information
+  regarding copyright ownership.  The ASF licenses this file
+  to you under the Apache License, Version 2.0 (the
+  "License"); you may not use this file except in compliance
+  with the License.  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+  -->
+
+<cluster colo="ua1" description="" name="corp" xmlns="uri:falcon:cluster:0.1" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
+	<interfaces>
+		<interface type="readonly" endpoint="http://gs1001.grid.corp.inmobi.com:50070"
+			version="0.20.2" />
+		<interface type="write" endpoint="hdfs://gs1001.grid.corp.inmobi.com:54310"
+			version="0.20.2" />
+		<interface type="execute" endpoint="hdfs://gs1001.grid.corp.inmobi.com:54311" version="0.20.2" />
+		<interface type="workflow" endpoint="http://gs1001.grid.corp.inmobi.com:11000/oozie/"
+			version="3.1" />
+		<interface type="messaging" endpoint="tcp://gs1001.grid.corp.inmobi.com:61616?daemon=true"
+			version="5.1.6" />
+	</interfaces>
+	<locations>
+		<location name="staging" path="/projects/ivory/staging" />
+		<location name="temp" path="/tmp" />
+		<location name="working" path="/projectsTest/ivory/working" />
+	</locations>
+	<properties>
+		<property name="field1" value="value1" />
+		<property name="field2" value="value2" />
+	</properties>
+</cluster>

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/bdcf001f/falcon-regression/merlin/src/test/resources/EmptyInputTagProcess/feed-template1.xml
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin/src/test/resources/EmptyInputTagProcess/feed-template1.xml b/falcon-regression/merlin/src/test/resources/EmptyInputTagProcess/feed-template1.xml
new file mode 100644
index 0000000..2da8e1c
--- /dev/null
+++ b/falcon-regression/merlin/src/test/resources/EmptyInputTagProcess/feed-template1.xml
@@ -0,0 +1,52 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  Licensed to the Apache Software Foundation (ASF) under one
+  or more contributor license agreements.  See the NOTICE file
+  distributed with this work for additional information
+  regarding copyright ownership.  The ASF licenses this file
+  to you under the Apache License, Version 2.0 (the
+  "License"); you may not use this file except in compliance
+  with the License.  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+  -->
+<feed description="clicks log" name="raaw-logs16" xmlns="uri:falcon:feed:0.1" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
+   <partitions>
+        <partition name="country" />
+        <partition name="colo" />
+    </partitions>
+<!--
+    <groups>online,bi</groups>-->
+
+    <frequency>minutes(20)</frequency>
+    <timezone>UTC</timezone>
+    <late-arrival cut-off="hours(6)" />
+
+    <clusters>
+        <cluster name="corp" type="source">
+            <validity start="2009-02-01T00:00Z" end="2099-05-01T00:00Z"
+     />
+            <retention limit="months(9000)" action="delete" /> <!-- Limit can be in Time or Instances 100, Action ENUM DELETE,ARCHIVE -->
+        </cluster>
+    </clusters>
+
+    <locations>
+        <location type="data" path="/samarth/input-data/rawLogs/${YEAR}/${MONTH}/${DAY}/${HOUR}/${MINUTE}" />
+        <location type="stats" path="/projects/ivory/clicksStats" />
+        <location type="meta" path="/projects/ivory/clicksMetaData" />
+    </locations>
+    
+    <ACL owner="testuser" group="group" permission="0x755" />
+    <schema location="/schema/clicks" provider="protobuf" />
+
+    <properties>
+        <property name="field1" value="value1" />
+        <property name="field2" value="value2" />
+    </properties>
+</feed>

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/bdcf001f/falcon-regression/merlin/src/test/resources/EmptyInputTagProcess/feed-template2.xml
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin/src/test/resources/EmptyInputTagProcess/feed-template2.xml b/falcon-regression/merlin/src/test/resources/EmptyInputTagProcess/feed-template2.xml
new file mode 100644
index 0000000..1ddac37
--- /dev/null
+++ b/falcon-regression/merlin/src/test/resources/EmptyInputTagProcess/feed-template2.xml
@@ -0,0 +1,52 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  Licensed to the Apache Software Foundation (ASF) under one
+  or more contributor license agreements.  See the NOTICE file
+  distributed with this work for additional information
+  regarding copyright ownership.  The ASF licenses this file
+  to you under the Apache License, Version 2.0 (the
+  "License"); you may not use this file except in compliance
+  with the License.  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+  -->
+<feed description="clicks log" name="agregated-logs16" xmlns="uri:falcon:feed:0.1" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
+<!--   <partitions>
+        <partition name="fraud" />
+        <partition name="good" />
+    </partitions>
+
+    <groups>online,bi</groups>-->
+
+    <frequency>hours(1)</frequency>
+<timezone>UTC</timezone>
+    <late-arrival cut-off="hours(6)" />
+
+    <clusters>
+        <cluster name="corp" type="source">
+            <validity start="2009-02-01T01:00Z" end="2099-05-01T00:00Z"
+           />
+            <retention limit="hours(6)" action="delete" /> <!-- Limit can be in Time or Instances 100, Action ENUM DELETE,ARCHIVE -->
+        </cluster>
+    </clusters>
+
+    <locations>
+        <location type="data" path="/examples/output-data/aggregator/aggregatedLogs/${YEAR}/${MONTH}/${DAY}/${HOUR}/${MINUTE}" />
+        <location type="stats" path="/projects/ivory/clicksStats" />
+        <location type="meta" path="/projects/ivory/clicksMetaData" />
+    </locations>
+    
+    <ACL owner="testuser" group="group" permission="0x755" />
+    <schema location="/schema/clicks" provider="protobuf" />
+
+    <properties>
+        <property name="field1" value="value1" />
+        <property name="field2" value="value2" />
+    </properties>
+</feed>

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/bdcf001f/falcon-regression/merlin/src/test/resources/EmptyInputTagProcess/process-agg.xml
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin/src/test/resources/EmptyInputTagProcess/process-agg.xml b/falcon-regression/merlin/src/test/resources/EmptyInputTagProcess/process-agg.xml
new file mode 100644
index 0000000..3f23ffe
--- /dev/null
+++ b/falcon-regression/merlin/src/test/resources/EmptyInputTagProcess/process-agg.xml
@@ -0,0 +1,50 @@
+<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+<!--
+  Licensed to the Apache Software Foundation (ASF) under one
+  or more contributor license agreements.  See the NOTICE file
+  distributed with this work for additional information
+  regarding copyright ownership.  The ASF licenses this file
+  to you under the Apache License, Version 2.0 (the
+  "License"); you may not use this file except in compliance
+  with the License.  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+  -->
+
+<process name="agregator-coord16" xmlns="uri:falcon:process:0.1" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
+
+       <clusters>
+        <cluster name="corp">
+           <validity end="2011-01-03T03:00Z" start="2010-01-02T01:00Z" />  
+        </cluster>
+      </clusters>
+	<parallel>1</parallel>
+	<order>FIFO</order>
+       <frequency>minutes(5)</frequency>
+	<timezone>UTC</timezone>
+	
+	<inputs>
+
+	</inputs>
+	<outputs>
+		<output instance="now(0,0)" feed="agregated-logs16"
+			name="outputData" />
+	</outputs>
+	<properties>
+		<property name="queueName" value="default"/>
+		
+       <property name="fileTime" value="${formatTime(dateOffset(instanceTime(), 1, 'DAY'), 'yyyy-MMM-dd')}"/>
+       <property name="user" value="${user()}"/>
+   
+	</properties>
+	<workflow path="/examples/apps/aggregator"/>
+	<retry policy="periodic" delay="minutes(3)" attempts="3" />
+
+</process>
+

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/bdcf001f/falcon-regression/merlin/src/test/resources/EmptyOutputTagProcess/cluster-0.1.xml
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin/src/test/resources/EmptyOutputTagProcess/cluster-0.1.xml b/falcon-regression/merlin/src/test/resources/EmptyOutputTagProcess/cluster-0.1.xml
new file mode 100644
index 0000000..cdb14ae
--- /dev/null
+++ b/falcon-regression/merlin/src/test/resources/EmptyOutputTagProcess/cluster-0.1.xml
@@ -0,0 +1,41 @@
+<?xml version="1.0"?>
+<!--
+  Licensed to the Apache Software Foundation (ASF) under one
+  or more contributor license agreements.  See the NOTICE file
+  distributed with this work for additional information
+  regarding copyright ownership.  The ASF licenses this file
+  to you under the Apache License, Version 2.0 (the
+  "License"); you may not use this file except in compliance
+  with the License.  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+  -->
+
+<cluster colo="ua1" description="" name="corp" xmlns="uri:falcon:cluster:0.1" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
+	<interfaces>
+		<interface type="readonly" endpoint="http://gs1001.grid.corp.inmobi.com:50070"
+			version="0.20.2" />
+		<interface type="write" endpoint="hdfs://gs1001.grid.corp.inmobi.com:54310"
+			version="0.20.2" />
+		<interface type="execute" endpoint="hdfs://gs1001.grid.corp.inmobi.com:54311" version="0.20.2" />
+		<interface type="workflow" endpoint="http://gs1001.grid.corp.inmobi.com:11000/oozie/"
+			version="3.1" />
+		<interface type="messaging" endpoint="tcp://gs1001.grid.corp.inmobi.com:61616?daemon=true"
+			version="5.1.6" />
+	</interfaces>
+	<locations>
+		<location name="staging" path="/projects/ivory/staging" />
+		<location name="temp" path="/tmp" />
+		<location name="working" path="/projectsTest/ivory/working" />
+	</locations>
+	<properties>
+		<property name="field1" value="value1" />
+		<property name="field2" value="value2" />
+	</properties>
+</cluster>

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/bdcf001f/falcon-regression/merlin/src/test/resources/EmptyOutputTagProcess/feed-template1.xml
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin/src/test/resources/EmptyOutputTagProcess/feed-template1.xml b/falcon-regression/merlin/src/test/resources/EmptyOutputTagProcess/feed-template1.xml
new file mode 100644
index 0000000..2da8e1c
--- /dev/null
+++ b/falcon-regression/merlin/src/test/resources/EmptyOutputTagProcess/feed-template1.xml
@@ -0,0 +1,52 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  Licensed to the Apache Software Foundation (ASF) under one
+  or more contributor license agreements.  See the NOTICE file
+  distributed with this work for additional information
+  regarding copyright ownership.  The ASF licenses this file
+  to you under the Apache License, Version 2.0 (the
+  "License"); you may not use this file except in compliance
+  with the License.  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+  -->
+<feed description="clicks log" name="raaw-logs16" xmlns="uri:falcon:feed:0.1" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
+   <partitions>
+        <partition name="country" />
+        <partition name="colo" />
+    </partitions>
+<!--
+    <groups>online,bi</groups>-->
+
+    <frequency>minutes(20)</frequency>
+    <timezone>UTC</timezone>
+    <late-arrival cut-off="hours(6)" />
+
+    <clusters>
+        <cluster name="corp" type="source">
+            <validity start="2009-02-01T00:00Z" end="2099-05-01T00:00Z"
+     />
+            <retention limit="months(9000)" action="delete" /> <!-- Limit can be in Time or Instances 100, Action ENUM DELETE,ARCHIVE -->
+        </cluster>
+    </clusters>
+
+    <locations>
+        <location type="data" path="/samarth/input-data/rawLogs/${YEAR}/${MONTH}/${DAY}/${HOUR}/${MINUTE}" />
+        <location type="stats" path="/projects/ivory/clicksStats" />
+        <location type="meta" path="/projects/ivory/clicksMetaData" />
+    </locations>
+    
+    <ACL owner="testuser" group="group" permission="0x755" />
+    <schema location="/schema/clicks" provider="protobuf" />
+
+    <properties>
+        <property name="field1" value="value1" />
+        <property name="field2" value="value2" />
+    </properties>
+</feed>

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/bdcf001f/falcon-regression/merlin/src/test/resources/EmptyOutputTagProcess/feed-template2.xml
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin/src/test/resources/EmptyOutputTagProcess/feed-template2.xml b/falcon-regression/merlin/src/test/resources/EmptyOutputTagProcess/feed-template2.xml
new file mode 100644
index 0000000..1ddac37
--- /dev/null
+++ b/falcon-regression/merlin/src/test/resources/EmptyOutputTagProcess/feed-template2.xml
@@ -0,0 +1,52 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  Licensed to the Apache Software Foundation (ASF) under one
+  or more contributor license agreements.  See the NOTICE file
+  distributed with this work for additional information
+  regarding copyright ownership.  The ASF licenses this file
+  to you under the Apache License, Version 2.0 (the
+  "License"); you may not use this file except in compliance
+  with the License.  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+  -->
+<feed description="clicks log" name="agregated-logs16" xmlns="uri:falcon:feed:0.1" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
+<!--   <partitions>
+        <partition name="fraud" />
+        <partition name="good" />
+    </partitions>
+
+    <groups>online,bi</groups>-->
+
+    <frequency>hours(1)</frequency>
+<timezone>UTC</timezone>
+    <late-arrival cut-off="hours(6)" />
+
+    <clusters>
+        <cluster name="corp" type="source">
+            <validity start="2009-02-01T01:00Z" end="2099-05-01T00:00Z"
+           />
+            <retention limit="hours(6)" action="delete" /> <!-- Limit can be in Time or Instances 100, Action ENUM DELETE,ARCHIVE -->
+        </cluster>
+    </clusters>
+
+    <locations>
+        <location type="data" path="/examples/output-data/aggregator/aggregatedLogs/${YEAR}/${MONTH}/${DAY}/${HOUR}/${MINUTE}" />
+        <location type="stats" path="/projects/ivory/clicksStats" />
+        <location type="meta" path="/projects/ivory/clicksMetaData" />
+    </locations>
+    
+    <ACL owner="testuser" group="group" permission="0x755" />
+    <schema location="/schema/clicks" provider="protobuf" />
+
+    <properties>
+        <property name="field1" value="value1" />
+        <property name="field2" value="value2" />
+    </properties>
+</feed>

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/bdcf001f/falcon-regression/merlin/src/test/resources/EmptyOutputTagProcess/process-agg.xml
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin/src/test/resources/EmptyOutputTagProcess/process-agg.xml b/falcon-regression/merlin/src/test/resources/EmptyOutputTagProcess/process-agg.xml
new file mode 100644
index 0000000..2175e24
--- /dev/null
+++ b/falcon-regression/merlin/src/test/resources/EmptyOutputTagProcess/process-agg.xml
@@ -0,0 +1,50 @@
+<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+<!--
+  Licensed to the Apache Software Foundation (ASF) under one
+  or more contributor license agreements.  See the NOTICE file
+  distributed with this work for additional information
+  regarding copyright ownership.  The ASF licenses this file
+  to you under the Apache License, Version 2.0 (the
+  "License"); you may not use this file except in compliance
+  with the License.  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+  -->
+
+<process name="agregator-coord16" xmlns="uri:falcon:process:0.1" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
+
+       <clusters>
+        <cluster name="corp">
+           <validity end="2011-01-03T03:00Z" start="2010-01-02T01:00Z" />  
+        </cluster>
+      </clusters>
+	<parallel>1</parallel>
+	<order>FIFO</order>
+       <frequency>minutes(5)</frequency>
+	<timezone>UTC</timezone>
+	
+	<inputs>
+		<input end="now(0,0)" start="now(0,-20)"
+			feed="raaw-logs16" name="inputData"/>
+	</inputs>
+	<outputs>
+
+	</outputs>
+	<properties>
+		<property name="queueName" value="default"/>
+		
+       <property name="fileTime" value="${formatTime(dateOffset(instanceTime(), 1, 'DAY'), 'yyyy-MMM-dd')}"/>
+       <property name="user" value="${user()}"/>
+   
+	</properties>
+	<workflow path="/examples/apps/aggregator"/>
+	<retry policy="periodic" delay="minutes(3)" attempts="3" />
+
+</process>
+

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/bdcf001f/falcon-regression/merlin/src/test/resources/IvoryClient/IvoryCLI.jar
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin/src/test/resources/IvoryClient/IvoryCLI.jar b/falcon-regression/merlin/src/test/resources/IvoryClient/IvoryCLI.jar
new file mode 100644
index 0000000..319bf5a
Binary files /dev/null and b/falcon-regression/merlin/src/test/resources/IvoryClient/IvoryCLI.jar differ

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/bdcf001f/falcon-regression/merlin/src/test/resources/LateDataBundles/valid1/bundle1/cluster-0.1.xml
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin/src/test/resources/LateDataBundles/valid1/bundle1/cluster-0.1.xml b/falcon-regression/merlin/src/test/resources/LateDataBundles/valid1/bundle1/cluster-0.1.xml
new file mode 100644
index 0000000..6a00752
--- /dev/null
+++ b/falcon-regression/merlin/src/test/resources/LateDataBundles/valid1/bundle1/cluster-0.1.xml
@@ -0,0 +1,41 @@
+<?xml version="1.0"?>
+<!--
+  Licensed to the Apache Software Foundation (ASF) under one
+  or more contributor license agreements.  See the NOTICE file
+  distributed with this work for additional information
+  regarding copyright ownership.  The ASF licenses this file
+  to you under the Apache License, Version 2.0 (the
+  "License"); you may not use this file except in compliance
+  with the License.  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+  -->
+
+<cluster colo="gs" description="" name="corp" xmlns="uri:falcon:cluster:0.1" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
+	<interfaces>
+		<interface type="readonly" endpoint="hftp://gs1001.grid.corp.inmobi.com:50070"
+			version="0.20.2" />
+		<interface type="write" endpoint="hdfs://gs1001.grid.corp.inmobi.com:54310"
+			version="0.20.2" />
+		<interface type="execute" endpoint="hdfs://gs1001.grid.corp.inmobi.com:54311" version="0.20.2" />
+		<interface type="workflow" endpoint="http://gs1001.grid.corp.inmobi.com:11000/oozie/"
+			version="3.1" />
+		<interface type="messaging" endpoint="tcp://gs1001.grid.corp.inmobi.com:61616?daemon=true"
+			version="5.1.6" />
+	</interfaces>
+	<locations>
+		<location name="staging" path="/projects/ivory/staging" />
+		<location name="temp" path="/tmp" />
+		<location name="working" path="/projects/ivory/working" />
+	</locations>
+	<properties>
+		<property name="field1" value="value1" />
+		<property name="field2" value="value2" />
+	</properties>
+</cluster>
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/bdcf001f/falcon-regression/merlin/src/test/resources/LateDataBundles/valid1/bundle1/feed-template1.xml
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin/src/test/resources/LateDataBundles/valid1/bundle1/feed-template1.xml b/falcon-regression/merlin/src/test/resources/LateDataBundles/valid1/bundle1/feed-template1.xml
new file mode 100644
index 0000000..5350d0a
--- /dev/null
+++ b/falcon-regression/merlin/src/test/resources/LateDataBundles/valid1/bundle1/feed-template1.xml
@@ -0,0 +1,51 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  Licensed to the Apache Software Foundation (ASF) under one
+  or more contributor license agreements.  See the NOTICE file
+  distributed with this work for additional information
+  regarding copyright ownership.  The ASF licenses this file
+  to you under the Apache License, Version 2.0 (the
+  "License"); you may not use this file except in compliance
+  with the License.  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+  -->
+<feed description="clicks log" name="raaw-logs16" xmlns="uri:falcon:feed:0.1" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
+<!--   <partitions>
+        <partition name="fraud" />
+        <partition name="good" />
+    </partitions>
+
+    <groups>online,bi</groups>-->
+
+   <frequency>minutes(1)</frequency>
+   <timezone>UTC</timezone>
+   <late-arrival cut-off="hours(6)"/>
+    
+    <clusters>
+        <cluster name="corp" type="source">
+            <validity start="2009-01-01T00:00Z" end="2099-12-31T23:59Z"/>
+            <retention limit="months(6)" action="delete" /> <!-- Limit can be in Time or Instances 100, Action ENUM DELETE,ARCHIVE -->
+        </cluster>
+    </clusters>
+
+    <locations>
+        <location type="data" path="/lateDataTest/testFolders/${YEAR}/${MONTH}/${DAY}/${HOUR}/${MINUTE}" />
+        <location type="stats" path="/projects/ivory/clicksStats" />
+        <location type="meta" path="/projects/ivory/clicksMetaData" />
+    </locations>
+    
+    <ACL owner="testuser" group="group" permission="0x755" />
+    <schema location="/schema/clicks" provider="protobuf" />
+
+    <properties>
+        <property name="field3" value="value1" />
+        <property name="field4" value="value2" />
+    </properties>
+</feed>

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/bdcf001f/falcon-regression/merlin/src/test/resources/LateDataBundles/valid1/bundle1/feed-template2.xml
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin/src/test/resources/LateDataBundles/valid1/bundle1/feed-template2.xml b/falcon-regression/merlin/src/test/resources/LateDataBundles/valid1/bundle1/feed-template2.xml
new file mode 100644
index 0000000..d9e4e32
--- /dev/null
+++ b/falcon-regression/merlin/src/test/resources/LateDataBundles/valid1/bundle1/feed-template2.xml
@@ -0,0 +1,51 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  Licensed to the Apache Software Foundation (ASF) under one
+  or more contributor license agreements.  See the NOTICE file
+  distributed with this work for additional information
+  regarding copyright ownership.  The ASF licenses this file
+  to you under the Apache License, Version 2.0 (the
+  "License"); you may not use this file except in compliance
+  with the License.  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+  -->
+<feed description="clicks log" name="agregated-logs16" xmlns="uri:falcon:feed:0.1" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
+<!--   <partitions>
+        <partition name="fraud" />
+        <partition name="good" />
+    </partitions>
+
+    <groups>online,bi</groups>-->
+
+     <frequency>minutes(1)</frequency>
+     <timezone>UTC</timezone>
+     <late-arrival cut-off="hours(6)" />
+
+    <clusters>
+        <cluster name="corp" type="source">
+            <validity start="2009-01-01T01:00Z" end="2099-12-31T23:59Z"/>
+            <retention limit="months(6)" action="delete" /> <!-- Limit can be in Time or Instances 100, Action ENUM DELETE,ARCHIVE -->
+        </cluster>
+    </clusters>
+
+    <locations>
+        <location type="data" path="/lateDataTest/testFolders/output-data/${YEAR}/${MONTH}/${DAY}/${HOUR}/${MINUTE}" />
+        <location type="stats" path="/projects/ivory/clicksStats" />
+        <location type="meta" path="/projects/ivory/clicksMetaData" />
+    </locations>
+    
+    <ACL owner="testuser" group="group" permission="0x755" />
+    <schema location="/schema/clicks" provider="protobuf" />
+
+    <properties>
+        <property name="field5" value="value1" />
+        <property name="field6" value="value2" />
+    </properties>
+</feed>

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/bdcf001f/falcon-regression/merlin/src/test/resources/LateDataBundles/valid1/bundle1/process-agg.xml
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin/src/test/resources/LateDataBundles/valid1/bundle1/process-agg.xml b/falcon-regression/merlin/src/test/resources/LateDataBundles/valid1/bundle1/process-agg.xml
new file mode 100644
index 0000000..09e8a09
--- /dev/null
+++ b/falcon-regression/merlin/src/test/resources/LateDataBundles/valid1/bundle1/process-agg.xml
@@ -0,0 +1,47 @@
+<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+<!--
+  Licensed to the Apache Software Foundation (ASF) under one
+  or more contributor license agreements.  See the NOTICE file
+  distributed with this work for additional information
+  regarding copyright ownership.  The ASF licenses this file
+  to you under the Apache License, Version 2.0 (the
+  "License"); you may not use this file except in compliance
+  with the License.  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+  -->
+
+<process name="agregator-coord16" xmlns="uri:falcon:process:0.1">
+    <clusters>
+        <cluster name="corp">
+            <validity end="2010-01-01T01:03Z" start="2010-01-01T01:01Z"/>
+        </cluster>
+    </clusters>
+    <parallel>3</parallel>
+
+    <order>FIFO</order>
+    <frequency>minutes(1)</frequency>
+    <timezone>UTC</timezone>
+
+    <inputs>
+        <input end="now(0,0)" start="now(0,-1)" feed="raaw-logs16" name="inputData"/>
+    </inputs>
+    <outputs>
+        <output instance="now(0,0)" feed="agregated-logs16" name="outputData"/>
+    </outputs>
+    <properties>
+        <property name="queueName" value="default"/>
+    </properties>
+    <workflow path="/examples/apps/aggregator"/>
+    <retry policy="periodic" delay="minutes(10)" attempts="3"/>
+
+    <late-process policy="periodic" delay="hours(1)">
+        <late-input input="inputData" workflow-path="hdfs://impression/late/workflow"/>
+    </late-process>
+</process>

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/bdcf001f/falcon-regression/merlin/src/test/resources/LocalDC_feedReplicaltion_BillingRC/FETL-BillingRC.xml
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin/src/test/resources/LocalDC_feedReplicaltion_BillingRC/FETL-BillingRC.xml b/falcon-regression/merlin/src/test/resources/LocalDC_feedReplicaltion_BillingRC/FETL-BillingRC.xml
new file mode 100755
index 0000000..d7e0a91
--- /dev/null
+++ b/falcon-regression/merlin/src/test/resources/LocalDC_feedReplicaltion_BillingRC/FETL-BillingRC.xml
@@ -0,0 +1,56 @@
+<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+<!--
+  Licensed to the Apache Software Foundation (ASF) under one
+  or more contributor license agreements.  See the NOTICE file
+  distributed with this work for additional information
+  regarding copyright ownership.  The ASF licenses this file
+  to you under the Apache License, Version 2.0 (the
+  "License"); you may not use this file except in compliance
+  with the License.  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+  -->
+
+<feed name="FETL-BillingRC" description="billing RC File" xmlns="uri:falcon:feed:0.1">
+    <partitions>
+        <partition name="colo"/>
+        <partition name="eventTime"/>
+        <partition name="impressionHour"/>
+        <partition name="pricingModel"/>
+    </partitions>
+    <frequency>minutes(5)</frequency>
+    <late-arrival cut-off="days(100000)"/>
+    <clusters>
+        <cluster name="lhr1-emerald" type="target" partition="${cluster.colo}">
+            <validity start="2012-07-20T00:00Z" end="2099-07-16T00:00Z"/>
+            <retention limit="days(10000)" action="delete"/>
+        </cluster>
+        <cluster name="ua2-ruby" type="target" partition="${cluster.colo}">
+            <validity start="2012-07-16T00:00Z" end="2099-07-16T00:00Z"/>
+            <retention limit="days(10000)" action="delete"/>
+        </cluster>
+         <cluster name="prod-gloabl" type="source">
+            <validity start="2012-07-20T03:00Z" end="2099-07-16T00:00Z"/>
+            <retention limit="days(10000)" action="delete"/>
+        </cluster>
+    </clusters>
+    <locations>
+        <location type="data" path="/data/regression/fetlrc/billing/${YEAR}/${MONTH}/${DAY}/${HOUR}/${MINUTE}"/>
+        <location type="stats" path="/data/regression/fetlrc/billing/stats"/>
+        <location type="meta" path="/data/regression/fetlrc/billing/metadata"/>
+    </locations>
+    <ACL owner="fetl" group="group" permission="0x755"/>
+    <schema location="/databus/streams_local/click_rr/schema/" provider="protobuf"/>
+
+    <properties>
+    <property name="field1" value="value1" />
+    <property name="field2" value="value2" />
+  </properties>
+</feed>
+

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/bdcf001f/falcon-regression/merlin/src/test/resources/LocalDC_feedReplicaltion_BillingRC/cluster-0.1.xml
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin/src/test/resources/LocalDC_feedReplicaltion_BillingRC/cluster-0.1.xml b/falcon-regression/merlin/src/test/resources/LocalDC_feedReplicaltion_BillingRC/cluster-0.1.xml
new file mode 100755
index 0000000..afa86c4
--- /dev/null
+++ b/falcon-regression/merlin/src/test/resources/LocalDC_feedReplicaltion_BillingRC/cluster-0.1.xml
@@ -0,0 +1,38 @@
+<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+<!--
+  Licensed to the Apache Software Foundation (ASF) under one
+  or more contributor license agreements.  See the NOTICE file
+  distributed with this work for additional information
+  regarding copyright ownership.  The ASF licenses this file
+  to you under the Apache License, Version 2.0 (the
+  "License"); you may not use this file except in compliance
+  with the License.  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+  -->
+
+<cluster name="ivory2-stg4" description="" colo="stg" xmlns="uri:falcon:cluster:0.1">
+    <interfaces>
+        <interface type="readonly" endpoint="hftp://stg-nn.blue.ua2.inmobi.com:50070" version="0.20.2-cdh3u0"/>
+        <interface type="execute" endpoint="stg-jt.blue.ua2.inmobi.com:54311" version="0.20.2-cdh3u0"/>
+        <interface type="write" endpoint="hdfs://stg-nn.blue.ua2.inmobi.com:54310" version="0.20.2-cdh3u0"/>
+        <interface type="messaging" endpoint="tcp://gs1134.blue.ua2.inmobi.com:61618?daemon=true" version="5.1.6"/>
+        <interface type="workflow" endpoint="http://gs1134.blue.ua2.inmobi.com:11002/oozie/" version="3.1.4"/>
+    </interfaces>
+    <locations>
+        <location name="staging" path="/projects/ivory2/stg/staging"/>
+        <location name="temp" path="/tmp"/>
+        <location name="working" path="/projects/ivory2/stg/working"/>
+    </locations>
+    <properties>
+        <property name="colo.name" value="ua2"/>
+        <property name="hbase.zookeeper.quorum" value="gs1001.grid.corp.inmobi.com"/>
+        <property name="hbase.zookeeper.property.clientPort" value="2181"/>
+    </properties>
+</cluster>

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/bdcf001f/falcon-regression/merlin/src/test/resources/OozieExampleInputData/2ndLateData/Configuration.java
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin/src/test/resources/OozieExampleInputData/2ndLateData/Configuration.java b/falcon-regression/merlin/src/test/resources/OozieExampleInputData/2ndLateData/Configuration.java
new file mode 100644
index 0000000..1ab0140
--- /dev/null
+++ b/falcon-regression/merlin/src/test/resources/OozieExampleInputData/2ndLateData/Configuration.java
@@ -0,0 +1,73 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ivory.entity.common;
+
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.concurrent.ConcurrentHashMap;
+
+public class Configuration implements Iterable<Map.Entry<String, String>>, Cloneable {
+
+  private final Map<String, String> properties;
+
+  public Configuration() {
+    properties = new ConcurrentHashMap<String, String>();
+  }
+
+  public Configuration(Map<String, String> properties) {
+    this.properties = properties;
+  }
+
+  public void addConfiguration(Configuration config) {
+    for (Entry<String, String> entry : config) {
+      properties.put(entry.getKey(), entry.getValue());
+    }
+  }
+
+  public Configuration addAndReturnNewConfiguration(Configuration config) {
+    Map<String, String> newProperties = new ConcurrentHashMap<String, String>(properties);
+    for (Entry<String, String> entry : config) {
+      newProperties.put(entry.getKey(), entry.getValue());
+    }
+    return new Configuration(newProperties);
+  }
+
+  public String getConf(String name) {
+    return properties.get(name);
+  }
+
+  public void setConf(String name, String value) {
+    properties.put(name, value);
+  }
+
+  public void setConf(String name, String value, String defaultValue) {
+    if (value == null) {
+      properties.put(name, defaultValue);
+    } else {
+      properties.put(name, value);
+    }
+  }
+
+  @Override
+  public Iterator<Entry<String, String>> iterator() {
+    return properties.entrySet().iterator();
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/bdcf001f/falcon-regression/merlin/src/test/resources/OozieExampleInputData/2ndLateData/DateValidator.java
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin/src/test/resources/OozieExampleInputData/2ndLateData/DateValidator.java b/falcon-regression/merlin/src/test/resources/OozieExampleInputData/2ndLateData/DateValidator.java
new file mode 100644
index 0000000..f02ed34
--- /dev/null
+++ b/falcon-regression/merlin/src/test/resources/OozieExampleInputData/2ndLateData/DateValidator.java
@@ -0,0 +1,87 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ivory.entity.common;
+
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+public class DateValidator {
+
+	private Pattern pattern;
+	private Matcher matcher;
+
+	private static final String DATE_PATTERN = "(2\\d\\d\\d|19\\d\\d)-(0[1-9]|1[012])-(0[1-9]|1[0-9]|2[0-9]|3[01])T([0-1][0-9]|2[0-3]):([0-5][0-9])Z";
+
+	public DateValidator() {
+		pattern = Pattern.compile(DATE_PATTERN);
+	}
+
+	/**
+	 * Validate date format with regular expression
+	 * 
+	 * @param date
+	 *            date address for validation
+	 * @return true valid date fromat, false invalid date format
+	 */
+	public boolean validate(final String date) {
+
+		matcher = pattern.matcher(date);
+
+		if (matcher.matches()) {
+
+			matcher.reset();
+
+			if (matcher.find()) {
+
+				int year = Integer.parseInt(matcher.group(1));
+				String month = matcher.group(2);
+				String day = matcher.group(3);
+
+				if (day.equals("31")
+						&& (month.equals("4") || month.equals("6")
+								|| month.equals("9") || month.equals("11")
+								|| month.equals("04") || month.equals("06") || month
+									.equals("09"))) {
+					return false; // only 1,3,5,7,8,10,12 has 31 days
+				} else if (month.equals("2") || month.equals("02")) {
+					// leap year
+					if (year % 4 == 0) {
+						if (day.equals("30") || day.equals("31")) {
+							return false;
+						} else {
+							return true;
+						}
+					} else {
+						if (day.equals("29") || day.equals("30")
+								|| day.equals("31")) {
+							return false;
+						} else {
+							return true;
+						}
+					}
+				} else {
+					return true;
+				}
+			} else {
+				return false;
+			}
+		} else {
+			return false;
+		}
+	}
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/bdcf001f/falcon-regression/merlin/src/test/resources/OozieExampleInputData/2ndLateData/TimeUnit.java
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin/src/test/resources/OozieExampleInputData/2ndLateData/TimeUnit.java b/falcon-regression/merlin/src/test/resources/OozieExampleInputData/2ndLateData/TimeUnit.java
new file mode 100644
index 0000000..67f9d4c
--- /dev/null
+++ b/falcon-regression/merlin/src/test/resources/OozieExampleInputData/2ndLateData/TimeUnit.java
@@ -0,0 +1,36 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ivory.entity.common;
+
+import java.util.Calendar;
+
+public enum TimeUnit {
+    MINUTE(Calendar.MINUTE), HOUR(Calendar.HOUR), DAY(Calendar.DATE), MONTH(Calendar.MONTH), END_OF_DAY(Calendar.DATE), END_OF_MONTH(
+        Calendar.MONTH), NONE(-1);
+
+    private int calendarUnit;
+
+    private TimeUnit(int calendarUnit) {
+        this.calendarUnit = calendarUnit;
+    }
+
+    public int getCalendarUnit() {
+        return calendarUnit;
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/bdcf001f/falcon-regression/merlin/src/test/resources/OozieExampleInputData/SingleFile/minimumText.txt
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin/src/test/resources/OozieExampleInputData/SingleFile/minimumText.txt b/falcon-regression/merlin/src/test/resources/OozieExampleInputData/SingleFile/minimumText.txt
new file mode 100644
index 0000000..3df60b0
--- /dev/null
+++ b/falcon-regression/merlin/src/test/resources/OozieExampleInputData/SingleFile/minimumText.txt
@@ -0,0 +1,17 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/bdcf001f/falcon-regression/merlin/src/test/resources/OozieExampleInputData/lateData/log_15.txt
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin/src/test/resources/OozieExampleInputData/lateData/log_15.txt b/falcon-regression/merlin/src/test/resources/OozieExampleInputData/lateData/log_15.txt
new file mode 100644
index 0000000..f3f94c5
--- /dev/null
+++ b/falcon-regression/merlin/src/test/resources/OozieExampleInputData/lateData/log_15.txt
@@ -0,0 +1,19 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+[15] LOG!!!!!!
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/bdcf001f/falcon-regression/merlin/src/test/resources/OozieExampleInputData/normalInput/EntityInstanceMessage.java.ignore.java
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin/src/test/resources/OozieExampleInputData/normalInput/EntityInstanceMessage.java.ignore.java b/falcon-regression/merlin/src/test/resources/OozieExampleInputData/normalInput/EntityInstanceMessage.java.ignore.java
new file mode 100644
index 0000000..f992f21
--- /dev/null
+++ b/falcon-regression/merlin/src/test/resources/OozieExampleInputData/normalInput/EntityInstanceMessage.java.ignore.java
@@ -0,0 +1,412 @@
+///**
+// * Licensed to the Apache Software Foundation (ASF) under one
+// * or more contributor license agreements.  See the NOTICE file
+// * distributed with this work for additional information
+// * regarding copyright ownership.  The ASF licenses this file
+// * to you under the Apache License, Version 2.0 (the
+// * "License"); you may not use this file except in compliance
+// * with the License.  You may obtain a copy of the License at
+// *
+// *     http://www.apache.org/licenses/LICENSE-2.0
+// *
+// * Unless required by applicable law or agreed to in writing, software
+// * distributed under the License is distributed on an "AS IS" BASIS,
+// * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// * See the License for the specific language governing permissions and
+// * limitations under the License.
+// */
+//
+//package org.apache.ivory.messaging;
+//
+//import org.apache.hadoop.conf.Configuration;
+//import org.apache.hadoop.fs.FileSystem;
+//import org.apache.hadoop.fs.Path;
+//import org.apache.hadoop.io.IOUtils;
+//import org.apache.log4j.Logger;
+//
+//import java.io.ByteArrayOutputStream;
+//import java.io.IOException;
+//import java.io.InputStream;
+//import java.util.HashMap;
+//import java.util.Map;
+//
+///**
+// * Value Object which is stored in JMS Topic as TextMessage
+// *
+// */
+//public class EntityInstanceMessage {
+//
+//	private final Map<ARG, String> msgs = new HashMap<ARG, String>();
+//	private static final String MSG_SEPERATOR = "$";
+//	private static final int ARG_LENGTH = EntityInstanceMessage.ARG.values().length;
+//	private static final Logger LOG = Logger
+//			.getLogger(EntityInstanceMessage.class);
+//	private static final String IVORY_PROCESS_TOPIC_NAME = "IVORY.PROCESS.TOPIC";
+//
+//	/**
+//	 * Enum for arguments that are used in coordinators to pass arguments to
+//	 * parent workflow
+//	 */
+//
+//	public enum entityOperation {
+//		GENERATE, DELETE, ARCHIVE, REPLICATE, CHMOD
+//	}
+//
+//	public enum ARG {
+//		PROCESS_NAME(0, "entityName"), FEED_NAME(1, "feedNames"), FEED_INSTANCE_PATH(
+//				2, "feedInstancePaths"), WORKFLOW_ID(3, "workflowId"), RUN_ID(
+//				4, "runId"), NOMINAL_TIME(5, "nominalTime"), TIME_STAMP(6,
+//				"timeStamp"), BROKER_URL(7, "brokerUrl"), BROKER_IMPL_CLASS(8,
+//				"brokerImplClass"), ENTITY_TYPE(9, "entityType"), OPERATION(10,
+//				"operation"), LOG_FILE(11, "logFile"), TOPIC_NAME(12,
+//				"topicName"), STATUS(13, "status"), BROKER_TTL(14, "broker.ttlInMins");
+//
+//		private int argOrder;
+//		private String argName;
+//
+//		private ARG(int argOrder, String argName) {
+//			this.argOrder = argOrder;
+//			this.argName = argName;
+//		}
+//
+//		public int ORDER() {
+//			return this.argOrder;
+//		}
+//
+//		public String NAME() {
+//			return this.argName;
+//		}
+//	}
+//
+//	public String getProcessName() {
+//		return this.msgs.get(ARG.PROCESS_NAME);
+//	}
+//
+//	public void setProcessName(String processName) {
+//		this.msgs.put(ARG.PROCESS_NAME, processName);
+//	}
+//
+//	public String getTopicName() {
+//		return this.msgs.get(ARG.TOPIC_NAME);
+//	}
+//
+//	public void setTopicName(String topicName) {
+//		this.msgs.put(ARG.TOPIC_NAME, topicName);
+//	}
+//
+//	public String getFeedName() {
+//		return this.msgs.get(ARG.FEED_NAME);
+//	}
+//
+//	public void setFeedName(String feedName) {
+//		this.msgs.put(ARG.FEED_NAME, feedName);
+//	}
+//
+//	public String getFeedInstancePath() {
+//		return this.msgs.get(ARG.FEED_INSTANCE_PATH);
+//	}
+//
+//	public void setFeedInstancePath(String feedInstancePath) {
+//		this.msgs.put(ARG.FEED_INSTANCE_PATH, feedInstancePath);
+//	}
+//
+//	public String getWorkflowId() {
+//		return this.msgs.get(ARG.WORKFLOW_ID);
+//	}
+//
+//	public void setWorkflowId(String workflowId) {
+//		this.msgs.put(ARG.WORKFLOW_ID, workflowId);
+//	}
+//
+//	public String getRunId() {
+//		return this.msgs.get(ARG.RUN_ID);
+//	}
+//
+//	public void setRunId(String runId) {
+//		this.msgs.put(ARG.RUN_ID, runId);
+//	}
+//
+//	public String getNominalTime() {
+//		return this.msgs.get(ARG.NOMINAL_TIME);
+//	}
+//
+//	public void setNominalTime(String nominalTime) {
+//		this.msgs.put(ARG.NOMINAL_TIME, nominalTime);
+//	}
+//
+//	public String getTimeStamp() {
+//		return this.msgs.get(ARG.TIME_STAMP);
+//	}
+//
+//	public void setTimeStamp(String timeStamp) {
+//		this.msgs.put(ARG.TIME_STAMP, timeStamp);
+//	}
+//
+//	public String getBrokerUrl() {
+//		return this.msgs.get(ARG.BROKER_URL);
+//	}
+//
+//	public void setBrokerUrl(String brokerUrl) {
+//		this.msgs.put(ARG.BROKER_URL, brokerUrl);
+//	}
+//
+//	public String getBrokerImplClass() {
+//		return this.msgs.get(ARG.BROKER_IMPL_CLASS);
+//	}
+//
+//	public void setBrokerImplClass(String brokerImplClass) {
+//		this.msgs.put(ARG.BROKER_IMPL_CLASS, brokerImplClass);
+//	}
+//
+//	public String getEntityType() {
+//		return this.msgs.get(ARG.ENTITY_TYPE);
+//	}
+//
+//	public void setEntityType(String entityType) {
+//		this.msgs.put(ARG.ENTITY_TYPE, entityType);
+//	}
+//
+//	public String getOperation() {
+//		return this.msgs.get(ARG.OPERATION);
+//	}
+//
+//	public void setOperation(String operation) {
+//		this.msgs.put(ARG.OPERATION, operation);
+//	}
+//
+//	public String getLogFile() {
+//		return this.msgs.get(ARG.LOG_FILE);
+//	}
+//
+//	public void setLogFile(String logFile) {
+//		this.msgs.put(ARG.LOG_FILE, logFile);
+//	}
+//
+//	public String getStatus() {
+//		return this.msgs.get(ARG.STATUS);
+//	}
+//
+//	public void setStatus(String status) {
+//		this.msgs.put(ARG.STATUS, status);
+//	}
+//
+//	public String getBrokerTTL() {
+//		return this.msgs.get(ARG.BROKER_TTL);
+//	}
+//
+//	public void setBrokerTTL(String brokerTTL) {
+//		this.msgs.put(ARG.BROKER_TTL, brokerTTL);
+//	}
+//
+//	@Override
+//	public String toString() {
+//		if (getEntityType().equalsIgnoreCase("PROCESS")
+//				&& getTopicName().equals(IVORY_PROCESS_TOPIC_NAME)) {
+//			return getIvoryMessage();
+//		}
+//		if (getEntityType().equalsIgnoreCase("FEED")) {
+//			return getFeedMessage();
+//		}
+//		return getProcessMessage();
+//
+//	}
+//
+//	private String getProcessMessage() {
+//		return getProcessName() + MSG_SEPERATOR + getFeedName() + MSG_SEPERATOR
+//				+ getFeedInstancePath() + MSG_SEPERATOR + getWorkflowId()
+//				+ MSG_SEPERATOR + getRunId() + MSG_SEPERATOR + getNominalTime()
+//				+ MSG_SEPERATOR + getTimeStamp();
+//	}
+//
+//	private String getIvoryMessage() {
+//		return getProcessName() + MSG_SEPERATOR + getFeedName() + MSG_SEPERATOR
+//				+ getFeedInstancePath() + MSG_SEPERATOR + getWorkflowId()
+//				+ MSG_SEPERATOR + getRunId() + MSG_SEPERATOR + getNominalTime()
+//				+ MSG_SEPERATOR + getTimeStamp() + MSG_SEPERATOR + getStatus();
+//	}
+//
+//	private String getFeedMessage() {
+//		return getFeedName() + MSG_SEPERATOR
+//				+ getFeedInstancePath() + MSG_SEPERATOR + getOperation()
+//				+ MSG_SEPERATOR + getWorkflowId() + MSG_SEPERATOR + getRunId()
+//				+ MSG_SEPERATOR + getNominalTime() + MSG_SEPERATOR
+//				+ getTimeStamp();
+//	}
+//
+//	/**
+//	 *
+//	 * @param args
+//	 *            - String array passed from oozie action for jms messaging
+//	 * @return ProcessMessage - Value object which is stored in JMS topic
+//	 */
+//	public static EntityInstanceMessage[] argsToMessage(String[] args) {
+//
+//		assert args.length == ARG_LENGTH : "Required number of arguments: "
+//				+ ARG_LENGTH;
+//
+//		String[] feedNames = getFeedNames(args);
+//
+//		String[] feedPaths;
+//		try {
+//			feedPaths = getFeedPaths(args);
+//		} catch (IOException e) {
+//			LOG.error("Error getting instance paths: ", e);
+//			throw new RuntimeException(e);
+//		}
+//
+//		EntityInstanceMessage[] processMessages = new EntityInstanceMessage[feedPaths.length];
+//		for (int i = 0; i < feedPaths.length; i++) {
+//			EntityInstanceMessage instanceMessage = new EntityInstanceMessage();
+//			instanceMessage
+//					.setProcessName(args[EntityInstanceMessage.ARG.PROCESS_NAME
+//							.ORDER()]);
+//			if (args[EntityInstanceMessage.ARG.ENTITY_TYPE.ORDER()]
+//					.equalsIgnoreCase("PROCESS")) {
+//				instanceMessage.setFeedName(feedNames[i]);
+//			} else {
+//				instanceMessage
+//						.setFeedName(args[EntityInstanceMessage.ARG.FEED_NAME
+//								.ORDER()]);
+//			}
+//			instanceMessage.setFeedInstancePath(feedPaths[i]);
+//			instanceMessage
+//					.setWorkflowId(args[EntityInstanceMessage.ARG.WORKFLOW_ID
+//							.ORDER()]);
+//			instanceMessage.setRunId(args[EntityInstanceMessage.ARG.RUN_ID
+//					.ORDER()]);
+//			instanceMessage
+//					.setNominalTime(args[EntityInstanceMessage.ARG.NOMINAL_TIME
+//							.ORDER()]);
+//			instanceMessage
+//					.setTimeStamp(args[EntityInstanceMessage.ARG.TIME_STAMP
+//							.ORDER()]);
+//			instanceMessage
+//					.setBrokerUrl(args[EntityInstanceMessage.ARG.BROKER_URL
+//							.ORDER()]);
+//			instanceMessage
+//					.setBrokerImplClass(args[EntityInstanceMessage.ARG.BROKER_IMPL_CLASS
+//							.ORDER()]);
+//			instanceMessage
+//					.setEntityType(args[EntityInstanceMessage.ARG.ENTITY_TYPE
+//							.ORDER()]);
+//			instanceMessage
+//					.setOperation(args[EntityInstanceMessage.ARG.OPERATION
+//							.ORDER()]);
+//			instanceMessage.setLogFile(args[EntityInstanceMessage.ARG.LOG_FILE
+//					.ORDER()]);
+//			instanceMessage
+//					.setTopicName(args[EntityInstanceMessage.ARG.TOPIC_NAME
+//							.ORDER()]);
+//			instanceMessage.setStatus(args[EntityInstanceMessage.ARG.STATUS
+//					.ORDER()]);
+//			instanceMessage.setBrokerTTL(args[EntityInstanceMessage.ARG.BROKER_TTL
+//			           					.ORDER()]);
+//
+//			processMessages[i] = instanceMessage;
+//		}
+//		return processMessages;
+//	}
+//
+//	private static String[] getFeedNames(String[] args) {
+//		String topicName = args[ARG.TOPIC_NAME.argOrder];
+//		if (topicName.equals(IVORY_PROCESS_TOPIC_NAME)) {
+//			return new String[] { args[EntityInstanceMessage.ARG.FEED_NAME
+//					.ORDER()] };
+//		}
+//		return args[EntityInstanceMessage.ARG.FEED_NAME.ORDER()].split(",");
+//	}
+//
+//	private static String[] getFeedPaths(String[] args) throws IOException {
+//		String entityType = args[EntityInstanceMessage.ARG.ENTITY_TYPE.ORDER()];
+//		String topicName = args[EntityInstanceMessage.ARG.TOPIC_NAME.ORDER()];
+//
+//		if (entityType.equalsIgnoreCase("PROCESS")
+//				&& topicName.equals(IVORY_PROCESS_TOPIC_NAME)) {
+//			LOG.debug("Returning instance paths for Ivory Topic: "
+//					+ args[EntityInstanceMessage.ARG.FEED_INSTANCE_PATH.ORDER()]);
+//			return new String[] { args[EntityInstanceMessage.ARG.FEED_INSTANCE_PATH
+//					.ORDER()] };
+//		}
+//
+//		if (entityType.equalsIgnoreCase("PROCESS")) {
+//			LOG.debug("Returning instance paths for process: "
+//					+ args[EntityInstanceMessage.ARG.FEED_INSTANCE_PATH.ORDER()]);
+//			return args[EntityInstanceMessage.ARG.FEED_INSTANCE_PATH.ORDER()]
+//					.split(",");
+//		}
+//		//
+//		Path logFile = new Path(
+//				args[EntityInstanceMessage.ARG.LOG_FILE.ORDER()]);
+//		FileSystem fs = FileSystem.get(logFile.toUri(), new Configuration());
+//		ByteArrayOutputStream writer = new ByteArrayOutputStream();
+//		InputStream instance = fs.open(logFile);
+//		IOUtils.copyBytes(instance, writer, 4096, true);
+//		String[] instancePaths = writer.toString().split("=");
+//		if (instancePaths.length == 1) {
+//			LOG.debug("Returning 0 instance paths for feed ");
+//			return new String[0];
+//		} else {
+//			LOG.debug("Returning instance paths for feed " + instancePaths[1]);
+//			return instancePaths[1].split(",");
+//		}
+//
+//	}
+//
+//	/**
+//	 *
+//	 * @param instanceMessages
+//	 *            - value object which is stored in JMS topic as TextMessage
+//	 * @return - String array.
+//	 */
+//	public static String[] messageToArgs(
+//			EntityInstanceMessage[] instanceMessages) {
+//		String[] args = new String[ARG_LENGTH];
+//
+//		args[EntityInstanceMessage.ARG.PROCESS_NAME.ORDER()] = instanceMessages[0]
+//				.getProcessName();
+//		StringBuilder feedNames = new StringBuilder();
+//		StringBuilder feedPaths = new StringBuilder();
+//
+//		for (EntityInstanceMessage instanceMessage : instanceMessages) {
+//			feedNames.append(instanceMessage.getFeedName()).append(",");
+//			feedPaths.append(instanceMessage.getFeedInstancePath()).append(",");
+//		}
+//		if (instanceMessages[0].getEntityType().equalsIgnoreCase("PROCESS")) {
+//			args[EntityInstanceMessage.ARG.FEED_NAME.ORDER()] = feedNames
+//					.toString();
+//
+//		} else {
+//			args[EntityInstanceMessage.ARG.FEED_NAME.ORDER()] = instanceMessages[0]
+//					.getFeedName();
+//		}
+//		args[EntityInstanceMessage.ARG.FEED_INSTANCE_PATH.ORDER()] = feedPaths
+//				.toString();
+//		args[EntityInstanceMessage.ARG.WORKFLOW_ID.ORDER()] = instanceMessages[0]
+//				.getWorkflowId();
+//		args[EntityInstanceMessage.ARG.RUN_ID.ORDER()] = instanceMessages[0]
+//				.getRunId();
+//		args[EntityInstanceMessage.ARG.NOMINAL_TIME.ORDER()] = instanceMessages[0]
+//				.getNominalTime();
+//		args[EntityInstanceMessage.ARG.TIME_STAMP.ORDER()] = instanceMessages[0]
+//				.getTimeStamp();
+//		args[EntityInstanceMessage.ARG.BROKER_URL.ORDER()] = instanceMessages[0]
+//				.getBrokerUrl();
+//		args[EntityInstanceMessage.ARG.BROKER_IMPL_CLASS.ORDER()] = instanceMessages[0]
+//				.getBrokerImplClass();
+//		args[EntityInstanceMessage.ARG.ENTITY_TYPE.ORDER()] = instanceMessages[0]
+//				.getEntityType();
+//		args[EntityInstanceMessage.ARG.OPERATION.ORDER()] = instanceMessages[0]
+//				.getOperation();
+//		args[EntityInstanceMessage.ARG.LOG_FILE.ORDER()] = instanceMessages[0]
+//				.getLogFile();
+//		args[EntityInstanceMessage.ARG.TOPIC_NAME.ORDER()] = instanceMessages[0]
+//				.getTopicName();
+//		args[EntityInstanceMessage.ARG.STATUS.ORDER()] = instanceMessages[0]
+//				.getStatus();
+//		args[EntityInstanceMessage.ARG.BROKER_TTL.ORDER()] = instanceMessages[0]
+//				.getBrokerTTL();
+//
+//		return args;
+//	}
+//
+//}

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/bdcf001f/falcon-regression/merlin/src/test/resources/OozieExampleInputData/normalInput/EntityInstanceMessageCreator.java.ignore.java
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin/src/test/resources/OozieExampleInputData/normalInput/EntityInstanceMessageCreator.java.ignore.java b/falcon-regression/merlin/src/test/resources/OozieExampleInputData/normalInput/EntityInstanceMessageCreator.java.ignore.java
new file mode 100644
index 0000000..f7e8bbb
--- /dev/null
+++ b/falcon-regression/merlin/src/test/resources/OozieExampleInputData/normalInput/EntityInstanceMessageCreator.java.ignore.java
@@ -0,0 +1,61 @@
+///**
+// * Licensed to the Apache Software Foundation (ASF) under one
+// * or more contributor license agreements.  See the NOTICE file
+// * distributed with this work for additional information
+// * regarding copyright ownership.  The ASF licenses this file
+// * to you under the Apache License, Version 2.0 (the
+// * "License"); you may not use this file except in compliance
+// * with the License.  You may obtain a copy of the License at
+// *
+// *     http://www.apache.org/licenses/LICENSE-2.0
+// *
+// * Unless required by applicable law or agreed to in writing, software
+// * distributed under the License is distributed on an "AS IS" BASIS,
+// * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// * See the License for the specific language governing permissions and
+// * limitations under the License.
+// */
+//
+//package org.apache.ivory.messaging;
+//
+//import javax.jms.JMSException;
+//import javax.jms.Session;
+//import javax.jms.TextMessage;
+//
+//import org.apache.log4j.Logger;
+//
+///**
+// * Ivory JMS message creator- creates JMS TextMessage
+// */
+//public class EntityInstanceMessageCreator  {
+//
+//	private static final Logger LOG = Logger
+//			.getLogger(EntityInstanceMessageCreator.class);
+//
+//	private TextMessage textMessage;
+//
+//	private final EntityInstanceMessage args;
+//
+//	public EntityInstanceMessageCreator(EntityInstanceMessage args) {
+//		this.args = args;
+//	}
+//
+//	public TextMessage createMessage(Session session) throws JMSException {
+//		this.textMessage = session.createTextMessage();
+//		this.textMessage.setText(this.args.toString());
+//		LOG.debug("Sending Message: " + this.textMessage.getText());
+//		// System.out.println("Sending Message: " + this.textMessage);
+//		return this.textMessage;
+//	}
+//
+//	@Override
+//	public String toString() {
+//		try {
+//			return this.textMessage.getText();
+//		} catch (JMSException e) {
+//			return e.getMessage();
+//		}
+//
+//	}
+//
+//}

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/bdcf001f/falcon-regression/merlin/src/test/resources/OozieExampleInputData/normalInput/MessageProducer.java.ignore.java
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin/src/test/resources/OozieExampleInputData/normalInput/MessageProducer.java.ignore.java b/falcon-regression/merlin/src/test/resources/OozieExampleInputData/normalInput/MessageProducer.java.ignore.java
new file mode 100644
index 0000000..7234f52
--- /dev/null
+++ b/falcon-regression/merlin/src/test/resources/OozieExampleInputData/normalInput/MessageProducer.java.ignore.java
@@ -0,0 +1,139 @@
+///**
+// * Licensed to the Apache Software Foundation (ASF) under one
+// * or more contributor license agreements.  See the NOTICE file
+// * distributed with this work for additional information
+// * regarding copyright ownership.  The ASF licenses this file
+// * to you under the Apache License, Version 2.0 (the
+// * "License"); you may not use this file except in compliance
+// * with the License.  You may obtain a copy of the License at
+// *
+// *     http://www.apache.org/licenses/LICENSE-2.0
+// *
+// * Unless required by applicable law or agreed to in writing, software
+// * distributed under the License is distributed on an "AS IS" BASIS,
+// * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// * See the License for the specific language governing permissions and
+// * limitations under the License.
+// */
+//
+//package org.apache.ivory.messaging;
+//
+//import java.lang.reflect.InvocationTargetException;
+//
+//import javax.jms.Connection;
+//import javax.jms.ConnectionFactory;
+//import javax.jms.DeliveryMode;
+//import javax.jms.JMSException;
+//import javax.jms.Session;
+//import javax.jms.Topic;
+//
+//import org.apache.log4j.Logger;
+//
+///**
+// * Default Ivory Message Producer The configuration are loaded from
+// * jms-beans.xml
+// */
+//public class MessageProducer {
+//
+//	private Connection connection;
+//	private static final Logger LOG = Logger.getLogger(MessageProducer.class);
+//	private static final long DEFAULT_TTL = 3 * 24 * 60 * 60 * 1000;
+//
+//	/**
+//	 *
+//	 * @param arguments
+//	 *            - Accepts a Message to be send to JMS topic, creates a new
+//	 *            Topic based on topic name if it does not exist or else
+//	 *            existing topic with the same name is used to send the message.
+//	 * @throws JMSException
+//	 */
+//	protected void sendMessage(EntityInstanceMessage entityInstanceMessage)
+//			throws JMSException {
+//
+//		Session session = connection.createSession(false,
+//				Session.AUTO_ACKNOWLEDGE);
+//		Topic entityTopic = session.createTopic(entityInstanceMessage
+//				.getTopicName());
+//		javax.jms.MessageProducer producer = session
+//				.createProducer(entityTopic);
+//		producer.setDeliveryMode(DeliveryMode.PERSISTENT);
+//		long messageTTL = DEFAULT_TTL;
+//		try {
+//			long messageTTLinMins = Long.parseLong(entityInstanceMessage
+//					.getBrokerTTL());
+//			messageTTL = messageTTLinMins * 60 * 1000;
+//		} catch (NumberFormatException e) {
+//			LOG.error("Error in parsing broker.ttl, setting TTL to:"
+//					+ DEFAULT_TTL+ " milli-seconds");
+//		}
+//		producer.setTimeToLive(messageTTL);
+//		producer.send(new EntityInstanceMessageCreator(entityInstanceMessage)
+//				.createMessage(session));
+//
+//	}
+//
+//	/**
+//	 *
+//	 * @param args
+//	 *            - array of Strings, which will be used to create TextMessage
+//	 */
+//	public static void main(String[] args) {
+//		debug(args);
+//		EntityInstanceMessage[] entityInstanceMessage = EntityInstanceMessage
+//				.argsToMessage(args);
+//		if (entityInstanceMessage.length == 0) {
+//			LOG.warn("No operation on output feed");
+//			return;
+//		}
+//
+//		MessageProducer ivoryMessageProducer = new MessageProducer();
+//		try {
+//			ivoryMessageProducer.createAndStartConnection(
+//					args[EntityInstanceMessage.ARG.BROKER_IMPL_CLASS.ORDER()],
+//					"", "", entityInstanceMessage[0].getBrokerUrl());
+//			for (EntityInstanceMessage processMessage : entityInstanceMessage) {
+//				ivoryMessageProducer.sendMessage(processMessage);
+//			}
+//		} catch (JMSException e) {
+//			LOG.error(e);
+//			e.printStackTrace();
+//		} catch (Exception e) {
+//			LOG.error(e);
+//			e.printStackTrace();
+//		} finally {
+//			try {
+//				ivoryMessageProducer.connection.close();
+//			} catch (JMSException e) {
+//				e.printStackTrace();
+//			}
+//		}
+//
+//	}
+//
+//	private static void debug(String[] args) {
+//		if (LOG.isDebugEnabled()) {
+//			for (int i = 0; i < args.length; i++) {
+//				LOG.debug(args[i] + "::");
+//			}
+//		}
+//
+//	}
+//
+//	private void createAndStartConnection(String implementation,
+//			String userName, String password, String url) throws JMSException,
+//			ClassNotFoundException, IllegalArgumentException,
+//			SecurityException, InstantiationException, IllegalAccessException,
+//			InvocationTargetException, NoSuchMethodException {
+//
+//		Class<ConnectionFactory> clazz = (Class<ConnectionFactory>) MessageProducer.class
+//				.getClassLoader().loadClass(implementation);
+//
+//		ConnectionFactory connectionFactory = clazz.getConstructor(
+//				String.class, String.class, String.class).newInstance(userName,
+//				password, url);
+//
+//		connection = connectionFactory.createConnection();
+//		connection.start();
+//	}
+//
+//}

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/bdcf001f/falcon-regression/merlin/src/test/resources/OozieExampleInputData/normalInput/_SUCCESS
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin/src/test/resources/OozieExampleInputData/normalInput/_SUCCESS b/falcon-regression/merlin/src/test/resources/OozieExampleInputData/normalInput/_SUCCESS
new file mode 100644
index 0000000..4c61a15
--- /dev/null
+++ b/falcon-regression/merlin/src/test/resources/OozieExampleInputData/normalInput/_SUCCESS
@@ -0,0 +1,19 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+test file needed 
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/bdcf001f/falcon-regression/merlin/src/test/resources/OozieExampleInputData/normalInput/jms-config.properties
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin/src/test/resources/OozieExampleInputData/normalInput/jms-config.properties b/falcon-regression/merlin/src/test/resources/OozieExampleInputData/normalInput/jms-config.properties
new file mode 100644
index 0000000..4d3d972
--- /dev/null
+++ b/falcon-regression/merlin/src/test/resources/OozieExampleInputData/normalInput/jms-config.properties
@@ -0,0 +1,31 @@
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+
+
+#Global configuration for Ivory JMS, default to use embedded ActiveMQ.
+
+#Defualt Active MQ url
+#ivory.broker.url = tcp://localhost:61616?daemon=true
+
+#Embedded
+ivory.broker.url = vm://localhost?broker.useJmx=false&broker.persistent=true
+
+
+


Mime
View raw message