falcon-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ajayyad...@apache.org
Subject [02/12] falcon git commit: FALCON-1188 Falcon support for Hive Replication. Contributed by Venkat Ranganathan.
Date Thu, 13 Aug 2015 09:08:46 GMT
http://git-wip-us.apache.org/repos/asf/falcon/blob/cc1d3840/src/main/assemblies/distributed-package.xml
----------------------------------------------------------------------
diff --git a/src/main/assemblies/distributed-package.xml b/src/main/assemblies/distributed-package.xml
index 1e48c94..0b56572 100644
--- a/src/main/assemblies/distributed-package.xml
+++ b/src/main/assemblies/distributed-package.xml
@@ -21,6 +21,7 @@
     xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2
http://maven.apache.org/xsd/assembly-1.1.2.xsd">
     <formats>
         <format>tar.gz</format>
+        <format>zip</format>
     </formats>
     <id>server</id>
     <baseDirectory>falcon-distributed-${project.version}</baseDirectory>
@@ -126,6 +127,20 @@
             <source>oozie-el-extensions/src/main/conf/oozie-site.xml</source>
             <outputDirectory>oozie/conf</outputDirectory>
         </file>
+
+        <file>
+            <source>addons/recipes/hdfs-replication/src/main/resources/hdfs-replication-workflow.xml</source>
+            <outputDirectory>data-mirroring/workflows</outputDirectory>
+        </file>
+
+        <file>
+            <source>addons/recipes/hive-disaster-recovery/src/main/resources/hive-disaster-recovery-workflow.xml</source>
+            <outputDirectory>data-mirroring/workflows</outputDirectory>
+        </file>
+        <file>
+            <source>addons/recipes/hive-disaster-recovery/src/main/resources/hive-disaster-recovery-secure-workflow.xml</source>
+            <outputDirectory>data-mirroring/workflows</outputDirectory>
+        </file>
     </files>
 </assembly>
     

http://git-wip-us.apache.org/repos/asf/falcon/blob/cc1d3840/src/main/assemblies/standalone-package.xml
----------------------------------------------------------------------
diff --git a/src/main/assemblies/standalone-package.xml b/src/main/assemblies/standalone-package.xml
index 682d52f..40e6b99 100644
--- a/src/main/assemblies/standalone-package.xml
+++ b/src/main/assemblies/standalone-package.xml
@@ -21,6 +21,7 @@
     xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2
http://maven.apache.org/xsd/assembly-1.1.2.xsd">
     <formats>
         <format>tar.gz</format>
+        <format>zip</format>
     </formats>
     <id>bin</id>
     <baseDirectory>falcon-${project.version}</baseDirectory>
@@ -115,6 +116,21 @@
         </file>
 
         <file>
+            <source>addons/recipes/hdfs-replication/src/main/resources/hdfs-replication-workflow.xml</source>
+            <outputDirectory>data-mirroring/workflows</outputDirectory>
+        </file>
+
+        <file>
+            <source>addons/recipes/hive-disaster-recovery/src/main/resources/hive-disaster-recovery-workflow.xml</source>
+            <outputDirectory>data-mirroring/workflows</outputDirectory>
+        </file>
+
+        <file>
+            <source>addons/recipes/hive-disaster-recovery/src/main/resources/hive-disaster-recovery-secure-workflow.xml</source>
+            <outputDirectory>data-mirroring/workflows</outputDirectory>
+        </file>
+
+        <file>
             <source>webapp/target/falcon-webapp-${project.version}.war</source>
             <outputDirectory>server/webapp</outputDirectory>
             <destName>falcon.war</destName>

http://git-wip-us.apache.org/repos/asf/falcon/blob/cc1d3840/test-tools/hadoop-webapp/pom.xml
----------------------------------------------------------------------
diff --git a/test-tools/hadoop-webapp/pom.xml b/test-tools/hadoop-webapp/pom.xml
index cadc7f5..c122a89 100644
--- a/test-tools/hadoop-webapp/pom.xml
+++ b/test-tools/hadoop-webapp/pom.xml
@@ -53,6 +53,12 @@
 
                 <dependency>
                     <groupId>org.apache.hadoop</groupId>
+                    <artifactId>hadoop-common</artifactId>
+                    <classifier>tests</classifier>
+                </dependency>
+
+                <dependency>
+                    <groupId>org.apache.hadoop</groupId>
                     <artifactId>hadoop-hdfs</artifactId>
                     <scope>compile</scope>
                 </dependency>
@@ -71,12 +77,24 @@
 
                 <dependency>
                     <groupId>org.apache.hadoop</groupId>
+                    <artifactId>hadoop-distcp</artifactId>
+                    <scope>compile</scope>
+                </dependency>
+
+                <dependency>
+                    <groupId>org.apache.hadoop</groupId>
                     <artifactId>hadoop-mapreduce-client-common</artifactId>
                     <scope>compile</scope>
                 </dependency>
 
                 <dependency>
                     <groupId>org.apache.hadoop</groupId>
+                    <artifactId>hadoop-mapreduce-client-core</artifactId>
+                    <scope>compile</scope>
+                </dependency>
+
+                <dependency>
+                    <groupId>org.apache.hadoop</groupId>
                     <artifactId>hadoop-yarn-server-resourcemanager</artifactId>
                     <scope>compile</scope>
                 </dependency>
@@ -192,6 +210,11 @@
 
         <!-- Oozie dependencies -->
         <dependency>
+            <groupId>org.apache.hive</groupId>
+            <artifactId>hive-exec</artifactId>
+        </dependency>
+
+        <dependency>
             <groupId>org.apache.hive.hcatalog</groupId>
             <artifactId>hive-webhcat-java-client</artifactId>
         </dependency>

http://git-wip-us.apache.org/repos/asf/falcon/blob/cc1d3840/webapp/pom.xml
----------------------------------------------------------------------
diff --git a/webapp/pom.xml b/webapp/pom.xml
index 063d42c..5a9e1da 100644
--- a/webapp/pom.xml
+++ b/webapp/pom.xml
@@ -45,7 +45,6 @@
                             <webResources>
                                 <resource>
                                     <directory>../falcon-ui/dist</directory>
-                                    <targetPath>public</targetPath>
                                 </resource>
                                 <resource>
                                     <directory>src/main/webapp/WEB-INF/distributed</directory>
@@ -70,6 +69,16 @@
                 </dependency>
             </dependencies>
         </profile>
+        <profile>
+            <id>hivedr</id>
+            <dependencies>
+                <dependency>
+                    <groupId>org.apache.falcon</groupId>
+                    <artifactId>falcon-hive-replication</artifactId>
+                    <version>${project.version}</version>
+                </dependency>
+            </dependencies>
+        </profile>
     </profiles>
 
     <dependencies>
@@ -113,7 +122,8 @@
 
         <dependency>
             <groupId>org.apache.falcon</groupId>
-            <artifactId>falcon-replication</artifactId>
+            <artifactId>falcon-distcp-replication</artifactId>
+            <version>${project.version}</version>
         </dependency>
 
         <dependency>

http://git-wip-us.apache.org/repos/asf/falcon/blob/cc1d3840/webapp/src/test/java/org/apache/falcon/cli/FalconCLIIT.java
----------------------------------------------------------------------
diff --git a/webapp/src/test/java/org/apache/falcon/cli/FalconCLIIT.java b/webapp/src/test/java/org/apache/falcon/cli/FalconCLIIT.java
index 90acb59..1e0a096 100644
--- a/webapp/src/test/java/org/apache/falcon/cli/FalconCLIIT.java
+++ b/webapp/src/test/java/org/apache/falcon/cli/FalconCLIIT.java
@@ -44,7 +44,7 @@ import java.util.Properties;
  */
 @Test(groups = {"exhaustive"})
 public class FalconCLIIT {
-    private static final String RECIPE_PROPERTIES_FILE_XML = "/process.properties";
+    private static final String RECIPE_PROPERTIES_FILE_XML = "/hdfs-replication.properties";
 
     private InMemoryWriter stream = new InMemoryWriter(System.out);
     private String recipePropertiesFilePath;
@@ -217,23 +217,23 @@ public class FalconCLIIT {
         Assert.assertEquals(executeWithURL("entity -status -type process -name " + overlay.get("processName")),
0);
 
         Assert.assertEquals(executeWithURL("entity -summary -type feed -cluster "+ overlay.get("cluster")
-                        + " -fields status,tags -start " + START_INSTANCE
-                        + " -filterBy TYPE:FEED -orderBy name -sortOrder asc "
-                        + " -offset 0 -numResults 1 -numInstances 5"), 0);
+                + " -fields status,tags -start " + START_INSTANCE
+                + " -filterBy TYPE:FEED -orderBy name -sortOrder asc "
+                + " -offset 0 -numResults 1 -numInstances 5"), 0);
 
         Assert.assertEquals(executeWithURL("entity -summary -type process -fields status,pipelines"
-                        + " -cluster " + overlay.get("cluster")
-                        + " -start " + SchemaHelper.getDateFormat().format(new Date(0))
-                        + " -end " + SchemaHelper.getDateFormat().format(new Date())
-                        + " -filterBy TYPE:PROCESS -orderBy name -sortOrder desc "
-                        + " -offset 0 -numResults 1 -numInstances 7"), 0);
+                + " -cluster " + overlay.get("cluster")
+                + " -start " + SchemaHelper.getDateFormat().format(new Date(0))
+                + " -end " + SchemaHelper.getDateFormat().format(new Date())
+                + " -filterBy TYPE:PROCESS -orderBy name -sortOrder desc "
+                + " -offset 0 -numResults 1 -numInstances 7"), 0);
 
         Assert.assertEquals(executeWithURL("entity -summary -type process -fields status,pipelines"
-                        + " -cluster " + overlay.get("cluster")
-                        + " -start " + SchemaHelper.getDateFormat().format(new Date(0))
-                        + " -end " + SchemaHelper.getDateFormat().format(new Date())
-                        + " -filterBy TYPE:PROCESS -orderBy name -sortOrder invalid "
-                        + " -offset 0 -numResults 1 -numInstances 7"), -1);
+                + " -cluster " + overlay.get("cluster")
+                + " -start " + SchemaHelper.getDateFormat().format(new Date(0))
+                + " -end " + SchemaHelper.getDateFormat().format(new Date())
+                + " -filterBy TYPE:PROCESS -orderBy name -sortOrder invalid "
+                + " -offset 0 -numResults 1 -numInstances 7"), -1);
 
         // No start or end date
         Assert.assertEquals(executeWithURL("entity -summary -type process -fields status,pipelines"
@@ -329,9 +329,9 @@ public class FalconCLIIT {
                 + " -orderBy startTime -sortOrder asc -offset 0 -numResults 1"), 0);
 
         Assert.assertEquals(executeWithURL("instance -running -type feed -lifecycle eviction
-name "
-                        + overlay.get("outputFeedName")
-                        + " -start " + SchemaHelper.getDateFormat().format(new Date())
-                        + " -orderBy INVALID -offset 0 -numResults 1"), -1);
+                + overlay.get("outputFeedName")
+                + " -start " + SchemaHelper.getDateFormat().format(new Date())
+                + " -orderBy INVALID -offset 0 -numResults 1"), -1);
 
         Assert.assertEquals(executeWithURL("instance -running -type feed -lifecycle eviction
-name "
                 + overlay.get("outputFeedName")
@@ -339,14 +339,14 @@ public class FalconCLIIT {
                 + " -orderBy startTime -sortOrder desc -offset 0 -numResults 1"), 0);
 
         Assert.assertEquals(executeWithURL("instance -running -type feed -lifecycle eviction
-name "
-                        + overlay.get("outputFeedName")
-                        + " -start " + startTimeString
-                        + " -orderBy startTime -sortOrder invalid -offset 0 -numResults 1"),
-1);
+                + overlay.get("outputFeedName")
+                + " -start " + startTimeString
+                + " -orderBy startTime -sortOrder invalid -offset 0 -numResults 1"), -1);
 
         Assert.assertEquals(executeWithURL("instance -running -type feed -lifecycle eviction
-name "
-                        + overlay.get("outputFeedName")
-                        + " -start " + SchemaHelper.getDateFormat().format(new Date())
-                        + " -filterBy INVALID:FILTER -offset 0 -numResults 1"), -1);
+                + overlay.get("outputFeedName")
+                + " -start " + SchemaHelper.getDateFormat().format(new Date())
+                + " -filterBy INVALID:FILTER -offset 0 -numResults 1"), -1);
 
         // testcase : start str is older than entity schedule time.
         Assert.assertEquals(executeWithURL("instance -running -type feed -lifecycle eviction
-name "
@@ -389,14 +389,14 @@ public class FalconCLIIT {
                 + " -orderBy startTime -offset 0 -numResults 1"), 0);
 
         Assert.assertEquals(executeWithURL("instance -status -type feed -lifecycle eviction
-name "
-                        + overlay.get("outputFeedName")
-                        + " -start "+ SchemaHelper.getDateFormat().format(new Date())
-                        +" -filterBy INVALID:FILTER -orderBy startTime -offset 0 -numResults
1"), -1);
+                + overlay.get("outputFeedName")
+                + " -start "+ SchemaHelper.getDateFormat().format(new Date())
+                +" -filterBy INVALID:FILTER -orderBy startTime -offset 0 -numResults 1"),
-1);
 
         Assert.assertEquals(executeWithURL("instance -list -type feed -lifecycle eviction
-name "
-                        + overlay.get("outputFeedName")
-                        + " -start "+ SchemaHelper.getDateFormat().format(new Date())
-                        +" -filterBy STATUS:SUCCEEDED -orderBy INVALID -offset 0 -numResults
1"), -1);
+                + overlay.get("outputFeedName")
+                + " -start "+ SchemaHelper.getDateFormat().format(new Date())
+                +" -filterBy STATUS:SUCCEEDED -orderBy INVALID -offset 0 -numResults 1"),
-1);
         Assert.assertEquals(executeWithURL("instance -status -type feed -lifecycle eviction
-name "
                 + overlay.get("outputFeedName")
                 + " -start " + SchemaHelper.getDateFormat().format(new Date())
@@ -453,8 +453,8 @@ public class FalconCLIIT {
 
         // No end date, should fail.
         Assert.assertEquals(executeWithURL("instance -suspend -type feed -lifecycle eviction
-name "
-                        + overlay.get("outputFeedName")
-                        + " -start "+ SchemaHelper.getDateFormat().format(new Date())), -1);
+                + overlay.get("outputFeedName")
+                + " -start "+ SchemaHelper.getDateFormat().format(new Date())), -1);
 
         Assert.assertEquals(executeWithURL("instance -resume -type process -name "
                 + overlay.get("processName")
@@ -479,13 +479,13 @@ public class FalconCLIIT {
 
         OozieTestUtils.waitForProcessWFtoStart(context);
         Assert.assertEquals(executeWithURL("instance -kill -type process -name "
-                        + overlay.get("processName")
-                        + " -start " + START_INSTANCE + " -end " + START_INSTANCE), 0);
+                + overlay.get("processName")
+                + " -start " + START_INSTANCE + " -end " + START_INSTANCE), 0);
 
         // Fail due to no end date
         Assert.assertEquals(executeWithURL("instance -kill -type feed -lifecycle eviction
-name "
-                        + overlay.get("outputFeedName")
-                        + " -start "+ SchemaHelper.getDateFormat().format(new Date())), -1);
+                + overlay.get("outputFeedName")
+                + " -start "+ SchemaHelper.getDateFormat().format(new Date())), -1);
 
         Assert.assertEquals(executeWithURL("instance -rerun -type process -name "
                 + overlay.get("processName")
@@ -575,13 +575,13 @@ public class FalconCLIIT {
                 + " -tags owner=producer@xyz.com,department=forecasting "
                 + " -filterBy STATUS:SUBMITTED,type:process -orderBy name -offset 1 -numResults
1"), 0);
         Assert.assertEquals(executeWithURL("entity -list -type process -fields status "
-                        + " -filterBy STATUS:SUCCEEDED,TYPE:process -orderBy INVALID -offset
0 -numResults 1"), -1);
+                + " -filterBy STATUS:SUCCEEDED,TYPE:process -orderBy INVALID -offset 0 -numResults
1"), -1);
 
         Assert.assertEquals(executeWithURL("entity -list -type process -fields INVALID "
-                        + " -filterBy STATUS:SUCCEEDED,TYPE:process -orderBy name -offset
1 -numResults 1"), -1);
+                + " -filterBy STATUS:SUCCEEDED,TYPE:process -orderBy name -offset 1 -numResults
1"), -1);
 
         Assert.assertEquals(executeWithURL("entity -list -type process -fields status "
-                        + " -filterBy INVALID:FILTER,TYPE:process -orderBy name -offset 1
-numResults 1"), -1);
+                + " -filterBy INVALID:FILTER,TYPE:process -orderBy name -offset 1 -numResults
1"), -1);
 
         Assert.assertEquals(executeWithURL("entity -definition -type cluster -name " + overlay.get("cluster")),
0);
 
@@ -678,26 +678,26 @@ public class FalconCLIIT {
         OozieTestUtils.waitForProcessWFtoStart(context);
 
         Assert.assertEquals(executeWithURL("instance -kill -type process -name "
-                        + overlay.get("processName")
-                        + " -start " + START_INSTANCE + " -end " + START_INSTANCE), 0);
+                + overlay.get("processName")
+                + " -start " + START_INSTANCE + " -end " + START_INSTANCE), 0);
 
         Assert.assertEquals(executeWithURL("instance -kill -type feed -lifecycle eviction
-name "
-                        + overlay.get("outputFeedName")
-                        + " -start "+ SchemaHelper.getDateFormat().format(new Date())
-                        + " -end " +  SchemaHelper.getDateFormat().format(new Date())), 0);
+                + overlay.get("outputFeedName")
+                + " -start "+ SchemaHelper.getDateFormat().format(new Date())
+                + " -end " +  SchemaHelper.getDateFormat().format(new Date())), 0);
 
         Assert.assertEquals(executeWithURL("instance -rerun -type process -name "
-                        + overlay.get("processName")
-                        + " -start " + START_INSTANCE), -1);
+                + overlay.get("processName")
+                + " -start " + START_INSTANCE), -1);
 
         Assert.assertEquals(executeWithURL("instance -rerun -type process -name "
-                        + overlay.get("processName")
-                        + " -start " + START_INSTANCE + " -end " + START_INSTANCE), 0);
+                + overlay.get("processName")
+                + " -start " + START_INSTANCE + " -end " + START_INSTANCE), 0);
 
         Assert.assertEquals(executeWithURL("instance -rerun -type feed -lifecycle eviction
-name "
-                        + overlay.get("outputFeedName")
-                        + " -start "+ SchemaHelper.getDateFormat().format(new Date())
-                        + " -end " + SchemaHelper.getDateFormat().format(new Date())), 0);
+                + overlay.get("outputFeedName")
+                + " -start "+ SchemaHelper.getDateFormat().format(new Date())
+                + " -end " + SchemaHelper.getDateFormat().format(new Date())), 0);
     }
 
     public void testInvalidCLIInstanceCommands() throws Exception {
@@ -712,7 +712,7 @@ public class FalconCLIIT {
                 + " -end 2010-01-01T03:00Z"), -1);
 
         Assert.assertEquals(executeWithURL("instance -kill -type process -name "
-                        + " -start 2010-01-01T01:00Z  -end 2010-01-01T03:00Z"), -1);
+                + " -start 2010-01-01T01:00Z  -end 2010-01-01T03:00Z"), -1);
     }
 
     public void testFalconURL() throws Exception {
@@ -734,12 +734,12 @@ public class FalconCLIIT {
         submitTestFiles(context, overlay);
 
         Assert.assertEquals(new FalconCLI().run(("entity -schedule -type feed -name "
-                        + overlay.get("outputFeedName") + " -url "
-                        + TestContext.BASE_URL).split("\\s+")), 0);
+                + overlay.get("outputFeedName") + " -url "
+                + TestContext.BASE_URL).split("\\s+")), 0);
 
         Assert.assertEquals(new FalconCLI().run(("entity -schedule -type process -name "
-                        + overlay.get("processName")+ " -url "
-                        + TestContext.BASE_URL).split("\\s+")), 0);
+                + overlay.get("processName")+ " -url "
+                + TestContext.BASE_URL).split("\\s+")), 0);
     }
 
     public void testGetVersion() throws Exception {
@@ -766,52 +766,52 @@ public class FalconCLIIT {
         Thread.sleep(500);
 
         Assert.assertEquals(executeWithURL("instance -logs -type process -name "
-                        + overlay.get("processName")
-                        + " -start " + START_INSTANCE + " -end " + START_INSTANCE), 0);
+                + overlay.get("processName")
+                + " -start " + START_INSTANCE + " -end " + START_INSTANCE), 0);
 
         Assert.assertEquals(executeWithURL("instance -logs -type feed -lifecycle eviction
-name "
-                        + overlay.get("outputFeedName")
-                        + " -start "+ SchemaHelper.getDateFormat().format(new Date())), 0);
+                + overlay.get("outputFeedName")
+                + " -start "+ SchemaHelper.getDateFormat().format(new Date())), 0);
 
         // test filterBy, orderBy, offset, numResults
         Assert.assertEquals(executeWithURL("instance -logs -type process -name "
-                        + overlay.get("processName")
-                        + " -start " + START_INSTANCE + " -end " + START_INSTANCE
-                        + " -filterBy STATUS:SUCCEEDED -orderBy endtime "
-                        + " -sortOrder asc -offset 0 -numResults 1"), 0);
+                + overlay.get("processName")
+                + " -start " + START_INSTANCE + " -end " + START_INSTANCE
+                + " -filterBy STATUS:SUCCEEDED -orderBy endtime "
+                + " -sortOrder asc -offset 0 -numResults 1"), 0);
         Assert.assertEquals(executeWithURL("instance -logs -type process -name "
-                        + overlay.get("processName")
-                        + " -start " + START_INSTANCE + " -end " + START_INSTANCE
-                        + " -filterBy STATUS:SUCCEEDED -orderBy starttime "
-                        + " -sortOrder asc -offset 0 -numResults 1"), 0);
+                + overlay.get("processName")
+                + " -start " + START_INSTANCE + " -end " + START_INSTANCE
+                + " -filterBy STATUS:SUCCEEDED -orderBy starttime "
+                + " -sortOrder asc -offset 0 -numResults 1"), 0);
         Assert.assertEquals(executeWithURL("instance -logs -type process -name "
-                        + overlay.get("processName")
-                        + " -start " + START_INSTANCE + " -end " + START_INSTANCE
-                        + " -filterBy STATUS:SUCCEEDED -orderBy cluster "
-                        + " -sortOrder asc -offset 0 -numResults 1"), 0);
+                + overlay.get("processName")
+                + " -start " + START_INSTANCE + " -end " + START_INSTANCE
+                + " -filterBy STATUS:SUCCEEDED -orderBy cluster "
+                + " -sortOrder asc -offset 0 -numResults 1"), 0);
 
         Assert.assertEquals(executeWithURL("instance -logs -type process -name "
-                        + overlay.get("processName")
-                        + " -start " + START_INSTANCE + " -end " + START_INSTANCE
-                        + " -filterBy STATUS:WAITING -orderBy startTime -offset 0 -numResults
1"), 0);
+                + overlay.get("processName")
+                + " -start " + START_INSTANCE + " -end " + START_INSTANCE
+                + " -filterBy STATUS:WAITING -orderBy startTime -offset 0 -numResults 1"),
0);
 
         Assert.assertEquals(executeWithURL("instance -logs -type process -name "
-                        + overlay.get("processName")
-                        + " -start " + START_INSTANCE + " -end " + START_INSTANCE
-                        + " -filterBy STATUS:SUCCEEDED -orderBy endtime "
-                        + " -sortOrder invalid -offset 0 -numResults 1"), -1);
+                + overlay.get("processName")
+                + " -start " + START_INSTANCE + " -end " + START_INSTANCE
+                + " -filterBy STATUS:SUCCEEDED -orderBy endtime "
+                + " -sortOrder invalid -offset 0 -numResults 1"), -1);
         Assert.assertEquals(executeWithURL("instance -logs -type process -name "
-                        + overlay.get("processName")
-                        + " -start " + START_INSTANCE + " -end " + START_INSTANCE
-                        + " -filterBy STATUS:SUCCEEDED,STARTEDAFTER:"+START_INSTANCE+" -offset
1 -numResults 1"), 0);
+                + overlay.get("processName")
+                + " -start " + START_INSTANCE + " -end " + START_INSTANCE
+                + " -filterBy STATUS:SUCCEEDED,STARTEDAFTER:"+START_INSTANCE+" -offset 1
-numResults 1"), 0);
         Assert.assertEquals(executeWithURL("instance -logs -type process -name "
-                        + overlay.get("processName")
-                        + " -start " + START_INSTANCE + " -end " + START_INSTANCE
-                        + " -filterBy INVALID:FILTER -orderBy startTime -offset 0 -numResults
1"), -1);
+                + overlay.get("processName")
+                + " -start " + START_INSTANCE + " -end " + START_INSTANCE
+                + " -filterBy INVALID:FILTER -orderBy startTime -offset 0 -numResults 1"),
-1);
         Assert.assertEquals(executeWithURL("instance -logs -type process -name "
-                        + overlay.get("processName")
-                        + " -start " + START_INSTANCE + " -end " + START_INSTANCE
-                        + " -filterBy STATUS:SUCCEEDED -orderBy wrongOrder -offset 0 -numResults
1"), -1);
+                + overlay.get("processName")
+                + " -start " + START_INSTANCE + " -end " + START_INSTANCE
+                + " -filterBy STATUS:SUCCEEDED -orderBy wrongOrder -offset 0 -numResults
1"), -1);
     }
 
     @SuppressWarnings("ResultOfMethodCallIgnored")
@@ -819,7 +819,8 @@ public class FalconCLIIT {
     public void testRecipeCommand() throws Exception {
         recipeSetup();
         try {
-            Assert.assertEquals(executeWithURL("recipe -name " + "process"), 0);
+            Assert.assertEquals(executeWithURL("recipe -name " + "hdfs-replication"
+                    + " -operation HDFS_REPLICATION"), 0);
         } finally {
             if (recipePropertiesFilePath != null) {
                 new File(recipePropertiesFilePath).delete();
@@ -836,12 +837,6 @@ public class FalconCLIIT {
                 overlay);
         Assert.assertEquals(executeWithURL("entity -submit -type cluster -file " + filePath),
0);
         context.setCluster(overlay.get("cluster"));
-
-        filePath = TestContext.overlayParametersOverTemplate(TestContext.FEED_TEMPLATE1,
overlay);
-        Assert.assertEquals(executeWithURL("entity -submit -type feed -file " + filePath),
0);
-
-        filePath = TestContext.overlayParametersOverTemplate(TestContext.FEED_TEMPLATE2,
overlay);
-        Assert.assertEquals(executeWithURL("entity -submit -type feed -file " + filePath),
0);
     }
 
     private void createPropertiesFile(TestContext context) throws Exception  {
@@ -854,16 +849,14 @@ public class FalconCLIIT {
         String resourcePath = FilenameUtils.getFullPathNoEndSeparator(wfFile);
         String libPath = TestContext.getTempFile("target/lib", "recipe", ".jar").getAbsolutePath();
 
-        File file = new File(resourcePath, "process.properties");
+        File file = new File(resourcePath, "hdfs-replication.properties");
         OutputStream out = new FileOutputStream(file);
-        props.setProperty("falcon.recipe.processName", context.getProcessName());
-        props.setProperty("falcon.recipe.src.cluster.name", context.getClusterName());
-        props.setProperty("falcon.recipe.processEndDate", context.getProcessEndTime());
-        props.setProperty("falcon.recipe.inputFeedName", context.getInputFeedName());
-        props.setProperty("falcon.recipe.outputFeedName", context.getOutputFeedName());
+        props.setProperty("falcon.recipe.name", context.getProcessName());
+        props.setProperty("falcon.recipe.cluster.name", context.getClusterName());
+        props.setProperty("falcon.recipe.cluster.validity.end", context.getProcessEndTime());
         props.setProperty("falcon.recipe.workflow.path", TestContext.class.getResource("/fs-workflow.xml").getPath());
         props.setProperty("falcon.recipe.workflow.lib.path", new File(libPath).getParent());
-        props.setProperty("falcon.recipe.src.cluster.hdfs.writeEndPoint", "jail://global:00");
+        props.setProperty("falcon.recipe.cluster.hdfs.writeEndPoint", "jail://global:00");
 
         props.store(out, null);
         out.close();

http://git-wip-us.apache.org/repos/asf/falcon/blob/cc1d3840/webapp/src/test/resources/hdfs-replication-template.xml
----------------------------------------------------------------------
diff --git a/webapp/src/test/resources/hdfs-replication-template.xml b/webapp/src/test/resources/hdfs-replication-template.xml
new file mode 100644
index 0000000..b93cc0b
--- /dev/null
+++ b/webapp/src/test/resources/hdfs-replication-template.xml
@@ -0,0 +1,43 @@
+<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+<!--
+  Licensed to the Apache Software Foundation (ASF) under one
+  or more contributor license agreements.  See the NOTICE file
+  distributed with this work for additional information
+  regarding copyright ownership.  The ASF licenses this file
+  to you under the Apache License, Version 2.0 (the
+  "License"); you may not use this file except in compliance
+  with the License.  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+  -->
+
+<process name="hdfs-replication" xmlns="uri:falcon:process:0.1">
+    <clusters>
+        <!--  source  -->
+        <cluster name="##falcon.recipe.cluster.name##">
+            <validity end="##falcon.recipe.cluster.validity.end##" start="##falcon.recipe.cluster.validity.start##"/>
+        </cluster>
+    </clusters>
+
+    <tags>_falcon_mirroring_type=HDFS</tags>
+
+    <parallel>1</parallel>
+    <!-- Dir replication needs to run only once to catch up -->
+    <order>LAST_ONLY</order>
+    <frequency>##falcon.recipe.frequency##</frequency>
+    <timezone>UTC</timezone>
+
+    <properties>
+        <property name="oozie.wf.subworkflow.classpath.inheritance" value="true"/>
+    </properties>
+
+    <workflow name="##falcon.recipe.workflow.name##" engine="oozie" path="/apps/data-mirroring/workflows/hdfs-replication-workflow.xml"
lib="##workflow.lib.path##"/>
+    <retry policy="##falcon.recipe.retry.policy##" delay="##falcon.recipe.retry.delay##"
attempts="3"/>
+    <ACL/>
+</process>

http://git-wip-us.apache.org/repos/asf/falcon/blob/cc1d3840/webapp/src/test/resources/hdfs-replication.properties
----------------------------------------------------------------------
diff --git a/webapp/src/test/resources/hdfs-replication.properties b/webapp/src/test/resources/hdfs-replication.properties
new file mode 100644
index 0000000..1b485b4
--- /dev/null
+++ b/webapp/src/test/resources/hdfs-replication.properties
@@ -0,0 +1,45 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+falcon.recipe.cluster.hdfs.writeEndPoint=jail://global:00
+falcon.recipe.workflow.path=
+falcon.recipe.processName=
+falcon.recipe.cluster.name=
+falcon.recipe.cluster.validity.end=
+falcon.recipe.cluster.validity.start=2012-04-20T00:00Z
+falcon.recipe.workflow.name=hdfs-dr-workflow
+falcon.recipe.process.frequency=minutes(5)
+
+##### Retry policy properties
+
+falcon.recipe.retry.policy=periodic
+falcon.recipe.retry.delay=minutes(30)
+falcon.recipe.retry.attempts=3
+
+drSourceDir=/tmp/test1
+drSourceClusterFS=jail://global:00
+drTargetDir=/tmp/test1
+drTargetClusterFS=jail://global:00
+
+# Change it to specify the maximum number of mappers for DistCP
+distcpMaxMaps=1
+# Change it to specify the bandwidth in MB for each mapper in DistCP
+distcpMapBandwidth=100
+
+##### Email on failure
+drNotificationReceivers=NA
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/falcon/blob/cc1d3840/webapp/src/test/resources/process.properties
----------------------------------------------------------------------
diff --git a/webapp/src/test/resources/process.properties b/webapp/src/test/resources/process.properties
deleted file mode 100644
index 1eb282c..0000000
--- a/webapp/src/test/resources/process.properties
+++ /dev/null
@@ -1,25 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-falcon.recipe.src.cluster.hdfs.writeEndPoint=jail://global:00
-falcon.recipe.workflow.path=
-falcon.recipe.processName=
-falcon.recipe.src.cluster.name=
-falcon.recipe.inputFeedName=
-falcon.recipe.outputFeedName=
-


Mime
View raw message