falcon-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From pisayc...@apache.org
Subject falcon git commit: FALCON-1766 Add CLI metrics check for HiveDR, HDFS and feed replication. Contributed by Paul Isaychuk
Date Tue, 02 Feb 2016 15:04:47 GMT
Repository: falcon
Updated Branches:
  refs/heads/master b5c0b6139 -> 4a76b11be


FALCON-1766 Add CLI metrics check for HiveDR, HDFS and feed replication. Contributed by Paul
Isaychuk


Project: http://git-wip-us.apache.org/repos/asf/falcon/repo
Commit: http://git-wip-us.apache.org/repos/asf/falcon/commit/4a76b11b
Tree: http://git-wip-us.apache.org/repos/asf/falcon/tree/4a76b11b
Diff: http://git-wip-us.apache.org/repos/asf/falcon/diff/4a76b11b

Branch: refs/heads/master
Commit: 4a76b11be2a6f7831929831d91ecc5ac46e25260
Parents: b5c0b61
Author: Paul Isaychuk <pisaychuk@apache.org>
Authored: Thu Jan 21 17:03:55 2016 +0200
Committer: Paul Isaychuk <pisaychuk@apache.org>
Committed: Tue Feb 2 15:07:18 2016 +0200

----------------------------------------------------------------------
 falcon-regression/CHANGES.txt                   |  2 +
 .../core/helpers/FalconClientBuilder.java       | 13 ++++
 .../helpers/entity/AbstractEntityHelper.java    | 12 +++
 .../falcon/regression/core/util/AssertUtil.java | 42 +++++++++-
 .../falcon/regression/FeedReplicationTest.java  | 30 ++++---
 .../regression/hive/dr/HdfsRecipeTest.java      |  5 ++
 .../falcon/regression/hive/dr/HiveDRTest.java   |  5 ++
 .../HdfsRecipe/hdfs-replication-template.xml    | 44 +++++++++++
 .../HdfsRecipe/hdfs-replication-workflow.xml    | 82 ++++++++++++++++++++
 .../HdfsRecipe/hdfs-replication.properties      | 79 +++++++++++++++++++
 .../hive-disaster-recovery-template.xml         | 44 -----------
 .../hive-disaster-recovery-workflow.xml         | 82 --------------------
 .../hive-disaster-recovery.properties           | 79 -------------------
 .../hive-disaster-recovery-workflow.xml         |  2 +
 14 files changed, 303 insertions(+), 218 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/falcon/blob/4a76b11b/falcon-regression/CHANGES.txt
----------------------------------------------------------------------
diff --git a/falcon-regression/CHANGES.txt b/falcon-regression/CHANGES.txt
index e6664f3..ccd9ceb 100644
--- a/falcon-regression/CHANGES.txt
+++ b/falcon-regression/CHANGES.txt
@@ -5,6 +5,8 @@ Trunk (Unreleased)
   INCOMPATIBLE CHANGES
 
   NEW FEATURES
+   FALCON-1766 Add CLI metrics check for HiveDR, HDFS and feed replication (Paul Isaychuk)
+
    FALCON-1777 Add regression for HDFS replication (recipe) (Paul Isaychuk)
 
    FALCON-1699 Test fixes for RetentionTest, LineageApiTest, TouchAPIPrismAndServerTest,
FeedReplicationTest and few fortifications(Paul Isaychuk via Pragya Mittal)

http://git-wip-us.apache.org/repos/asf/falcon/blob/4a76b11b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/helpers/FalconClientBuilder.java
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/helpers/FalconClientBuilder.java
b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/helpers/FalconClientBuilder.java
index 2e6fca9..fcbb7f9 100644
--- a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/helpers/FalconClientBuilder.java
+++ b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/helpers/FalconClientBuilder.java
@@ -139,6 +139,19 @@ public final class FalconClientBuilder implements Builder<CommandLine>
{
     }
 
     /**
+     * Create get metrics command.
+     * @param entityType type of the entity
+     * @param entityName name of the entity to be deleted
+     * @return this
+     */
+    public FalconClientBuilder getMetricsCommand(String entityType, String entityName) {
+        addArg("metadata").addArg("-list");
+        addArg("-type").addArg("replication_metrics");
+        addArg("-" + entityType).addArg(entityName);
+        return this;
+    }
+
+    /**
      * Create submit command.
      * @param entityType type of the entity
      * @param fileName file containing the entity to be submitted

http://git-wip-us.apache.org/repos/asf/falcon/blob/4a76b11b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/helpers/entity/AbstractEntityHelper.java
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/helpers/entity/AbstractEntityHelper.java
b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/helpers/entity/AbstractEntityHelper.java
index e406cae..29c97b2 100644
--- a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/helpers/entity/AbstractEntityHelper.java
+++ b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/helpers/entity/AbstractEntityHelper.java
@@ -593,6 +593,18 @@ public abstract class AbstractEntityHelper {
     }
 
     /**
+     * Get CLI metrics for recipe based process or feed replication.
+     * @param entityName
+     * @return
+     */
+    public ExecResult getCLIMetrics(String entityName) {
+        LOGGER.info("Getting CLI metrics for " + getEntityType()+ " " + entityName);
+        final CommandLine commandLine = FalconClientBuilder.getBuilder()
+            .getMetricsCommand(getEntityType(), entityName).build();
+        return ExecUtil.executeCommand(commandLine);
+    }
+
+    /**
      * Delete an entity through falcon client.
      * @param entityStr string of the entity to be submitted
      * @throws IOException

http://git-wip-us.apache.org/repos/asf/falcon/blob/4a76b11b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/AssertUtil.java
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/AssertUtil.java
b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/AssertUtil.java
index 9d3b802..0af0c1e 100644
--- a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/AssertUtil.java
+++ b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/AssertUtil.java
@@ -33,6 +33,8 @@ import org.apache.log4j.Logger;
 import org.apache.oozie.client.Job;
 import org.apache.oozie.client.OozieClient;
 import org.apache.oozie.client.OozieClientException;
+import org.json.JSONArray;
+import org.json.JSONObject;
 import org.testng.Assert;
 
 import javax.xml.bind.JAXBException;
@@ -52,6 +54,38 @@ public final class AssertUtil {
     private static final Logger LOGGER = Logger.getLogger(AssertUtil.class);
 
     /**
+     * Asserts correctness of CLI metrics for recipe based process or feed replication.
+     * @param execResult CLI metrics exec result to be checked
+     * @param entityName name of recipe process or replication feed
+     * @param instanceNum expected number of process/feed instances in metrics output
+     * @param withData is data expected to be replicated
+     * @throws Exception
+     */
+    public static void assertCLIMetrics(ExecResult execResult, String entityName, int instanceNum,
boolean withData)
+        throws Exception {
+        String output = execResult.getOutput();
+        Assert.assertTrue(StringUtils.isNotBlank(output), "Exec result output is blank.");
+        JSONObject jsonObject = new JSONObject(output);
+        int totalSize = jsonObject.getInt("totalSize");
+        Assert.assertEquals(totalSize, instanceNum);
+        JSONArray array = jsonObject.getJSONArray("results");
+        for (int i = 0; i < array.length(); i++) {
+            String name = array.getJSONObject(i).getString("name");
+            Assert.assertTrue(name.contains(entityName));
+            int timeTaken = array.getJSONObject(i).getInt("TIMETAKEN");
+            Assert.assertTrue(timeTaken > 0, "TIMETAKEN metric should be greater then
zero.");
+            int bytescopied = array.getJSONObject(i).getInt("BYTESCOPIED");
+            Assert.assertTrue(bytescopied >= 0, "BYTESCOPIED metric should be greater
or equal to zero.");
+            int copy = array.getJSONObject(i).getInt("COPY");
+            if (withData) {
+                Assert.assertTrue(copy > 0, "COPY metric should be greater then zero.");
+            } else {
+                Assert.assertEquals(copy, 0, "COPY metric should be equal to zero as data
was absent.");
+            }
+        }
+    }
+
+    /**
      * Checks that any path in list doesn't contains a string.
      *
      * @param paths list of paths
@@ -96,7 +130,7 @@ public final class AssertUtil {
             LOGGER.info("elements = " + elements);
         }
         Assert.assertEquals(elements.size(), expectedSize,
-                "Size of expected and actual list don't match.");
+            "Size of expected and actual list don't match.");
     }
 
     /**
@@ -218,7 +252,7 @@ public final class AssertUtil {
         Assert.assertFalse(execResult.hasSuceeded(),
             "Unexpectedly succeeded execResult: " + execResult);
         Assert.assertTrue((execResult.getError() + execResult.getOutput()).contains(expectedMessage),
-                "Expected error: " + expectedMessage + " in execResult: " + execResult);
+            "Expected error: " + expectedMessage + " in execResult: " + execResult);
     }
 
     /**
@@ -258,7 +292,7 @@ public final class AssertUtil {
      */
     public static void assertFailed(ServiceResponse response) throws JAXBException {
         Assert.assertNotEquals(response.getMessage(), "null",
-                "response message should not be null");
+            "response message should not be null");
 
         Assert.assertEquals(Util.parseResponse(response).getStatus(), APIResult.Status.FAILED);
         Assert.assertEquals(response.getCode(), 400);
@@ -271,7 +305,7 @@ public final class AssertUtil {
      */
     public static void assertFailed(APIResult response) {
         Assert.assertNotEquals(response.getMessage(), "null",
-                "response message should not be null");
+            "response message should not be null");
         Assert.assertEquals(response.getStatus(), APIResult.Status.FAILED,
                 "Status should be FAILED. Message: " + response.getMessage());
     }

http://git-wip-us.apache.org/repos/asf/falcon/blob/4a76b11b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/FeedReplicationTest.java
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/FeedReplicationTest.java
b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/FeedReplicationTest.java
index 6728edf..a936aa1 100644
--- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/FeedReplicationTest.java
+++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/FeedReplicationTest.java
@@ -24,6 +24,7 @@ import org.apache.falcon.entity.v0.feed.ClusterType;
 import org.apache.falcon.regression.Entities.FeedMerlin;
 import org.apache.falcon.regression.core.bundle.Bundle;
 import org.apache.falcon.regression.core.helpers.ColoHelper;
+import org.apache.falcon.regression.core.supportClasses.ExecResult;
 import org.apache.falcon.regression.core.util.AssertUtil;
 import org.apache.falcon.regression.core.util.BundleUtil;
 import org.apache.falcon.regression.core.util.HadoopUtil;
@@ -36,11 +37,9 @@ import org.apache.falcon.regression.testHelper.BaseTestClass;
 import org.apache.falcon.resource.InstancesResult;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.security.authentication.client.AuthenticationException;
 import org.apache.log4j.Logger;
 import org.apache.oozie.client.CoordinatorAction;
 import org.apache.oozie.client.OozieClient;
-import org.apache.oozie.client.OozieClientException;
 import org.joda.time.DateTime;
 import org.joda.time.DateTimeZone;
 import org.joda.time.format.DateTimeFormat;
@@ -53,7 +52,6 @@ import org.testng.annotations.Test;
 
 import javax.xml.bind.JAXBException;
 import java.io.IOException;
-import java.net.URISyntaxException;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -133,6 +131,7 @@ public class FeedReplicationTest extends BaseTestClass {
                 .withClusterType(ClusterType.TARGET)
                 .withDataLocation(targetDataLocation)
                 .build());
+        feed.withProperty("job.counter", "true");
 
         //submit and schedule feed
         LOGGER.info("Feed : " + Util.prettyPrintXml(feed.toString()));
@@ -178,7 +177,10 @@ public class FeedReplicationTest extends BaseTestClass {
         Assert.assertEquals(HadoopUtil.getSuccessFolder(cluster2FS, toTarget, ""), true);
 
         AssertUtil.assertLogMoverPath(true, Util.readEntityName(feed.toString()),
-                cluster2FS, "feed", "Success logs are not present");
+            cluster2FS, "feed", "Success logs are not present");
+
+        ExecResult execResult = cluster1.getFeedHelper().getCLIMetrics(feed.getName());
+        AssertUtil.assertCLIMetrics(execResult, feed.getName(), 1, dataFlag);
     }
 
     /**
@@ -222,6 +224,7 @@ public class FeedReplicationTest extends BaseTestClass {
                         .withClusterType(ClusterType.TARGET)
                         .withDataLocation(targetDataLocation)
                         .build());
+        feed.withProperty("job.counter", "true");
 
         //submit and schedule feed
         LOGGER.info("Feed : " + Util.prettyPrintXml(feed.toString()));
@@ -278,7 +281,10 @@ public class FeedReplicationTest extends BaseTestClass {
         Assert.assertEquals(HadoopUtil.getSuccessFolder(cluster3FS, toTarget, ""), true);
 
         AssertUtil.assertLogMoverPath(true, Util.readEntityName(feed.toString()),
-                cluster2FS, "feed", "Success logs are not present");
+            cluster2FS, "feed", "Success logs are not present");
+
+        ExecResult execResult = cluster1.getFeedHelper().getCLIMetrics(feed.getName());
+        AssertUtil.assertCLIMetrics(execResult, feed.getName(), 1, dataFlag);
     }
 
     /**
@@ -322,6 +328,7 @@ public class FeedReplicationTest extends BaseTestClass {
                 .withClusterType(ClusterType.TARGET)
                 .withDataLocation(targetDataLocation)
                 .build());
+        feed.withProperty("job.counter", "true");
 
         //submit and schedule feed
         LOGGER.info("Feed : " + Util.prettyPrintXml(feed.toString()));
@@ -384,7 +391,10 @@ public class FeedReplicationTest extends BaseTestClass {
         Assert.assertEquals(HadoopUtil.getSuccessFolder(cluster2FS, toTarget, availabilityFlagName),
true);
 
         AssertUtil.assertLogMoverPath(true, Util.readEntityName(feed.toString()),
-                cluster2FS, "feed", "Success logs are not present");
+            cluster2FS, "feed", "Success logs are not present");
+
+        ExecResult execResult = cluster1.getFeedHelper().getCLIMetrics(feed.getName());
+        AssertUtil.assertCLIMetrics(execResult, feed.getName(), 1, dataFlag);
     }
 
     /**
@@ -392,9 +402,7 @@ public class FeedReplicationTest extends BaseTestClass {
      * Check that new DistCp options are allowed.
      */
     @Test
-    public void testNewDistCpOptions()
-        throws URISyntaxException, AuthenticationException, InterruptedException, IOException,
JAXBException,
-        OozieClientException {
+    public void testNewDistCpOptions() throws Exception {
         Bundle.submitCluster(bundles[0], bundles[1]);
         String startTime = TimeUtil.getTimeWrtSystemTime(0);
         String endTime = TimeUtil.addMinsToTime(startTime, 5);
@@ -422,6 +430,7 @@ public class FeedReplicationTest extends BaseTestClass {
                 .withClusterType(ClusterType.TARGET)
                 .withDataLocation(targetDataLocation)
                 .build());
+        feed.withProperty("job.counter", "true");
 
         //add custom properties to feed
         HashMap<String, String> propMap = new HashMap<>();
@@ -485,6 +494,9 @@ public class FeedReplicationTest extends BaseTestClass {
         List<Path> finalFiles = HadoopUtil.getAllFilesRecursivelyHDFS(cluster2FS, new
Path(targetPath));
         Assert.assertEquals(finalFiles.size(), 2, "Only replicated files should be present
on target "
             + "because of 'removeDeletedFiles' distCp property.");
+
+        ExecResult execResult = cluster1.getFeedHelper().getCLIMetrics(feed.getName());
+        AssertUtil.assertCLIMetrics(execResult, feed.getName(), 1, true);
     }
 
     /**

http://git-wip-us.apache.org/repos/asf/falcon/blob/4a76b11b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/hive/dr/HdfsRecipeTest.java
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/hive/dr/HdfsRecipeTest.java
b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/hive/dr/HdfsRecipeTest.java
index 05b9cf4..4a2d913 100644
--- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/hive/dr/HdfsRecipeTest.java
+++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/hive/dr/HdfsRecipeTest.java
@@ -25,6 +25,7 @@ import org.apache.falcon.regression.Entities.ClusterMerlin;
 import org.apache.falcon.regression.Entities.RecipeMerlin;
 import org.apache.falcon.regression.core.bundle.Bundle;
 import org.apache.falcon.regression.core.helpers.ColoHelper;
+import org.apache.falcon.regression.core.supportClasses.ExecResult;
 import org.apache.falcon.regression.core.util.AssertUtil;
 import org.apache.falcon.regression.core.util.BundleUtil;
 import org.apache.falcon.regression.core.util.HadoopUtil;
@@ -111,6 +112,10 @@ public class HdfsRecipeTest extends BaseTestClass {
             .getAllFilesRecursivelyHDFS(clusterFS2, new Path(targetDataLocation));
 
         AssertUtil.checkForListSizes(cluster1ReplicatedData, cluster2ReplicatedData);
+
+        //particular check for https://issues.apache.org/jira/browse/FALCON-1643
+        ExecResult execResult = cluster.getProcessHelper().getCLIMetrics(hdfsRecipe.getName());
+        AssertUtil.assertCLIMetrics(execResult, hdfsRecipe.getName(), 1, true);
     }
 
     @AfterMethod(alwaysRun = true)

http://git-wip-us.apache.org/repos/asf/falcon/blob/4a76b11b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/hive/dr/HiveDRTest.java
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/hive/dr/HiveDRTest.java
b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/hive/dr/HiveDRTest.java
index 7660e04..4dab0db 100644
--- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/hive/dr/HiveDRTest.java
+++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/hive/dr/HiveDRTest.java
@@ -26,6 +26,8 @@ import org.apache.falcon.regression.Entities.RecipeMerlin;
 import org.apache.falcon.regression.core.bundle.Bundle;
 import org.apache.falcon.regression.core.enumsAndConstants.MerlinConstants;
 import org.apache.falcon.regression.core.helpers.ColoHelper;
+import org.apache.falcon.regression.core.supportClasses.ExecResult;
+import org.apache.falcon.regression.core.util.AssertUtil;
 import org.apache.falcon.regression.core.supportClasses.NotifyingAssert;
 import org.apache.falcon.regression.core.util.BundleUtil;
 import org.apache.falcon.regression.core.util.HiveAssert;
@@ -164,6 +166,9 @@ public class HiveDRTest extends BaseTestClass {
         HiveAssert.assertTableEqual(cluster, clusterHC.getTable(DB_NAME, tblName),
             cluster2, clusterHC2.getTable(DB_NAME, tblName), new NotifyingAssert(true)
         ).assertAll();
+
+        ExecResult execResult = cluster.getProcessHelper().getCLIMetrics(recipeMerlin.getName());
+        AssertUtil.assertCLIMetrics(execResult, recipeMerlin.getName(), 1, true);
     }
 
     @Test

http://git-wip-us.apache.org/repos/asf/falcon/blob/4a76b11b/falcon-regression/merlin/src/test/resources/HdfsRecipe/hdfs-replication-template.xml
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin/src/test/resources/HdfsRecipe/hdfs-replication-template.xml
b/falcon-regression/merlin/src/test/resources/HdfsRecipe/hdfs-replication-template.xml
new file mode 100644
index 0000000..46546ce
--- /dev/null
+++ b/falcon-regression/merlin/src/test/resources/HdfsRecipe/hdfs-replication-template.xml
@@ -0,0 +1,44 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  ~ Licensed to the Apache Software Foundation (ASF) under one
+  ~ or more contributor license agreements.  See the NOTICE file
+  ~ distributed with this work for additional information
+  ~ regarding copyright ownership.  The ASF licenses this file
+  ~ to you under the Apache License, Version 2.0 (the
+  ~ "License"); you may not use this file except in compliance
+  ~ with the License.  You may obtain a copy of the License at
+  ~
+  ~     http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing, software
+  ~ distributed under the License is distributed on an "AS IS" BASIS,
+  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  ~ See the License for the specific language governing permissions and
+  ~ limitations under the License.
+  -->
+
+<process name="##falcon.recipe.job.name##" xmlns="uri:falcon:process:0.1">
+    <clusters>
+        <!--  source  -->
+        <cluster name="##falcon.recipe.cluster.name##">
+            <validity end="##falcon.recipe.cluster.validity.end##" start="##falcon.recipe.cluster.validity.start##"/>
+        </cluster>
+    </clusters>
+
+    <tags>_falcon_mirroring_type=HDFS</tags>
+
+    <parallel>1</parallel>
+    <!-- Dir replication needs to run only once to catch up -->
+    <order>LAST_ONLY</order>
+    <frequency>##falcon.recipe.frequency##</frequency>
+    <timezone>UTC</timezone>
+
+    <properties>
+        <property name="oozie.wf.subworkflow.classpath.inheritance" value="true"/>
+    </properties>
+
+    <workflow name="##falcon.recipe.workflow.name##" engine="oozie" path="/apps/data-mirroring/workflows/hdfs-replication-workflow.xml"
lib="##workflow.lib.path##"/>
+    <retry policy="##falcon.recipe.retry.policy##" delay="##falcon.recipe.retry.delay##"
attempts="3"/>
+    <notification type="##falcon.recipe.notification.type##" to="##falcon.recipe.notification.receivers##"/>
+    <ACL/>
+</process>

http://git-wip-us.apache.org/repos/asf/falcon/blob/4a76b11b/falcon-regression/merlin/src/test/resources/HdfsRecipe/hdfs-replication-workflow.xml
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin/src/test/resources/HdfsRecipe/hdfs-replication-workflow.xml
b/falcon-regression/merlin/src/test/resources/HdfsRecipe/hdfs-replication-workflow.xml
new file mode 100644
index 0000000..aa820d0
--- /dev/null
+++ b/falcon-regression/merlin/src/test/resources/HdfsRecipe/hdfs-replication-workflow.xml
@@ -0,0 +1,82 @@
+<!--
+  ~ Licensed to the Apache Software Foundation (ASF) under one
+  ~ or more contributor license agreements.  See the NOTICE file
+  ~ distributed with this work for additional information
+  ~ regarding copyright ownership.  The ASF licenses this file
+  ~ to you under the Apache License, Version 2.0 (the
+  ~ "License"); you may not use this file except in compliance
+  ~ with the License.  You may obtain a copy of the License at
+  ~
+  ~     http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing, software
+  ~ distributed under the License is distributed on an "AS IS" BASIS,
+  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  ~ See the License for the specific language governing permissions and
+  ~ limitations under the License.
+  -->
+<workflow-app xmlns='uri:oozie:workflow:0.3' name='falcon-dr-fs-workflow'>
+    <start to='dr-replication'/>
+    <!-- Replication action -->
+    <action name="dr-replication">
+        <java>
+            <job-tracker>${jobTracker}</job-tracker>
+            <name-node>${nameNode}</name-node>
+            <configuration>
+                <property> <!-- hadoop 2 parameter -->
+                    <name>oozie.launcher.mapreduce.job.user.classpath.first</name>
+                    <value>true</value>
+                </property>
+                <property>
+                    <name>mapred.job.queue.name</name>
+                    <value>${queueName}</value>
+                </property>
+                <property>
+                    <name>oozie.launcher.mapred.job.priority</name>
+                    <value>${jobPriority}</value>
+                </property>
+                <property>
+                    <name>oozie.use.system.libpath</name>
+                    <value>true</value>
+                </property>
+                <property>
+                    <name>oozie.action.sharelib.for.java</name>
+                    <value>distcp</value>
+                </property>
+                <property>
+                    <name>oozie.launcher.oozie.libpath</name>
+                    <value>${wf:conf("falcon.libpath")}</value>
+                </property>
+                <property>
+                    <name>oozie.launcher.mapreduce.job.hdfs-servers</name>
+                    <value>${drSourceClusterFS},${drTargetClusterFS}</value>
+                </property>
+            </configuration>
+            <main-class>org.apache.falcon.replication.FeedReplicator</main-class>
+            <arg>-Dmapred.job.queue.name=${queueName}</arg>
+            <arg>-Dmapred.job.priority=${jobPriority}</arg>
+            <arg>-maxMaps</arg>
+            <arg>${distcpMaxMaps}</arg>
+            <arg>-mapBandwidth</arg>
+            <arg>${distcpMapBandwidth}</arg>
+            <arg>-sourcePaths</arg>
+            <arg>${drSourceDir}</arg>
+            <arg>-targetPath</arg>
+            <arg>${drTargetClusterFS}${drTargetDir}</arg>
+            <arg>-falconFeedStorageType</arg>
+            <arg>FILESYSTEM</arg>
+            <arg>-availabilityFlag</arg>
+            <arg>${availabilityFlag == 'NA' ? "NA" : availabilityFlag}</arg>
+            <arg>-counterLogDir</arg>
+            <arg>${logDir}/job-${nominalTime}/${srcClusterName == 'NA' ? '' : srcClusterName}</arg>
+        </java>
+        <ok to="end"/>
+        <error to="fail"/>
+    </action>
+    <kill name="fail">
+        <message>
+            Workflow action failed, error message[${wf:errorMessage(wf:lastErrorNode())}]
+        </message>
+    </kill>
+    <end name="end"/>
+</workflow-app>

http://git-wip-us.apache.org/repos/asf/falcon/blob/4a76b11b/falcon-regression/merlin/src/test/resources/HdfsRecipe/hdfs-replication.properties
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin/src/test/resources/HdfsRecipe/hdfs-replication.properties
b/falcon-regression/merlin/src/test/resources/HdfsRecipe/hdfs-replication.properties
new file mode 100644
index 0000000..6c715f3
--- /dev/null
+++ b/falcon-regression/merlin/src/test/resources/HdfsRecipe/hdfs-replication.properties
@@ -0,0 +1,79 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+##### NOTE: This is a TEMPLATE file which can be copied and edited
+
+##### Recipe properties
+##### Unique recipe job name
+falcon.recipe.name=hdfs-replication-monthly
+
+##### Workflow properties
+falcon.recipe.workflow.name=hdfs-dr-workflow
+# Provide Wf absolute path. This can be HDFS or local FS path. If WF is on local FS it will
be copied to HDFS
+falcon.recipe.workflow.path=/apps/data-mirroring/workflows/hdfs-replication-workflow.xml
+
+##### Cluster properties
+
+# Cluster where job should run
+falcon.recipe.cluster.name=primaryCluster
+# Change the cluster hdfs write end point here. This is mandatory.
+falcon.recipe.cluster.hdfs.writeEndPoint=hdfs://localhost:8020
+# Change the cluster validity start time here
+falcon.recipe.cluster.validity.start=2015-03-13T00:00Z
+# Change the cluster validity end time here
+falcon.recipe.cluster.validity.end=2016-12-30T00:00Z
+
+##### Scheduling properties
+# Change the recipe frequency here. Valid frequency type are minutes, hours, days, months
+falcon.recipe.process.frequency=minutes(5)
+
+##### Retry policy properties
+
+falcon.recipe.retry.policy=periodic
+falcon.recipe.retry.delay=minutes(30)
+falcon.recipe.retry.attempts=3
+falcon.recipe.retry.onTimeout=false
+
+##### Tag properties - An optional list of comma separated tags, Key Value Pairs, separated
by comma
+##### Uncomment to add tags
+#falcon.recipe.tags=
+
+##### ACL properties - Uncomment and change ACL if authorization is enabled
+
+#falcon.recipe.acl.owner=ambari-qa
+#falcon.recipe.acl.group=users
+#falcon.recipe.acl.permission=0x755
+#falcon.recipe.nn.principal=nn/_HOST@EXAMPLE.COM
+
+##### Custom Job properties
+
+# Specify multiple comma separated source directories
+drSourceDir=/user/falcon_qa/dr/test/primaryCluster/input
+drSourceClusterFS=hdfs://240.0.0.10:8020
+drTargetDir=/user/falcon_qa/dr/test/backupCluster/input
+drTargetClusterFS=hdfs://240.0.0.11:8020
+
+# Change it to specify the maximum number of mappers for DistCP
+distcpMaxMaps=1
+# Change it to specify the bandwidth in MB for each mapper in DistCP
+distcpMapBandwidth=100
+
+##### Email Notification for Falcon instance completion
+falcon.recipe.notification.type=email
+falcon.recipe.notification.receivers=NA
+

http://git-wip-us.apache.org/repos/asf/falcon/blob/4a76b11b/falcon-regression/merlin/src/test/resources/HdfsRecipe/hive-disaster-recovery-template.xml
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin/src/test/resources/HdfsRecipe/hive-disaster-recovery-template.xml
b/falcon-regression/merlin/src/test/resources/HdfsRecipe/hive-disaster-recovery-template.xml
deleted file mode 100644
index 46546ce..0000000
--- a/falcon-regression/merlin/src/test/resources/HdfsRecipe/hive-disaster-recovery-template.xml
+++ /dev/null
@@ -1,44 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
-  ~ Licensed to the Apache Software Foundation (ASF) under one
-  ~ or more contributor license agreements.  See the NOTICE file
-  ~ distributed with this work for additional information
-  ~ regarding copyright ownership.  The ASF licenses this file
-  ~ to you under the Apache License, Version 2.0 (the
-  ~ "License"); you may not use this file except in compliance
-  ~ with the License.  You may obtain a copy of the License at
-  ~
-  ~     http://www.apache.org/licenses/LICENSE-2.0
-  ~
-  ~ Unless required by applicable law or agreed to in writing, software
-  ~ distributed under the License is distributed on an "AS IS" BASIS,
-  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  ~ See the License for the specific language governing permissions and
-  ~ limitations under the License.
-  -->
-
-<process name="##falcon.recipe.job.name##" xmlns="uri:falcon:process:0.1">
-    <clusters>
-        <!--  source  -->
-        <cluster name="##falcon.recipe.cluster.name##">
-            <validity end="##falcon.recipe.cluster.validity.end##" start="##falcon.recipe.cluster.validity.start##"/>
-        </cluster>
-    </clusters>
-
-    <tags>_falcon_mirroring_type=HDFS</tags>
-
-    <parallel>1</parallel>
-    <!-- Dir replication needs to run only once to catch up -->
-    <order>LAST_ONLY</order>
-    <frequency>##falcon.recipe.frequency##</frequency>
-    <timezone>UTC</timezone>
-
-    <properties>
-        <property name="oozie.wf.subworkflow.classpath.inheritance" value="true"/>
-    </properties>
-
-    <workflow name="##falcon.recipe.workflow.name##" engine="oozie" path="/apps/data-mirroring/workflows/hdfs-replication-workflow.xml"
lib="##workflow.lib.path##"/>
-    <retry policy="##falcon.recipe.retry.policy##" delay="##falcon.recipe.retry.delay##"
attempts="3"/>
-    <notification type="##falcon.recipe.notification.type##" to="##falcon.recipe.notification.receivers##"/>
-    <ACL/>
-</process>

http://git-wip-us.apache.org/repos/asf/falcon/blob/4a76b11b/falcon-regression/merlin/src/test/resources/HdfsRecipe/hive-disaster-recovery-workflow.xml
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin/src/test/resources/HdfsRecipe/hive-disaster-recovery-workflow.xml
b/falcon-regression/merlin/src/test/resources/HdfsRecipe/hive-disaster-recovery-workflow.xml
deleted file mode 100644
index aa820d0..0000000
--- a/falcon-regression/merlin/src/test/resources/HdfsRecipe/hive-disaster-recovery-workflow.xml
+++ /dev/null
@@ -1,82 +0,0 @@
-<!--
-  ~ Licensed to the Apache Software Foundation (ASF) under one
-  ~ or more contributor license agreements.  See the NOTICE file
-  ~ distributed with this work for additional information
-  ~ regarding copyright ownership.  The ASF licenses this file
-  ~ to you under the Apache License, Version 2.0 (the
-  ~ "License"); you may not use this file except in compliance
-  ~ with the License.  You may obtain a copy of the License at
-  ~
-  ~     http://www.apache.org/licenses/LICENSE-2.0
-  ~
-  ~ Unless required by applicable law or agreed to in writing, software
-  ~ distributed under the License is distributed on an "AS IS" BASIS,
-  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  ~ See the License for the specific language governing permissions and
-  ~ limitations under the License.
-  -->
-<workflow-app xmlns='uri:oozie:workflow:0.3' name='falcon-dr-fs-workflow'>
-    <start to='dr-replication'/>
-    <!-- Replication action -->
-    <action name="dr-replication">
-        <java>
-            <job-tracker>${jobTracker}</job-tracker>
-            <name-node>${nameNode}</name-node>
-            <configuration>
-                <property> <!-- hadoop 2 parameter -->
-                    <name>oozie.launcher.mapreduce.job.user.classpath.first</name>
-                    <value>true</value>
-                </property>
-                <property>
-                    <name>mapred.job.queue.name</name>
-                    <value>${queueName}</value>
-                </property>
-                <property>
-                    <name>oozie.launcher.mapred.job.priority</name>
-                    <value>${jobPriority}</value>
-                </property>
-                <property>
-                    <name>oozie.use.system.libpath</name>
-                    <value>true</value>
-                </property>
-                <property>
-                    <name>oozie.action.sharelib.for.java</name>
-                    <value>distcp</value>
-                </property>
-                <property>
-                    <name>oozie.launcher.oozie.libpath</name>
-                    <value>${wf:conf("falcon.libpath")}</value>
-                </property>
-                <property>
-                    <name>oozie.launcher.mapreduce.job.hdfs-servers</name>
-                    <value>${drSourceClusterFS},${drTargetClusterFS}</value>
-                </property>
-            </configuration>
-            <main-class>org.apache.falcon.replication.FeedReplicator</main-class>
-            <arg>-Dmapred.job.queue.name=${queueName}</arg>
-            <arg>-Dmapred.job.priority=${jobPriority}</arg>
-            <arg>-maxMaps</arg>
-            <arg>${distcpMaxMaps}</arg>
-            <arg>-mapBandwidth</arg>
-            <arg>${distcpMapBandwidth}</arg>
-            <arg>-sourcePaths</arg>
-            <arg>${drSourceDir}</arg>
-            <arg>-targetPath</arg>
-            <arg>${drTargetClusterFS}${drTargetDir}</arg>
-            <arg>-falconFeedStorageType</arg>
-            <arg>FILESYSTEM</arg>
-            <arg>-availabilityFlag</arg>
-            <arg>${availabilityFlag == 'NA' ? "NA" : availabilityFlag}</arg>
-            <arg>-counterLogDir</arg>
-            <arg>${logDir}/job-${nominalTime}/${srcClusterName == 'NA' ? '' : srcClusterName}</arg>
-        </java>
-        <ok to="end"/>
-        <error to="fail"/>
-    </action>
-    <kill name="fail">
-        <message>
-            Workflow action failed, error message[${wf:errorMessage(wf:lastErrorNode())}]
-        </message>
-    </kill>
-    <end name="end"/>
-</workflow-app>

http://git-wip-us.apache.org/repos/asf/falcon/blob/4a76b11b/falcon-regression/merlin/src/test/resources/HdfsRecipe/hive-disaster-recovery.properties
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin/src/test/resources/HdfsRecipe/hive-disaster-recovery.properties
b/falcon-regression/merlin/src/test/resources/HdfsRecipe/hive-disaster-recovery.properties
deleted file mode 100644
index 6c715f3..0000000
--- a/falcon-regression/merlin/src/test/resources/HdfsRecipe/hive-disaster-recovery.properties
+++ /dev/null
@@ -1,79 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-##### NOTE: This is a TEMPLATE file which can be copied and edited
-
-##### Recipe properties
-##### Unique recipe job name
-falcon.recipe.name=hdfs-replication-monthly
-
-##### Workflow properties
-falcon.recipe.workflow.name=hdfs-dr-workflow
-# Provide Wf absolute path. This can be HDFS or local FS path. If WF is on local FS it will
be copied to HDFS
-falcon.recipe.workflow.path=/apps/data-mirroring/workflows/hdfs-replication-workflow.xml
-
-##### Cluster properties
-
-# Cluster where job should run
-falcon.recipe.cluster.name=primaryCluster
-# Change the cluster hdfs write end point here. This is mandatory.
-falcon.recipe.cluster.hdfs.writeEndPoint=hdfs://localhost:8020
-# Change the cluster validity start time here
-falcon.recipe.cluster.validity.start=2015-03-13T00:00Z
-# Change the cluster validity end time here
-falcon.recipe.cluster.validity.end=2016-12-30T00:00Z
-
-##### Scheduling properties
-# Change the recipe frequency here. Valid frequency type are minutes, hours, days, months
-falcon.recipe.process.frequency=minutes(5)
-
-##### Retry policy properties
-
-falcon.recipe.retry.policy=periodic
-falcon.recipe.retry.delay=minutes(30)
-falcon.recipe.retry.attempts=3
-falcon.recipe.retry.onTimeout=false
-
-##### Tag properties - An optional list of comma separated tags, Key Value Pairs, separated
by comma
-##### Uncomment to add tags
-#falcon.recipe.tags=
-
-##### ACL properties - Uncomment and change ACL if authorization is enabled
-
-#falcon.recipe.acl.owner=ambari-qa
-#falcon.recipe.acl.group=users
-#falcon.recipe.acl.permission=0x755
-#falcon.recipe.nn.principal=nn/_HOST@EXAMPLE.COM
-
-##### Custom Job properties
-
-# Specify multiple comma separated source directories
-drSourceDir=/user/falcon_qa/dr/test/primaryCluster/input
-drSourceClusterFS=hdfs://240.0.0.10:8020
-drTargetDir=/user/falcon_qa/dr/test/backupCluster/input
-drTargetClusterFS=hdfs://240.0.0.11:8020
-
-# Change it to specify the maximum number of mappers for DistCP
-distcpMaxMaps=1
-# Change it to specify the bandwidth in MB for each mapper in DistCP
-distcpMapBandwidth=100
-
-##### Email Notification for Falcon instance completion
-falcon.recipe.notification.type=email
-falcon.recipe.notification.receivers=NA
-

http://git-wip-us.apache.org/repos/asf/falcon/blob/4a76b11b/falcon-regression/merlin/src/test/resources/HiveDrRecipe/hive-disaster-recovery-workflow.xml
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin/src/test/resources/HiveDrRecipe/hive-disaster-recovery-workflow.xml
b/falcon-regression/merlin/src/test/resources/HiveDrRecipe/hive-disaster-recovery-workflow.xml
index 72d40a3..66f97c4 100644
--- a/falcon-regression/merlin/src/test/resources/HiveDrRecipe/hive-disaster-recovery-workflow.xml
+++ b/falcon-regression/merlin/src/test/resources/HiveDrRecipe/hive-disaster-recovery-workflow.xml
@@ -158,6 +158,8 @@
             <arg>${drJobName}-${nominalTime}</arg>
             <arg>-executionStage</arg>
             <arg>export</arg>
+            <arg>-counterLogDir</arg>
+            <arg>${logDir}/job-${nominalTime}/${srcClusterName == 'NA' ? '' : srcClusterName}</arg>
         </java>
         <ok to="import-dr-replication"/>
         <error to="fail"/>


Mime
View raw message