falcon-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From b...@apache.org
Subject [2/3] falcon git commit: FALCON-1107 Move trusted extensions processing to server side
Date Tue, 12 Apr 2016 23:05:56 GMT
http://git-wip-us.apache.org/repos/asf/falcon/blob/95bf312f/addons/recipes/hdfs-replication/src/main/resources/hdfs-replication.properties
----------------------------------------------------------------------
diff --git a/addons/recipes/hdfs-replication/src/main/resources/hdfs-replication.properties b/addons/recipes/hdfs-replication/src/main/resources/hdfs-replication.properties
deleted file mode 100644
index 4642835..0000000
--- a/addons/recipes/hdfs-replication/src/main/resources/hdfs-replication.properties
+++ /dev/null
@@ -1,79 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-##### NOTE: This is a TEMPLATE file which can be copied and edited
-
-##### Recipe properties
-##### Unique recipe job name
-falcon.recipe.name=sales-monthly
-
-##### Workflow properties
-falcon.recipe.workflow.name=hdfs-dr-workflow
-# Provide Wf absolute path. This can be HDFS or local FS path. If WF is on local FS it will be copied to HDFS
-falcon.recipe.workflow.path=/apps/data-mirroring/workflows/hdfs-replication-workflow.xml
-# Provide Wf lib absolute path. This can be HDFS or local FS path. If libs are on local FS it will be copied to HDFS
-#falcon.recipe.workflow.lib.path=/recipes/hdfs-replication/lib
-
-##### Cluster properties
-# Cluster where job should run
-falcon.recipe.cluster.name=primaryCluster
-# Change the cluster hdfs write end point here. This is mandatory.
-falcon.recipe.cluster.hdfs.writeEndPoint=hdfs://240.0.0.10:8020
-# Change the cluster validity start time here
-falcon.recipe.cluster.validity.start=2015-03-13T00:00Z
-# Change the cluster validity end time here
-falcon.recipe.cluster.validity.end=2016-12-30T00:00Z
-
-##### Scheduling properties
-# Change the recipe frequency here. Valid frequency type are minutes, hours, days, months
-falcon.recipe.process.frequency=minutes(5)
-
-##### Tag properties - An optional list of comma separated tags, Key Value Pairs, separated by comma
-##### Uncomment to add tags
-#falcon.recipe.tags=
-
-##### Retry policy properties
-
-falcon.recipe.retry.policy=periodic
-falcon.recipe.retry.delay=minutes(30)
-falcon.recipe.retry.attempts=3
-falcon.recipe.retry.onTimeout=false
-
-##### ACL properties - Uncomment and change ACL if authorization is enabled
-
-falcon.recipe.acl.owner=ambari-qa
-falcon.recipe.acl.group=users
-falcon.recipe.acl.permission=0x755
-falcon.recipe.nn.principal=nn/_HOST@EXAMPLE.COM
-
-##### Custom Job properties
-
-# Specify multiple comma separated source directories
-drSourceDir=/user/hrt_qa/dr/test/primaryCluster/input
-drSourceClusterFS=hdfs://240.0.0.10:8020
-drTargetDir=/user/hrt_qa/dr/test/backupCluster/input
-drTargetClusterFS=hdfs://240.0.0.11:8020
-
-# Change it to specify the maximum number of mappers for DistCP
-distcpMaxMaps=1
-# Change it to specify the bandwidth in MB for each mapper in DistCP
-distcpMapBandwidth=100
-
-##### Email Notification for Falcon instance completion
-falcon.recipe.notification.type=email
-falcon.recipe.notification.receivers=NA
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/falcon/blob/95bf312f/addons/recipes/hive-disaster-recovery/README.txt
----------------------------------------------------------------------
diff --git a/addons/recipes/hive-disaster-recovery/README.txt b/addons/recipes/hive-disaster-recovery/README.txt
deleted file mode 100644
index ab393b1..0000000
--- a/addons/recipes/hive-disaster-recovery/README.txt
+++ /dev/null
@@ -1,58 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-Hive Metastore Disaster Recovery Recipe
-
-Overview
-This recipe implements replicating hive metadata and data from one
-Hadoop cluster to another Hadoop cluster.
-This piggy backs on replication solution in Falcon which uses the DistCp tool.
-
-Use Case
-*
-*
-
-Limitations
-*
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-Hive Metastore Disaster Recovery Recipe
-
-Overview
-This recipe implements replicating hive metadata and data from one
-Hadoop cluster to another Hadoop cluster.
-This piggy backs on replication solution in Falcon which uses the DistCp tool.
-
-Use Case
-*
-*
-
-Limitations
-*

http://git-wip-us.apache.org/repos/asf/falcon/blob/95bf312f/addons/recipes/hive-disaster-recovery/pom.xml
----------------------------------------------------------------------
diff --git a/addons/recipes/hive-disaster-recovery/pom.xml b/addons/recipes/hive-disaster-recovery/pom.xml
deleted file mode 100644
index 0f782d2..0000000
--- a/addons/recipes/hive-disaster-recovery/pom.xml
+++ /dev/null
@@ -1,32 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-
-<!--
-  Licensed to the Apache Software Foundation (ASF) under one
-  or more contributor license agreements.  See the NOTICE file
-  distributed with this work for additional information
-  regarding copyright ownership.  The ASF licenses this file
-  to you under the Apache License, Version 2.0 (the
-  "License"); you may not use this file except in compliance
-  with the License.  You may obtain a copy of the License at
-
-      http://www.apache.org/licenses/LICENSE-2.0
-
-  Unless required by applicable law or agreed to in writing, software
-  distributed under the License is distributed on an "AS IS" BASIS,
-  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  See the License for the specific language governing permissions and
-  limitations under the License.
-  -->
-
-<project xmlns="http://maven.apache.org/POM/4.0.0"
-         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
-
-    <modelVersion>4.0.0</modelVersion>
-    <groupId>org.apache.falcon.recipes</groupId>
-    <artifactId>falcon-hive-replication-recipe</artifactId>
-    <version>0.10-SNAPSHOT</version>
-    <description>Apache Falcon Hive Disaster Recovery Recipe</description>
-    <name>Apache Falcon Sample Hive Disaster Recovery Recipe</name>
-    <packaging>jar</packaging>
-</project>

http://git-wip-us.apache.org/repos/asf/falcon/blob/95bf312f/addons/recipes/hive-disaster-recovery/src/main/resources/hive-disaster-recovery-secure-template.xml
----------------------------------------------------------------------
diff --git a/addons/recipes/hive-disaster-recovery/src/main/resources/hive-disaster-recovery-secure-template.xml b/addons/recipes/hive-disaster-recovery/src/main/resources/hive-disaster-recovery-secure-template.xml
deleted file mode 100644
index f0de091..0000000
--- a/addons/recipes/hive-disaster-recovery/src/main/resources/hive-disaster-recovery-secure-template.xml
+++ /dev/null
@@ -1,45 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
-  Licensed to the Apache Software Foundation (ASF) under one
-  or more contributor license agreements.  See the NOTICE file
-  distributed with this work for additional information
-  regarding copyright ownership.  The ASF licenses this file
-  to you under the Apache License, Version 2.0 (the
-  "License"); you may not use this file except in compliance
-  with the License.  You may obtain a copy of the License at
-
-      http://www.apache.org/licenses/LICENSE-2.0
-
-  Unless required by applicable law or agreed to in writing, software
-  distributed under the License is distributed on an "AS IS" BASIS,
-  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  See the License for the specific language governing permissions and
-  limitations under the License.
-  -->
-
-<process name="##name##" xmlns="uri:falcon:process:0.1">
-    <clusters>
-        <!--  source  -->
-        <cluster name="##cluster.name##">
-            <validity end="##cluster.validity.end##" start="##cluster.validity.start##"/>
-        </cluster>
-    </clusters>
-
-    <tags>_falcon_mirroring_type=HIVE</tags>
-
-    <parallel>1</parallel>
-    <!-- Replication needs to run only once to catch up -->
-    <order>LAST_ONLY</order>
-    <frequency>##process.frequency##</frequency>
-    <timezone>UTC</timezone>
-
-    <properties>
-        <property name="oozie.wf.subworkflow.classpath.inheritance" value="true"/>
-    </properties>
-
-    <workflow name="##workflow.name##" engine="oozie"
-              path="/apps/data-mirroring/workflows/hive-disaster-recovery-workflow.xml" lib="##workflow.lib.path##"/>
-    <retry policy="##retry.policy##" delay="##retry.delay##" attempts="3"/>
-    <notification type="##notification.type##" to="##notification.receivers##"/>
-    <ACL/>
-</process>

http://git-wip-us.apache.org/repos/asf/falcon/blob/95bf312f/addons/recipes/hive-disaster-recovery/src/main/resources/hive-disaster-recovery-secure-workflow.xml
----------------------------------------------------------------------
diff --git a/addons/recipes/hive-disaster-recovery/src/main/resources/hive-disaster-recovery-secure-workflow.xml b/addons/recipes/hive-disaster-recovery/src/main/resources/hive-disaster-recovery-secure-workflow.xml
deleted file mode 100644
index 2d6b8be..0000000
--- a/addons/recipes/hive-disaster-recovery/src/main/resources/hive-disaster-recovery-secure-workflow.xml
+++ /dev/null
@@ -1,363 +0,0 @@
-<!--
-  Licensed to the Apache Software Foundation (ASF) under one
-  or more contributor license agreements.  See the NOTICE file
-  distributed with this work for additional information
-  regarding copyright ownership.  The ASF licenses this file
-  to you under the Apache License, Version 2.0 (the
-  "License"); you may not use this file except in compliance
-  with the License.  You may obtain a copy of the License at
-
-      http://www.apache.org/licenses/LICENSE-2.0
-
-  Unless required by applicable law or agreed to in writing, software
-  distributed under the License is distributed on an "AS IS" BASIS,
-  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  See the License for the specific language governing permissions and
-  limitations under the License.
-  -->
-<workflow-app xmlns='uri:oozie:workflow:0.3' name='falcon-dr-hive-workflow'>
-    <credentials>
-        <credential name='hive_src_credentials' type='hcat'>
-            <property>
-                <name>hcat.metastore.uri</name>
-                <value>${sourceMetastoreUri}</value>
-            </property>
-            <property>
-                <name>hcat.metastore.principal</name>
-                <value>${sourceHiveMetastoreKerberosPrincipal}</value>
-            </property>
-        </credential>
-        <credential name='hive_tgt_credentials' type='hcat'>
-            <property>
-                <name>hcat.metastore.uri</name>
-                <value>${targetMetastoreUri}</value>
-            </property>
-            <property>
-                <name>hcat.metastore.principal</name>
-                <value>${targetHiveMetastoreKerberosPrincipal}</value>
-            </property>
-        </credential>
-        <credential name="hive2_src_credentials" type="hive2">
-            <property>
-                <name>hive2.server.principal</name>
-                <value>${sourceHive2KerberosPrincipal}</value>
-            </property>
-            <property>
-                <name>hive2.jdbc.url</name>
-                <value>jdbc:${sourceHiveServer2Uri}/${sourceDatabase}</value>
-            </property>
-        </credential>
-        <credential name="hive2_tgt_credentials" type="hive2">
-            <property>
-                <name>hive2.server.principal</name>
-                <value>${targetHive2KerberosPrincipal}</value>
-            </property>
-            <property>
-                <name>hive2.jdbc.url</name>
-                <value>jdbc:${targetHiveServer2Uri}/${sourceDatabase}</value>
-            </property>
-        </credential>
-    </credentials>
-    <start to='last-event'/>
-    <action name="last-event" cred="hive_tgt_credentials">
-        <java>
-            <job-tracker>${jobTracker}</job-tracker>
-            <name-node>${nameNode}</name-node>
-            <configuration>
-                <property> <!-- hadoop 2 parameter -->
-                    <name>oozie.launcher.mapreduce.job.user.classpath.first</name>
-                    <value>true</value>
-                </property>
-                <property>
-                    <name>mapred.job.queue.name</name>
-                    <value>${queueName}</value>
-                </property>
-                <property>
-                    <name>oozie.launcher.mapred.job.priority</name>
-                    <value>${jobPriority}</value>
-                </property>
-                <property>
-                    <name>oozie.use.system.libpath</name>
-                    <value>true</value>
-                </property>
-                <property>
-                    <name>oozie.action.sharelib.for.java</name>
-                    <value>distcp,hive,hive2,hcatalog</value>
-                </property>
-                <property>
-                    <name>oozie.launcher.mapreduce.job.hdfs-servers</name>
-                    <value>${sourceNN},${targetNN}</value>
-                </property>
-                <property>
-                    <name>mapreduce.job.hdfs-servers</name>
-                    <value>${sourceNN},${targetNN}</value>
-                </property>
-            </configuration>
-            <main-class>org.apache.falcon.hive.HiveDRTool</main-class>
-            <arg>-Dmapred.job.queue.name=${queueName}</arg>
-            <arg>-Dmapred.job.priority=${jobPriority}</arg>
-            <arg>-falconLibPath</arg>
-            <arg>${wf:conf("falcon.libpath")}</arg>
-            <arg>-sourceCluster</arg>
-            <arg>${sourceCluster}</arg>
-            <arg>-sourceMetastoreUri</arg>
-            <arg>${sourceMetastoreUri}</arg>
-            <arg>-sourceHiveServer2Uri</arg>
-            <arg>${sourceHiveServer2Uri}</arg>
-            <arg>-sourceDatabase</arg>
-            <arg>${sourceDatabase}</arg>
-            <arg>-sourceTable</arg>
-            <arg>${sourceTable}</arg>
-            <arg>-sourceStagingPath</arg>
-            <arg>${sourceStagingPath}</arg>
-            <arg>-sourceNN</arg>
-            <arg>${sourceNN}</arg>
-            <arg>-sourceNNKerberosPrincipal</arg>
-            <arg>${sourceNNKerberosPrincipal}</arg>
-            <arg>-sourceHiveMetastoreKerberosPrincipal</arg>
-            <arg>${sourceHiveMetastoreKerberosPrincipal}</arg>
-            <arg>-sourceHive2KerberosPrincipal</arg>
-            <arg>${sourceHive2KerberosPrincipal}</arg>
-            <arg>-targetCluster</arg>
-            <arg>${targetCluster}</arg>
-            <arg>-targetMetastoreUri</arg>
-            <arg>${targetMetastoreUri}</arg>
-            <arg>-targetHiveServer2Uri</arg>
-            <arg>${targetHiveServer2Uri}</arg>
-            <arg>-targetStagingPath</arg>
-            <arg>${targetStagingPath}</arg>
-            <arg>-targetNN</arg>
-            <arg>${targetNN}</arg>
-            <arg>-targetNNKerberosPrincipal</arg>
-            <arg>${targetNNKerberosPrincipal}</arg>
-            <arg>-targetHiveMetastoreKerberosPrincipal</arg>
-            <arg>${targetHiveMetastoreKerberosPrincipal}</arg>
-            <arg>-targetHive2KerberosPrincipal</arg>
-            <arg>${targetHive2KerberosPrincipal}</arg>
-            <arg>-maxEvents</arg>
-            <arg>${maxEvents}</arg>
-            <arg>-clusterForJobRun</arg>
-            <arg>${clusterForJobRun}</arg>
-            <arg>-clusterForJobRunWriteEP</arg>
-            <arg>${clusterForJobRunWriteEP}</arg>
-            <arg>-clusterForJobNNKerberosPrincipal</arg>
-            <arg>${clusterForJobNNKerberosPrincipal}</arg>
-            <arg>-tdeEncryptionEnabled</arg>
-            <arg>${tdeEncryptionEnabled}</arg>
-            <arg>-drJobName</arg>
-            <arg>${drJobName}-${nominalTime}</arg>
-            <arg>-executionStage</arg>
-            <arg>lastevents</arg>
-        </java>
-        <ok to="export-dr-replication"/>
-        <error to="fail"/>
-    </action>
-    <!-- Export Replication action -->
-    <action name="export-dr-replication" cred="hive_src_credentials,hive2_src_credentials">
-        <java>
-            <job-tracker>${jobTracker}</job-tracker>
-            <name-node>${nameNode}</name-node>
-            <configuration>
-                <property> <!-- hadoop 2 parameter -->
-                    <name>oozie.launcher.mapreduce.job.user.classpath.first</name>
-                    <value>true</value>
-                </property>
-                <property>
-                    <name>mapred.job.queue.name</name>
-                    <value>${queueName}</value>
-                </property>
-                <property>
-                    <name>oozie.launcher.mapred.job.priority</name>
-                    <value>${jobPriority}</value>
-                </property>
-                <property>
-                    <name>oozie.use.system.libpath</name>
-                    <value>true</value>
-                </property>
-                <property>
-                    <name>oozie.action.sharelib.for.java</name>
-                    <value>distcp,hive,hive2,hcatalog</value>
-                </property>
-                <property>
-                    <name>oozie.launcher.mapreduce.job.hdfs-servers</name>
-                    <value>${sourceNN},${targetNN}</value>
-                </property>
-                <property>
-                    <name>mapreduce.job.hdfs-servers</name>
-                    <value>${sourceNN},${targetNN}</value>
-                </property>
-            </configuration>
-            <main-class>org.apache.falcon.hive.HiveDRTool</main-class>
-            <arg>-Dmapred.job.queue.name=${queueName}</arg>
-            <arg>-Dmapred.job.priority=${jobPriority}</arg>
-            <arg>-falconLibPath</arg>
-            <arg>${wf:conf("falcon.libpath")}</arg>
-            <arg>-replicationMaxMaps</arg>
-            <arg>${replicationMaxMaps}</arg>
-            <arg>-distcpMaxMaps</arg>
-            <arg>${distcpMaxMaps}</arg>
-            <arg>-sourceCluster</arg>
-            <arg>${sourceCluster}</arg>
-            <arg>-sourceMetastoreUri</arg>
-            <arg>${sourceMetastoreUri}</arg>
-            <arg>-sourceHiveServer2Uri</arg>
-            <arg>${sourceHiveServer2Uri}</arg>
-            <arg>-sourceDatabase</arg>
-            <arg>${sourceDatabase}</arg>
-            <arg>-sourceTable</arg>
-            <arg>${sourceTable}</arg>
-            <arg>-sourceStagingPath</arg>
-            <arg>${sourceStagingPath}</arg>
-            <arg>-sourceNN</arg>
-            <arg>${sourceNN}</arg>
-            <arg>-sourceNNKerberosPrincipal</arg>
-            <arg>${sourceNNKerberosPrincipal}</arg>
-            <arg>-sourceHiveMetastoreKerberosPrincipal</arg>
-            <arg>${sourceHiveMetastoreKerberosPrincipal}</arg>
-            <arg>-sourceHive2KerberosPrincipal</arg>
-            <arg>${sourceHive2KerberosPrincipal}</arg>
-            <arg>-targetCluster</arg>
-            <arg>${targetCluster}</arg>
-            <arg>-targetMetastoreUri</arg>
-            <arg>${targetMetastoreUri}</arg>
-            <arg>-targetHiveServer2Uri</arg>
-            <arg>${targetHiveServer2Uri}</arg>
-            <arg>-targetStagingPath</arg>
-            <arg>${targetStagingPath}</arg>
-            <arg>-targetNN</arg>
-            <arg>${targetNN}</arg>
-            <arg>-targetNNKerberosPrincipal</arg>
-            <arg>${targetNNKerberosPrincipal}</arg>
-            <arg>-targetHiveMetastoreKerberosPrincipal</arg>
-            <arg>${targetHiveMetastoreKerberosPrincipal}</arg>
-            <arg>-targetHive2KerberosPrincipal</arg>
-            <arg>${targetHive2KerberosPrincipal}</arg>
-            <arg>-maxEvents</arg>
-            <arg>${maxEvents}</arg>
-            <arg>-distcpMapBandwidth</arg>
-            <arg>${distcpMapBandwidth}</arg>
-            <arg>-clusterForJobRun</arg>
-            <arg>${clusterForJobRun}</arg>
-            <arg>-clusterForJobRunWriteEP</arg>
-            <arg>${clusterForJobRunWriteEP}</arg>
-            <arg>-clusterForJobNNKerberosPrincipal</arg>
-            <arg>${clusterForJobNNKerberosPrincipal}</arg>
-            <arg>-tdeEncryptionEnabled</arg>
-            <arg>${tdeEncryptionEnabled}</arg>
-            <arg>-drJobName</arg>
-            <arg>${drJobName}-${nominalTime}</arg>
-            <arg>-executionStage</arg>
-            <arg>export</arg>
-            <arg>-counterLogDir</arg>
-            <arg>${logDir}/job-${nominalTime}/${srcClusterName == 'NA' ? '' : srcClusterName}/</arg>
-        </java>
-        <ok to="import-dr-replication"/>
-        <error to="fail"/>
-    </action>
-    <!-- Import Replication action -->
-    <action name="import-dr-replication" cred="hive_tgt_credentials,hive2_tgt_credentials">
-        <java>
-            <job-tracker>${jobTracker}</job-tracker>
-            <name-node>${nameNode}</name-node>
-            <configuration>
-                <property> <!-- hadoop 2 parameter -->
-                    <name>oozie.launcher.mapreduce.job.user.classpath.first</name>
-                    <value>true</value>
-                </property>
-                <property>
-                    <name>mapred.job.queue.name</name>
-                    <value>${queueName}</value>
-                </property>
-                <property>
-                    <name>oozie.launcher.mapred.job.priority</name>
-                    <value>${jobPriority}</value>
-                </property>
-                <property>
-                    <name>oozie.use.system.libpath</name>
-                    <value>true</value>
-                </property>
-                <property>
-                    <name>oozie.action.sharelib.for.java</name>
-                    <value>distcp,hive,hive2,hcatalog</value>
-                </property>
-                <property>
-                    <name>oozie.launcher.mapreduce.job.hdfs-servers</name>
-                    <value>${sourceNN},${targetNN}</value>
-                </property>
-                <property>
-                    <name>mapreduce.job.hdfs-servers</name>
-                    <value>${sourceNN},${targetNN}</value>
-                </property>
-            </configuration>
-            <main-class>org.apache.falcon.hive.HiveDRTool</main-class>
-            <arg>-Dmapred.job.queue.name=${queueName}</arg>
-            <arg>-Dmapred.job.priority=${jobPriority}</arg>
-            <arg>-falconLibPath</arg>
-            <arg>${wf:conf("falcon.libpath")}</arg>
-            <arg>-replicationMaxMaps</arg>
-            <arg>${replicationMaxMaps}</arg>
-            <arg>-distcpMaxMaps</arg>
-            <arg>${distcpMaxMaps}</arg>
-            <arg>-sourceCluster</arg>
-            <arg>${sourceCluster}</arg>
-            <arg>-sourceMetastoreUri</arg>
-            <arg>${sourceMetastoreUri}</arg>
-            <arg>-sourceHiveServer2Uri</arg>
-            <arg>${sourceHiveServer2Uri}</arg>
-            <arg>-sourceDatabase</arg>
-            <arg>${sourceDatabase}</arg>
-            <arg>-sourceTable</arg>
-            <arg>${sourceTable}</arg>
-            <arg>-sourceStagingPath</arg>
-            <arg>${sourceStagingPath}</arg>
-            <arg>-sourceNN</arg>
-            <arg>${sourceNN}</arg>
-            <arg>-sourceNNKerberosPrincipal</arg>
-            <arg>${sourceNNKerberosPrincipal}</arg>
-            <arg>-sourceHiveMetastoreKerberosPrincipal</arg>
-            <arg>${sourceHiveMetastoreKerberosPrincipal}</arg>
-            <arg>-sourceHive2KerberosPrincipal</arg>
-            <arg>${sourceHive2KerberosPrincipal}</arg>
-            <arg>-targetCluster</arg>
-            <arg>${targetCluster}</arg>
-            <arg>-targetMetastoreUri</arg>
-            <arg>${targetMetastoreUri}</arg>
-            <arg>-targetHiveServer2Uri</arg>
-            <arg>${targetHiveServer2Uri}</arg>
-            <arg>-targetStagingPath</arg>
-            <arg>${targetStagingPath}</arg>
-            <arg>-targetNN</arg>
-            <arg>${targetNN}</arg>
-            <arg>-targetNNKerberosPrincipal</arg>
-            <arg>${targetNNKerberosPrincipal}</arg>
-            <arg>-targetHiveMetastoreKerberosPrincipal</arg>
-            <arg>${targetHiveMetastoreKerberosPrincipal}</arg>
-            <arg>-targetHive2KerberosPrincipal</arg>
-            <arg>${targetHive2KerberosPrincipal}</arg>
-            <arg>-maxEvents</arg>
-            <arg>${maxEvents}</arg>
-            <arg>-distcpMapBandwidth</arg>
-            <arg>${distcpMapBandwidth}</arg>
-            <arg>-clusterForJobRun</arg>
-            <arg>${clusterForJobRun}</arg>
-            <arg>-clusterForJobRunWriteEP</arg>
-            <arg>${clusterForJobRunWriteEP}</arg>
-            <arg>-clusterForJobNNKerberosPrincipal</arg>
-            <arg>${clusterForJobNNKerberosPrincipal}</arg>
-            <arg>-tdeEncryptionEnabled</arg>
-            <arg>${tdeEncryptionEnabled}</arg>
-            <arg>-drJobName</arg>
-            <arg>${drJobName}-${nominalTime}</arg>
-            <arg>-executionStage</arg>
-            <arg>import</arg>
-        </java>
-        <ok to="end"/>
-        <error to="fail"/>
-    </action>
-    <kill name="fail">
-        <message>
-            Workflow action failed, error message[${wf:errorMessage(wf:lastErrorNode())}]
-        </message>
-    </kill>
-    <end name="end"/>
-</workflow-app>

http://git-wip-us.apache.org/repos/asf/falcon/blob/95bf312f/addons/recipes/hive-disaster-recovery/src/main/resources/hive-disaster-recovery-secure.properties
----------------------------------------------------------------------
diff --git a/addons/recipes/hive-disaster-recovery/src/main/resources/hive-disaster-recovery-secure.properties b/addons/recipes/hive-disaster-recovery/src/main/resources/hive-disaster-recovery-secure.properties
deleted file mode 100644
index 331d57e..0000000
--- a/addons/recipes/hive-disaster-recovery/src/main/resources/hive-disaster-recovery-secure.properties
+++ /dev/null
@@ -1,110 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-##### NOTE: This is a TEMPLATE file which can be copied and edited
-
-##### Recipe properties
-falcon.recipe.name=hive-disaster-recovery
-
-
-##### Workflow properties
-falcon.recipe.workflow.name=hive-dr-workflow
-# Provide Wf absolute path. This can be HDFS or local FS path. If WF is on local FS it will be copied to HDFS
-falcon.recipe.workflow.path=/recipes/hive-replication/hive-disaster-recovery-secure-workflow.xml
-
-##### Cluster properties
-
-# Change the cluster name where replication job should run here
-falcon.recipe.cluster.name=backupCluster
-# Change the cluster hdfs write end point here. This is mandatory.
-falcon.recipe.cluster.hdfs.writeEndPoint=hdfs://localhost:8020
-# Change the cluster validity start time here
-falcon.recipe.cluster.validity.start=2014-10-01T00:00Z
-# Change the cluster validity end time here
-falcon.recipe.cluster.validity.end=2016-12-30T00:00Z
-# Change the cluster namenode kerberos principal. This is mandatory on secure clusters.
-falcon.recipe.nn.principal=nn/_HOST@EXAMPLE.COM
-
-##### Scheduling properties
-
-# Change the process frequency here. Valid frequency type are minutes, hours, days, months
-falcon.recipe.process.frequency=minutes(60)
-
-##### Retry policy properties
-
-falcon.recipe.retry.policy=periodic
-falcon.recipe.retry.delay=minutes(30)
-falcon.recipe.retry.attempts=3
-falcon.recipe.retry.onTimeout=false
-
-##### Tag properties - An optional list of comma separated tags, Key Value Pairs, separated by comma
-##### Uncomment to add tags
-#falcon.recipe.tags=owner=landing,pipeline=adtech
-
-##### ACL properties - Uncomment and change ACL if authorization is enabled
-
-#falcon.recipe.acl.owner=testuser
-#falcon.recipe.acl.group=group
-#falcon.recipe.acl.permission=0x755
-
-##### Custom Job properties
-
-##### Source Cluster DR properties
-sourceCluster=primaryCluster
-sourceMetastoreUri=thrift://localhost:9083
-sourceHiveServer2Uri=hive2://localhost:10000
-# For DB level replicaiton to replicate multiple databases specify comma separated list of tables
-sourceDatabase=default
-# For DB level replication specify * for sourceTable.
-# For table level replication to replicate multiple tables specify comma separated list of tables
-sourceTable=testtable_dr
-## Please specify staging dir in the source without fully qualified domain name.
-sourceStagingPath=/apps/hive/tools/dr
-sourceNN=hdfs://localhost:8020
-# Specify kerberos principal required to access source namenode and hive servers, optional on non-secure cluster.
-sourceNNKerberosPrincipal=nn/_HOST@EXAMPLE.COM
-sourceHiveMetastoreKerberosPrincipal=hive/_HOST@EXAMPLE.COM
-sourceHive2KerberosPrincipal=hive/_HOST@EXAMPLE.COM
-
-##### Target Cluster DR properties
-targetCluster=backupCluster
-targetMetastoreUri=thrift://localhost:9083
-targetHiveServer2Uri=hive2://localhost:10000
-## Please specify staging dir in the target without fully qualified domain name.
-targetStagingPath=/apps/hive/tools/dr
-targetNN=hdfs://localhost:8020
-# Specify kerberos principal required to access target namenode and hive servers, optional on non-secure cluster.
-targetNNKerberosPrincipal=nn/_HOST@EXAMPLE.COM
-targetHiveMetastoreKerberosPrincipal=hive/_HOST@EXAMPLE.COM
-targetHive2KerberosPrincipal=hive/_HOST@EXAMPLE.COM
-
-# To ceil the max events processed each time job runs. Set it to max value depending on your bandwidth limit.
-# Setting it to -1 will process all the events but can hog up the bandwidth. Use it judiciously!
-maxEvents=-1
-# Change it to specify the maximum number of mappers for replication
-replicationMaxMaps=5
-# Change it to specify the maximum number of mappers for DistCP
-distcpMaxMaps=1
-# Change it to specify the bandwidth in MB for each mapper in DistCP
-distcpMapBandwidth=100
-# Set this flag to true if TDE encryption is enabled on source and target
-tdeEncryptionEnabled=false
-
-##### Email Notification for Falcon instance completion
-falcon.recipe.notification.type=email
-falcon.recipe.notification.receivers=NA
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/falcon/blob/95bf312f/addons/recipes/hive-disaster-recovery/src/main/resources/hive-disaster-recovery-template.xml
----------------------------------------------------------------------
diff --git a/addons/recipes/hive-disaster-recovery/src/main/resources/hive-disaster-recovery-template.xml b/addons/recipes/hive-disaster-recovery/src/main/resources/hive-disaster-recovery-template.xml
deleted file mode 100644
index f0de091..0000000
--- a/addons/recipes/hive-disaster-recovery/src/main/resources/hive-disaster-recovery-template.xml
+++ /dev/null
@@ -1,45 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
-  Licensed to the Apache Software Foundation (ASF) under one
-  or more contributor license agreements.  See the NOTICE file
-  distributed with this work for additional information
-  regarding copyright ownership.  The ASF licenses this file
-  to you under the Apache License, Version 2.0 (the
-  "License"); you may not use this file except in compliance
-  with the License.  You may obtain a copy of the License at
-
-      http://www.apache.org/licenses/LICENSE-2.0
-
-  Unless required by applicable law or agreed to in writing, software
-  distributed under the License is distributed on an "AS IS" BASIS,
-  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  See the License for the specific language governing permissions and
-  limitations under the License.
-  -->
-
-<process name="##name##" xmlns="uri:falcon:process:0.1">
-    <clusters>
-        <!--  source  -->
-        <cluster name="##cluster.name##">
-            <validity end="##cluster.validity.end##" start="##cluster.validity.start##"/>
-        </cluster>
-    </clusters>
-
-    <tags>_falcon_mirroring_type=HIVE</tags>
-
-    <parallel>1</parallel>
-    <!-- Replication needs to run only once to catch up -->
-    <order>LAST_ONLY</order>
-    <frequency>##process.frequency##</frequency>
-    <timezone>UTC</timezone>
-
-    <properties>
-        <property name="oozie.wf.subworkflow.classpath.inheritance" value="true"/>
-    </properties>
-
-    <workflow name="##workflow.name##" engine="oozie"
-              path="/apps/data-mirroring/workflows/hive-disaster-recovery-workflow.xml" lib="##workflow.lib.path##"/>
-    <retry policy="##retry.policy##" delay="##retry.delay##" attempts="3"/>
-    <notification type="##notification.type##" to="##notification.receivers##"/>
-    <ACL/>
-</process>

http://git-wip-us.apache.org/repos/asf/falcon/blob/95bf312f/addons/recipes/hive-disaster-recovery/src/main/resources/hive-disaster-recovery-workflow.xml
----------------------------------------------------------------------
diff --git a/addons/recipes/hive-disaster-recovery/src/main/resources/hive-disaster-recovery-workflow.xml b/addons/recipes/hive-disaster-recovery/src/main/resources/hive-disaster-recovery-workflow.xml
deleted file mode 100644
index 296e049..0000000
--- a/addons/recipes/hive-disaster-recovery/src/main/resources/hive-disaster-recovery-workflow.xml
+++ /dev/null
@@ -1,249 +0,0 @@
-<!--
-  Licensed to the Apache Software Foundation (ASF) under one
-  or more contributor license agreements.  See the NOTICE file
-  distributed with this work for additional information
-  regarding copyright ownership.  The ASF licenses this file
-  to you under the Apache License, Version 2.0 (the
-  "License"); you may not use this file except in compliance
-  with the License.  You may obtain a copy of the License at
-
-      http://www.apache.org/licenses/LICENSE-2.0
-
-  Unless required by applicable law or agreed to in writing, software
-  distributed under the License is distributed on an "AS IS" BASIS,
-  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  See the License for the specific language governing permissions and
-  limitations under the License.
-  -->
-<workflow-app xmlns='uri:oozie:workflow:0.3' name='falcon-dr-hive-workflow'>
-    <start to='last-event'/>
-    <action name="last-event">
-        <java>
-            <job-tracker>${jobTracker}</job-tracker>
-            <name-node>${nameNode}</name-node>
-            <configuration>
-                <property> <!-- hadoop 2 parameter -->
-                    <name>oozie.launcher.mapreduce.job.user.classpath.first</name>
-                    <value>true</value>
-                </property>
-                <property>
-                    <name>mapred.job.queue.name</name>
-                    <value>${queueName}</value>
-                </property>
-                <property>
-                    <name>oozie.launcher.mapred.job.priority</name>
-                    <value>${jobPriority}</value>
-                </property>
-                <property>
-                    <name>oozie.use.system.libpath</name>
-                    <value>true</value>
-                </property>
-                <property>
-                    <name>oozie.action.sharelib.for.java</name>
-                    <value>distcp,hive,hive2,hcatalog</value>
-                </property>
-            </configuration>
-            <main-class>org.apache.falcon.hive.HiveDRTool</main-class>
-            <arg>-Dmapred.job.queue.name=${queueName}</arg>
-            <arg>-Dmapred.job.priority=${jobPriority}</arg>
-            <arg>-falconLibPath</arg>
-            <arg>${wf:conf("falcon.libpath")}</arg>
-            <arg>-sourceCluster</arg>
-            <arg>${sourceCluster}</arg>
-            <arg>-sourceMetastoreUri</arg>
-            <arg>${sourceMetastoreUri}</arg>
-            <arg>-sourceHiveServer2Uri</arg>
-            <arg>${sourceHiveServer2Uri}</arg>
-            <arg>-sourceDatabase</arg>
-            <arg>${sourceDatabase}</arg>
-            <arg>-sourceTable</arg>
-            <arg>${sourceTable}</arg>
-            <arg>-sourceStagingPath</arg>
-            <arg>${sourceStagingPath}</arg>
-            <arg>-sourceNN</arg>
-            <arg>${sourceNN}</arg>
-            <arg>-targetCluster</arg>
-            <arg>${targetCluster}</arg>
-            <arg>-targetMetastoreUri</arg>
-            <arg>${targetMetastoreUri}</arg>
-            <arg>-targetHiveServer2Uri</arg>
-            <arg>${targetHiveServer2Uri}</arg>
-            <arg>-targetStagingPath</arg>
-            <arg>${targetStagingPath}</arg>
-            <arg>-targetNN</arg>
-            <arg>${targetNN}</arg>
-            <arg>-maxEvents</arg>
-            <arg>${maxEvents}</arg>
-            <arg>-clusterForJobRun</arg>
-            <arg>${clusterForJobRun}</arg>
-            <arg>-clusterForJobRunWriteEP</arg>
-            <arg>${clusterForJobRunWriteEP}</arg>
-            <arg>-drJobName</arg>
-            <arg>${drJobName}-${nominalTime}</arg>
-            <arg>-executionStage</arg>
-            <arg>lastevents</arg>
-        </java>
-        <ok to="export-dr-replication"/>
-        <error to="fail"/>
-    </action>
-    <!-- Export Replication action -->
-    <action name="export-dr-replication">
-        <java>
-            <job-tracker>${jobTracker}</job-tracker>
-            <name-node>${nameNode}</name-node>
-            <configuration>
-                <property> <!-- hadoop 2 parameter -->
-                    <name>oozie.launcher.mapreduce.job.user.classpath.first</name>
-                    <value>true</value>
-                </property>
-                <property>
-                    <name>mapred.job.queue.name</name>
-                    <value>${queueName}</value>
-                </property>
-                <property>
-                    <name>oozie.launcher.mapred.job.priority</name>
-                    <value>${jobPriority}</value>
-                </property>
-                <property>
-                    <name>oozie.use.system.libpath</name>
-                    <value>true</value>
-                </property>
-                <property>
-                    <name>oozie.action.sharelib.for.java</name>
-                    <value>distcp,hive,hive2,hcatalog</value>
-                </property>
-            </configuration>
-            <main-class>org.apache.falcon.hive.HiveDRTool</main-class>
-            <arg>-Dmapred.job.queue.name=${queueName}</arg>
-            <arg>-Dmapred.job.priority=${jobPriority}</arg>
-            <arg>-falconLibPath</arg>
-            <arg>${wf:conf("falcon.libpath")}</arg>
-            <arg>-replicationMaxMaps</arg>
-            <arg>${replicationMaxMaps}</arg>
-            <arg>-distcpMaxMaps</arg>
-            <arg>${distcpMaxMaps}</arg>
-            <arg>-sourceCluster</arg>
-            <arg>${sourceCluster}</arg>
-            <arg>-sourceMetastoreUri</arg>
-            <arg>${sourceMetastoreUri}</arg>
-            <arg>-sourceHiveServer2Uri</arg>
-            <arg>${sourceHiveServer2Uri}</arg>
-            <arg>-sourceDatabase</arg>
-            <arg>${sourceDatabase}</arg>
-            <arg>-sourceTable</arg>
-            <arg>${sourceTable}</arg>
-            <arg>-sourceStagingPath</arg>
-            <arg>${sourceStagingPath}</arg>
-            <arg>-sourceNN</arg>
-            <arg>${sourceNN}</arg>
-            <arg>-targetCluster</arg>
-            <arg>${targetCluster}</arg>
-            <arg>-targetMetastoreUri</arg>
-            <arg>${targetMetastoreUri}</arg>
-            <arg>-targetHiveServer2Uri</arg>
-            <arg>${targetHiveServer2Uri}</arg>
-            <arg>-targetStagingPath</arg>
-            <arg>${targetStagingPath}</arg>
-            <arg>-targetNN</arg>
-            <arg>${targetNN}</arg>
-            <arg>-maxEvents</arg>
-            <arg>${maxEvents}</arg>
-            <arg>-distcpMapBandwidth</arg>
-            <arg>${distcpMapBandwidth}</arg>
-            <arg>-clusterForJobRun</arg>
-            <arg>${clusterForJobRun}</arg>
-            <arg>-clusterForJobRunWriteEP</arg>
-            <arg>${clusterForJobRunWriteEP}</arg>
-            <arg>-drJobName</arg>
-            <arg>${drJobName}-${nominalTime}</arg>
-            <arg>-executionStage</arg>
-            <arg>export</arg>
-            <arg>-counterLogDir</arg>
-            <arg>${logDir}/job-${nominalTime}/${srcClusterName == 'NA' ? '' : srcClusterName}/</arg>
-        </java>
-        <ok to="import-dr-replication"/>
-        <error to="fail"/>
-    </action>
-    <!-- Import Replication action -->
-    <action name="import-dr-replication">
-        <java>
-            <job-tracker>${jobTracker}</job-tracker>
-            <name-node>${nameNode}</name-node>
-            <configuration>
-                <property> <!-- hadoop 2 parameter -->
-                    <name>oozie.launcher.mapreduce.job.user.classpath.first</name>
-                    <value>true</value>
-                </property>
-                <property>
-                    <name>mapred.job.queue.name</name>
-                    <value>${queueName}</value>
-                </property>
-                <property>
-                    <name>oozie.launcher.mapred.job.priority</name>
-                    <value>${jobPriority}</value>
-                </property>
-                <property>
-                    <name>oozie.use.system.libpath</name>
-                    <value>true</value>
-                </property>
-                <property>
-                    <name>oozie.action.sharelib.for.java</name>
-                    <value>distcp,hive,hive2,hcatalog</value>
-                </property>
-            </configuration>
-            <main-class>org.apache.falcon.hive.HiveDRTool</main-class>
-            <arg>-Dmapred.job.queue.name=${queueName}</arg>
-            <arg>-Dmapred.job.priority=${jobPriority}</arg>
-            <arg>-falconLibPath</arg>
-            <arg>${wf:conf("falcon.libpath")}</arg>
-            <arg>-replicationMaxMaps</arg>
-            <arg>${replicationMaxMaps}</arg>
-            <arg>-distcpMaxMaps</arg>
-            <arg>${distcpMaxMaps}</arg>
-            <arg>-sourceCluster</arg>
-            <arg>${sourceCluster}</arg>
-            <arg>-sourceMetastoreUri</arg>
-            <arg>${sourceMetastoreUri}</arg>
-            <arg>-sourceHiveServer2Uri</arg>
-            <arg>${sourceHiveServer2Uri}</arg>
-            <arg>-sourceDatabase</arg>
-            <arg>${sourceDatabase}</arg>
-            <arg>-sourceTable</arg>
-            <arg>${sourceTable}</arg>
-            <arg>-sourceStagingPath</arg>
-            <arg>${sourceStagingPath}</arg>
-            <arg>-sourceNN</arg>
-            <arg>${sourceNN}</arg>
-            <arg>-targetCluster</arg>
-            <arg>${targetCluster}</arg>
-            <arg>-targetMetastoreUri</arg>
-            <arg>${targetMetastoreUri}</arg>
-            <arg>-targetHiveServer2Uri</arg>
-            <arg>${targetHiveServer2Uri}</arg>
-            <arg>-targetStagingPath</arg>
-            <arg>${targetStagingPath}</arg>
-            <arg>-targetNN</arg>
-            <arg>${targetNN}</arg>
-            <arg>-maxEvents</arg>
-            <arg>${maxEvents}</arg>
-            <arg>-distcpMapBandwidth</arg>
-            <arg>${distcpMapBandwidth}</arg>
-            <arg>-clusterForJobRun</arg>
-            <arg>${clusterForJobRun}</arg>
-            <arg>-clusterForJobRunWriteEP</arg>
-            <arg>${clusterForJobRunWriteEP}</arg>
-            <arg>-drJobName</arg>
-            <arg>${drJobName}-${nominalTime}</arg>
-            <arg>-executionStage</arg>
-            <arg>import</arg>
-        </java>
-        <ok to="end"/>
-        <error to="fail"/>
-    </action>
-    <kill name="fail">
-        <message>
-            Workflow action failed, error message[${wf:errorMessage(wf:lastErrorNode())}]
-        </message>
-    </kill>
-    <end name="end"/>
-</workflow-app>

http://git-wip-us.apache.org/repos/asf/falcon/blob/95bf312f/addons/recipes/hive-disaster-recovery/src/main/resources/hive-disaster-recovery.properties
----------------------------------------------------------------------
diff --git a/addons/recipes/hive-disaster-recovery/src/main/resources/hive-disaster-recovery.properties b/addons/recipes/hive-disaster-recovery/src/main/resources/hive-disaster-recovery.properties
deleted file mode 100644
index b14ec7c..0000000
--- a/addons/recipes/hive-disaster-recovery/src/main/resources/hive-disaster-recovery.properties
+++ /dev/null
@@ -1,98 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-##### NOTE: This is a TEMPLATE file which can be copied and edited
-
-##### Recipe properties
-falcon.recipe.name=hive-disaster-recovery
-
-
-##### Workflow properties
-falcon.recipe.workflow.name=hive-dr-workflow
-# Provide Wf absolute path. This can be HDFS or local FS path. If WF is on local FS it will be copied to HDFS
-falcon.recipe.workflow.path=/recipes/hive-replication/hive-disaster-recovery-workflow.xml
-
-##### Cluster properties
-
-# Change the cluster name where replication job should run here
-falcon.recipe.cluster.name=backupCluster
-# Change the cluster hdfs write end point here. This is mandatory.
-falcon.recipe.cluster.hdfs.writeEndPoint=hdfs://localhost:8020
-# Change the cluster validity start time here
-falcon.recipe.cluster.validity.start=2014-10-01T00:00Z
-# Change the cluster validity end time here
-falcon.recipe.cluster.validity.end=2016-12-30T00:00Z
-
-##### Scheduling properties
-
-# Change the process frequency here. Valid frequency type are minutes, hours, days, months
-falcon.recipe.process.frequency=minutes(60)
-
-##### Retry policy properties
-
-falcon.recipe.retry.policy=periodic
-falcon.recipe.retry.delay=minutes(30)
-falcon.recipe.retry.attempts=3
-falcon.recipe.retry.onTimeout=false
-
-##### Tag properties - An optional list of comma separated tags, Key Value Pairs, separated by comma
-##### Uncomment to add tags
-#falcon.recipe.tags=owner=landing,pipeline=adtech
-
-##### ACL properties - Uncomment and change ACL if authorization is enabled
-
-#falcon.recipe.acl.owner=testuser
-#falcon.recipe.acl.group=group
-#falcon.recipe.acl.permission=0x755
-
-##### Custom Job properties
-
-##### Source Cluster DR properties
-sourceCluster=primaryCluster
-sourceMetastoreUri=thrift://localhost:9083
-sourceHiveServer2Uri=hive2://localhost:10000
-# For DB level replicaiton to replicate multiple databases specify comma separated list of tables
-sourceDatabase=default
-# For DB level replication specify * for sourceTable.
-# For table level replication to replicate multiple tables specify comma separated list of tables
-sourceTable=testtable_dr
-## Please specify staging dir in the source without fully qualified domain name.
-sourceStagingPath=/apps/hive/tools/dr
-sourceNN=hdfs://localhost:8020
-
-##### Target Cluster DR properties
-targetCluster=backupCluster
-targetMetastoreUri=thrift://localhost:9083
-targetHiveServer2Uri=hive2://localhost:10000
-## Please specify staging dir in the target without fully qualified domain name.
-targetStagingPath=/apps/hive/tools/dr
-targetNN=hdfs://localhost:8020
-
-# To ceil the max events processed each time job runs. Set it to max value depending on your bandwidth limit.
-# Setting it to -1 will process all the events but can hog up the bandwidth. Use it judiciously!
-maxEvents=-1
-# Change it to specify the maximum number of mappers for replication
-replicationMaxMaps=5
-# Change it to specify the maximum number of mappers for DistCP
-distcpMaxMaps=1
-# Change it to specify the bandwidth in MB for each mapper in DistCP
-distcpMapBandwidth=100
-
-##### Email Notification for Falcon instance completion
-falcon.recipe.notification.type=email
-falcon.recipe.notification.receivers=NA
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/falcon/blob/95bf312f/common/src/main/java/org/apache/falcon/catalog/AbstractCatalogService.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/falcon/catalog/AbstractCatalogService.java b/common/src/main/java/org/apache/falcon/catalog/AbstractCatalogService.java
index 41d50df..30a0118 100644
--- a/common/src/main/java/org/apache/falcon/catalog/AbstractCatalogService.java
+++ b/common/src/main/java/org/apache/falcon/catalog/AbstractCatalogService.java
@@ -40,6 +40,18 @@ public abstract class AbstractCatalogService {
     public abstract boolean isAlive(Configuration conf, String catalogUrl) throws FalconException;
 
     /**
+     * This method checks if the given db exists in the catalog.
+     *
+     * @param conf  conf
+     * @param catalogUrl url for the catalog service
+     * @param database database the table belongs to
+     * @return if the db exists
+     * @throws FalconException exception
+     */
+    public abstract boolean dbExists(Configuration conf, String catalogUrl,
+                                     String databaseName) throws FalconException;
+
+    /**
      * This method checks if the given table exists in the catalog.
      *
      * @param conf  conf

http://git-wip-us.apache.org/repos/asf/falcon/blob/95bf312f/common/src/main/java/org/apache/falcon/catalog/HiveCatalogService.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/falcon/catalog/HiveCatalogService.java b/common/src/main/java/org/apache/falcon/catalog/HiveCatalogService.java
index 872f91f..4ad62d7 100644
--- a/common/src/main/java/org/apache/falcon/catalog/HiveCatalogService.java
+++ b/common/src/main/java/org/apache/falcon/catalog/HiveCatalogService.java
@@ -208,6 +208,22 @@ public class HiveCatalogService extends AbstractCatalogService {
     }
 
     @Override
+    public boolean dbExists(Configuration conf, final String catalogUrl,
+                            final String databaseName) throws FalconException {
+        LOG.info("Checking if the db exists: {}", databaseName);
+
+        try {
+            HiveMetaStoreClient client = createProxiedClient(conf, catalogUrl);
+            Database db = client.getDatabase(databaseName);
+            return db != null;
+        } catch (NoSuchObjectException e) {
+            return false;
+        } catch (Exception e) {
+            throw new FalconException("Exception checking if the db exists:" + e.getMessage(), e);
+        }
+    }
+
+    @Override
     public boolean tableExists(Configuration conf, final String catalogUrl, final String database,
                                final String tableName) throws FalconException {
         LOG.info("Checking if the table exists: {}", tableName);

http://git-wip-us.apache.org/repos/asf/falcon/blob/95bf312f/common/src/main/resources/startup.properties
----------------------------------------------------------------------
diff --git a/common/src/main/resources/startup.properties b/common/src/main/resources/startup.properties
index 87a74bf..dd51963 100644
--- a/common/src/main/resources/startup.properties
+++ b/common/src/main/resources/startup.properties
@@ -100,6 +100,8 @@ debug.libext.feed.retention.paths=${falcon.libext}
 debug.libext.feed.replication.paths=${falcon.libext}
 debug.libext.process.paths=${falcon.libext}
 
+debug.extension.store.uri=file://${user.dir}/target/recipe/store
+
 #Configurations used in ITs
 it.config.store.uri=file://${user.dir}/target/store
 it.config.oozie.conf.uri=${user.dir}/target/oozie

http://git-wip-us.apache.org/repos/asf/falcon/blob/95bf312f/extensions/pom.xml
----------------------------------------------------------------------
diff --git a/extensions/pom.xml b/extensions/pom.xml
new file mode 100644
index 0000000..6a0725a
--- /dev/null
+++ b/extensions/pom.xml
@@ -0,0 +1,112 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<!--
+   Licensed to the Apache Software Foundation (ASF) under one or more
+   contributor license agreements.  See the NOTICE file distributed with
+   this work for additional information regarding copyright ownership.
+   The ASF licenses this file to You under the Apache License, Version 2.0
+   (the "License"); you may not use this file except in compliance with
+   the License.  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+  -->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+
+    <modelVersion>4.0.0</modelVersion>
+    <parent>
+        <groupId>org.apache.falcon</groupId>
+        <artifactId>falcon-main</artifactId>
+        <version>0.10-SNAPSHOT</version>
+    </parent>
+    <artifactId>falcon-extensions</artifactId>
+    <description>Apache Falcon server side extensions Module</description>
+    <name>Apache Falcon extensions</name>
+    <packaging>jar</packaging>
+
+    <profiles>
+        <profile>
+            <id>hadoop-2</id>
+            <activation>
+                <activeByDefault>true</activeByDefault>
+            </activation>
+            <dependencies>
+                <dependency>
+                    <groupId>org.apache.hadoop</groupId>
+                    <artifactId>hadoop-client</artifactId>
+                </dependency>
+            </dependencies>
+        </profile>
+    </profiles>
+
+    <dependencies>
+        <dependency>
+            <groupId>org.apache.falcon</groupId>
+            <artifactId>falcon-client</artifactId>
+        </dependency>
+
+        <dependency>
+            <groupId>org.apache.falcon</groupId>
+            <artifactId>falcon-common</artifactId>
+        </dependency>
+
+        <dependency>
+            <groupId>org.apache.commons</groupId>
+            <artifactId>commons-lang3</artifactId>
+        </dependency>
+
+        <dependency>
+            <groupId>org.testng</groupId>
+            <artifactId>testng</artifactId>
+        </dependency>
+
+        <dependency>
+            <groupId>org.slf4j</groupId>
+            <artifactId>slf4j-log4j12</artifactId>
+        </dependency>
+
+        <dependency>
+            <groupId>org.slf4j</groupId>
+            <artifactId>slf4j-api</artifactId>
+        </dependency>
+    </dependencies>
+
+    <build>
+        <plugins>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-compiler-plugin</artifactId>
+                <configuration>
+                    <source>1.7</source>
+                    <target>1.7</target>
+                </configuration>
+            </plugin>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-jar-plugin</artifactId>
+                <version>2.4</version>
+                <configuration>
+                    <excludes>
+                        <exclude>**/log4j.xml</exclude>
+                    </excludes>
+                </configuration>
+                <executions>
+                    <execution>
+                        <goals>
+                            <goal>test-jar</goal>
+                        </goals>
+                    </execution>
+                </executions>
+            </plugin>
+        </plugins>
+    </build>
+
+</project>

http://git-wip-us.apache.org/repos/asf/falcon/blob/95bf312f/extensions/src/main/java/org/apache/falcon/extensions/AbstractExtension.java
----------------------------------------------------------------------
diff --git a/extensions/src/main/java/org/apache/falcon/extensions/AbstractExtension.java b/extensions/src/main/java/org/apache/falcon/extensions/AbstractExtension.java
new file mode 100644
index 0000000..11b3725
--- /dev/null
+++ b/extensions/src/main/java/org/apache/falcon/extensions/AbstractExtension.java
@@ -0,0 +1,58 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.falcon.extensions;
+
+import org.apache.falcon.FalconException;
+import org.apache.falcon.extensions.mirroring.hdfs.HdfsMirroringExtension;
+import org.apache.falcon.extensions.mirroring.hive.HiveMirroringExtension;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Properties;
+
+/**
+ * Abstract extension class to be extended by all trusted and custom extensions.
+ */
+public abstract class AbstractExtension {
+    private static final List<String> TRUSTED_EXTENSIONS = Arrays.asList(
+            new HdfsMirroringExtension().getName().toUpperCase(),
+            new HiveMirroringExtension().getName().toUpperCase());
+    private static List<AbstractExtension> extensions = new ArrayList<>();
+
+    public static List<AbstractExtension> getExtensions() {
+        if (extensions.isEmpty()) {
+            extensions.add(new HdfsMirroringExtension());
+            extensions.add(new HiveMirroringExtension());
+        }
+        return extensions;
+    }
+
+    public static boolean isExtensionTrusted(final String extensionName) {
+        return TRUSTED_EXTENSIONS.contains(extensionName.toUpperCase());
+    }
+
+    /* Name cannot be null */
+    public abstract String getName();
+
+    public abstract void validate(final Properties extensionProperties) throws FalconException;
+
+    public abstract Properties getAdditionalProperties(final Properties extensionProperties) throws FalconException;
+}
+

http://git-wip-us.apache.org/repos/asf/falcon/blob/95bf312f/extensions/src/main/java/org/apache/falcon/extensions/Extension.java
----------------------------------------------------------------------
diff --git a/extensions/src/main/java/org/apache/falcon/extensions/Extension.java b/extensions/src/main/java/org/apache/falcon/extensions/Extension.java
new file mode 100644
index 0000000..4ab9f5d
--- /dev/null
+++ b/extensions/src/main/java/org/apache/falcon/extensions/Extension.java
@@ -0,0 +1,102 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.falcon.extensions;
+
+import org.apache.commons.lang3.StringUtils;
+import org.apache.falcon.FalconException;
+import org.apache.falcon.entity.v0.Entity;
+import org.apache.falcon.extensions.store.ExtensionStore;
+import org.apache.falcon.extensions.util.ExtensionProcessBuilderUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+
+/**
+ * Extension class to construct well formed entities from the templates for trusted extensions.
+ */
+public class Extension implements ExtensionBuilder {
+    private static final Logger LOG = LoggerFactory.getLogger(Extension.class);
+    private static final String EXTENSION_WF_SUFFIX = "-workflow.xml";
+    private static final String EXTENSION_TEMPLATE_SUFFIX = "-template.xml";
+
+    private static void validateProperties(final Properties extensionProperties) throws FalconException {
+        for (ExtensionProperties option : ExtensionProperties.values()) {
+            if (extensionProperties.getProperty(option.getName()) == null && option.isRequired()) {
+                throw new FalconException("Missing extension property: " + option.getName());
+            }
+        }
+    }
+
+    private static String getExtensionTemplate(final ExtensionStore store,
+                                            final Map<String, String> extensionResources,
+                                            final String extensionName) throws FalconException {
+        return store.getExtensionResource(extensionResources.get(extensionName.toLowerCase()
+                + EXTENSION_TEMPLATE_SUFFIX));
+    }
+
+    private static String getWFPath(final Map<String, String> extensionResources,
+                                    final String extensionName) {
+        return extensionResources.get(extensionName.toLowerCase() + EXTENSION_WF_SUFFIX);
+    }
+
+    @Override
+    public List<Entity> getEntities(final String extensionName, final Properties extensionProperties)
+        throws FalconException {
+        if (StringUtils.isBlank(extensionName)) {
+            throw new FalconException("Extension name cannot be null or empty");
+        }
+        validateProperties(extensionProperties);
+
+        String name = extensionName.toLowerCase();
+        AbstractExtension extension = ExtensionFactory.getExtensionType(name);
+        if (extension != null) {
+            extension.validate(extensionProperties);
+            Properties props = extension.getAdditionalProperties(extensionProperties);
+            if (props != null && !props.isEmpty()) {
+                extensionProperties.putAll(props);
+            }
+        }
+
+        ExtensionStore store = ExtensionService.getExtensionStore();
+
+        String resourceName = extensionProperties.getProperty(ExtensionProperties.RESOURCE_NAME.getName());
+        if (StringUtils.isBlank(resourceName)) {
+            resourceName = name;
+        }
+
+        Map<String, String> extensionResources = store.getExtensionResources(name);
+        /* Get the resources */
+        String extensionTemplate = getExtensionTemplate(store, extensionResources, resourceName);
+        String wfPath = getWFPath(extensionResources, resourceName);
+
+        /* Get Lib path */
+        String wfLibPath = store.getExtensionLibPath(name);
+        Entity entity = ExtensionProcessBuilderUtils.createProcessFromTemplate(extensionTemplate,
+                name, extensionProperties, wfPath, wfLibPath);
+        if (entity == null) {
+            throw new FalconException("Entity created from the extension template cannot be null");
+        }
+        LOG.info("Extension processing complete");
+        return Arrays.asList(entity);
+    }
+}

http://git-wip-us.apache.org/repos/asf/falcon/blob/95bf312f/extensions/src/main/java/org/apache/falcon/extensions/ExtensionBuilder.java
----------------------------------------------------------------------
diff --git a/extensions/src/main/java/org/apache/falcon/extensions/ExtensionBuilder.java b/extensions/src/main/java/org/apache/falcon/extensions/ExtensionBuilder.java
new file mode 100644
index 0000000..bd52ed4
--- /dev/null
+++ b/extensions/src/main/java/org/apache/falcon/extensions/ExtensionBuilder.java
@@ -0,0 +1,32 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.falcon.extensions;
+
+import org.apache.falcon.FalconException;
+import org.apache.falcon.entity.v0.Entity;
+
+import java.util.Properties;
+import java.util.List;
+
+/**
+ * Extension interface to be implemented by all trusted and custom extensions.
+ */
+public interface ExtensionBuilder {
+    List<Entity> getEntities(final String extensionName, final Properties extensionProperties) throws FalconException;
+}

http://git-wip-us.apache.org/repos/asf/falcon/blob/95bf312f/extensions/src/main/java/org/apache/falcon/extensions/ExtensionFactory.java
----------------------------------------------------------------------
diff --git a/extensions/src/main/java/org/apache/falcon/extensions/ExtensionFactory.java b/extensions/src/main/java/org/apache/falcon/extensions/ExtensionFactory.java
new file mode 100644
index 0000000..41e0648
--- /dev/null
+++ b/extensions/src/main/java/org/apache/falcon/extensions/ExtensionFactory.java
@@ -0,0 +1,48 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.falcon.extensions;
+
+import org.apache.commons.lang3.StringUtils;
+import org.apache.falcon.FalconException;
+
+import java.util.List;
+
+/**
+ * Extension Factory class to get the corresponding extension.
+ */
+public final class ExtensionFactory {
+
+    private ExtensionFactory() {
+    }
+
+    public static AbstractExtension getExtensionType(final String extensionName) throws FalconException {
+        if (StringUtils.isBlank(extensionName)) {
+            return null;
+        }
+
+        List<AbstractExtension> extensions = AbstractExtension.getExtensions();
+
+        for (AbstractExtension extension : extensions) {
+            if (extensionName.equalsIgnoreCase(extension.getName())) {
+                return extension;
+            }
+        }
+        return null;
+    }
+}

http://git-wip-us.apache.org/repos/asf/falcon/blob/95bf312f/extensions/src/main/java/org/apache/falcon/extensions/ExtensionProperties.java
----------------------------------------------------------------------
diff --git a/extensions/src/main/java/org/apache/falcon/extensions/ExtensionProperties.java b/extensions/src/main/java/org/apache/falcon/extensions/ExtensionProperties.java
new file mode 100644
index 0000000..b543d73
--- /dev/null
+++ b/extensions/src/main/java/org/apache/falcon/extensions/ExtensionProperties.java
@@ -0,0 +1,89 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.falcon.extensions;
+
+import java.util.Map;
+import java.util.HashMap;
+
+/**
+ * Extension options.
+ */
+public enum ExtensionProperties {
+    JOB_NAME("jobName", "job name"),
+    RESOURCE_NAME("resourceName", "resource name", false),
+    //Can't use default as target as extension is generic and need not be used only for replication
+    CLUSTER_NAME("jobClusterName", "Cluster name where job should run"),
+    VALIDITY_START("jobValidityStart", "Job validity start"),
+    VALIDITY_END("jobValidityEnd", "Job validity end"),
+    FREQUENCY("jobFrequency", "Job frequency"),
+    TIMEZONE("jobTimezone", "Time zone", false),
+    // Use defaults for retry
+    RETRY_POLICY("jobRetryPolicy", "Retry policy", false),
+    RETRY_DELAY("jobRetryDelay", "Retry delay", false),
+    RETRY_ATTEMPTS("jobRetryAttempts", "Retry attempts", false),
+    RETRY_ON_TIMEOUT("jobRetryOnTimeout", "Retry onTimeout", false),
+    JOB_TAGS("jobTags", "Job tags", false),
+    JOB_ACL_OWNER("jobAclOwner", "Job acl owner", false),
+    JOB_ACL_GROUP("jobAclGroup", "Job acl group", false),
+    JOB_ACL_PERMISSION("jobAclPermission", "Job acl permission", false),
+    JOB_NOTIFICATION_TYPE("jobNotificationType", "Notification Type", false),
+    JOB_NOTIFICATION_ADDRESS("jobNotificationReceivers", "Email Notification receivers", false);
+
+    private final String name;
+    private final String description;
+    private final boolean isRequired;
+
+    private static Map<String, ExtensionProperties> optionsMap = new HashMap<>();
+    static {
+        for (ExtensionProperties c : ExtensionProperties.values()) {
+            optionsMap.put(c.getName(), c);
+        }
+    }
+
+    public static Map<String, ExtensionProperties> getOptionsMap() {
+        return optionsMap;
+    }
+
+    ExtensionProperties(String name, String description) {
+        this(name, description, true);
+    }
+
+    ExtensionProperties(String name, String description, boolean isRequired) {
+        this.name = name;
+        this.description = description;
+        this.isRequired = isRequired;
+    }
+
+    public String getName() {
+        return this.name;
+    }
+
+    public String getDescription() {
+        return description;
+    }
+
+    public boolean isRequired() {
+        return isRequired;
+    }
+
+    @Override
+    public String toString() {
+        return getName();
+    }
+}

http://git-wip-us.apache.org/repos/asf/falcon/blob/95bf312f/extensions/src/main/java/org/apache/falcon/extensions/ExtensionService.java
----------------------------------------------------------------------
diff --git a/extensions/src/main/java/org/apache/falcon/extensions/ExtensionService.java b/extensions/src/main/java/org/apache/falcon/extensions/ExtensionService.java
new file mode 100644
index 0000000..ffed6ca
--- /dev/null
+++ b/extensions/src/main/java/org/apache/falcon/extensions/ExtensionService.java
@@ -0,0 +1,49 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.falcon.extensions;
+
+import org.apache.falcon.FalconException;
+import org.apache.falcon.extensions.store.ExtensionStore;
+import org.apache.falcon.service.FalconService;
+
+/**
+ * Extension service.
+ */
+public class ExtensionService implements FalconService {
+    private static ExtensionStore extensionStore = ExtensionStore.get();
+
+    public static final String SERVICE_NAME = ExtensionService.class.getSimpleName();
+
+    @Override
+    public String getName() {
+        return SERVICE_NAME;
+    }
+
+    @Override
+    public void init() throws FalconException {
+    }
+
+    @Override
+    public void destroy() throws FalconException {
+    }
+
+    public static ExtensionStore getExtensionStore() {
+        return extensionStore;
+    }
+}

http://git-wip-us.apache.org/repos/asf/falcon/blob/95bf312f/extensions/src/main/java/org/apache/falcon/extensions/mirroring/hdfs/HdfsMirroringExtension.java
----------------------------------------------------------------------
diff --git a/extensions/src/main/java/org/apache/falcon/extensions/mirroring/hdfs/HdfsMirroringExtension.java b/extensions/src/main/java/org/apache/falcon/extensions/mirroring/hdfs/HdfsMirroringExtension.java
new file mode 100644
index 0000000..f1acae2
--- /dev/null
+++ b/extensions/src/main/java/org/apache/falcon/extensions/mirroring/hdfs/HdfsMirroringExtension.java
@@ -0,0 +1,111 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.falcon.extensions.mirroring.hdfs;
+
+import org.apache.commons.lang3.StringUtils;
+import org.apache.falcon.FalconException;
+import org.apache.falcon.entity.ClusterHelper;
+import org.apache.falcon.entity.v0.cluster.Cluster;
+import org.apache.falcon.extensions.AbstractExtension;
+
+import java.util.Properties;
+
+/**
+ * Hdfs mirroring extension.
+ */
+public class HdfsMirroringExtension extends AbstractExtension {
+
+    private static final String COMMA_SEPARATOR = ",";
+    private static final String EXTENSION_NAME = "HDFS-MIRRORING";
+
+    @Override
+    public String getName() {
+        return EXTENSION_NAME;
+    }
+
+    @Override
+    public void validate(final Properties extensionProperties) throws FalconException {
+        for (HdfsMirroringExtensionProperties option : HdfsMirroringExtensionProperties.values()) {
+            if (extensionProperties.getProperty(option.getName()) == null && option.isRequired()) {
+                throw new FalconException("Missing extension property: " + option.getName());
+            }
+        }
+    }
+
+    @Override
+    public Properties getAdditionalProperties(final Properties extensionProperties) throws FalconException {
+        Properties additionalProperties = new Properties();
+
+        // Add default properties if not passed
+        String distcpMaxMaps = extensionProperties.getProperty(HdfsMirroringExtensionProperties.MAX_MAPS.getName());
+        if (StringUtils.isBlank(distcpMaxMaps)) {
+            additionalProperties.put(HdfsMirroringExtensionProperties.MAX_MAPS.getName(), "1");
+        }
+
+        String distcpMapBandwidth = extensionProperties.getProperty(
+                HdfsMirroringExtensionProperties.MAP_BANDWIDTH_IN_MB.getName());
+        if (StringUtils.isBlank(distcpMapBandwidth)) {
+            additionalProperties.put(HdfsMirroringExtensionProperties.MAP_BANDWIDTH_IN_MB.getName(), "100");
+        }
+
+        // Construct fully qualified hdfs src path
+        String srcPaths = extensionProperties.getProperty(HdfsMirroringExtensionProperties
+                .SOURCE_DIR.getName());
+        StringBuilder absoluteSrcPaths = new StringBuilder();
+        String sourceClusterName = extensionProperties.getProperty(
+                HdfsMirroringExtensionProperties.SOURCE_CLUSTER.getName());
+
+        // Since source cluster get read interface
+        Cluster srcCluster = ClusterHelper.getCluster(sourceClusterName);
+        if (srcCluster == null) {
+            throw new FalconException("Cluster entity " + sourceClusterName + " not found");
+        }
+        String srcClusterEndPoint = ClusterHelper.getReadOnlyStorageUrl(srcCluster);
+
+        if (StringUtils.isNotBlank(srcPaths)) {
+            String[] paths = srcPaths.split(COMMA_SEPARATOR);
+
+            for (String path : paths) {
+                StringBuilder srcpath = new StringBuilder(srcClusterEndPoint);
+                srcpath.append(path.trim());
+                srcpath.append(COMMA_SEPARATOR);
+                absoluteSrcPaths.append(srcpath);
+            }
+        }
+
+        additionalProperties.put(HdfsMirroringExtensionProperties.SOURCE_DIR.getName(),
+                StringUtils.removeEnd(absoluteSrcPaths.toString(), COMMA_SEPARATOR));
+
+        // add sourceClusterFS and targetClusterFS
+        additionalProperties.put(HdfsMirroringExtensionProperties.SOURCE_CLUSTER_FS_WRITE_ENDPOINT.getName(),
+                ClusterHelper.getStorageUrl(srcCluster));
+
+        String targetClusterName = extensionProperties.getProperty(
+                HdfsMirroringExtensionProperties.TARGET_CLUSTER.getName());
+
+        Cluster targetCluster = ClusterHelper.getCluster(targetClusterName);
+        if (targetCluster == null) {
+            throw new FalconException("Cluster entity " + targetClusterName + " not found");
+        }
+        additionalProperties.put(HdfsMirroringExtensionProperties.TARGET_CLUSTER_FS_WRITE_ENDPOINT.getName(),
+                ClusterHelper.getStorageUrl(targetCluster));
+        return additionalProperties;
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/falcon/blob/95bf312f/extensions/src/main/java/org/apache/falcon/extensions/mirroring/hdfs/HdfsMirroringExtensionProperties.java
----------------------------------------------------------------------
diff --git a/extensions/src/main/java/org/apache/falcon/extensions/mirroring/hdfs/HdfsMirroringExtensionProperties.java b/extensions/src/main/java/org/apache/falcon/extensions/mirroring/hdfs/HdfsMirroringExtensionProperties.java
new file mode 100644
index 0000000..7d24b45
--- /dev/null
+++ b/extensions/src/main/java/org/apache/falcon/extensions/mirroring/hdfs/HdfsMirroringExtensionProperties.java
@@ -0,0 +1,65 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.falcon.extensions.mirroring.hdfs;
+
+/**
+ * Hdfs Extension properties.
+ */
+public enum HdfsMirroringExtensionProperties {
+    SOURCE_DIR("sourceDir", "Location of source data to replicate"),
+    SOURCE_CLUSTER("sourceCluster", "Source cluster"),
+    SOURCE_CLUSTER_FS_WRITE_ENDPOINT("sourceClusterFS", "Source cluster end point", false),
+    TARGET_DIR("targetDir", "Location on target cluster for replication"),
+    TARGET_CLUSTER("targetCluster", "Target cluster"),
+    TARGET_CLUSTER_FS_WRITE_ENDPOINT("targetClusterFS", "Target cluster end point", false),
+    MAX_MAPS("distcpMaxMaps", "Maximum number of maps used during replication", false),
+    MAP_BANDWIDTH_IN_MB("distcpMapBandwidth", "Bandwidth in MB/s used by each mapper during replication",
+            false);
+
+    private final String name;
+    private final String description;
+    private final boolean isRequired;
+
+    HdfsMirroringExtensionProperties(String name, String description) {
+        this(name, description, true);
+    }
+
+    HdfsMirroringExtensionProperties(String name, String description, boolean isRequired) {
+        this.name = name;
+        this.description = description;
+        this.isRequired = isRequired;
+    }
+
+    public String getName() {
+        return this.name;
+    }
+
+    public String getDescription() {
+        return description;
+    }
+
+    public boolean isRequired() {
+        return isRequired;
+    }
+
+    @Override
+    public String toString() {
+        return getName();
+    }
+}


Mime
View raw message