falcon-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From venkat...@apache.org
Subject [2/5] git commit: FALCON-636 Add a sample recipe for disaster recovery of hdfs dirs/files. Contributed by Sowmya Ramesh
Date Fri, 12 Sep 2014 20:29:30 GMT
FALCON-636 Add a sample recipe for disaster recovery of hdfs dirs/files. Contributed by Sowmya
Ramesh


Project: http://git-wip-us.apache.org/repos/asf/incubator-falcon/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-falcon/commit/42a1b767
Tree: http://git-wip-us.apache.org/repos/asf/incubator-falcon/tree/42a1b767
Diff: http://git-wip-us.apache.org/repos/asf/incubator-falcon/diff/42a1b767

Branch: refs/heads/master
Commit: 42a1b76778274eb1cc289b7e4225db63944a7572
Parents: c54c1ef
Author: Venkatesh Seetharam <venkatesh@apache.org>
Authored: Fri Sep 12 13:26:08 2014 -0700
Committer: Venkatesh Seetharam <venkatesh@apache.org>
Committed: Fri Sep 12 13:26:08 2014 -0700

----------------------------------------------------------------------
 CHANGES.txt                                     |  3 +
 addons/recipes/hdfs-replication/pom.xml         | 32 +++++++++
 .../resources/hdfs-replication-template.xml     | 45 +++++++++++++
 .../resources/hdfs-replication-workflow.xml     | 69 ++++++++++++++++++++
 .../main/resources/hdfs-replication.properties  | 63 ++++++++++++++++++
 5 files changed, 212 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/42a1b767/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index b85ca8b..5764c47 100755
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -5,6 +5,9 @@ Trunk (Unreleased)
   INCOMPATIBLE CHANGES
 
   NEW FEATURES
+   FALCON-636 Add a sample recipe for disaster recovery of hdfs dirs/files
+   (Sowmya Ramesh via Venkatesh Seetharam)
+
    FALCON-635 Add recipe option in falcon client (Sowmya Ramesh via
    Venkatesh Seetharam)
 

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/42a1b767/addons/recipes/hdfs-replication/pom.xml
----------------------------------------------------------------------
diff --git a/addons/recipes/hdfs-replication/pom.xml b/addons/recipes/hdfs-replication/pom.xml
new file mode 100644
index 0000000..24ce24a
--- /dev/null
+++ b/addons/recipes/hdfs-replication/pom.xml
@@ -0,0 +1,32 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<!--
+  Licensed to the Apache Software Foundation (ASF) under one
+  or more contributor license agreements.  See the NOTICE file
+  distributed with this work for additional information
+  regarding copyright ownership.  The ASF licenses this file
+  to you under the Apache License, Version 2.0 (the
+  "License"); you may not use this file except in compliance
+  with the License.  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+  -->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+
+    <modelVersion>4.0.0</modelVersion>
+    <groupId>org.apache.falcon.recipes</groupId>
+    <artifactId>falcon-hdfs-replication-recipe</artifactId>
+    <version>0.6-incubating-SNAPSHOT</version>
+    <description>Apache Falcon Sample Hdfs Replicaiton Recipe</description>
+    <name>Apache Falcon Sample Hdfs Replication Recipe</name>
+    <packaging>jar</packaging>
+</project>

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/42a1b767/addons/recipes/hdfs-replication/src/main/resources/hdfs-replication-template.xml
----------------------------------------------------------------------
diff --git a/addons/recipes/hdfs-replication/src/main/resources/hdfs-replication-template.xml
b/addons/recipes/hdfs-replication/src/main/resources/hdfs-replication-template.xml
new file mode 100644
index 0000000..67c82db
--- /dev/null
+++ b/addons/recipes/hdfs-replication/src/main/resources/hdfs-replication-template.xml
@@ -0,0 +1,45 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  Licensed to the Apache Software Foundation (ASF) under one
+  or more contributor license agreements.  See the NOTICE file
+  distributed with this work for additional information
+  regarding copyright ownership.  The ASF licenses this file
+  to you under the Apache License, Version 2.0 (the
+  "License"); you may not use this file except in compliance
+  with the License.  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+  -->
+
+<process name="##name##" xmlns="uri:falcon:process:0.1">
+    <clusters>
+        <!--  source  -->
+        <cluster name="##src.cluster.name##">
+            <validity end="##src.cluster.validity.end##" start="##src.cluster.validity.start##"/>
+        </cluster>
+    </clusters>
+
+    <parallel>1</parallel>
+    <order>LAST_ONLY</order>
+    <frequency>##process.frequency##</frequency>
+    <timezone>UTC</timezone>
+
+    <properties>
+        <property name="oozie.wf.subworkflow.classpath.inheritance" value="true"/>
+        <property name="##process.property2.name##" value="##process.property2.value##"/>
+        <property name="##process.property3.name##" value="##process.property3.value##"/>
+        <property name="##process.property4.name##" value="##process.property4.value##"/>
+        <property name="##process.property5.name##" value="##process.property5.value##"/>
+        <property name="##process.property6.name##" value="##process.property6.value##"/>
+        <property name="##process.property7.name##" value="##process.property7.value##"/>
+    </properties>
+
+    <workflow name="##workflow.name##" engine="oozie" path="##workflow.path##"/>
+    <retry policy="periodic" delay="minutes(10)" attempts="3"/>
+</process>

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/42a1b767/addons/recipes/hdfs-replication/src/main/resources/hdfs-replication-workflow.xml
----------------------------------------------------------------------
diff --git a/addons/recipes/hdfs-replication/src/main/resources/hdfs-replication-workflow.xml
b/addons/recipes/hdfs-replication/src/main/resources/hdfs-replication-workflow.xml
new file mode 100644
index 0000000..7043e42
--- /dev/null
+++ b/addons/recipes/hdfs-replication/src/main/resources/hdfs-replication-workflow.xml
@@ -0,0 +1,69 @@
+<!--
+  Licensed to the Apache Software Foundation (ASF) under one
+  or more contributor license agreements.  See the NOTICE file
+  distributed with this work for additional information
+  regarding copyright ownership.  The ASF licenses this file
+  to you under the Apache License, Version 2.0 (the
+  "License"); you may not use this file except in compliance
+  with the License.  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+  -->
+<workflow-app xmlns='uri:oozie:workflow:0.3' name='falcon-dr-fs-workflow'>
+    <start to='dr-replication'/>
+    <!-- Replication action -->
+    <action name="dr-replication">
+        <java>
+            <job-tracker>${jobTracker}</job-tracker>
+            <name-node>${nameNode}</name-node>
+            <configuration>
+                <property> <!-- hadoop 2 parameter -->
+                    <name>oozie.launcher.mapreduce.job.user.classpath.first</name>
+                    <value>true</value>
+                </property>
+                <property> <!-- hadoop 1 parameter -->
+                    <name>oozie.launcher.mapreduce.user.classpath.first</name>
+                    <value>true</value>
+                </property>
+                <property>
+                    <name>mapred.job.queue.name</name>
+                    <value>${queueName}</value>
+                </property>
+                <property>
+                    <name>oozie.launcher.mapred.job.priority</name>
+                    <value>${jobPriority}</value>
+                </property>
+                <property>
+                    <name>oozie.use.system.libpath</name>
+                    <value>true</value>
+                </property>
+            </configuration>
+            <main-class>org.apache.falcon.replication.FeedReplicator</main-class>
+            <arg>-Dmapred.job.queue.name=${queueName}</arg>
+            <arg>-Dmapred.job.priority=${jobPriority}</arg>
+            <arg>-maxMaps</arg>
+            <arg>${maxMaps}</arg>
+            <arg>-sourcePaths</arg>
+            <arg>${nameNode}${drSourceDir}</arg>
+            <arg>-targetPath</arg>
+            <arg>${drTargetClusterFS}${drTargetDir}</arg>
+            <arg>-falconFeedStorageType</arg>
+            <arg>FILESYSTEM</arg>
+            <file>${wf:conf("falcon.libpath")}/hadoop-distcp.jar</file>
+        </java>
+        <ok to="end"/>
+        <error to="fail"/>
+    </action>
+    <kill name="fail">
+        <message>
+            Workflow failed, error message[${wf:errorMessage(wf:lastErrorNode())}]
+        </message>
+    </kill>
+    <end name='end'/>
+</workflow-app>

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/42a1b767/addons/recipes/hdfs-replication/src/main/resources/hdfs-replication.properties
----------------------------------------------------------------------
diff --git a/addons/recipes/hdfs-replication/src/main/resources/hdfs-replication.properties
b/addons/recipes/hdfs-replication/src/main/resources/hdfs-replication.properties
new file mode 100644
index 0000000..34d1843
--- /dev/null
+++ b/addons/recipes/hdfs-replication/src/main/resources/hdfs-replication.properties
@@ -0,0 +1,63 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+##### Recipe properties
+falcon.recipe.name=hdfs-replication
+
+
+##### Workflow properties
+
+falcon.recipe.workflow.name=hdfs-dr-workflow
+# If artifacts are present on local FS, provide paths here
+falcon.recipe.workflow.path=/recipes/hdfs-replication/hdfs-replication-workflow.xml
+falcon.recipe.workflow.lib.path=/recipes/hdfs-replication/lib
+
+
+##### Cluster properties
+
+# Change the src cluster name here
+falcon.recipe.src.cluster.name=test
+# Change the src cluster hdfs write end point here. This is mandatory.
+falcon.recipe.src.cluster.hdfs.writeEndPoint=hdfs://sandbox.hortonworks.com:8020
+# Change the src cluster validity start time here
+falcon.recipe.src.cluster.validity.start=2012-04-20T00:00Z
+# Change the src cluster validity end time here
+falcon.recipe.src.cluster.validity.end=2014-04-20T00:00Z
+
+
+##### Scheduling properties
+
+# Change the process here. Valid frequency type are minutes, hours, days, months
+falcon.recipe.process.frequency=minutes(60)
+
+
+##### Custom Job properties
+
+# Specify property names and values for properties defined in recipe template
+falcon.recipe.process.property2.name=drSourceDir
+falcon.recipe.process.property2.value=/falcon/test/srcCluster/input
+falcon.recipe.process.property3.name=drTargetClusterFS
+falcon.recipe.process.property3.value=hdfs://sandbox.hortonworks.com:8020
+falcon.recipe.process.property4.name=drTargetDir
+falcon.recipe.process.property4.value=/falcon/test/targetCluster/input
+falcon.recipe.process.property5.name=drTargetCluster
+falcon.recipe.process.property5.value=backupCluster
+falcon.recipe.process.property6.name=maxMaps
+falcon.recipe.process.property6.value=5
+falcon.recipe.process.property7.name=mapBandwidth
+falcon.recipe.process.property7.value=102400


Mime
View raw message