falcon-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From sowmya...@apache.org
Subject falcon git commit: FALCON-1437 Change DR recipes notification with Falcon notification. Contributed by Peeyush Bishnoi.
Date Tue, 22 Sep 2015 18:46:40 GMT
Repository: falcon
Updated Branches:
  refs/heads/master 48d0723cc -> 8600e9465


FALCON-1437 Change DR recipes notification with Falcon notification. Contributed by Peeyush
Bishnoi.


Project: http://git-wip-us.apache.org/repos/asf/falcon/repo
Commit: http://git-wip-us.apache.org/repos/asf/falcon/commit/8600e946
Tree: http://git-wip-us.apache.org/repos/asf/falcon/tree/8600e946
Diff: http://git-wip-us.apache.org/repos/asf/falcon/diff/8600e946

Branch: refs/heads/master
Commit: 8600e94654ea21b5cff1ea363560ffc17078aa53
Parents: 48d0723
Author: Sowmya Ramesh <sramesh@hortonworks.com>
Authored: Tue Sep 22 11:46:37 2015 -0700
Committer: Sowmya Ramesh <sramesh@hortonworks.com>
Committed: Tue Sep 22 11:46:37 2015 -0700

----------------------------------------------------------------------
 CHANGES.txt                                     |  2 +
 .../resources/hdfs-replication-template.xml     |  1 +
 .../resources/hdfs-replication-workflow.xml     | 42 ----------------
 .../main/resources/hdfs-replication.properties  |  5 +-
 .../hive-disaster-recovery-secure-template.xml  |  1 +
 .../hive-disaster-recovery-secure-workflow.xml  | 50 +-------------------
 .../hive-disaster-recovery-secure.properties    |  5 +-
 .../hive-disaster-recovery-template.xml         |  1 +
 .../hive-disaster-recovery-workflow.xml         | 50 +-------------------
 .../resources/hive-disaster-recovery.properties |  5 +-
 .../apache/falcon/recipe/RecipeToolOptions.java |  4 +-
 .../recipe/util/RecipeProcessBuilderUtils.java  | 16 +++++++
 .../resources/hdfs-replication-template.xml     |  1 +
 .../test/resources/hdfs-replication.properties  |  5 +-
 14 files changed, 41 insertions(+), 147 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/falcon/blob/8600e946/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 79d7548..5a7c46e 100755
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -6,6 +6,8 @@ Trunk (Unreleased)
     FALCON-1401 MetadataMappingService fails to add an edge for a process instance(Pallavi
Rao) 
 
   NEW FEATURES
+    FALCON-1437 Change DR recipes notification with Falcon notification(Peeyush Bishnoi via
Sowmya Ramesh)
+
     FALCON-1027 Falcon proxy user support(Sowmya Ramesh)
 
   IMPROVEMENTS

http://git-wip-us.apache.org/repos/asf/falcon/blob/8600e946/addons/recipes/hdfs-replication/src/main/resources/hdfs-replication-template.xml
----------------------------------------------------------------------
diff --git a/addons/recipes/hdfs-replication/src/main/resources/hdfs-replication-template.xml
b/addons/recipes/hdfs-replication/src/main/resources/hdfs-replication-template.xml
index de0f748..441a189 100644
--- a/addons/recipes/hdfs-replication/src/main/resources/hdfs-replication-template.xml
+++ b/addons/recipes/hdfs-replication/src/main/resources/hdfs-replication-template.xml
@@ -39,5 +39,6 @@
 
     <workflow name="##falcon.recipe.workflow.name##" engine="oozie" path="/apps/data-mirroring/workflows/hdfs-replication-workflow.xml"
lib="##workflow.lib.path##"/>
     <retry policy="##falcon.recipe.retry.policy##" delay="##falcon.recipe.retry.delay##"
attempts="3"/>
+    <notification type="##falcon.recipe.notification.type##" to="##falcon.recipe.notification.receivers##"/>
     <ACL/>
 </process>
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/falcon/blob/8600e946/addons/recipes/hdfs-replication/src/main/resources/hdfs-replication-workflow.xml
----------------------------------------------------------------------
diff --git a/addons/recipes/hdfs-replication/src/main/resources/hdfs-replication-workflow.xml
b/addons/recipes/hdfs-replication/src/main/resources/hdfs-replication-workflow.xml
index e35773f..942421f 100644
--- a/addons/recipes/hdfs-replication/src/main/resources/hdfs-replication-workflow.xml
+++ b/addons/recipes/hdfs-replication/src/main/resources/hdfs-replication-workflow.xml
@@ -68,48 +68,6 @@
             <arg>-availabilityFlag</arg>
             <arg>${availabilityFlag == 'NA' ? "NA" : availabilityFlag}</arg>
         </java>
-        <ok to="success"/>
-        <error to="failure"/>
-    </action>
-    <decision name="success">
-        <switch>
-            <case to="successAlert">
-                ${drNotificationReceivers ne 'NA'}
-            </case>
-            <default to="end"/>
-        </switch>
-    </decision>
-    <decision name="failure">
-        <switch>
-            <case to="failureAlert">
-                ${drNotificationReceivers ne 'NA'}
-            </case>
-            <default to="fail"/>
-        </switch>
-    </decision>
-    <action name="successAlert">
-        <email xmlns="uri:oozie:email-action:0.2">
-            <to>${drNotificationReceivers}</to>
-            <subject>INFO: HDFS DR workflow ${entityName} completed successfully</subject>
-            <body>
-                The HDFS DR workflow ${wf:id()} is successful.
-                Source      =   ${drSourceDir}
-                Target      =   ${drTargetClusterFS}${drTargetDir}
-            </body>
-        </email>
-        <ok to="end"/>
-        <error to="end"/>
-    </action>
-    <action name="failureAlert">
-        <email xmlns="uri:oozie:email-action:0.2">
-            <to>${drNotificationReceivers}</to>
-            <subject>ERROR: HDFS DR workflow ${entityName} failed</subject>
-            <body>
-                The workflow ${wf:id()} had issues and was killed.  The error message is:
${wf:errorMessage(wf:lastErrorNode())}
-                Source      =   ${drSourceDir}
-                Target      =   ${drTargetClusterFS}${drTargetDir}
-            </body>
-        </email>
         <ok to="end"/>
         <error to="fail"/>
     </action>

http://git-wip-us.apache.org/repos/asf/falcon/blob/8600e946/addons/recipes/hdfs-replication/src/main/resources/hdfs-replication.properties
----------------------------------------------------------------------
diff --git a/addons/recipes/hdfs-replication/src/main/resources/hdfs-replication.properties
b/addons/recipes/hdfs-replication/src/main/resources/hdfs-replication.properties
index 64ab6b8..7ef4095 100644
--- a/addons/recipes/hdfs-replication/src/main/resources/hdfs-replication.properties
+++ b/addons/recipes/hdfs-replication/src/main/resources/hdfs-replication.properties
@@ -73,5 +73,6 @@ distcpMaxMaps=1
 # Change it to specify the bandwidth in MB for each mapper in DistCP
 distcpMapBandwidth=100
 
-##### Email on failure
-drNotificationReceivers=NA
+##### Email Notification for Falcon instance completion
+falcon.recipe.notification.type=email
+falcon.recipe.notification.receivers=NA
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/falcon/blob/8600e946/addons/recipes/hive-disaster-recovery/src/main/resources/hive-disaster-recovery-secure-template.xml
----------------------------------------------------------------------
diff --git a/addons/recipes/hive-disaster-recovery/src/main/resources/hive-disaster-recovery-secure-template.xml
b/addons/recipes/hive-disaster-recovery/src/main/resources/hive-disaster-recovery-secure-template.xml
index 3afbef0..f0de091 100644
--- a/addons/recipes/hive-disaster-recovery/src/main/resources/hive-disaster-recovery-secure-template.xml
+++ b/addons/recipes/hive-disaster-recovery/src/main/resources/hive-disaster-recovery-secure-template.xml
@@ -40,5 +40,6 @@
     <workflow name="##workflow.name##" engine="oozie"
               path="/apps/data-mirroring/workflows/hive-disaster-recovery-workflow.xml" lib="##workflow.lib.path##"/>
     <retry policy="##retry.policy##" delay="##retry.delay##" attempts="3"/>
+    <notification type="##notification.type##" to="##notification.receivers##"/>
     <ACL/>
 </process>

http://git-wip-us.apache.org/repos/asf/falcon/blob/8600e946/addons/recipes/hive-disaster-recovery/src/main/resources/hive-disaster-recovery-secure-workflow.xml
----------------------------------------------------------------------
diff --git a/addons/recipes/hive-disaster-recovery/src/main/resources/hive-disaster-recovery-secure-workflow.xml
b/addons/recipes/hive-disaster-recovery/src/main/resources/hive-disaster-recovery-secure-workflow.xml
index 7362c2e..74902b4 100644
--- a/addons/recipes/hive-disaster-recovery/src/main/resources/hive-disaster-recovery-secure-workflow.xml
+++ b/addons/recipes/hive-disaster-recovery/src/main/resources/hive-disaster-recovery-secure-workflow.xml
@@ -148,7 +148,7 @@
             <arg>lastevents</arg>
         </java>
         <ok to="export-dr-replication"/>
-        <error to="failure"/>
+        <error to="fail"/>
     </action>
     <!-- Export Replication action -->
     <action name="export-dr-replication" cred="hive_src_credentials,hive2_src_credentials">
@@ -246,7 +246,7 @@
             <arg>export</arg>
         </java>
         <ok to="import-dr-replication"/>
-        <error to="failure"/>
+        <error to="fail"/>
     </action>
     <!-- Import Replication action -->
     <action name="import-dr-replication" cred="hive_tgt_credentials,hive2_tgt_credentials">
@@ -343,52 +343,6 @@
             <arg>-executionStage</arg>
             <arg>import</arg>
         </java>
-        <ok to="success"/>
-        <error to="failure"/>
-    </action>
-    <decision name="success">
-        <switch>
-            <case to="successAlert">
-                ${drNotificationReceivers ne 'NA'}
-            </case>
-            <default to="end"/>
-        </switch>
-    </decision>
-    <decision name="failure">
-        <switch>
-            <case to="failureAlert">
-                ${drNotificationReceivers ne 'NA'}
-            </case>
-            <default to="fail"/>
-        </switch>
-    </decision>
-    <action name="successAlert">
-        <email xmlns="uri:oozie:email-action:0.2">
-            <to>${drNotificationReceivers}</to>
-            <subject>INFO: Hive DR workflow ${drJobName} completed successfully</subject>
-            <body>
-                The Hive DR workflow ${wf:id()} is successful.
-                Source          = ${sourceCluster}
-                Target          = ${targetCluster}
-                DB Name         = ${sourceDatabase}
-                Table Name      = ${sourceTable}
-            </body>
-        </email>
-        <ok to="end"/>
-        <error to="end"/>
-    </action>
-    <action name="failureAlert">
-        <email xmlns="uri:oozie:email-action:0.2">
-            <to>${drNotificationReceivers}</to>
-            <subject>ERROR: Hive DR workflow ${drJobName} failed</subject>
-            <body>
-                The Hive DR workflow ${wf:id()} had issues and was killed.  The error message
is: ${wf:errorMessage(wf:lastErrorNode())}
-                Source          = ${sourceCluster}
-                Target          = ${targetCluster}
-                DB Name         = ${sourceDatabase}
-                Table Name      = ${sourceTable}
-            </body>
-        </email>
         <ok to="end"/>
         <error to="fail"/>
     </action>

http://git-wip-us.apache.org/repos/asf/falcon/blob/8600e946/addons/recipes/hive-disaster-recovery/src/main/resources/hive-disaster-recovery-secure.properties
----------------------------------------------------------------------
diff --git a/addons/recipes/hive-disaster-recovery/src/main/resources/hive-disaster-recovery-secure.properties
b/addons/recipes/hive-disaster-recovery/src/main/resources/hive-disaster-recovery-secure.properties
index cc70ac4..df4a8fd 100644
--- a/addons/recipes/hive-disaster-recovery/src/main/resources/hive-disaster-recovery-secure.properties
+++ b/addons/recipes/hive-disaster-recovery/src/main/resources/hive-disaster-recovery-secure.properties
@@ -102,5 +102,6 @@ distcpMaxMaps=1
 # Change it to specify the bandwidth in MB for each mapper in DistCP
 distcpMapBandwidth=100
 
-##### Email on failure
-drNotificationReceivers=NA
+##### Email Notification for Falcon instance completion
+falcon.recipe.notification.type=email
+falcon.recipe.notification.receivers=NA
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/falcon/blob/8600e946/addons/recipes/hive-disaster-recovery/src/main/resources/hive-disaster-recovery-template.xml
----------------------------------------------------------------------
diff --git a/addons/recipes/hive-disaster-recovery/src/main/resources/hive-disaster-recovery-template.xml
b/addons/recipes/hive-disaster-recovery/src/main/resources/hive-disaster-recovery-template.xml
index 3afbef0..f0de091 100644
--- a/addons/recipes/hive-disaster-recovery/src/main/resources/hive-disaster-recovery-template.xml
+++ b/addons/recipes/hive-disaster-recovery/src/main/resources/hive-disaster-recovery-template.xml
@@ -40,5 +40,6 @@
     <workflow name="##workflow.name##" engine="oozie"
               path="/apps/data-mirroring/workflows/hive-disaster-recovery-workflow.xml" lib="##workflow.lib.path##"/>
     <retry policy="##retry.policy##" delay="##retry.delay##" attempts="3"/>
+    <notification type="##notification.type##" to="##notification.receivers##"/>
     <ACL/>
 </process>

http://git-wip-us.apache.org/repos/asf/falcon/blob/8600e946/addons/recipes/hive-disaster-recovery/src/main/resources/hive-disaster-recovery-workflow.xml
----------------------------------------------------------------------
diff --git a/addons/recipes/hive-disaster-recovery/src/main/resources/hive-disaster-recovery-workflow.xml
b/addons/recipes/hive-disaster-recovery/src/main/resources/hive-disaster-recovery-workflow.xml
index c441998..72d40a3 100644
--- a/addons/recipes/hive-disaster-recovery/src/main/resources/hive-disaster-recovery-workflow.xml
+++ b/addons/recipes/hive-disaster-recovery/src/main/resources/hive-disaster-recovery-workflow.xml
@@ -84,7 +84,7 @@
             <arg>lastevents</arg>
         </java>
         <ok to="export-dr-replication"/>
-        <error to="failure"/>
+        <error to="fail"/>
     </action>
     <!-- Export Replication action -->
     <action name="export-dr-replication">
@@ -160,7 +160,7 @@
             <arg>export</arg>
         </java>
         <ok to="import-dr-replication"/>
-        <error to="failure"/>
+        <error to="fail"/>
     </action>
     <!-- Import Replication action -->
     <action name="import-dr-replication">
@@ -235,52 +235,6 @@
             <arg>-executionStage</arg>
             <arg>import</arg>
         </java>
-        <ok to="success"/>
-        <error to="failure"/>
-    </action>
-    <decision name="success">
-        <switch>
-            <case to="successAlert">
-                ${drNotificationReceivers ne 'NA'}
-            </case>
-            <default to="end"/>
-        </switch>
-    </decision>
-    <decision name="failure">
-        <switch>
-            <case to="failureAlert">
-                ${drNotificationReceivers ne 'NA'}
-            </case>
-            <default to="fail"/>
-        </switch>
-    </decision>
-    <action name="successAlert">
-        <email xmlns="uri:oozie:email-action:0.2">
-            <to>${drNotificationReceivers}</to>
-            <subject>INFO: Hive DR workflow ${drJobName} completed successfully</subject>
-            <body>
-                The Hive DR workflow ${wf:id()} is successful.
-                Source          = ${sourceCluster}
-                Target          = ${targetCluster}
-                DB Name         = ${sourceDatabase}
-                Table Name      = ${sourceTable}
-            </body>
-        </email>
-        <ok to="end"/>
-        <error to="end"/>
-    </action>
-    <action name="failureAlert">
-        <email xmlns="uri:oozie:email-action:0.2">
-            <to>${drNotificationReceivers}</to>
-            <subject>ERROR: Hive DR workflow ${drJobName} failed</subject>
-            <body>
-                The Hive DR workflow ${wf:id()} had issues and was killed.  The error message
is: ${wf:errorMessage(wf:lastErrorNode())}
-                Source          = ${sourceCluster}
-                Target          = ${targetCluster}
-                DB Name         = ${sourceDatabase}
-                Table Name      = ${sourceTable}
-            </body>
-        </email>
         <ok to="end"/>
         <error to="fail"/>
     </action>

http://git-wip-us.apache.org/repos/asf/falcon/blob/8600e946/addons/recipes/hive-disaster-recovery/src/main/resources/hive-disaster-recovery.properties
----------------------------------------------------------------------
diff --git a/addons/recipes/hive-disaster-recovery/src/main/resources/hive-disaster-recovery.properties
b/addons/recipes/hive-disaster-recovery/src/main/resources/hive-disaster-recovery.properties
index c2cbf5d..c2915fb 100644
--- a/addons/recipes/hive-disaster-recovery/src/main/resources/hive-disaster-recovery.properties
+++ b/addons/recipes/hive-disaster-recovery/src/main/resources/hive-disaster-recovery.properties
@@ -92,5 +92,6 @@ distcpMaxMaps=1
 # Change it to specify the bandwidth in MB for each mapper in DistCP
 distcpMapBandwidth=100
 
-##### Email on failure
-drNotificationReceivers=NA
+##### Email Notification for Falcon instance completion
+falcon.recipe.notification.type=email
+falcon.recipe.notification.receivers=NA
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/falcon/blob/8600e946/client/src/main/java/org/apache/falcon/recipe/RecipeToolOptions.java
----------------------------------------------------------------------
diff --git a/client/src/main/java/org/apache/falcon/recipe/RecipeToolOptions.java b/client/src/main/java/org/apache/falcon/recipe/RecipeToolOptions.java
index 5df9b0a..0ece90b 100644
--- a/client/src/main/java/org/apache/falcon/recipe/RecipeToolOptions.java
+++ b/client/src/main/java/org/apache/falcon/recipe/RecipeToolOptions.java
@@ -42,7 +42,9 @@ public enum RecipeToolOptions {
     RECIPE_ACL_OWNER("falcon.recipe.acl.owner", "Recipe acl owner", false),
     RECIPE_ACL_GROUP("falcon.recipe.acl.group", "Recipe acl group", false),
     RECIPE_ACL_PERMISSION("falcon.recipe.acl.permission", "Recipe acl permission", false),
-    RECIPE_NN_PRINCIPAL("falcon.recipe.nn.principal", "Recipe DFS NN principal", false);
+    RECIPE_NN_PRINCIPAL("falcon.recipe.nn.principal", "Recipe DFS NN principal", false),
+    RECIPE_NOTIFICATION_TYPE("falcon.recipe.notification.type", "Recipe Notification Type",
false),
+    RECIPE_NOTIFICATION_ADDRESS("falcon.recipe.notification.receivers", "Recipe Email Notification
receivers", false);
 
     private final String name;
     private final String description;

http://git-wip-us.apache.org/repos/asf/falcon/blob/8600e946/client/src/main/java/org/apache/falcon/recipe/util/RecipeProcessBuilderUtils.java
----------------------------------------------------------------------
diff --git a/client/src/main/java/org/apache/falcon/recipe/util/RecipeProcessBuilderUtils.java
b/client/src/main/java/org/apache/falcon/recipe/util/RecipeProcessBuilderUtils.java
index 9522816..9477bb4 100644
--- a/client/src/main/java/org/apache/falcon/recipe/util/RecipeProcessBuilderUtils.java
+++ b/client/src/main/java/org/apache/falcon/recipe/util/RecipeProcessBuilderUtils.java
@@ -26,6 +26,7 @@ import org.apache.falcon.entity.v0.Frequency;
 import org.apache.falcon.entity.v0.SchemaHelper;
 import org.apache.falcon.entity.v0.process.ACL;
 import org.apache.falcon.entity.v0.process.Cluster;
+import org.apache.falcon.entity.v0.process.Notification;
 import org.apache.falcon.entity.v0.process.PolicyType;
 import org.apache.falcon.entity.v0.process.Property;
 import org.apache.falcon.entity.v0.process.Retry;
@@ -111,6 +112,7 @@ public final class RecipeProcessBuilderUtils {
 
         bindWorkflowProperties(process.getWorkflow(), recipeProperties);
         bindRetryProperties(process.getRetry(), recipeProperties);
+        bindNotificationProperties(process.getNotification(), recipeProperties);
         bindACLProperties(process.getACL(), recipeProperties);
         bindTagsProperties(process, recipeProperties);
         bindCustomProperties(process.getProperties(), recipeProperties);
@@ -175,6 +177,20 @@ public final class RecipeProcessBuilderUtils {
         }
     }
 
+    private static void bindNotificationProperties(final Notification processNotification,
+                                                   final Properties recipeProperties) {
+        processNotification.setType(recipeProperties.getProperty(
+                RecipeToolOptions.RECIPE_NOTIFICATION_TYPE.getName()));
+
+        String notificationAddress = recipeProperties.getProperty(
+                RecipeToolOptions.RECIPE_NOTIFICATION_ADDRESS.getName());
+        if (StringUtils.isNotBlank(notificationAddress)) {
+            processNotification.setTo(notificationAddress);
+        } else {
+            processNotification.setTo("NA");
+        }
+    }
+
     private static void bindACLProperties(final ACL acl,
                                           final Properties recipeProperties) {
         String aclowner = recipeProperties.getProperty(RecipeToolOptions.RECIPE_ACL_OWNER.getName());

http://git-wip-us.apache.org/repos/asf/falcon/blob/8600e946/webapp/src/test/resources/hdfs-replication-template.xml
----------------------------------------------------------------------
diff --git a/webapp/src/test/resources/hdfs-replication-template.xml b/webapp/src/test/resources/hdfs-replication-template.xml
index b93cc0b..c038484 100644
--- a/webapp/src/test/resources/hdfs-replication-template.xml
+++ b/webapp/src/test/resources/hdfs-replication-template.xml
@@ -39,5 +39,6 @@
 
     <workflow name="##falcon.recipe.workflow.name##" engine="oozie" path="/apps/data-mirroring/workflows/hdfs-replication-workflow.xml"
lib="##workflow.lib.path##"/>
     <retry policy="##falcon.recipe.retry.policy##" delay="##falcon.recipe.retry.delay##"
attempts="3"/>
+    <notification type="##falcon.recipe.notification.type##" to="##falcon.recipe.notification.receivers##"/>
     <ACL/>
 </process>

http://git-wip-us.apache.org/repos/asf/falcon/blob/8600e946/webapp/src/test/resources/hdfs-replication.properties
----------------------------------------------------------------------
diff --git a/webapp/src/test/resources/hdfs-replication.properties b/webapp/src/test/resources/hdfs-replication.properties
index 1b485b4..bd77a1c 100644
--- a/webapp/src/test/resources/hdfs-replication.properties
+++ b/webapp/src/test/resources/hdfs-replication.properties
@@ -41,5 +41,6 @@ distcpMaxMaps=1
 # Change it to specify the bandwidth in MB for each mapper in DistCP
 distcpMapBandwidth=100
 
-##### Email on failure
-drNotificationReceivers=NA
\ No newline at end of file
+##### Email Notification for Falcon instance completion ####
+falcon.recipe.notification.type=email
+falcon.recipe.notification.receivers=NA
\ No newline at end of file


Mime
View raw message