falcon-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From pall...@apache.org
Subject falcon git commit: FALCON-1874 Import and Export fails with HDFS as src/dest
Date Mon, 28 Mar 2016 06:43:34 GMT
Repository: falcon
Updated Branches:
  refs/heads/master 8e79ba81a -> 10f3843ad


FALCON-1874 Import and Export fails with HDFS as src/dest

1. job-xml was pointing to hive-site.xml even for HDFS, cause workflow to fail.
2. datain element was repeated in export coordinator, causing workflow to fail.

Author: Pallavi Rao <pallavi.rao@inmobi.com>

Reviewers: @vramachan, @PraveenAdlakha

Closes #78 from pallavi-rao/FALCON-1874


Project: http://git-wip-us.apache.org/repos/asf/falcon/repo
Commit: http://git-wip-us.apache.org/repos/asf/falcon/commit/10f3843a
Tree: http://git-wip-us.apache.org/repos/asf/falcon/tree/10f3843a
Diff: http://git-wip-us.apache.org/repos/asf/falcon/diff/10f3843a

Branch: refs/heads/master
Commit: 10f3843addd509e3f77bad9737ee2ddb33681a94
Parents: 8e79ba8
Author: Pallavi Rao <pallavi.rao@inmobi.com>
Authored: Mon Mar 28 12:13:32 2016 +0530
Committer: Pallavi Rao <pallavi.rao@inmobi.com>
Committed: Mon Mar 28 12:13:32 2016 +0530

----------------------------------------------------------------------
 .../org/apache/falcon/oozie/DatabaseExportWorkflowBuilder.java  | 3 +--
 .../org/apache/falcon/oozie/DatabaseImportWorkflowBuilder.java  | 3 +--
 .../org/apache/falcon/oozie/FeedExportCoordinatorBuilder.java   | 1 -
 .../main/java/org/apache/falcon/oozie/ImportExportCommon.java   | 5 ++++-
 4 files changed, 6 insertions(+), 6 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/falcon/blob/10f3843a/oozie/src/main/java/org/apache/falcon/oozie/DatabaseExportWorkflowBuilder.java
----------------------------------------------------------------------
diff --git a/oozie/src/main/java/org/apache/falcon/oozie/DatabaseExportWorkflowBuilder.java
b/oozie/src/main/java/org/apache/falcon/oozie/DatabaseExportWorkflowBuilder.java
index 284c4a3..e7ca869 100644
--- a/oozie/src/main/java/org/apache/falcon/oozie/DatabaseExportWorkflowBuilder.java
+++ b/oozie/src/main/java/org/apache/falcon/oozie/DatabaseExportWorkflowBuilder.java
@@ -64,8 +64,7 @@ public class DatabaseExportWorkflowBuilder extends ExportWorkflowBuilder
{
         org.apache.falcon.oozie.sqoop.ACTION sqoopExport = actionJaxbElement.getValue();
 
         Properties props = new Properties();
-        ImportExportCommon.addHCatalogProperties(props, entity, cluster, workflow, this,
buildPath);
-        sqoopExport.getJobXml().add("${wf:appPath()}/conf/hive-site.xml");
+        ImportExportCommon.addHCatalogProperties(props, entity, cluster, workflow, this,
buildPath, sqoopExport);
         OozieUtils.marshalSqoopAction(action, actionJaxbElement);
 
         addTransition(action, SUCCESS_POSTPROCESS_ACTION_NAME, FAIL_POSTPROCESS_ACTION_NAME);

http://git-wip-us.apache.org/repos/asf/falcon/blob/10f3843a/oozie/src/main/java/org/apache/falcon/oozie/DatabaseImportWorkflowBuilder.java
----------------------------------------------------------------------
diff --git a/oozie/src/main/java/org/apache/falcon/oozie/DatabaseImportWorkflowBuilder.java
b/oozie/src/main/java/org/apache/falcon/oozie/DatabaseImportWorkflowBuilder.java
index 3e24428..1efa368 100644
--- a/oozie/src/main/java/org/apache/falcon/oozie/DatabaseImportWorkflowBuilder.java
+++ b/oozie/src/main/java/org/apache/falcon/oozie/DatabaseImportWorkflowBuilder.java
@@ -66,8 +66,7 @@ public class DatabaseImportWorkflowBuilder extends ImportWorkflowBuilder
{
         org.apache.falcon.oozie.sqoop.ACTION sqoopImport = actionJaxbElement.getValue();
 
         Properties props = new Properties();
-        ImportExportCommon.addHCatalogProperties(props, entity, cluster, workflow, this,
buildPath);
-        sqoopImport.getJobXml().add("${wf:appPath()}/conf/hive-site.xml");
+        ImportExportCommon.addHCatalogProperties(props, entity, cluster, workflow, this,
buildPath, sqoopImport);
         OozieUtils.marshalSqoopAction(action, actionJaxbElement);
 
         addTransition(action, SUCCESS_POSTPROCESS_ACTION_NAME, FAIL_POSTPROCESS_ACTION_NAME);

http://git-wip-us.apache.org/repos/asf/falcon/blob/10f3843a/oozie/src/main/java/org/apache/falcon/oozie/FeedExportCoordinatorBuilder.java
----------------------------------------------------------------------
diff --git a/oozie/src/main/java/org/apache/falcon/oozie/FeedExportCoordinatorBuilder.java
b/oozie/src/main/java/org/apache/falcon/oozie/FeedExportCoordinatorBuilder.java
index 4437d8b..e37accd 100644
--- a/oozie/src/main/java/org/apache/falcon/oozie/FeedExportCoordinatorBuilder.java
+++ b/oozie/src/main/java/org/apache/falcon/oozie/FeedExportCoordinatorBuilder.java
@@ -125,7 +125,6 @@ public class FeedExportCoordinatorBuilder extends OozieCoordinatorBuilder<Feed>
         datain.setDataset(EXPORT_DATASET_NAME);
         org.apache.falcon.entity.v0.feed.Cluster feedCluster = FeedHelper.getCluster(feed,
cluster.getName());
         datain.getInstance().add(SchemaHelper.formatDateUTC(feedCluster.getValidity().getStart()));
-        datain.getInstance().add(SchemaHelper.formatDateUTC(feedCluster.getValidity().getStart()));
         return datain;
     }
 

http://git-wip-us.apache.org/repos/asf/falcon/blob/10f3843a/oozie/src/main/java/org/apache/falcon/oozie/ImportExportCommon.java
----------------------------------------------------------------------
diff --git a/oozie/src/main/java/org/apache/falcon/oozie/ImportExportCommon.java b/oozie/src/main/java/org/apache/falcon/oozie/ImportExportCommon.java
index 52c7820..fafd56e 100644
--- a/oozie/src/main/java/org/apache/falcon/oozie/ImportExportCommon.java
+++ b/oozie/src/main/java/org/apache/falcon/oozie/ImportExportCommon.java
@@ -28,6 +28,7 @@ import org.apache.falcon.entity.v0.datasource.Credential;
 import org.apache.falcon.entity.v0.datasource.Credentialtype;
 import org.apache.falcon.entity.v0.datasource.Datasource;
 import org.apache.falcon.entity.v0.feed.Feed;
+import org.apache.falcon.oozie.sqoop.ACTION;
 import org.apache.falcon.oozie.workflow.WORKFLOWAPP;
 import org.apache.falcon.security.SecurityUtil;
 import org.apache.hadoop.fs.Path;
@@ -88,7 +89,8 @@ public final class ImportExportCommon {
     }
 
     public static void addHCatalogProperties(Properties props, Feed entity, Cluster cluster,
-        WORKFLOWAPP workflow, OozieOrchestrationWorkflowBuilder<Feed> wBuilder, Path
buildPath)
+                                             WORKFLOWAPP workflow, OozieOrchestrationWorkflowBuilder<Feed>
wBuilder,
+                                             Path buildPath, ACTION sqoopAction)
         throws FalconException {
         if (FeedHelper.getStorageType(entity, cluster) == Storage.TYPE.TABLE) {
             wBuilder.createHiveConfiguration(cluster, buildPath, "");
@@ -98,6 +100,7 @@ public final class ImportExportCommon {
                 wBuilder.addHCatalogCredentials(workflow, cluster,
                         OozieOrchestrationWorkflowBuilder.HIVE_CREDENTIAL_NAME, FALCON_IMPORT_SQOOP_ACTIONS);
             }
+            sqoopAction.getJobXml().add("${wf:appPath()}/conf/hive-site.xml");
         }
     }
     private static void addHCatalogShareLibs(Properties props) throws FalconException {


Mime
View raw message