Return-Path: X-Original-To: apmail-falcon-commits-archive@minotaur.apache.org Delivered-To: apmail-falcon-commits-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id AE7BE19E6A for ; Mon, 28 Mar 2016 06:43:34 +0000 (UTC) Received: (qmail 30865 invoked by uid 500); 28 Mar 2016 06:43:34 -0000 Delivered-To: apmail-falcon-commits-archive@falcon.apache.org Received: (qmail 30830 invoked by uid 500); 28 Mar 2016 06:43:34 -0000 Mailing-List: contact commits-help@falcon.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@falcon.apache.org Delivered-To: mailing list commits@falcon.apache.org Received: (qmail 30821 invoked by uid 99); 28 Mar 2016 06:43:34 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Mon, 28 Mar 2016 06:43:34 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 76A57E01F5; Mon, 28 Mar 2016 06:43:34 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: pallavi@apache.org To: commits@falcon.apache.org Message-Id: X-Mailer: ASF-Git Admin Mailer Subject: falcon git commit: FALCON-1874 Import and Export fails with HDFS as src/dest Date: Mon, 28 Mar 2016 06:43:34 +0000 (UTC) Repository: falcon Updated Branches: refs/heads/master 8e79ba81a -> 10f3843ad FALCON-1874 Import and Export fails with HDFS as src/dest 1. job-xml was pointing to hive-site.xml even for HDFS, cause workflow to fail. 2. datain element was repeated in export coordinator, causing workflow to fail. Author: Pallavi Rao Reviewers: @vramachan, @PraveenAdlakha Closes #78 from pallavi-rao/FALCON-1874 Project: http://git-wip-us.apache.org/repos/asf/falcon/repo Commit: http://git-wip-us.apache.org/repos/asf/falcon/commit/10f3843a Tree: http://git-wip-us.apache.org/repos/asf/falcon/tree/10f3843a Diff: http://git-wip-us.apache.org/repos/asf/falcon/diff/10f3843a Branch: refs/heads/master Commit: 10f3843addd509e3f77bad9737ee2ddb33681a94 Parents: 8e79ba8 Author: Pallavi Rao Authored: Mon Mar 28 12:13:32 2016 +0530 Committer: Pallavi Rao Committed: Mon Mar 28 12:13:32 2016 +0530 ---------------------------------------------------------------------- .../org/apache/falcon/oozie/DatabaseExportWorkflowBuilder.java | 3 +-- .../org/apache/falcon/oozie/DatabaseImportWorkflowBuilder.java | 3 +-- .../org/apache/falcon/oozie/FeedExportCoordinatorBuilder.java | 1 - .../main/java/org/apache/falcon/oozie/ImportExportCommon.java | 5 ++++- 4 files changed, 6 insertions(+), 6 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/falcon/blob/10f3843a/oozie/src/main/java/org/apache/falcon/oozie/DatabaseExportWorkflowBuilder.java ---------------------------------------------------------------------- diff --git a/oozie/src/main/java/org/apache/falcon/oozie/DatabaseExportWorkflowBuilder.java b/oozie/src/main/java/org/apache/falcon/oozie/DatabaseExportWorkflowBuilder.java index 284c4a3..e7ca869 100644 --- a/oozie/src/main/java/org/apache/falcon/oozie/DatabaseExportWorkflowBuilder.java +++ b/oozie/src/main/java/org/apache/falcon/oozie/DatabaseExportWorkflowBuilder.java @@ -64,8 +64,7 @@ public class DatabaseExportWorkflowBuilder extends ExportWorkflowBuilder { org.apache.falcon.oozie.sqoop.ACTION sqoopExport = actionJaxbElement.getValue(); Properties props = new Properties(); - ImportExportCommon.addHCatalogProperties(props, entity, cluster, workflow, this, buildPath); - sqoopExport.getJobXml().add("${wf:appPath()}/conf/hive-site.xml"); + ImportExportCommon.addHCatalogProperties(props, entity, cluster, workflow, this, buildPath, sqoopExport); OozieUtils.marshalSqoopAction(action, actionJaxbElement); addTransition(action, SUCCESS_POSTPROCESS_ACTION_NAME, FAIL_POSTPROCESS_ACTION_NAME); http://git-wip-us.apache.org/repos/asf/falcon/blob/10f3843a/oozie/src/main/java/org/apache/falcon/oozie/DatabaseImportWorkflowBuilder.java ---------------------------------------------------------------------- diff --git a/oozie/src/main/java/org/apache/falcon/oozie/DatabaseImportWorkflowBuilder.java b/oozie/src/main/java/org/apache/falcon/oozie/DatabaseImportWorkflowBuilder.java index 3e24428..1efa368 100644 --- a/oozie/src/main/java/org/apache/falcon/oozie/DatabaseImportWorkflowBuilder.java +++ b/oozie/src/main/java/org/apache/falcon/oozie/DatabaseImportWorkflowBuilder.java @@ -66,8 +66,7 @@ public class DatabaseImportWorkflowBuilder extends ImportWorkflowBuilder { org.apache.falcon.oozie.sqoop.ACTION sqoopImport = actionJaxbElement.getValue(); Properties props = new Properties(); - ImportExportCommon.addHCatalogProperties(props, entity, cluster, workflow, this, buildPath); - sqoopImport.getJobXml().add("${wf:appPath()}/conf/hive-site.xml"); + ImportExportCommon.addHCatalogProperties(props, entity, cluster, workflow, this, buildPath, sqoopImport); OozieUtils.marshalSqoopAction(action, actionJaxbElement); addTransition(action, SUCCESS_POSTPROCESS_ACTION_NAME, FAIL_POSTPROCESS_ACTION_NAME); http://git-wip-us.apache.org/repos/asf/falcon/blob/10f3843a/oozie/src/main/java/org/apache/falcon/oozie/FeedExportCoordinatorBuilder.java ---------------------------------------------------------------------- diff --git a/oozie/src/main/java/org/apache/falcon/oozie/FeedExportCoordinatorBuilder.java b/oozie/src/main/java/org/apache/falcon/oozie/FeedExportCoordinatorBuilder.java index 4437d8b..e37accd 100644 --- a/oozie/src/main/java/org/apache/falcon/oozie/FeedExportCoordinatorBuilder.java +++ b/oozie/src/main/java/org/apache/falcon/oozie/FeedExportCoordinatorBuilder.java @@ -125,7 +125,6 @@ public class FeedExportCoordinatorBuilder extends OozieCoordinatorBuilder datain.setDataset(EXPORT_DATASET_NAME); org.apache.falcon.entity.v0.feed.Cluster feedCluster = FeedHelper.getCluster(feed, cluster.getName()); datain.getInstance().add(SchemaHelper.formatDateUTC(feedCluster.getValidity().getStart())); - datain.getInstance().add(SchemaHelper.formatDateUTC(feedCluster.getValidity().getStart())); return datain; } http://git-wip-us.apache.org/repos/asf/falcon/blob/10f3843a/oozie/src/main/java/org/apache/falcon/oozie/ImportExportCommon.java ---------------------------------------------------------------------- diff --git a/oozie/src/main/java/org/apache/falcon/oozie/ImportExportCommon.java b/oozie/src/main/java/org/apache/falcon/oozie/ImportExportCommon.java index 52c7820..fafd56e 100644 --- a/oozie/src/main/java/org/apache/falcon/oozie/ImportExportCommon.java +++ b/oozie/src/main/java/org/apache/falcon/oozie/ImportExportCommon.java @@ -28,6 +28,7 @@ import org.apache.falcon.entity.v0.datasource.Credential; import org.apache.falcon.entity.v0.datasource.Credentialtype; import org.apache.falcon.entity.v0.datasource.Datasource; import org.apache.falcon.entity.v0.feed.Feed; +import org.apache.falcon.oozie.sqoop.ACTION; import org.apache.falcon.oozie.workflow.WORKFLOWAPP; import org.apache.falcon.security.SecurityUtil; import org.apache.hadoop.fs.Path; @@ -88,7 +89,8 @@ public final class ImportExportCommon { } public static void addHCatalogProperties(Properties props, Feed entity, Cluster cluster, - WORKFLOWAPP workflow, OozieOrchestrationWorkflowBuilder wBuilder, Path buildPath) + WORKFLOWAPP workflow, OozieOrchestrationWorkflowBuilder wBuilder, + Path buildPath, ACTION sqoopAction) throws FalconException { if (FeedHelper.getStorageType(entity, cluster) == Storage.TYPE.TABLE) { wBuilder.createHiveConfiguration(cluster, buildPath, ""); @@ -98,6 +100,7 @@ public final class ImportExportCommon { wBuilder.addHCatalogCredentials(workflow, cluster, OozieOrchestrationWorkflowBuilder.HIVE_CREDENTIAL_NAME, FALCON_IMPORT_SQOOP_ACTIONS); } + sqoopAction.getJobXml().add("${wf:appPath()}/conf/hive-site.xml"); } } private static void addHCatalogShareLibs(Properties props) throws FalconException {