Return-Path: X-Original-To: apmail-falcon-commits-archive@minotaur.apache.org Delivered-To: apmail-falcon-commits-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 2063F10A48 for ; Fri, 15 Nov 2013 22:29:28 +0000 (UTC) Received: (qmail 61085 invoked by uid 500); 15 Nov 2013 22:29:28 -0000 Delivered-To: apmail-falcon-commits-archive@falcon.apache.org Received: (qmail 61022 invoked by uid 500); 15 Nov 2013 22:29:28 -0000 Mailing-List: contact commits-help@falcon.incubator.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@falcon.incubator.apache.org Delivered-To: mailing list commits@falcon.incubator.apache.org Received: (qmail 61014 invoked by uid 99); 15 Nov 2013 22:29:27 -0000 Received: from nike.apache.org (HELO nike.apache.org) (192.87.106.230) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 15 Nov 2013 22:29:27 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=5.0 tests=ALL_TRUSTED,RP_MATCHES_RCVD X-Spam-Check-By: apache.org Received: from [140.211.11.3] (HELO mail.apache.org) (140.211.11.3) by apache.org (qpsmtpd/0.29) with SMTP; Fri, 15 Nov 2013 22:29:24 +0000 Received: (qmail 59322 invoked by uid 99); 15 Nov 2013 22:29:03 -0000 Received: from tyr.zones.apache.org (HELO tyr.zones.apache.org) (140.211.11.114) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 15 Nov 2013 22:29:03 +0000 Received: by tyr.zones.apache.org (Postfix, from userid 65534) id C8B1882F8D9; Fri, 15 Nov 2013 22:29:02 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: venkatesh@apache.org To: commits@falcon.incubator.apache.org Date: Fri, 15 Nov 2013 22:29:02 -0000 Message-Id: <415e262267da452a9325a68e09390a2c@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [1/3] git commit: FALCON-204 Process mapper generates params assuming table input and output as names literally. Contributed by Venkatesh Seetharam X-Virus-Checked: Checked by ClamAV on apache.org Updated Branches: refs/heads/branch-0.4 624259368 -> c2731dcfd FALCON-204 Process mapper generates params assuming table input and output as names literally. Contributed by Venkatesh Seetharam Project: http://git-wip-us.apache.org/repos/asf/incubator-falcon/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-falcon/commit/6a638c03 Tree: http://git-wip-us.apache.org/repos/asf/incubator-falcon/tree/6a638c03 Diff: http://git-wip-us.apache.org/repos/asf/incubator-falcon/diff/6a638c03 Branch: refs/heads/branch-0.4 Commit: 6a638c03fcc86308759ef81c9bf5deda5df15c8a Parents: 6242593 Author: Venkatesh Seetharam Authored: Fri Nov 15 14:20:19 2013 -0800 Committer: Venkatesh Seetharam Committed: Fri Nov 15 14:20:19 2013 -0800 ---------------------------------------------------------------------- CHANGES.txt | 3 +++ .../apache/falcon/converter/OozieProcessMapper.java | 14 ++++++++++---- .../falcon/converter/OozieProcessMapperTest.java | 6 +++--- 3 files changed, 16 insertions(+), 7 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/6a638c03/CHANGES.txt ---------------------------------------------------------------------- diff --git a/CHANGES.txt b/CHANGES.txt index 5cf9083..6bd84f3 100755 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -178,6 +178,9 @@ Release Version: 0.4-incubating FALCON-194 Import external libraries into the repository + FALCON-204 Process mapper generates params assuming table input and output + as names literally (Venkatesh Seetharam) + Release Version: 0.3-incubating http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/6a638c03/process/src/main/java/org/apache/falcon/converter/OozieProcessMapper.java ---------------------------------------------------------------------- diff --git a/process/src/main/java/org/apache/falcon/converter/OozieProcessMapper.java b/process/src/main/java/org/apache/falcon/converter/OozieProcessMapper.java index eed7fa3..8749f07 100644 --- a/process/src/main/java/org/apache/falcon/converter/OozieProcessMapper.java +++ b/process/src/main/java/org/apache/falcon/converter/OozieProcessMapper.java @@ -375,9 +375,12 @@ public class OozieProcessMapper extends AbstractOozieEntityMapper { propagateCommonCatalogTableProperties(tableStorage, props, prefix); - props.put(prefix + "_partition_filter_pig", "${coord:dataInPartitionFilter('input', 'pig')}"); - props.put(prefix + "_partition_filter_hive", "${coord:dataInPartitionFilter('input', 'hive')}"); - props.put(prefix + "_partition_filter_java", "${coord:dataInPartitionFilter('input', 'java')}"); + props.put(prefix + "_partition_filter_pig", + "${coord:dataInPartitionFilter('" + input.getName() + "', 'pig')}"); + props.put(prefix + "_partition_filter_hive", + "${coord:dataInPartitionFilter('" + input.getName() + "', 'hive')}"); + props.put(prefix + "_partition_filter_java", + "${coord:dataInPartitionFilter('" + input.getName() + "', 'java')}"); } private void propagateCatalogTableProperties(Output output, CatalogStorage tableStorage, @@ -386,7 +389,10 @@ public class OozieProcessMapper extends AbstractOozieEntityMapper { propagateCommonCatalogTableProperties(tableStorage, props, prefix); - props.put(prefix + "_dataout_partitions", "${coord:dataOutPartitions('output')}"); + props.put(prefix + "_dataout_partitions", + "${coord:dataOutPartitions('" + output.getName() + "')}"); + props.put(prefix + "_dated_partition_value", "${coord:dataOutPartitionValue('" + + output.getName() + "', '" + tableStorage.getDatedPartitionKey() + "')}"); } private String join(Iterator itr, char sep) { http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/6a638c03/process/src/test/java/org/apache/falcon/converter/OozieProcessMapperTest.java ---------------------------------------------------------------------- diff --git a/process/src/test/java/org/apache/falcon/converter/OozieProcessMapperTest.java b/process/src/test/java/org/apache/falcon/converter/OozieProcessMapperTest.java index b6f03e5..7d5f4d1 100644 --- a/process/src/test/java/org/apache/falcon/converter/OozieProcessMapperTest.java +++ b/process/src/test/java/org/apache/falcon/converter/OozieProcessMapperTest.java @@ -278,7 +278,7 @@ public class OozieProcessMapperTest extends AbstractTestBase { Assert.assertNull(hiveAction.getPrepare()); Assert.assertEquals(Collections.EMPTY_LIST, hiveAction.getArchive()); Assert.assertFalse(hiveAction.getParam().isEmpty()); - Assert.assertEquals(10, hiveAction.getParam().size()); + Assert.assertEquals(11, hiveAction.getParam().size()); } @Test @@ -356,11 +356,11 @@ public class OozieProcessMapperTest extends AbstractTestBase { props.put(prefix + "_database", tableStorage.getDatabase()); props.put(prefix + "_table", tableStorage.getTable()); - if (prefix.equals("input")) { + if (prefix.equals("falcon_input")) { props.put(prefix + "_partition_filter_pig", "${coord:dataInPartitionFilter('input', 'pig')}"); props.put(prefix + "_partition_filter_hive", "${coord:dataInPartitionFilter('input', 'hive')}"); props.put(prefix + "_partition_filter_java", "${coord:dataInPartitionFilter('input', 'java')}"); - } else if (prefix.equals("output")) { + } else if (prefix.equals("falcon_output")) { props.put(prefix + "_dataout_partitions", "${coord:dataOutPartitions('output')}"); } }