falcon-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From venkat...@apache.org
Subject [1/3] git commit: FALCON-204 Process mapper generates params assuming table input and output as names literally. Contributed by Venkatesh Seetharam
Date Fri, 15 Nov 2013 22:29:02 GMT
Updated Branches:
  refs/heads/branch-0.4 624259368 -> c2731dcfd


FALCON-204 Process mapper generates params assuming table input and output as names literally.
Contributed by Venkatesh Seetharam


Project: http://git-wip-us.apache.org/repos/asf/incubator-falcon/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-falcon/commit/6a638c03
Tree: http://git-wip-us.apache.org/repos/asf/incubator-falcon/tree/6a638c03
Diff: http://git-wip-us.apache.org/repos/asf/incubator-falcon/diff/6a638c03

Branch: refs/heads/branch-0.4
Commit: 6a638c03fcc86308759ef81c9bf5deda5df15c8a
Parents: 6242593
Author: Venkatesh Seetharam <venkatesh@apache.org>
Authored: Fri Nov 15 14:20:19 2013 -0800
Committer: Venkatesh Seetharam <venkatesh@apache.org>
Committed: Fri Nov 15 14:20:19 2013 -0800

----------------------------------------------------------------------
 CHANGES.txt                                           |  3 +++
 .../apache/falcon/converter/OozieProcessMapper.java   | 14 ++++++++++----
 .../falcon/converter/OozieProcessMapperTest.java      |  6 +++---
 3 files changed, 16 insertions(+), 7 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/6a638c03/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 5cf9083..6bd84f3 100755
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -178,6 +178,9 @@ Release Version: 0.4-incubating
 
     FALCON-194 Import external libraries into the repository
 
+    FALCON-204 Process mapper generates params assuming table input and output
+    as names literally (Venkatesh Seetharam)
+
 
 
 Release Version: 0.3-incubating

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/6a638c03/process/src/main/java/org/apache/falcon/converter/OozieProcessMapper.java
----------------------------------------------------------------------
diff --git a/process/src/main/java/org/apache/falcon/converter/OozieProcessMapper.java b/process/src/main/java/org/apache/falcon/converter/OozieProcessMapper.java
index eed7fa3..8749f07 100644
--- a/process/src/main/java/org/apache/falcon/converter/OozieProcessMapper.java
+++ b/process/src/main/java/org/apache/falcon/converter/OozieProcessMapper.java
@@ -375,9 +375,12 @@ public class OozieProcessMapper extends AbstractOozieEntityMapper<Process>
{
 
         propagateCommonCatalogTableProperties(tableStorage, props, prefix);
 
-        props.put(prefix + "_partition_filter_pig", "${coord:dataInPartitionFilter('input',
'pig')}");
-        props.put(prefix + "_partition_filter_hive", "${coord:dataInPartitionFilter('input',
'hive')}");
-        props.put(prefix + "_partition_filter_java", "${coord:dataInPartitionFilter('input',
'java')}");
+        props.put(prefix + "_partition_filter_pig",
+                "${coord:dataInPartitionFilter('" + input.getName() + "', 'pig')}");
+        props.put(prefix + "_partition_filter_hive",
+                "${coord:dataInPartitionFilter('" + input.getName() + "', 'hive')}");
+        props.put(prefix + "_partition_filter_java",
+                "${coord:dataInPartitionFilter('" + input.getName() + "', 'java')}");
     }
 
     private void propagateCatalogTableProperties(Output output, CatalogStorage tableStorage,
@@ -386,7 +389,10 @@ public class OozieProcessMapper extends AbstractOozieEntityMapper<Process>
{
 
         propagateCommonCatalogTableProperties(tableStorage, props, prefix);
 
-        props.put(prefix + "_dataout_partitions", "${coord:dataOutPartitions('output')}");
+        props.put(prefix + "_dataout_partitions",
+                "${coord:dataOutPartitions('" + output.getName() + "')}");
+        props.put(prefix + "_dated_partition_value", "${coord:dataOutPartitionValue('"
+                + output.getName() + "', '" + tableStorage.getDatedPartitionKey() + "')}");
     }
 
     private String join(Iterator<String> itr, char sep) {

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/6a638c03/process/src/test/java/org/apache/falcon/converter/OozieProcessMapperTest.java
----------------------------------------------------------------------
diff --git a/process/src/test/java/org/apache/falcon/converter/OozieProcessMapperTest.java
b/process/src/test/java/org/apache/falcon/converter/OozieProcessMapperTest.java
index b6f03e5..7d5f4d1 100644
--- a/process/src/test/java/org/apache/falcon/converter/OozieProcessMapperTest.java
+++ b/process/src/test/java/org/apache/falcon/converter/OozieProcessMapperTest.java
@@ -278,7 +278,7 @@ public class OozieProcessMapperTest extends AbstractTestBase {
         Assert.assertNull(hiveAction.getPrepare());
         Assert.assertEquals(Collections.EMPTY_LIST, hiveAction.getArchive());
         Assert.assertFalse(hiveAction.getParam().isEmpty());
-        Assert.assertEquals(10, hiveAction.getParam().size());
+        Assert.assertEquals(11, hiveAction.getParam().size());
     }
 
     @Test
@@ -356,11 +356,11 @@ public class OozieProcessMapperTest extends AbstractTestBase {
         props.put(prefix + "_database", tableStorage.getDatabase());
         props.put(prefix + "_table", tableStorage.getTable());
 
-        if (prefix.equals("input")) {
+        if (prefix.equals("falcon_input")) {
             props.put(prefix + "_partition_filter_pig", "${coord:dataInPartitionFilter('input',
'pig')}");
             props.put(prefix + "_partition_filter_hive", "${coord:dataInPartitionFilter('input',
'hive')}");
             props.put(prefix + "_partition_filter_java", "${coord:dataInPartitionFilter('input',
'java')}");
-        } else if (prefix.equals("output")) {
+        } else if (prefix.equals("falcon_output")) {
             props.put(prefix + "_dataout_partitions", "${coord:dataOutPartitions('output')}");
         }
     }


Mime
View raw message