falcon-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From shweth...@apache.org
Subject git commit: FALCON-357 HCatalog Feed replication: Hive export job fails when table partition contains multiple dated columns. Contributed by Satish Mittal
Date Tue, 08 Jul 2014 09:46:54 GMT
Repository: incubator-falcon
Updated Branches:
  refs/heads/master 64c2cb573 -> 5626b2dd1


FALCON-357 HCatalog Feed replication: Hive export job fails when table partition contains
multiple dated columns. Contributed by Satish Mittal


Project: http://git-wip-us.apache.org/repos/asf/incubator-falcon/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-falcon/commit/5626b2dd
Tree: http://git-wip-us.apache.org/repos/asf/incubator-falcon/tree/5626b2dd
Diff: http://git-wip-us.apache.org/repos/asf/incubator-falcon/diff/5626b2dd

Branch: refs/heads/master
Commit: 5626b2dd1d63a9762806e0e3a20d9624a46005c5
Parents: 64c2cb5
Author: Shwetha GS <shwetha.gs@inmobi.com>
Authored: Tue Jul 8 15:16:42 2014 +0530
Committer: Shwetha GS <shwetha.gs@inmobi.com>
Committed: Tue Jul 8 15:16:42 2014 +0530

----------------------------------------------------------------------
 CHANGES.txt                                     |   3 +
 build-tools/src/bin/build-oozie.sh              |   6 +-
 build-tools/src/patches/OOZIE-1741.patch        | 397 +++++++++++++++++++
 .../workflow/OozieFeedWorkflowBuilder.java      |   2 +-
 .../converter/OozieFeedWorkflowBuilderTest.java |   2 +-
 .../workflow/OozieProcessWorkflowBuilder.java   |   2 +
 .../OozieProcessWorkflowBuilderTest.java        |   1 +
 7 files changed, 410 insertions(+), 3 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/5626b2dd/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 6b7d148..56c9ff6 100755
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -20,6 +20,9 @@ Trunk (Unreleased)
   OPTIMIZATIONS
 
   BUG FIXES
+   FALCON-357 HCatalog Feed replication: Hive export job fails when table partition 
+   contains multiple dated columns. (Satish Mittal via Shwetha GS)
+
    FALCON-495 multi source single target feed replication failing in regression.
    (Satish Mittal via Shwetha GS)
 

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/5626b2dd/build-tools/src/bin/build-oozie.sh
----------------------------------------------------------------------
diff --git a/build-tools/src/bin/build-oozie.sh b/build-tools/src/bin/build-oozie.sh
index 625613c..d7e99ba 100755
--- a/build-tools/src/bin/build-oozie.sh
+++ b/build-tools/src/bin/build-oozie.sh
@@ -79,6 +79,10 @@ case $VERSION in
     ;;
 4.0.0 )
     patch -p1 --verbose < ../../build-tools/src/patches/OOZIE-1551-4.0.patch
+    patch -p0 < ../../build-tools/src/patches/OOZIE-1741.patch
+    ;;
+4.0.1 )
+    patch -p0 < ../../build-tools/src/patches/OOZIE-1741.patch
     ;;
 esac
 
@@ -86,4 +90,4 @@ rm `find . -name 'pom.xml.bak'`
 
 $MVN_CMD clean install -DskipTests
 
-popd
\ No newline at end of file
+popd

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/5626b2dd/build-tools/src/patches/OOZIE-1741.patch
----------------------------------------------------------------------
diff --git a/build-tools/src/patches/OOZIE-1741.patch b/build-tools/src/patches/OOZIE-1741.patch
new file mode 100644
index 0000000..e69b2d9
--- /dev/null
+++ b/build-tools/src/patches/OOZIE-1741.patch
@@ -0,0 +1,397 @@
+diff --git core/src/main/java/org/apache/oozie/coord/HCatELFunctions.java core/src/main/java/org/apache/oozie/coord/HCatELFunctions.java
+index e5f0146..9a36af0 100644
+--- core/src/main/java/org/apache/oozie/coord/HCatELFunctions.java
++++ core/src/main/java/org/apache/oozie/coord/HCatELFunctions.java
+@@ -115,6 +115,12 @@ public class HCatELFunctions {
+         return echoUnResolved("dataOutPartitions", "'" + dataOutName + "'");
+     }
+
++    public static String ph1_coord_dataInPartitions_echo(String dataInName, String type)
{
++        // Checking if the dataIn/dataOut is correct?
++        isValidDataEvent(dataInName);
++        return echoUnResolved("dataInPartitions", "'" + dataInName + "', '" + type + "'");
++    }
++
+     public static String ph1_coord_dataOutPartitionValue_echo(String dataOutName, String
partition) {
+         // Checking if the dataIn/dataOut is correct?
+         isValidDataEvent(dataOutName);
+@@ -266,6 +272,47 @@ public class HCatELFunctions {
+     }
+
+     /**
++     * Used to specify the entire HCat partition defining input for workflow job. <p/>
Look for two evaluator-level
++     * variables <p/> A) .datain.<DATAIN_NAME> B) .datain.<DATAIN_NAME>.unresolved
<p/> A defines the data-in HCat URI.
++     * <p/> B defines whether there are any unresolved EL-function (i.e latest) <p/>
If there are something unresolved,
++     * this function will echo back the original function <p/> otherwise it sends
the partition.
++     *
++     * @param dataInName : DataIn name
++     * @param type : for action type: hive-export
++     */
++    public static String ph3_coord_dataInPartitions(String dataInName, String type) {
++        ELEvaluator eval = ELEvaluator.getCurrent();
++        String uri = (String) eval.getVariable(".datain." + dataInName);
++        Boolean unresolved = (Boolean) eval.getVariable(".datain." + dataInName + ".unresolved");
++        if (unresolved != null && unresolved.booleanValue() == true) {
++            return "${coord:dataInPartitions('" + dataInName + "', '" + type + "')}";
++        }
++        String partitionValue = null;
++        if (uri != null) {
++            if (type.equals("hive-export")) {
++                String[] uriList = uri.split(CoordELFunctions.DIR_SEPARATOR);
++                if (uriList.length > 1) {
++                    throw new RuntimeException("Multiple partitions not supported for hive-export
type. Dataset name: "
++                        + dataInName + " URI: " + uri);
++                }
++                try {
++                    partitionValue = new HCatURI(uri).toPartitionValueString(type);
++                }
++                catch (URISyntaxException e) {
++                    throw new RuntimeException("Parsing exception for HCatURI " + uri, e);
++                }
++            } else {
++                  throw new RuntimeException("Unsupported type: " + type + " dataset name:
" + dataInName);
++            }
++        }
++        else {
++            XLog.getLog(HCatELFunctions.class).warn("URI is null");
++            return null;
++        }
++        return partitionValue;
++    }
++
++    /**
+      * Used to specify the MAXIMUM value of an HCat partition which is input dependency
for workflow job.<p/> Look for two evaluator-level
+      * variables <p/> A) .datain.<DATAIN_NAME> B) .datain.<DATAIN_NAME>.unresolved
<p/> A defines the current list of
+      * HCat URIs. <p/> B defines whether there are any unresolved EL-function (i.e
latest) <p/> If there are something
+diff --git core/src/main/resources/oozie-default.xml core/src/main/resources/oozie-default.xml
+index 455ef9d..889f10d 100644
+--- core/src/main/resources/oozie-default.xml
++++ core/src/main/resources/oozie-default.xml
+@@ -837,6 +837,7 @@
+             coord:dataInPartitionFilter=org.apache.oozie.coord.HCatELFunctions#ph1_coord_dataInPartitionFilter_echo,
+             coord:dataInPartitionMin=org.apache.oozie.coord.HCatELFunctions#ph1_coord_dataInPartitionMin_echo,
+             coord:dataInPartitionMax=org.apache.oozie.coord.HCatELFunctions#ph1_coord_dataInPartitionMax_echo,
++            coord:dataInPartitions=org.apache.oozie.coord.HCatELFunctions#ph1_coord_dataInPartitions_echo,
+             coord:dataOutPartitions=org.apache.oozie.coord.HCatELFunctions#ph1_coord_dataOutPartitions_echo,
+             coord:dataOutPartitionValue=org.apache.oozie.coord.HCatELFunctions#ph1_coord_dataOutPartitionValue_echo
+         </value>
+@@ -1101,6 +1102,7 @@
+             coord:dataInPartitionFilter=org.apache.oozie.coord.HCatELFunctions#ph3_coord_dataInPartitionFilter,
+             coord:dataInPartitionMin=org.apache.oozie.coord.HCatELFunctions#ph3_coord_dataInPartitionMin,
+             coord:dataInPartitionMax=org.apache.oozie.coord.HCatELFunctions#ph3_coord_dataInPartitionMax,
++            coord:dataInPartitions=org.apache.oozie.coord.HCatELFunctions#ph3_coord_dataInPartitions,
+             coord:dataOutPartitions=org.apache.oozie.coord.HCatELFunctions#ph3_coord_dataOutPartitions,
+             coord:dataOutPartitionValue=org.apache.oozie.coord.HCatELFunctions#ph3_coord_dataOutPartitionValue
+         </value>
+diff --git core/src/test/java/org/apache/oozie/coord/TestHCatELFunctions.java core/src/test/java/org/apache/oozie/coord/TestHCatELFunctions.java
+index f46f1ec..fac2177 100644
+--- core/src/test/java/org/apache/oozie/coord/TestHCatELFunctions.java
++++ core/src/test/java/org/apache/oozie/coord/TestHCatELFunctions.java
+@@ -264,6 +264,38 @@ public class TestHCatELFunctions extends XHCatTestCase {
+     }
+
+     /**
++     * Test HCat dataInPartition EL function (phase 1) which echo back the EL
++     * function itself
++     *
++     * @throws Exception
++     */
++    @Test
++    public void testDataInPartitionsPh1() throws Exception {
++        init("coord-job-submit-data");
++        String expr = "${coord:dataInPartitions('ABC', 'hive-export')}";
++        // +ve test
++        eval.setVariable("oozie.dataname.ABC", "data-in");
++        assertEquals("${coord:dataInPartitions('ABC', 'hive-export')}", CoordELFunctions.evalAndWrap(eval,
expr));
++        // -ve test
++        expr = "${coord:dataInPartitions('ABCD', 'hive-export')}";
++        try {
++            CoordELFunctions.evalAndWrap(eval, expr);
++            fail("should throw exception because Data-in is not defined");
++        }
++        catch (Exception ex) {
++        }
++        // -ve test
++        expr = "${coord:dataInPartitions('ABCD')}";
++        eval.setVariable("oozie.dataname.ABCD", "data-in");
++        try {
++            CoordELFunctions.evalAndWrap(eval, expr);
++            fail("should throw exception because EL function requires 2 parameters");
++        }
++        catch (Exception ex) {
++        }
++    }
++
++    /**
+      * Test HCat dataOutPartition EL function (phase 1) which echo back the EL
+      * function itself
+      *
+@@ -463,6 +495,31 @@ public class TestHCatELFunctions extends XHCatTestCase {
+         assertTrue(res.equals("20"));
+     }
+
++    /**
++     * Test dataInPartitions EL function (phase 3) which returns the complete partition
value string of a single partition
++     * in case of hive-export type.
++     *
++     * @throws Exception
++     */
++    @Test
++    public void testDataInPartitions() throws Exception {
++        init("coord-action-start");
++        String expr = "${coord:dataInPartitions('ABC', 'hive-export')}";
++        eval.setVariable(".datain.ABC", "hcat://hcat.server.com:5080/mydb/clicks/datastamp=20120230;region=us");
++        eval.setVariable(".datain.ABC.unresolved", Boolean.FALSE);
++        String res = CoordELFunctions.evalAndWrap(eval, expr);
++        assertTrue(res.equals("datastamp='20120230',region='us'") || res.equals("region='us',datastamp='20120230'"));
++        // -ve test; execute EL function with any other type than hive-export
++        try {
++            expr = "${coord:dataInPartitions('ABC', 'invalid-type')}";
++            eval.setVariable(".datain.ABC", "hcat://hcat.server.com:5080/mydb/clicks/datastamp=20120230;region=us");
++            eval.setVariable(".datain.ABC.unresolved", Boolean.FALSE);
++            res = CoordELFunctions.evalAndWrap(eval, expr);
++            fail("EL function should throw exception because of invalid type");
++        } catch (Exception e) {
++        }
++    }
++
+     private void init(String tag) throws Exception {
+         init(tag, "hdfs://localhost:9000/user/" + getTestUser() + "/US/${YEAR}/${MONTH}/${DAY}");
+     }
+diff --git docs/src/site/twiki/CoordinatorFunctionalSpec.twiki docs/src/site/twiki/CoordinatorFunctionalSpec.twiki
+index a5ecbc5..621bd3d 100644
+--- docs/src/site/twiki/CoordinatorFunctionalSpec.twiki
++++ docs/src/site/twiki/CoordinatorFunctionalSpec.twiki
+@@ -2608,6 +2608,192 @@ C = foreach B generate foo, bar;
+ store C into 'myOutputDatabase.myOutputTable' using org.apache.hcatalog.pig.HCatStorer('region=APAC,datestamp=20090102');
+ </blockquote>
+
++---++++ 6.8.8 coord:dataInPartitions(String name, String type) EL function
++
++The =${coord:dataInPartitions(String name, String type)}= EL function resolves to a list
of partition key-value
++pairs for the input-event dataset. Currently the only type supported is 'hive-export'. The
'hive-export' type
++supports only one partition instance and it can be used to create the complete partition
value string that can
++be used in a hive query for partition export/import.
++
++The example below illustrates a hive export-import job triggered by a coordinator, using
the EL functions for HCat database,
++table, input partitions. The example replicates the hourly processed data across hive tables.
++
++*%GREEN% Example: %ENDCOLOR%*
++
++#HCatHiveExampleOne
++
++*Coordinator application definition:*
++
++<blockquote>
++    <coordinator-app xmlns="uri:oozie:coordinator:0.3" name="app-coord"
++    frequency="${coord:hours(1)}" start="2014-03-28T08:00Z"
++    end="2030-01-01T00:00Z" timezone="UTC">
++
++    <datasets>
++        <dataset name="Stats-1" frequency="${coord:hours(1)}"
++            initial-instance="2014-03-28T08:00Z" timezone="UTC">
++            <uri-template>hcat://foo:11002/myInputDatabase1/myInputTable1/year=${YEAR};month=${MONTH};day=${DAY};hour=${HOUR}
++            </uri-template>
++        </dataset>
++        <dataset name="Stats-2" frequency="${coord:hours(1)}"
++            initial-instance="2014-03-28T08:00Z" timezone="UTC">
++            <uri-template>hcat://foo:11002/myInputDatabase2/myInputTable2/year=${YEAR};month=${MONTH};day=${DAY};hour=${HOUR}
++            </uri-template>
++        </dataset>
++    </datasets>
++    <input-events>
++        <data-in name="processed-logs-1" dataset="Stats-1">
++            <instance>${coord:current(0)}</instance>
++        </data-in>
++    </input-events>
++    <output-events>
++        <data-out name="processed-logs-2" dataset="Stats-2">
++            <instance>${coord:current(0)}</instance>
++        </data-out>
++    </output-events>
++    <action>
++      <workflow>
++        <app-path>hdfs://bar:8020/usr/joe/logsreplicator-wf</app-path>
++        <configuration>
++          <property>
++            <name>EXPORT_DB</name>
++            <value>${coord:databaseIn('processed-logs-1')}</value>
++          </property>
++          <property>
++            <name>EXPORT_TABLE</name>
++            <value>${coord:tableIn('processed-logs-1')}</value>
++          </property>
++          <property>
++            <name>IMPORT_DB</name>
++            <value>${coord:databaseOut('processed-logs-2')}</value>
++          </property>
++          <property>
++            <name>IMPORT_TABLE</name>
++            <value>${coord:tableOut('processed-logs-2')}</value>
++          </property>
++          <property>
++            <name>EXPORT_PARTITION</name>
++            <value>${coord:dataInPartitions('processed-logs-1', 'hive-export')}</value>
++          </property>
++          <property>
++            <name>EXPORT_PATH</name>
++            <value>hdfs://bar:8020/staging/${coord:formatTime(coord:nominalTime(),
'yyyy-MM-dd-HH')}/data</value>
++          </property>
++        </configuration>
++      </workflow>
++    </action>
++</coordinator-app>
++</blockquote>
++
++Parameterizing the input/output databases and tables using the corresponding EL function
as shown will make them
++available in the hive action of the workflow 'logsreplicator-wf'.
++
++Each coordinator action will use as input events the hourly instances of the 'processed-logs-1'
dataset. The
++=${coord:dataInPartitions(String name, String type)}= function enables the coordinator application
to pass the
++partition corresponding to hourly dataset instances to the workflow job triggered by the
coordinator action.
++The workflow passes this partition value to the hive export script that exports the hourly
partition from source
++database to the staging location referred as =EXPORT_PATH=. The hive import script imports
the hourly partition from
++=EXPORT_PATH= staging location into the target database.
++
++#HCatWorkflow
++
++*Workflow definition:*
++
++<blockquote>
++<workflow-app xmlns="uri:oozie:workflow:0.3" name="logsreplicator-wf">
++    <start to="table-export"/>
++    <action name="table-export">
++        <hive:hive xmlns:hive="uri:oozie:hive-action:0.2" xmlns="uri:oozie:hive-action:0.2">
++            <job-tracker>${jobTracker}</job-tracker>
++            <name-node>${nameNode}</name-node>
++            <job-xml>${wf:appPath()}/conf/hive-site.xml</job-xml>
++            <configuration>
++                <property>
++                    <name>mapred.job.queue.name</name>
++                    <value>${queueName}</value>
++                </property>
++                <property>
++                    <name>oozie.launcher.mapred.job.priority</name>
++                    <value>${jobPriority}</value>
++                </property>
++            </configuration>
++            <script>${wf:appPath()}/scripts/table-export.hql</script>
++            <param>sourceDatabase=${EXPORT_DB}</param>
++            <param>sourceTable=${EXPORT_TABLE}</param>
++            <param>sourcePartition=${EXPORT_PARTITION}</param>
++            <param>sourceStagingDir=${EXPORT_PATH}</param>
++        </hive:hive>
++        <ok to="table-import"/>
++        <error to="fail"/>
++    </action>
++    <action name="table-import">
++        <hive:hive xmlns:hive="uri:oozie:hive-action:0.2" xmlns="uri:oozie:hive-action:0.2">
++            <job-tracker>${jobTracker}</job-tracker>
++            <name-node>${nameNode}</name-node>
++            <job-xml>${wf:appPath()}/conf/hive-site.xml</job-xml>
++            <configuration>
++                <property>
++                    <name>mapred.job.queue.name</name>
++                    <value>${queueName}</value>
++                </property>
++                <property>
++                    <name>oozie.launcher.mapred.job.priority</name>
++                    <value>${jobPriority}</value>
++                </property>
++            </configuration>
++            <script>${wf:appPath()}/scripts/table-import.hql</script>
++            <param>targetDatabase=${IMPORT_DB}</param>
++            <param>targetTable=${IMPORT_TABLE}</param>
++            <param>targetPartition=${EXPORT_PARTITION}</param>
++            <param>sourceStagingDir=${EXPORT_PATH}</param>
++        </hive:hive>
++        <ok to="end"/>
++        <error to="fail"/>
++    </action>
++    <kill name="fail">
++        <message>
++            Workflow failed, error message[${wf:errorMessage(wf:lastErrorNode())}]
++        </message>
++    </kill>
++    <end name="end"/>
++</workflow-app>
++</blockquote>
++
++Ensure that the following jars are in classpath, with versions corresponding to hcatalog
installation:
++hcatalog-core.jar, webhcat-java-client.jar, hive-common.jar, hive-exec.jar, hive-metastore.jar,
hive-serde.jar,
++ libfb303.jar. The hive-site.xml needs to be present in classpath as well.
++
++*Example Hive Export script:*
++The following script exports a particular Hive table partition into staging location, where
the partition value
++ is computed through =${coord:dataInPartitions(String name, String type)}= EL function.
++<blockquote>
++export table ${sourceDatabase}.${sourceTable} partition (${sourcePartition}) to '${sourceStagingDir}';
++</blockquote>
++
++For example, for the 2014-03-28T08:00Z run with the given dataset instances and ${coord:dataInPartitions(
++'processed-logs-1', 'hive-export'), the above Hive script with resolved values would look
like:
++<blockquote>
++export table myInputDatabase1/myInputTable1 partition (year='2014',month='03',day='28',hour='08')
to 'hdfs://bar:8020/staging/2014-03-28-08';
++</blockquote>
++
++*Example Hive Import script:*
++The following script imports a particular Hive table partition from staging location, where
the partition value is computed
++ through =${coord:dataInPartitions(String name, String type)}= EL function.
++<blockquote>
++use ${targetDatabase};
++alter table ${targetTable} drop if exists partition ${targetPartition};
++import table ${targetTable} partition (${targetPartition}) from '${sourceStagingDir}';
++</blockquote>
++
++For example, for the 2014-03-28T08:00Z run with the given dataset instances and ${coord:dataInPartitions(
++'processed-logs-2', 'hive-export'), the above Hive script with resolved values would look
like:
++
++<blockquote>
++use myInputDatabase2;
++alter table myInputTable2 drop if exists partition (year='2014',month='03',day='28',hour='08');
++import table myInputTable2 partition (year='2014',month='03',day='28',hour='08') from 'hdfs://bar:8020/staging/2014-03-28-08';
++</blockquote>
++
+
+ ---+++ 6.9. Parameterization of Coordinator Application
+
+diff --git sharelib/hcatalog/src/main/java/org/apache/oozie/util/HCatURI.java sharelib/hcatalog/src/main/java/org/apache/oozie/util/HCatURI.java
+index d797f9b..4bc5048 100644
+--- sharelib/hcatalog/src/main/java/org/apache/oozie/util/HCatURI.java
++++ sharelib/hcatalog/src/main/java/org/apache/oozie/util/HCatURI.java
+@@ -260,6 +260,35 @@ public class HCatURI {
+         return filter.toString();
+     }
+
++    /**
++     * Get the entire partition value string from partition map.
++     * In case of type hive-export, it can be used to create entire partition value string
++     * that can be used in Hive query for partition export/import.
++     *
++     * type hive-export
++     * @return partition value string
++     */
++    public String toPartitionValueString(String type) {
++        StringBuilder value = new StringBuilder();
++        if (type.equals("hive-export")) {
++            String comparator = "=";
++            String separator = ",";
++            for (Map.Entry<String, String> entry : partitions.entrySet()) {
++                if (value.length() > 1) {
++                    value.append(separator);
++                }
++                value.append(entry.getKey());
++                value.append(comparator);
++                value.append(PARTITION_VALUE_QUOTE);
++                value.append(entry.getValue());
++                value.append(PARTITION_VALUE_QUOTE);
++            }
++        } else {
++            throw new RuntimeException("Unsupported type: " + type);
++        }
++        return value.toString();
++    }
++
+     @Override
+     public String toString() {
+         StringBuilder sb = new StringBuilder();

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/5626b2dd/feed/src/main/java/org/apache/falcon/workflow/OozieFeedWorkflowBuilder.java
----------------------------------------------------------------------
diff --git a/feed/src/main/java/org/apache/falcon/workflow/OozieFeedWorkflowBuilder.java b/feed/src/main/java/org/apache/falcon/workflow/OozieFeedWorkflowBuilder.java
index 6d36840..4e300bf 100644
--- a/feed/src/main/java/org/apache/falcon/workflow/OozieFeedWorkflowBuilder.java
+++ b/feed/src/main/java/org/apache/falcon/workflow/OozieFeedWorkflowBuilder.java
@@ -622,7 +622,7 @@ public class OozieFeedWorkflowBuilder extends OozieWorkflowBuilder<Feed>
{
 
             props.put(prefix + "Database", tableStorage.getDatabase());
             props.put(prefix + "Table", tableStorage.getTable());
-            props.put(prefix + "Partition", "${coord:dataInPartitionFilter('input', 'hive')}");
+            props.put(prefix + "Partition", "(${coord:dataInPartitions('input', 'hive-export')})");
         }
 
         private void setupHiveConfiguration(Cluster srcCluster, Cluster trgCluster,

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/5626b2dd/feed/src/test/java/org/apache/falcon/converter/OozieFeedWorkflowBuilderTest.java
----------------------------------------------------------------------
diff --git a/feed/src/test/java/org/apache/falcon/converter/OozieFeedWorkflowBuilderTest.java
b/feed/src/test/java/org/apache/falcon/converter/OozieFeedWorkflowBuilderTest.java
index 5d6879a..d793e65 100644
--- a/feed/src/test/java/org/apache/falcon/converter/OozieFeedWorkflowBuilderTest.java
+++ b/feed/src/test/java/org/apache/falcon/converter/OozieFeedWorkflowBuilderTest.java
@@ -537,7 +537,7 @@ public class OozieFeedWorkflowBuilderTest {
 
         Assert.assertEquals(props.get(prefix + "Database"), tableStorage.getDatabase());
         Assert.assertEquals(props.get(prefix + "Table"), tableStorage.getTable());
-        Assert.assertEquals(props.get(prefix + "Partition"), "${coord:dataInPartitionFilter('input',
'hive')}");
+        Assert.assertEquals(props.get(prefix + "Partition"), "(${coord:dataInPartitions('input',
'hive-export')})");
     }
 
     @Test

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/5626b2dd/process/src/main/java/org/apache/falcon/workflow/OozieProcessWorkflowBuilder.java
----------------------------------------------------------------------
diff --git a/process/src/main/java/org/apache/falcon/workflow/OozieProcessWorkflowBuilder.java
b/process/src/main/java/org/apache/falcon/workflow/OozieProcessWorkflowBuilder.java
index 70aeebd..3751f95 100644
--- a/process/src/main/java/org/apache/falcon/workflow/OozieProcessWorkflowBuilder.java
+++ b/process/src/main/java/org/apache/falcon/workflow/OozieProcessWorkflowBuilder.java
@@ -545,6 +545,8 @@ public class OozieProcessWorkflowBuilder extends OozieWorkflowBuilder<Process>
{
             "${coord:dataInPartitionFilter('" + input.getName() + "', 'hive')}");
         props.put(prefix + "_partition_filter_java",
             "${coord:dataInPartitionFilter('" + input.getName() + "', 'java')}");
+        props.put(prefix + "_datain_partitions_hive",
+            "${coord:dataInPartitions('" + input.getName() + "', 'hive-export')}");
     }
 
     private void propagateCatalogTableProperties(Output output, CatalogStorage tableStorage,

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/5626b2dd/process/src/test/java/org/apache/falcon/converter/OozieProcessWorkflowBuilderTest.java
----------------------------------------------------------------------
diff --git a/process/src/test/java/org/apache/falcon/converter/OozieProcessWorkflowBuilderTest.java
b/process/src/test/java/org/apache/falcon/converter/OozieProcessWorkflowBuilderTest.java
index 2522ca3..1eeadaf 100644
--- a/process/src/test/java/org/apache/falcon/converter/OozieProcessWorkflowBuilderTest.java
+++ b/process/src/test/java/org/apache/falcon/converter/OozieProcessWorkflowBuilderTest.java
@@ -617,6 +617,7 @@ public class OozieProcessWorkflowBuilderTest extends AbstractTestBase
{
             props.put(prefix + "_partition_filter_pig", "${coord:dataInPartitionFilter('input',
'pig')}");
             props.put(prefix + "_partition_filter_hive", "${coord:dataInPartitionFilter('input',
'hive')}");
             props.put(prefix + "_partition_filter_java", "${coord:dataInPartitionFilter('input',
'java')}");
+            props.put(prefix + "_datain_partitions_hive", "${coord:dataInPartitions('input',
'hive-export')}");
         } else if (prefix.equals("falcon_output")) {
             props.put(prefix + "_dataout_partitions", "${coord:dataOutPartitions('output')}");
         }


Mime
View raw message