falcon-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From b...@apache.org
Subject falcon git commit: FALCON-1881 Database export does not require fields tag in entity specification
Date Wed, 06 Apr 2016 20:40:07 GMT
Repository: falcon
Updated Branches:
  refs/heads/master 14b1bb8f4 -> fa66c17e9


FALCON-1881 Database export does not require fields tag in entity specification

Database export does not require fields tag in entity specification

Author: Venkatesan Ramachandran <vramachandran@hortonworks.com>

Reviewers: "Peeyush Bishnoi <peeyushb@apache.org>, Pallavi Rao <pallavi.rao@inmobi.com>,
Balu Vellanki <bvellanki@hortonworks.com>"

Closes #83 from vramachan/FALCON-1881.20160331-1.ExportFieldsList


Project: http://git-wip-us.apache.org/repos/asf/falcon/repo
Commit: http://git-wip-us.apache.org/repos/asf/falcon/commit/fa66c17e
Tree: http://git-wip-us.apache.org/repos/asf/falcon/tree/fa66c17e
Diff: http://git-wip-us.apache.org/repos/asf/falcon/diff/fa66c17e

Branch: refs/heads/master
Commit: fa66c17e967359c519c59717f52a705297595d3f
Parents: 14b1bb8
Author: Venkatesan Ramachandran <vramachandran@hortonworks.com>
Authored: Wed Apr 6 13:39:42 2016 -0700
Committer: bvellanki <bvellanki@hortonworks.com>
Committed: Wed Apr 6 13:39:42 2016 -0700

----------------------------------------------------------------------
 .../falcon/entity/parser/FeedEntityParser.java  | 16 +++--
 .../entity/parser/FeedEntityParserTest.java     |  4 +-
 .../feed/feed-export-exclude-fields-0.1.xml     | 66 --------------------
 .../config/feed/feed-export-fields-0.1.xml      | 66 ++++++++++++++++++++
 .../test/resources/feed-export-template6.xml    |  6 --
 5 files changed, 79 insertions(+), 79 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/falcon/blob/fa66c17e/common/src/main/java/org/apache/falcon/entity/parser/FeedEntityParser.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/falcon/entity/parser/FeedEntityParser.java b/common/src/main/java/org/apache/falcon/entity/parser/FeedEntityParser.java
index c942862..2a9a852 100644
--- a/common/src/main/java/org/apache/falcon/entity/parser/FeedEntityParser.java
+++ b/common/src/main/java/org/apache/falcon/entity/parser/FeedEntityParser.java
@@ -110,7 +110,7 @@ public class FeedEntityParser extends EntityParser<Feed> {
             if (FeedHelper.isExportEnabled(cluster)) {
                 validateEntityExists(EntityType.DATASOURCE, FeedHelper.getExportDatasourceName(cluster));
                 validateFeedExportArgs(cluster);
-                validateFeedExportFieldExcludes(cluster);
+                validateFeedExportFields(cluster);
             }
         }
 
@@ -709,10 +709,16 @@ public class FeedEntityParser extends EntityParser<Feed> {
         }
     }
 
-    private void validateFeedExportFieldExcludes(Cluster feedCluster) throws FalconException
{
-        if (FeedHelper.isFieldExcludes(feedCluster.getExport().getTarget())) {
-            throw new ValidationException(String.format("Field excludes are not supported
"
-                    + "currently in Feed import policy"));
+    /**
+     * Export infers the target fields from the destination. There is no need to enumerate
or exclude the fields
+     * in the feed entity definition.
+     *
+     * @param feedCluster feed's cluster
+     * @throws FalconException
+     */
+    private void validateFeedExportFields(Cluster feedCluster) throws FalconException {
+        if (feedCluster.getExport().getTarget().getFields() != null) {
+            throw new ValidationException(String.format("Feed Export does not expect Fields
specification"));
         }
     }
 

http://git-wip-us.apache.org/repos/asf/falcon/blob/fa66c17e/common/src/test/java/org/apache/falcon/entity/parser/FeedEntityParserTest.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/falcon/entity/parser/FeedEntityParserTest.java
b/common/src/test/java/org/apache/falcon/entity/parser/FeedEntityParserTest.java
index 382a8c4..ceec3c4 100644
--- a/common/src/test/java/org/apache/falcon/entity/parser/FeedEntityParserTest.java
+++ b/common/src/test/java/org/apache/falcon/entity/parser/FeedEntityParserTest.java
@@ -1213,9 +1213,9 @@ public class FeedEntityParserTest extends AbstractTestBase {
     public void testExportFeedSqoopExcludeFields() throws Exception {
 
         storeEntity(EntityType.CLUSTER, "testCluster");
-        InputStream feedStream = this.getClass().getResourceAsStream("/config/feed/feed-export-exclude-fields-0.1.xml");
+        InputStream feedStream = this.getClass().getResourceAsStream("/config/feed/feed-export-fields-0.1.xml");
         Feed feed = parser.parseAndValidate(feedStream);
-        Assert.fail("An exception should have been thrown: Feed Export policy not yet implement
Field exclusion.");
+        Assert.fail("An exception should have been thrown: Feed Export policy does not require
Fields specification.");
     }
 
     @Test (expectedExceptions = ValidationException.class)

http://git-wip-us.apache.org/repos/asf/falcon/blob/fa66c17e/common/src/test/resources/config/feed/feed-export-exclude-fields-0.1.xml
----------------------------------------------------------------------
diff --git a/common/src/test/resources/config/feed/feed-export-exclude-fields-0.1.xml b/common/src/test/resources/config/feed/feed-export-exclude-fields-0.1.xml
deleted file mode 100644
index 6753a00..0000000
--- a/common/src/test/resources/config/feed/feed-export-exclude-fields-0.1.xml
+++ /dev/null
@@ -1,66 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
-  Licensed to the Apache Software Foundation (ASF) under one
-  or more contributor license agreements.  See the NOTICE file
-  distributed with this work for additional information
-  regarding copyright ownership.  The ASF licenses this file
-  to you under the Apache License, Version 2.0 (the
-  "License"); you may not use this file except in compliance
-  with the License.  You may obtain a copy of the License at
-
-      http://www.apache.org/licenses/LICENSE-2.0
-
-  Unless required by applicable law or agreed to in writing, software
-  distributed under the License is distributed on an "AS IS" BASIS,
-  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  See the License for the specific language governing permissions and
-  limitations under the License.
-  -->
-<feed description="Customer data" name="CustomerFeed" xmlns="uri:falcon:feed:0.1">
-    <tags>consumer=consumer@xyz.com, owner=producer@xyz.com, _department_type=forecasting</tags>
-    <partitions>
-        <partition name="fraud"/>
-        <partition name="good"/>
-    </partitions>
-
-    <groups>online,bi</groups>
-    <availabilityFlag>_SUCCESS</availabilityFlag>
-
-    <frequency>hours(1)</frequency>
-    <sla slaLow="hours(2)" slaHigh="hours(3)"/>
-    <timezone>UTC</timezone>
-
-    <late-arrival cut-off="hours(6)"/>
-
-    <clusters>
-        <cluster name="testCluster" type="source">
-            <validity start="2011-11-01T00:00Z" end="2011-12-31T00:00Z"/>
-            <retention limit="hours(48)" action="delete"/>
-            <!-- Limit can be in Time or Instances 100, Action ENUM DELETE,ARCHIVE -->
-            <sla slaLow="hours(3)" slaHigh="hours(4)"/>
-            <export>
-                <target name="test-hsql-db" tableName="customer">
-                    <load type="updateonly"/>
-                    <fields>
-                        <excludes>
-                            <field>id</field>
-                            <field>name</field>
-                        </excludes>
-                    </fields>
-                </target>
-                <arguments>
-                    <argument name="--update-key" value="id"/>
-                </arguments>
-            </export>
-        </cluster>
-    </clusters>
-
-    <locations>
-        <location type="data" path="/projects/falcon/clicks"/>
-        <location type="stats" path="/projects/falcon/clicksStats"/>
-        <location type="meta" path="/projects/falcon/clicksMetaData"/>
-    </locations>
-
-    <ACL owner="testuser" group="group" permission="0x755"/>
-    <schema location="/schema/clicks" provider="protobuf"/>
-</feed>

http://git-wip-us.apache.org/repos/asf/falcon/blob/fa66c17e/common/src/test/resources/config/feed/feed-export-fields-0.1.xml
----------------------------------------------------------------------
diff --git a/common/src/test/resources/config/feed/feed-export-fields-0.1.xml b/common/src/test/resources/config/feed/feed-export-fields-0.1.xml
new file mode 100644
index 0000000..8fda43d
--- /dev/null
+++ b/common/src/test/resources/config/feed/feed-export-fields-0.1.xml
@@ -0,0 +1,66 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  Licensed to the Apache Software Foundation (ASF) under one
+  or more contributor license agreements.  See the NOTICE file
+  distributed with this work for additional information
+  regarding copyright ownership.  The ASF licenses this file
+  to you under the Apache License, Version 2.0 (the
+  "License"); you may not use this file except in compliance
+  with the License.  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+  -->
+<feed description="Customer data" name="CustomerFeed" xmlns="uri:falcon:feed:0.1">
+    <tags>consumer=consumer@xyz.com, owner=producer@xyz.com, _department_type=forecasting</tags>
+    <partitions>
+        <partition name="fraud"/>
+        <partition name="good"/>
+    </partitions>
+
+    <groups>online,bi</groups>
+    <availabilityFlag>_SUCCESS</availabilityFlag>
+
+    <frequency>hours(1)</frequency>
+    <sla slaLow="hours(2)" slaHigh="hours(3)"/>
+    <timezone>UTC</timezone>
+
+    <late-arrival cut-off="hours(6)"/>
+
+    <clusters>
+        <cluster name="testCluster" type="source">
+            <validity start="2011-11-01T00:00Z" end="2011-12-31T00:00Z"/>
+            <retention limit="hours(48)" action="delete"/>
+            <!-- Limit can be in Time or Instances 100, Action ENUM DELETE,ARCHIVE -->
+            <sla slaLow="hours(3)" slaHigh="hours(4)"/>
+            <export>
+                <target name="test-hsql-db" tableName="customer">
+                    <load type="updateonly"/>
+                    <fields>
+                        <includes>
+                            <field>id</field>
+                            <field>name</field>
+                        </includes>
+                    </fields>
+                </target>
+                <arguments>
+                    <argument name="--update-key" value="id"/>
+                </arguments>
+            </export>
+        </cluster>
+    </clusters>
+
+    <locations>
+        <location type="data" path="/projects/falcon/clicks"/>
+        <location type="stats" path="/projects/falcon/clicksStats"/>
+        <location type="meta" path="/projects/falcon/clicksMetaData"/>
+    </locations>
+
+    <ACL owner="testuser" group="group" permission="0x755"/>
+    <schema location="/schema/clicks" provider="protobuf"/>
+</feed>

http://git-wip-us.apache.org/repos/asf/falcon/blob/fa66c17e/webapp/src/test/resources/feed-export-template6.xml
----------------------------------------------------------------------
diff --git a/webapp/src/test/resources/feed-export-template6.xml b/webapp/src/test/resources/feed-export-template6.xml
index 0eb748b..0534257 100644
--- a/webapp/src/test/resources/feed-export-template6.xml
+++ b/webapp/src/test/resources/feed-export-template6.xml
@@ -31,12 +31,6 @@
             <export>
                 <target name="##datasourcename##" tableName="simple_export">
                 <load type="allowinsert"/>
-                <fields>
-                    <includes>
-                        <field>id</field>
-                        <field>name</field>
-                    </includes>
-                </fields>
             </target>
             <arguments>
                 <argument name="--update-key" value="id"/>


Mime
View raw message