falcon-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From srik...@apache.org
Subject git commit: FALCON-91 Handle feed updates in purview of tables. Contributed by Venkatesh Seetharam
Date Thu, 17 Oct 2013 17:31:42 GMT
Updated Branches:
  refs/heads/FALCON-85 fa41a1e78 -> f28977729


FALCON-91 Handle feed updates in purview of tables. Contributed by Venkatesh Seetharam


Project: http://git-wip-us.apache.org/repos/asf/incubator-falcon/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-falcon/commit/f2897772
Tree: http://git-wip-us.apache.org/repos/asf/incubator-falcon/tree/f2897772
Diff: http://git-wip-us.apache.org/repos/asf/incubator-falcon/diff/f2897772

Branch: refs/heads/FALCON-85
Commit: f2897772955f9ccd82beba3acac768800880f9ef
Parents: fa41a1e
Author: srikanth.sundarrajan <srikanth.sundarrajan@inmobi.com>
Authored: Thu Oct 17 23:01:15 2013 +0530
Committer: srikanth.sundarrajan <srikanth.sundarrajan@inmobi.com>
Committed: Thu Oct 17 23:01:15 2013 +0530

----------------------------------------------------------------------
 CHANGES.txt                                     |  3 ++
 .../apache/falcon/update/UpdateHelperTest.java  | 33 ++++++++++++++++++++
 .../resources/config/feed/hive-table-feed.xml   |  2 +-
 .../resources/config/process/process-table.xml  |  2 +-
 4 files changed, 38 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/f2897772/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 64961c1..405631f 100755
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -5,6 +5,9 @@ Trunk (Unreleased)
   INCOMPATIBLE CHANGES
 
   NEW FEATURES
+    FALCON-91 Handle feed updates in purview of tables. (Venkatesh
+    Seetharam via Srikanth Sundarrajan) 
+
     FALCON-104 FALCON-86 introduces a backward incompatible change.
     (Venkatesh Seetharam via Srikanth Sundarrajan)
 

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/f2897772/common/src/test/java/org/apache/falcon/update/UpdateHelperTest.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/falcon/update/UpdateHelperTest.java b/common/src/test/java/org/apache/falcon/update/UpdateHelperTest.java
index e8f80bd..42bcee0 100644
--- a/common/src/test/java/org/apache/falcon/update/UpdateHelperTest.java
+++ b/common/src/test/java/org/apache/falcon/update/UpdateHelperTest.java
@@ -27,6 +27,7 @@ import org.apache.falcon.entity.parser.ProcessEntityParser;
 import org.apache.falcon.entity.v0.EntityType;
 import org.apache.falcon.entity.v0.Frequency;
 import org.apache.falcon.entity.v0.SchemaHelper;
+import org.apache.falcon.entity.v0.feed.CatalogTable;
 import org.apache.falcon.entity.v0.feed.Feed;
 import org.apache.falcon.entity.v0.feed.Location;
 import org.apache.falcon.entity.v0.feed.LocationType;
@@ -42,6 +43,8 @@ import org.testng.annotations.BeforeClass;
 import org.testng.annotations.BeforeMethod;
 import org.testng.annotations.Test;
 
+import java.io.InputStream;
+
 /**
  * Test for Update helper methods.
  */
@@ -157,6 +160,36 @@ public class UpdateHelperTest extends AbstractTestBase {
         Assert.assertFalse(UpdateHelper.shouldUpdate(oldFeed, newFeed, process));
     }
 
+    @Test
+    public void testShouldUpdateTable() throws Exception {
+        InputStream inputStream = getClass().getResourceAsStream("/config/feed/hive-table-feed.xml");
+        Feed oldTableFeed = (Feed) EntityType.FEED.getUnmarshaller().unmarshal(inputStream);
+        getStore().publish(EntityType.FEED, oldTableFeed);
+
+        String cluster = "testCluster";
+        Feed newTableFeed = (Feed) oldTableFeed.copy();
+        Assert.assertFalse(UpdateHelper.shouldUpdate(oldTableFeed, newTableFeed, cluster));
+
+        newTableFeed.setGroups("newgroups");
+        Assert.assertFalse(UpdateHelper.shouldUpdate(oldTableFeed, newTableFeed, cluster));
+        newTableFeed.setFrequency(Frequency.fromString("days(1)"));
+        Assert.assertTrue(UpdateHelper.shouldUpdate(oldTableFeed, newTableFeed, cluster));
+
+        final CatalogTable table = new CatalogTable();
+        table.setUri("catalog:default:clicks-blah#ds=${YEAR}-${MONTH}-${DAY}-${HOUR}");
+        newTableFeed.setTable(table);
+        Assert.assertTrue(UpdateHelper.shouldUpdate(oldTableFeed, newTableFeed, cluster));
+
+        inputStream = getClass().getResourceAsStream("/config/process/process-table.xml");
+        Process oldProcess = (Process) EntityType.PROCESS.getUnmarshaller().unmarshal(inputStream);
+        Process newProcess = (Process) oldProcess.copy();
+
+        newProcess.getRetry().setPolicy(PolicyType.FINAL);
+        Assert.assertFalse(UpdateHelper.shouldUpdate(oldProcess, newProcess, cluster));
+        newProcess.setFrequency(Frequency.fromString("days(1)"));
+        Assert.assertTrue(UpdateHelper.shouldUpdate(oldProcess, newProcess, cluster));
+    }
+
     private static Location getLocation(Feed feed, LocationType type) {
         return getLocation(feed.getLocations(), type);
     }

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/f2897772/common/src/test/resources/config/feed/hive-table-feed.xml
----------------------------------------------------------------------
diff --git a/common/src/test/resources/config/feed/hive-table-feed.xml b/common/src/test/resources/config/feed/hive-table-feed.xml
index ef936d2..67b3f4e 100644
--- a/common/src/test/resources/config/feed/hive-table-feed.xml
+++ b/common/src/test/resources/config/feed/hive-table-feed.xml
@@ -16,7 +16,7 @@
   See the License for the specific language governing permissions and
   limitations under the License.
   -->
-<feed description="clicks log" name="clicks" xmlns="uri:falcon:feed:0.1">
+<feed description="clicks log table" name="clicks-table" xmlns="uri:falcon:feed:0.1">
     <groups>online,bi</groups>
 
     <frequency>hours(1)</frequency>

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/f2897772/common/src/test/resources/config/process/process-table.xml
----------------------------------------------------------------------
diff --git a/common/src/test/resources/config/process/process-table.xml b/common/src/test/resources/config/process/process-table.xml
index 9273b90..1d6a8f0 100644
--- a/common/src/test/resources/config/process/process-table.xml
+++ b/common/src/test/resources/config/process/process-table.xml
@@ -32,7 +32,7 @@
 
     <!-- what -->
     <inputs>
-        <input name="input" feed="clicks" start="today(0,0)" end="today(20,0)"/>
+        <input name="input" feed="clicks-table" start="today(0,0)" end="today(20,0)"/>
     </inputs>
 
     <outputs>


Mime
View raw message