falcon-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ajayyad...@apache.org
Subject falcon git commit: FALCON-1417 Make validity end date optional for feed / process. Contributed by Pragya Mittal.
Date Tue, 22 Sep 2015 14:45:42 GMT
Repository: falcon
Updated Branches:
  refs/heads/master 55bcc14af -> 75079c2e5


FALCON-1417 Make validity end date optional for feed / process. Contributed by Pragya Mittal.


Project: http://git-wip-us.apache.org/repos/asf/falcon/repo
Commit: http://git-wip-us.apache.org/repos/asf/falcon/commit/75079c2e
Tree: http://git-wip-us.apache.org/repos/asf/falcon/tree/75079c2e
Diff: http://git-wip-us.apache.org/repos/asf/falcon/diff/75079c2e

Branch: refs/heads/master
Commit: 75079c2e5166ef517e500109f0015fff6df47b90
Parents: 55bcc14
Author: Ajay Yadava <ajaynsit@gmail.com>
Authored: Tue Sep 22 19:52:49 2015 +0530
Committer: Ajay Yadava <ajaynsit@gmail.com>
Committed: Tue Sep 22 19:52:49 2015 +0530

----------------------------------------------------------------------
 CHANGES.txt                                      |  2 ++
 .../entity/parser/CrossEntityValidations.java    |  7 +++++++
 .../falcon/entity/parser/FeedEntityParser.java   |  7 +++++++
 .../entity/parser/ProcessEntityParser.java       |  7 +++++++
 .../java/org/apache/falcon/util/DateUtil.java    |  4 ++++
 .../entity/parser/FeedEntityParserTest.java      |  8 ++++++++
 .../entity/parser/ProcessEntityParserTest.java   | 19 +++++++++++++++++++
 7 files changed, 54 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/falcon/blob/75079c2e/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 445b56f..6836690 100755
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -9,6 +9,8 @@ Trunk (Unreleased)
     FALCON-1027 Falcon proxy user support(Sowmya Ramesh)
 
   IMPROVEMENTS
+    FALCON-1417 Make validity end date optional for feed / process(Pragya Mittal via Ajay
Yadava)
+
     FALCON-1434 Enhance schedule API to accept key-value properties(Pallavi Rao)   
 
     FALCON-1426 Add documentation for Falcon Email notification (Peeyush Bishnoi via Pallavi
Rao)

http://git-wip-us.apache.org/repos/asf/falcon/blob/75079c2e/common/src/main/java/org/apache/falcon/entity/parser/CrossEntityValidations.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/falcon/entity/parser/CrossEntityValidations.java
b/common/src/main/java/org/apache/falcon/entity/parser/CrossEntityValidations.java
index 2696552..18ae754 100644
--- a/common/src/main/java/org/apache/falcon/entity/parser/CrossEntityValidations.java
+++ b/common/src/main/java/org/apache/falcon/entity/parser/CrossEntityValidations.java
@@ -28,6 +28,7 @@ import org.apache.falcon.entity.v0.process.Output;
 import org.apache.falcon.entity.v0.process.Process;
 import org.apache.falcon.entity.v0.process.Validity;
 import org.apache.falcon.expression.ExpressionHelper;
+import org.apache.falcon.util.DateUtil;
 
 import java.util.Date;
 
@@ -45,6 +46,12 @@ public final class CrossEntityValidations {
                 String clusterName = cluster.getName();
                 org.apache.falcon.entity.v0.feed.Validity feedValidity = FeedHelper.getCluster(feed,
                         clusterName).getValidity();
+
+                // Optinal end_date
+                if (feedValidity.getEnd() == null) {
+                    feedValidity.setEnd(DateUtil.NEVER);
+                }
+
                 Date feedStart = feedValidity.getStart();
                 Date feedEnd = feedValidity.getEnd();
 

http://git-wip-us.apache.org/repos/asf/falcon/blob/75079c2e/common/src/main/java/org/apache/falcon/entity/parser/FeedEntityParser.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/falcon/entity/parser/FeedEntityParser.java b/common/src/main/java/org/apache/falcon/entity/parser/FeedEntityParser.java
index 992fc51..4f5599e 100644
--- a/common/src/main/java/org/apache/falcon/entity/parser/FeedEntityParser.java
+++ b/common/src/main/java/org/apache/falcon/entity/parser/FeedEntityParser.java
@@ -47,6 +47,7 @@ import org.apache.falcon.entity.v0.process.Process;
 import org.apache.falcon.expression.ExpressionHelper;
 import org.apache.falcon.group.FeedGroup;
 import org.apache.falcon.group.FeedGroupMap;
+import org.apache.falcon.util.DateUtil;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.security.authorize.AuthorizationException;
 import org.slf4j.Logger;
@@ -82,6 +83,12 @@ public class FeedEntityParser extends EntityParser<Feed> {
         validateACL(feed);
         for (Cluster cluster : feed.getClusters().getClusters()) {
             validateEntityExists(EntityType.CLUSTER, cluster.getName());
+
+            // Optinal end_date
+            if (cluster.getValidity().getEnd() == null) {
+                cluster.getValidity().setEnd(DateUtil.NEVER);
+            }
+
             validateClusterValidity(cluster.getValidity().getStart(), cluster.getValidity().getEnd(),
                     cluster.getName());
             validateClusterHasRegistry(feed, cluster);

http://git-wip-us.apache.org/repos/asf/falcon/blob/75079c2e/common/src/main/java/org/apache/falcon/entity/parser/ProcessEntityParser.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/falcon/entity/parser/ProcessEntityParser.java
b/common/src/main/java/org/apache/falcon/entity/parser/ProcessEntityParser.java
index 48a4286..10a5265 100644
--- a/common/src/main/java/org/apache/falcon/entity/parser/ProcessEntityParser.java
+++ b/common/src/main/java/org/apache/falcon/entity/parser/ProcessEntityParser.java
@@ -40,6 +40,7 @@ import org.apache.falcon.entity.v0.process.Outputs;
 import org.apache.falcon.entity.v0.process.Process;
 import org.apache.falcon.expression.ExpressionHelper;
 import org.apache.falcon.hadoop.HadoopClientFactory;
+import org.apache.falcon.util.DateUtil;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
@@ -79,6 +80,12 @@ public class ProcessEntityParser extends EntityParser<Process> {
                         + " is defined more than once for process: " + process.getName());
             }
             validateEntityExists(EntityType.CLUSTER, clusterName);
+
+            // Optinal end_date
+            if (cluster.getValidity().getEnd() == null) {
+                cluster.getValidity().setEnd(DateUtil.NEVER);
+            }
+
             validateProcessValidity(cluster.getValidity().getStart(), cluster.getValidity().getEnd());
             validateHDFSPaths(process, clusterName);
             validateProperties(process);

http://git-wip-us.apache.org/repos/asf/falcon/blob/75079c2e/common/src/main/java/org/apache/falcon/util/DateUtil.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/falcon/util/DateUtil.java b/common/src/main/java/org/apache/falcon/util/DateUtil.java
index e736340..f89ef64 100644
--- a/common/src/main/java/org/apache/falcon/util/DateUtil.java
+++ b/common/src/main/java/org/apache/falcon/util/DateUtil.java
@@ -27,6 +27,9 @@ import java.util.TimeZone;
  */
 public final class DateUtil {
 
+    //Friday, April 16, 9999 7:12:55 AM UTC corresponding date
+    public static final Date NEVER = new Date(Long.parseLong("253379862775000"));
+
     private DateUtil() {}
 
     public static Date getNextMinute(Date time) throws Exception {
@@ -36,4 +39,5 @@ public final class DateUtil {
         insCal.add(Calendar.MINUTE, 1);
         return insCal.getTime();
     }
+
 }

http://git-wip-us.apache.org/repos/asf/falcon/blob/75079c2e/common/src/test/java/org/apache/falcon/entity/parser/FeedEntityParserTest.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/falcon/entity/parser/FeedEntityParserTest.java
b/common/src/test/java/org/apache/falcon/entity/parser/FeedEntityParserTest.java
index 1e9b72f..b6fdb13 100644
--- a/common/src/test/java/org/apache/falcon/entity/parser/FeedEntityParserTest.java
+++ b/common/src/test/java/org/apache/falcon/entity/parser/FeedEntityParserTest.java
@@ -974,4 +974,12 @@ public class FeedEntityParserTest extends AbstractTestBase {
         }
     }
 
+    @Test
+    public void testFeedEndTimeOptional() throws Exception {
+        Feed feed = parser.parseAndValidate(ProcessEntityParserTest.class
+                .getResourceAsStream(FEED_XML));
+        feed.getClusters().getClusters().get(0).getValidity().setEnd(null);
+        parser.validate(feed);
+    }
+
 }

http://git-wip-us.apache.org/repos/asf/falcon/blob/75079c2e/common/src/test/java/org/apache/falcon/entity/parser/ProcessEntityParserTest.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/falcon/entity/parser/ProcessEntityParserTest.java
b/common/src/test/java/org/apache/falcon/entity/parser/ProcessEntityParserTest.java
index a935db3..da22d56 100644
--- a/common/src/test/java/org/apache/falcon/entity/parser/ProcessEntityParserTest.java
+++ b/common/src/test/java/org/apache/falcon/entity/parser/ProcessEntityParserTest.java
@@ -21,6 +21,7 @@ package org.apache.falcon.entity.parser;
 import org.apache.falcon.FalconException;
 import org.apache.falcon.cluster.util.EmbeddedCluster;
 import org.apache.falcon.entity.AbstractTestBase;
+import org.apache.falcon.entity.EntityUtil;
 import org.apache.falcon.entity.v0.EntityType;
 import org.apache.falcon.entity.v0.Frequency;
 import org.apache.falcon.entity.v0.SchemaHelper;
@@ -601,4 +602,22 @@ public class ProcessEntityParserTest extends AbstractTestBase {
         }
     }
 
+    @Test
+    public void testProcessEndTimeOptional() throws FalconException {
+        Process process = parser.parseAndValidate((ProcessEntityParserTest.class
+                .getResourceAsStream(PROCESS_XML)));
+        process.getClusters().getClusters().get(0).getValidity().setEnd(null);
+        parser.validate(process);
+    }
+
+    @Test
+    public void testProcessEndTime() throws FalconException {
+        Process process = parser.parseAndValidate((ProcessEntityParserTest.class
+                .getResourceAsStream(PROCESS_XML)));
+        String feedName = process.getInputs().getInputs().get(0).getFeed();
+        Feed feedEntity = EntityUtil.getEntity(EntityType.FEED, feedName);
+        feedEntity.getClusters().getClusters().get(0).getValidity().setEnd(null);
+        process.getClusters().getClusters().get(0).getValidity().setEnd(null);
+        parser.validate(process);
+    }
 }


Mime
View raw message