falcon-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From venkat...@apache.org
Subject [4/4] git commit: FALCON-359 Instance id's captured are of different formats in process and feed. Contributed by Venkatesh Seetharam
Date Wed, 19 Mar 2014 23:14:36 GMT
FALCON-359 Instance id's captured are of different formats in process and feed. Contributed
by Venkatesh Seetharam


Project: http://git-wip-us.apache.org/repos/asf/incubator-falcon/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-falcon/commit/652e4a15
Tree: http://git-wip-us.apache.org/repos/asf/incubator-falcon/tree/652e4a15
Diff: http://git-wip-us.apache.org/repos/asf/incubator-falcon/diff/652e4a15

Branch: refs/heads/master
Commit: 652e4a15702cb273485b2c480cc758ce9967f089
Parents: abc97fb
Author: Venkatesh Seetharam <venkatesh@hortonworks.com>
Authored: Wed Mar 19 11:07:17 2014 -0700
Committer: Venkatesh Seetharam <venkatesh@hortonworks.com>
Committed: Wed Mar 19 11:07:17 2014 -0700

----------------------------------------------------------------------
 CHANGES.txt                                          |  3 +++
 .../org/apache/falcon/entity/v0/SchemaHelper.java    | 15 ++++++++++++++-
 .../metadata/InstanceRelationshipGraphBuilder.java   | 13 ++++++++++---
 .../falcon/metadata/MetadataMappingServiceTest.java  | 12 +++++++-----
 4 files changed, 34 insertions(+), 9 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/652e4a15/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index c75f04c..adf5c7b 100755
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -262,6 +262,9 @@ Release Version: 0.4-incubating
 
     FALCON-358 REST API does not conform to Rexster (Venkatesh Seetharam)
 
+    FALCON-359 Instance id's captured are of different formats in process and feed
+    (Venkatesh Seetharam)
+
   BUG FIXES
 
     FALCON-223: hive-exec bundles protobuf-2.4.1 which is incompatible with hadoop-2 requiring
protobuf-2.5

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/652e4a15/client/src/main/java/org/apache/falcon/entity/v0/SchemaHelper.java
----------------------------------------------------------------------
diff --git a/client/src/main/java/org/apache/falcon/entity/v0/SchemaHelper.java b/client/src/main/java/org/apache/falcon/entity/v0/SchemaHelper.java
index c6de668..9061a9e 100644
--- a/client/src/main/java/org/apache/falcon/entity/v0/SchemaHelper.java
+++ b/client/src/main/java/org/apache/falcon/entity/v0/SchemaHelper.java
@@ -29,6 +29,8 @@ import java.util.TimeZone;
  */
 public final class SchemaHelper {
 
+    public static final String ISO8601_FORMAT = "yyyy-MM-dd'T'HH:mm'Z'";
+
     private SchemaHelper() {}
 
     public static String getTimeZoneId(TimeZone tz) {
@@ -36,7 +38,7 @@ public final class SchemaHelper {
     }
 
     private static DateFormat getDateFormat() {
-        DateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm'Z'");
+        DateFormat dateFormat = new SimpleDateFormat(ISO8601_FORMAT);
         dateFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
         return dateFormat;
     }
@@ -55,4 +57,15 @@ public final class SchemaHelper {
             throw new RuntimeException(e);
         }
     }
+
+    public static String formatDateUTCToISO8601(final String dateString, final String dateStringFormat)
{
+
+        try {
+            DateFormat dateFormat = new SimpleDateFormat(dateStringFormat.substring(0, dateString.length()));
+            dateFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
+            return SchemaHelper.formatDateUTC(dateFormat.parse(dateString));
+        } catch (ParseException e) {
+            throw new RuntimeException(e);
+        }
+    }
 }

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/652e4a15/common/src/main/java/org/apache/falcon/metadata/InstanceRelationshipGraphBuilder.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/falcon/metadata/InstanceRelationshipGraphBuilder.java
b/common/src/main/java/org/apache/falcon/metadata/InstanceRelationshipGraphBuilder.java
index b02ba9a..7efa74a 100644
--- a/common/src/main/java/org/apache/falcon/metadata/InstanceRelationshipGraphBuilder.java
+++ b/common/src/main/java/org/apache/falcon/metadata/InstanceRelationshipGraphBuilder.java
@@ -27,6 +27,7 @@ import org.apache.falcon.entity.Storage;
 import org.apache.falcon.entity.common.FeedDataPath;
 import org.apache.falcon.entity.store.ConfigurationStore;
 import org.apache.falcon.entity.v0.EntityType;
+import org.apache.falcon.entity.v0.SchemaHelper;
 import org.apache.falcon.entity.v0.cluster.Cluster;
 import org.apache.falcon.entity.v0.feed.Feed;
 import org.apache.falcon.entity.v0.feed.LocationType;
@@ -43,6 +44,9 @@ public class InstanceRelationshipGraphBuilder extends RelationshipGraphBuilder
{
 
     private static final Logger LOG = Logger.getLogger(InstanceRelationshipGraphBuilder.class);
 
+    private static final String PROCESS_INSTANCE_FORMAT = "yyyy-MM-dd-HH-mm"; // nominal
time
+    private static final String FEED_INSTANCE_FORMAT = "yyyyMMddHHmm"; // computed
+
     // instance vertex types
     public static final String FEED_INSTANCE_TYPE = "feed-instance";
     public static final String PROCESS_INSTANCE_TYPE = "process-instance";
@@ -99,8 +103,10 @@ public class InstanceRelationshipGraphBuilder extends RelationshipGraphBuilder
{
                 lineageMetadata.get(LineageArgs.USER_WORKFLOW_VERSION.getOptionName()));
     }
 
-    public String getProcessInstanceName(String entityName, String nominalTime) {
-        return entityName + "/" + nominalTime;
+    public String getProcessInstanceName(String entityName,
+                                         String nominalTime) {
+        return entityName + "/"
+                + SchemaHelper.formatDateUTCToISO8601(nominalTime, PROCESS_INSTANCE_FORMAT);
     }
 
     public void addInstanceToEntity(Vertex instanceVertex, String entityName,
@@ -214,6 +220,7 @@ public class InstanceRelationshipGraphBuilder extends RelationshipGraphBuilder
{
             instance = instance.replaceFirst(element, "");
         }
 
-        return feed.getName() + "/" + instance;
+        return feed.getName() + "/"
+                + SchemaHelper.formatDateUTCToISO8601(instance, FEED_INSTANCE_FORMAT);
     }
 }

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/652e4a15/common/src/test/java/org/apache/falcon/metadata/MetadataMappingServiceTest.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/falcon/metadata/MetadataMappingServiceTest.java
b/common/src/test/java/org/apache/falcon/metadata/MetadataMappingServiceTest.java
index f573392..dab308d 100644
--- a/common/src/test/java/org/apache/falcon/metadata/MetadataMappingServiceTest.java
+++ b/common/src/test/java/org/apache/falcon/metadata/MetadataMappingServiceTest.java
@@ -652,13 +652,13 @@ public class MetadataMappingServiceTest {
     private void verifyLineageGraph(String feedType) {
         // feeds owned by a user
         List<String> feedNamesOwnedByUser = getFeedsOwnedByAUser(feedType);
-        List<String> expected = Arrays.asList("impression-feed/20140101", "clicks-feed/20140101",
-                "imp-click-join1/20140101", "imp-click-join2/20140101");
+        List<String> expected = Arrays.asList("impression-feed/2014-01-01T00:00Z",
"clicks-feed/2014-01-01T00:00Z",
+                "imp-click-join1/2014-01-01T00:00Z", "imp-click-join2/2014-01-01T00:00Z");
         Assert.assertTrue(feedNamesOwnedByUser.containsAll(expected));
 
         Graph graph = service.getGraph();
 
-        Iterator<Vertex> vertices = graph.getVertices("name", "impression-feed/20140101").iterator();
+        Iterator<Vertex> vertices = graph.getVertices("name", "impression-feed/2014-01-01T00:00Z").iterator();
         Assert.assertTrue(vertices.hasNext());
         Vertex feedInstanceVertex = vertices.next();
         Assert.assertEquals(
@@ -671,11 +671,13 @@ public class MetadataMappingServiceTest {
 
         // feeds classified as secure
         verifyFeedsClassifiedAsSecure(feedType,
-                Arrays.asList("impression-feed/20140101", "clicks-feed/20140101", "imp-click-join2/20140101"));
+                Arrays.asList("impression-feed/2014-01-01T00:00Z",
+                        "clicks-feed/2014-01-01T00:00Z", "imp-click-join2/2014-01-01T00:00Z"));
 
         // feeds owned by a user and classified as secure
         verifyFeedsOwnedByUserAndClassification(feedType, "Financial",
-                Arrays.asList("clicks-feed/20140101", "imp-click-join1/20140101", "imp-click-join2/20140101"));
+                Arrays.asList("clicks-feed/2014-01-01T00:00Z",
+                        "imp-click-join1/2014-01-01T00:00Z", "imp-click-join2/2014-01-01T00:00Z"));
     }
 
     private static String[] getTestMessageArgs() {


Mime
View raw message