falcon-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From peeyu...@apache.org
Subject falcon git commit: FALCON-1840 Archive older definition in case of update
Date Wed, 23 Mar 2016 15:13:46 GMT
Repository: falcon
Updated Branches:
  refs/heads/master c2f7a2a5e -> 820572d8a


FALCON-1840 Archive older definition in case of update

Author: Praveen Adlakha <adlakha.praveen@gmail.com>

Reviewers: Peeyush Bishnoi<peeyushb@apache.org>, Balu<bvellanki@hortonworks.com>,
Pallavi<pallavi.rao@inmobi.com>, Sandeep<sandysmdl@gmail.com>

Closes #75 from PraveenAdlakha/1840 and squashes the following commits:

c9c19d8 [Praveen Adlakha] documentation added for archival in delete and update
6c570e3 [Praveen Adlakha] comments addressed
b475fc4 [Praveen Adlakha] IT added
7ba81af [Praveen Adlakha] FALCON-1840 Archive older definition in case of update


Project: http://git-wip-us.apache.org/repos/asf/falcon/repo
Commit: http://git-wip-us.apache.org/repos/asf/falcon/commit/820572d8
Tree: http://git-wip-us.apache.org/repos/asf/falcon/tree/820572d8
Diff: http://git-wip-us.apache.org/repos/asf/falcon/diff/820572d8

Branch: refs/heads/master
Commit: 820572d8afb5b1244af7f8c36c810c4efd0b749d
Parents: c2f7a2a
Author: Praveen Adlakha <adlakha.praveen@gmail.com>
Authored: Wed Mar 23 20:43:12 2016 +0530
Committer: peeyush b <pbishnoi@hortonworks.com>
Committed: Wed Mar 23 20:43:12 2016 +0530

----------------------------------------------------------------------
 .../falcon/entity/store/ConfigurationStore.java | 15 ++++-
 .../entity/store/FeedLocationStoreTest.java     | 58 ++++++++++++++++++--
 .../src/site/twiki/falconcli/DeleteEntity.twiki |  2 +-
 .../src/site/twiki/falconcli/UpdateEntity.twiki |  3 +-
 4 files changed, 68 insertions(+), 10 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/falcon/blob/820572d8/common/src/main/java/org/apache/falcon/entity/store/ConfigurationStore.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/falcon/entity/store/ConfigurationStore.java b/common/src/main/java/org/apache/falcon/entity/store/ConfigurationStore.java
index 9c7a932..bdcd1af 100644
--- a/common/src/main/java/org/apache/falcon/entity/store/ConfigurationStore.java
+++ b/common/src/main/java/org/apache/falcon/entity/store/ConfigurationStore.java
@@ -98,10 +98,18 @@ public final class ConfigurationStore implements FalconService {
     public static ConfigurationStore get() {
         return STORE;
     }
-
     private FileSystem fs;
     private Path storePath;
 
+    public FileSystem getFs() {
+        return fs;
+    }
+
+    public Path getStorePath() {
+        return storePath;
+    }
+
+
     private ConfigurationStore() {
         for (EntityType type : EntityType.values()) {
             dictionary.put(type, new ConcurrentHashMap<String, Entity>());
@@ -250,6 +258,11 @@ public final class ConfigurationStore implements FalconService {
 
     public synchronized void update(EntityType type, Entity entity) throws FalconException
{
         if (updatesInProgress.get() == entity) {
+            try {
+                archive(type, entity.getName());
+            } catch (IOException e) {
+                throw new StoreAccessException(e);
+            }
             updateInternal(type, entity);
         } else {
             throw new FalconException(entity.toShortString() + " is not initialized for update");

http://git-wip-us.apache.org/repos/asf/falcon/blob/820572d8/common/src/test/java/org/apache/falcon/entity/store/FeedLocationStoreTest.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/falcon/entity/store/FeedLocationStoreTest.java
b/common/src/test/java/org/apache/falcon/entity/store/FeedLocationStoreTest.java
index 033a55b..40c077e 100644
--- a/common/src/test/java/org/apache/falcon/entity/store/FeedLocationStoreTest.java
+++ b/common/src/test/java/org/apache/falcon/entity/store/FeedLocationStoreTest.java
@@ -33,6 +33,8 @@ import org.apache.falcon.security.CurrentUser;
 import org.apache.falcon.util.FalconRadixUtils;
 import org.apache.falcon.util.FalconTestUtil;
 import org.apache.falcon.util.StartupProperties;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.Path;
 import org.testng.Assert;
 import org.testng.annotations.AfterMethod;
 import org.testng.annotations.BeforeClass;
@@ -40,6 +42,7 @@ import org.testng.annotations.BeforeMethod;
 import org.testng.annotations.Test;
 
 import java.io.File;
+import java.io.IOException;
 import java.net.URI;
 import java.util.Collection;
 
@@ -50,7 +53,6 @@ import java.util.Collection;
 public class FeedLocationStoreTest extends AbstractTestBase {
     private ConfigurationStore store;
 
-
     @BeforeClass
     public void initConfigStore() throws Exception {
         String configPath = new URI(StartupProperties.get().getProperty("config.store.uri")).getPath();
@@ -65,7 +67,6 @@ public class FeedLocationStoreTest extends AbstractTestBase {
         StartupProperties.get().setProperty("configstore.listeners", listeners);
         store = ConfigurationStore.get();
         store.init();
-
         CurrentUser.authenticate(FalconTestUtil.TEST_USER_2);
 
     }
@@ -102,10 +103,41 @@ public class FeedLocationStoreTest extends AbstractTestBase {
     }
 
     @Test
+    public void testOnUpdate() throws FalconException{
+        Feed f1 = createFeed("f1");
+        f1.getLocations().getLocations().add(createLocation(LocationType.DATA,
+                "/projects/cas/data/hourly/2014/09/09/09"));
+        store.publish(EntityType.FEED, f1);
+
+        Feed f2 = createFeed("f1");
+        f2.getLocations().getLocations().add(createLocation(LocationType.DATA,
+                "/projects/cas/data/monthly"));
+        store.initiateUpdate(f2);
+        store.update(EntityType.FEED, f2);
+        store.cleanupUpdateInit();
+        boolean isArchived = false;
+        try {
+            Path archivePath = new Path(store.getStorePath(), "archive" + Path.SEPARATOR
+ "FEED");
+            FileStatus [] files= store.getFs().listStatus(archivePath);
+            for(FileStatus f:files){
+                String name = f.getPath().getName();
+                if (name.startsWith(f2.getName())){
+                    isArchived= true;
+                    break;
+                }
+            }
+        }catch (IOException e){
+            e.printStackTrace();
+        }
+        Assert.assertTrue(isArchived);
+    }
+
+
+    @Test
     public void testOnRemove() throws FalconException{
         int initialSize = FeedLocationStore.get().store.getSize();
-
-        Feed f1 = createFeed("f1ForRemove");
+        String feedName = "f1ForRemove";
+        Feed f1 = createFeed(feedName);
         f1.getLocations().getLocations().add(createLocation(LocationType.DATA,
                 "/projects/cas/data/hourly/2014/09/09/09"));
         f1.getLocations().getLocations().add(createLocation(LocationType.STATS,
@@ -113,9 +145,23 @@ public class FeedLocationStoreTest extends AbstractTestBase {
 
         store.publish(EntityType.FEED, f1);
         Assert.assertEquals(FeedLocationStore.get().store.getSize() - initialSize, 4);
-        store.remove(EntityType.FEED, "f1ForRemove");
+        store.remove(EntityType.FEED, feedName);
+        boolean isArchived = false;
+        try {
+            Path archivePath = new Path(store.getStorePath(), "archive" + Path.SEPARATOR
+ "FEED");
+            FileStatus [] files= store.getFs().listStatus(archivePath);
+            for(FileStatus f:files){
+                String name = f.getPath().getName();
+                if (name.startsWith(feedName)){
+                    isArchived= true;
+                    break;
+                }
+            }
+        }catch (IOException e){
+            e.printStackTrace();
+        }
+        Assert.assertTrue(isArchived);
         Assert.assertEquals(FeedLocationStore.get().store.getSize(), initialSize);
-
     }
 
 

http://git-wip-us.apache.org/repos/asf/falcon/blob/820572d8/docs/src/site/twiki/falconcli/DeleteEntity.twiki
----------------------------------------------------------------------
diff --git a/docs/src/site/twiki/falconcli/DeleteEntity.twiki b/docs/src/site/twiki/falconcli/DeleteEntity.twiki
index f2b3080..cc07406 100644
--- a/docs/src/site/twiki/falconcli/DeleteEntity.twiki
+++ b/docs/src/site/twiki/falconcli/DeleteEntity.twiki
@@ -2,7 +2,7 @@
 
 [[CommonCLI][Common CLI Options]]
 
-Delete removes the submitted entity definition for the specified entity and put it into the
archive.
+Delete removes the submitted entity definition for the specified entity and put it into the
archive.Archive path is defined in startup.properties in variable "config.store.uri".
 
 Usage:
 $FALCON_HOME/bin/falcon entity  -type [cluster|datasource|feed|process] -name <<name>>
-delete

http://git-wip-us.apache.org/repos/asf/falcon/blob/820572d8/docs/src/site/twiki/falconcli/UpdateEntity.twiki
----------------------------------------------------------------------
diff --git a/docs/src/site/twiki/falconcli/UpdateEntity.twiki b/docs/src/site/twiki/falconcli/UpdateEntity.twiki
index ae60559..5d49a76 100644
--- a/docs/src/site/twiki/falconcli/UpdateEntity.twiki
+++ b/docs/src/site/twiki/falconcli/UpdateEntity.twiki
@@ -2,8 +2,7 @@
 
 [[CommonCLI][Common CLI Options]]
 
-Update operation allows an already submitted/scheduled entity to be updated. Cluster and
datasource updates are
-currently not allowed.
+Update operation allows an already submitted/scheduled entity to be updated and put it into
the archive.Archive path is defined in startup.properties in variable "config.store.uri".
Cluster and datasource updates are currently not allowed.
 
 Usage:
 $FALCON_HOME/bin/falcon entity  -type [feed|process] -name <<name>> -update -file
<<path_to_file>>


Mime
View raw message