falcon-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From samar...@apache.org
Subject [1/2] falcon git commit: FALCON-1017 FeedReplicationTest modified to check for _SUCCESS getting created on target directory. Contributed by Pragya M
Date Fri, 27 Feb 2015 05:07:02 GMT
Repository: falcon
Updated Branches:
  refs/heads/master f63a47f51 -> 833458ac1


FALCON-1017 FeedReplicationTest modified to check for _SUCCESS getting created on target directory.
Contributed by Pragya M


Project: http://git-wip-us.apache.org/repos/asf/falcon/repo
Commit: http://git-wip-us.apache.org/repos/asf/falcon/commit/444e31e3
Tree: http://git-wip-us.apache.org/repos/asf/falcon/tree/444e31e3
Diff: http://git-wip-us.apache.org/repos/asf/falcon/diff/444e31e3

Branch: refs/heads/master
Commit: 444e31e37425cd25c49afaa57fc68a5152b2090d
Parents: f63a47f
Author: samarthg <samarthg@apacge.org>
Authored: Fri Feb 27 10:26:06 2015 +0530
Committer: samarthg <samarthg@apacge.org>
Committed: Fri Feb 27 10:26:06 2015 +0530

----------------------------------------------------------------------
 falcon-regression/CHANGES.txt                   |  4 ++-
 .../falcon/regression/core/util/HadoopUtil.java | 26 ++++++++++++++++++++
 .../falcon/regression/FeedReplicationTest.java  | 22 +++++++++++++++++
 3 files changed, 51 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/falcon/blob/444e31e3/falcon-regression/CHANGES.txt
----------------------------------------------------------------------
diff --git a/falcon-regression/CHANGES.txt b/falcon-regression/CHANGES.txt
index 6e6ac7f..aa33831 100644
--- a/falcon-regression/CHANGES.txt
+++ b/falcon-regression/CHANGES.txt
@@ -51,7 +51,9 @@ Trunk (Unreleased)
    via Samarth Gupta)
 
   IMPROVEMENTS
-   
+
+   FALCON-1017 FeedReplicationTest modified to check for _SUCCESS getting created on 
+   target directory(Pragya M via Samarth G)   
    FALCON-1040 Modifying ProcessInstanceStatusTest to expose job id for running jobs in 
    Falcon. (Pragya M via Samarth G)
 

http://git-wip-us.apache.org/repos/asf/falcon/blob/444e31e3/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/HadoopUtil.java
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/HadoopUtil.java
b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/HadoopUtil.java
index 64574a5..7122274 100644
--- a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/HadoopUtil.java
+++ b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/HadoopUtil.java
@@ -136,6 +136,32 @@ public final class HadoopUtil {
         return returnList;
     }
 
+    /**
+     * Recursively retrieves all data file names from a given location and looks for presence
of availabilityFlag.
+     * If availabilityFlag is null then it looks for _SUCCESS file(set as default).
+     * @param fs filesystem
+     * @param location given location
+     * @param availabilityFlag value of availability flag set in entity
+     * @return
+     * @throws IOException
+     */
+    public static boolean getSuccessFolder(
+            FileSystem fs, Path location, String availabilityFlag) throws IOException {
+        LOGGER.info("location : " + location);
+        for(FileStatus stat : fs.listStatus(location)) {
+            if (availabilityFlag.isEmpty()) {
+                if (stat.getPath().getName().equals("_SUCCESS")) {
+                    return true;
+                }
+            } else {
+                if (stat.getPath().getName().equals(availabilityFlag)) {
+                    return true;
+                }
+            }
+        }
+        return false;
+    }
+
     @SuppressWarnings("deprecation")
     private static boolean isDir(FileStatus stat) {
         return stat.isDir();

http://git-wip-us.apache.org/repos/asf/falcon/blob/444e31e3/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/FeedReplicationTest.java
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/FeedReplicationTest.java
b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/FeedReplicationTest.java
index b88cffc..3930c8c 100644
--- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/FeedReplicationTest.java
+++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/FeedReplicationTest.java
@@ -97,6 +97,7 @@ public class FeedReplicationTest extends BaseTestClass {
      * Test demonstrates replication of stored data from one source cluster to one target
cluster.
      * It checks the lifecycle of replication workflow instance including its creation. When
      * replication ends test checks if data was replicated correctly.
+     * Also checks for presence of _SUCCESS file in target directory.
      */
     @Test
     public void replicate1Source1Target()
@@ -164,12 +165,19 @@ public class FeedReplicationTest extends BaseTestClass {
             .getAllFilesRecursivelyHDFS(cluster2FS, toTarget);
 
         AssertUtil.checkForListSizes(cluster1ReplicatedData, cluster2ReplicatedData);
+
+        //_SUCCESS does not exist in source
+        Assert.assertEquals(HadoopUtil.getSuccessFolder(cluster1FS, toSource, ""), false);
+
+        //_SUCCESS should exist in target
+        Assert.assertEquals(HadoopUtil.getSuccessFolder(cluster2FS, toTarget, ""), true);
     }
 
     /**
      * Test demonstrates replication of stored data from one source cluster to two target
clusters.
      * It checks the lifecycle of replication workflow instances including their creation
on both
      * targets. When replication ends test checks if data was replicated correctly.
+     * Also checks for presence of _SUCCESS file in target directory.
      */
     @Test
     public void replicate1Source2Targets() throws Exception {
@@ -254,6 +262,13 @@ public class FeedReplicationTest extends BaseTestClass {
 
         AssertUtil.checkForListSizes(cluster1ReplicatedData, cluster2ReplicatedData);
         AssertUtil.checkForListSizes(cluster1ReplicatedData, cluster3ReplicatedData);
+
+        //_SUCCESS does not exist in source
+        Assert.assertEquals(HadoopUtil.getSuccessFolder(cluster1FS, toSource, ""), false);
+
+        //_SUCCESS should exist in target
+        Assert.assertEquals(HadoopUtil.getSuccessFolder(cluster2FS, toTarget, ""), true);
+        Assert.assertEquals(HadoopUtil.getSuccessFolder(cluster3FS, toTarget, ""), true);
     }
 
     /**
@@ -262,6 +277,7 @@ public class FeedReplicationTest extends BaseTestClass {
      * feed still waits for availability flag (file which name is defined as availability
flag in
      * feed definition). As soon as mentioned file is got uploaded in data directory,
      * replication starts and when it ends test checks if data was replicated correctly.
+     * Also checks for presence of availability flag in target directory.
      */
     @Test
     public void availabilityFlagTest() throws Exception {
@@ -349,5 +365,11 @@ public class FeedReplicationTest extends BaseTestClass {
             .getAllFilesRecursivelyHDFS(cluster2FS, toTarget);
         LOGGER.info("Data on target cluster: " + cluster2ReplicatedData);
         AssertUtil.checkForListSizes(cluster1ReplicatedData, cluster2ReplicatedData);
+
+        //availabilityFlag exists in source
+        Assert.assertEquals(HadoopUtil.getSuccessFolder(cluster1FS, toSource, availabilityFlagName),
true);
+
+        //availabilityFlag should exist in target
+        Assert.assertEquals(HadoopUtil.getSuccessFolder(cluster2FS, toTarget, availabilityFlagName),
true);
     }
 }


Mime
View raw message