falcon-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From rag...@apache.org
Subject incubator-falcon git commit: FALCON-953 Change wasb replication to run with single cluster. Contributed by Ruslan Ostafiychuk
Date Wed, 17 Dec 2014 20:38:28 GMT
Repository: incubator-falcon
Updated Branches:
  refs/heads/master 59e036398 -> 952de6517


FALCON-953 Change wasb replication to run with single cluster. Contributed by Ruslan Ostafiychuk


Project: http://git-wip-us.apache.org/repos/asf/incubator-falcon/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-falcon/commit/952de651
Tree: http://git-wip-us.apache.org/repos/asf/incubator-falcon/tree/952de651
Diff: http://git-wip-us.apache.org/repos/asf/incubator-falcon/diff/952de651

Branch: refs/heads/master
Commit: 952de65174bc612775f94cfa424c8950ff38066f
Parents: 59e0363
Author: Raghav Kumar Gautam <raghav@apache.org>
Authored: Wed Dec 17 12:38:17 2014 -0800
Committer: Raghav Kumar Gautam <raghav@apache.org>
Committed: Wed Dec 17 12:38:17 2014 -0800

----------------------------------------------------------------------
 falcon-regression/CHANGES.txt                   |  3 +++
 .../falcon/regression/ExternalFSTest.java       | 21 ++++++++++----------
 2 files changed, 14 insertions(+), 10 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/952de651/falcon-regression/CHANGES.txt
----------------------------------------------------------------------
diff --git a/falcon-regression/CHANGES.txt b/falcon-regression/CHANGES.txt
index 8927790..7afceef 100644
--- a/falcon-regression/CHANGES.txt
+++ b/falcon-regression/CHANGES.txt
@@ -36,6 +36,9 @@ Trunk (Unreleased)
    via Samarth Gupta)
 
   IMPROVEMENTS
+   FALCON-953 Change wasb replication to run with single cluster (Ruslan Ostafiychuk
+   via Raghav Kumar Gautam)
+
    FALCON-947 Ignore hadoop configs in merlin/src/test/resources/hadoop-conf (Ruslan Ostafiychuk)
 
    FALCON-940 Avoid NPE in getAllEntitiesOfOneType() for zero elements

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/952de651/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ExternalFSTest.java
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ExternalFSTest.java
b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ExternalFSTest.java
index 9c2a42b..d6d9854 100644
--- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ExternalFSTest.java
+++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ExternalFSTest.java
@@ -69,10 +69,10 @@ public class ExternalFSTest extends BaseTestClass{
     public static final String WASB_END_POINT =
             "wasb://" + MerlinConstants.WASB_CONTAINER + "@" + MerlinConstants.WASB_ACCOUNT;
     private ColoHelper cluster = servers.get(0);
-    private ColoHelper cluster2 = servers.get(1);
     private FileSystem clusterFS = serverFS.get(0);
-    private OozieClient cluster2OC = serverOC.get(1);
+    private OozieClient clusterOC = serverOC.get(0);
     private FileSystem wasbFS;
+    private Bundle externalBundle;
 
 
     private String baseTestDir = baseHDFSDir + "/ExternalFSTest";
@@ -84,6 +84,7 @@ public class ExternalFSTest extends BaseTestClass{
 
     @BeforeClass
     public void setUpClass() throws IOException {
+        HadoopUtil.recreateDir(clusterFS, baseTestDir);
         Configuration conf = new Configuration();
         conf.set("fs.defaultFS", WASB_END_POINT);
         conf.set("fs.azure.account.key." + MerlinConstants.WASB_ACCOUNT,
@@ -98,10 +99,10 @@ public class ExternalFSTest extends BaseTestClass{
         Bundle bundle = BundleUtil.readFeedReplicationBundle();
 
         bundles[0] = new Bundle(bundle, cluster);
-        bundles[1] = new Bundle(bundle, cluster2);
+        externalBundle = new Bundle(bundle, cluster);
 
         bundles[0].generateUniqueBundle();
-        bundles[1].generateUniqueBundle();
+        externalBundle.generateUniqueBundle();
 
         LOGGER.info("checking wasb credentials with location: " + testWasbTargetDir);
         wasbFS.create(new Path(testWasbTargetDir));
@@ -110,7 +111,7 @@ public class ExternalFSTest extends BaseTestClass{
 
     @AfterMethod
     public void tearDown() throws IOException {
-        removeBundles();
+        removeBundles(externalBundle);
         wasbFS.delete(new Path(testWasbTargetDir), true);
     }
 
@@ -136,7 +137,7 @@ public class ExternalFSTest extends BaseTestClass{
     public void replicateToExternalFS(final FileSystem externalFS,
         final String separator, final boolean withData) throws Exception {
         final String endpoint = externalFS.getUri().toString();
-        Bundle.submitCluster(bundles[0], bundles[1]);
+        Bundle.submitCluster(bundles[0], externalBundle);
         String startTime = TimeUtil.getTimeWrtSystemTime(0);
         String endTime = TimeUtil.addMinsToTime(startTime, 5);
         LOGGER.info("Time range between : " + startTime + " and " + endTime);
@@ -158,7 +159,7 @@ public class ExternalFSTest extends BaseTestClass{
                 .build()).toString();
         //set externalFS cluster as target
         feed = FeedMerlin.fromString(feed).addFeedCluster(
-            new FeedMerlin.FeedClusterBuilder(Util.readEntityName(bundles[1].getClusters().get(0)))
+            new FeedMerlin.FeedClusterBuilder(Util.readEntityName(externalBundle.getClusters().get(0)))
                 .withRetention("days(1000000)", ActionType.DELETE)
                 .withValidity(startTime, endTime)
                 .withClusterType(ClusterType.TARGET)
@@ -182,15 +183,15 @@ public class ExternalFSTest extends BaseTestClass{
         Path dstPath = new Path(endpoint + testWasbTargetDir + '/' + timePattern);
 
         //check if coordinator exists
-        InstanceUtil.waitTillInstancesAreCreated(cluster2, feed, 0);
+        InstanceUtil.waitTillInstancesAreCreated(cluster, feed, 0);
 
         Assert.assertEquals(InstanceUtil
-            .checkIfFeedCoordExist(cluster2.getFeedHelper(), Util.readEntityName(feed),
+            .checkIfFeedCoordExist(cluster.getFeedHelper(), Util.readEntityName(feed),
                 "REPLICATION"), 1);
 
         TimeUtil.sleepSeconds(10);
         //replication should start, wait while it ends
-        InstanceUtil.waitTillInstanceReachState(cluster2OC, Util.readEntityName(feed), 1,
+        InstanceUtil.waitTillInstanceReachState(clusterOC, Util.readEntityName(feed), 1,
             CoordinatorAction.Status.SUCCEEDED, EntityType.FEED);
 
         //check if data has been replicated correctly


Mime
View raw message