falcon-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From shaikid...@apache.org
Subject falcon git commit: FALCON-1195 ClusterEntityParserTest fails intermittently & fixed Checkstyle errors. Contributed by Balu Vellanki
Date Wed, 13 May 2015 10:00:12 GMT
Repository: falcon
Updated Branches:
  refs/heads/0.6.1 c6ff55f80 -> bd1ac1e2e


FALCON-1195 ClusterEntityParserTest fails intermittently & fixed Checkstyle errors. Contributed
by Balu Vellanki


Project: http://git-wip-us.apache.org/repos/asf/falcon/repo
Commit: http://git-wip-us.apache.org/repos/asf/falcon/commit/bd1ac1e2
Tree: http://git-wip-us.apache.org/repos/asf/falcon/tree/bd1ac1e2
Diff: http://git-wip-us.apache.org/repos/asf/falcon/diff/bd1ac1e2

Branch: refs/heads/0.6.1
Commit: bd1ac1e2eefe1e252006b43210e8c8c1b75aadb0
Parents: c6ff55f
Author: shaik.idris <shaik.idris@inmobi.com>
Authored: Wed May 13 15:30:04 2015 +0530
Committer: shaik.idris <shaik.idris@inmobi.com>
Committed: Wed May 13 15:30:04 2015 +0530

----------------------------------------------------------------------
 CHANGES.txt                                     |  3 +
 .../entity/parser/ClusterEntityParser.java      |  2 +-
 .../entity/parser/ClusterEntityParserTest.java  | 93 ++++++++++++--------
 3 files changed, 62 insertions(+), 36 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/falcon/blob/bd1ac1e2/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index e6e4d41..8057105 100755
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -142,6 +142,9 @@ Branch: 0.6.1 (Proposed Release Version: 0.6.1)
    (Suhas vasu)
 
   BUG FIXES
+   FALCON-1195 ClusterEntityParserTest fails intermittently. 
+   (Balu Vellanki via Shaik Idris Ali)
+
    FALCON-1162 Cluster submit succeeds when staging HDFS dir does not
    have 777 (ALL) permission (Venkat Ramachandran via Suhas Vasu)
 

http://git-wip-us.apache.org/repos/asf/falcon/blob/bd1ac1e2/common/src/main/java/org/apache/falcon/entity/parser/ClusterEntityParser.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/falcon/entity/parser/ClusterEntityParser.java
b/common/src/main/java/org/apache/falcon/entity/parser/ClusterEntityParser.java
index 4555cb0..5c3ce4f 100644
--- a/common/src/main/java/org/apache/falcon/entity/parser/ClusterEntityParser.java
+++ b/common/src/main/java/org/apache/falcon/entity/parser/ClusterEntityParser.java
@@ -294,7 +294,7 @@ public class ClusterEntityParser extends EntityParser<Cluster> {
                 if (stagingLocation.getPath().equals(workingLocation.getPath())) {
                     throw new ValidationException(
                             "Location with name: " + stagingLocation.getName().value() +
" and " + workingLocation
-                                    .getName().value() + " cannot of same path: " + stagingLocation.getPath()
+                                    .getName().value() + " cannot have same path: " + stagingLocation.getPath()
                                     + " for cluster :" + cluster.getName());
                 } else {
 

http://git-wip-us.apache.org/repos/asf/falcon/blob/bd1ac1e2/common/src/test/java/org/apache/falcon/entity/parser/ClusterEntityParserTest.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/falcon/entity/parser/ClusterEntityParserTest.java
b/common/src/test/java/org/apache/falcon/entity/parser/ClusterEntityParserTest.java
index 4920d03..69230b2 100644
--- a/common/src/test/java/org/apache/falcon/entity/parser/ClusterEntityParserTest.java
+++ b/common/src/test/java/org/apache/falcon/entity/parser/ClusterEntityParserTest.java
@@ -18,6 +18,7 @@
 
 package org.apache.falcon.entity.parser;
 
+import org.apache.commons.lang3.StringUtils;
 import org.apache.falcon.FalconException;
 import org.apache.falcon.catalog.CatalogServiceFactory;
 import org.apache.falcon.cluster.util.EmbeddedCluster;
@@ -53,6 +54,7 @@ import java.io.StringWriter;
 public class ClusterEntityParserTest extends AbstractTestBase {
 
     private final ClusterEntityParser parser = (ClusterEntityParser) EntityParserFactory.getParser(EntityType.CLUSTER);
+    private static final String CLUSTER_LOCATIONS_BASE_DIR = "/projects/falcon/ClusterEntityParserTestLocations/";
 
     @Test
     public void testParse() throws IOException, FalconException, JAXBException {
@@ -62,6 +64,8 @@ public class ClusterEntityParserTest extends AbstractTestBase {
         Cluster cluster = parser.parse(stream);
         ClusterHelper.getInterface(cluster, Interfacetype.WRITE)
                 .setEndpoint(conf.get(HadoopClientFactory.FS_DEFAULT_NAME_KEY));
+        Locations locations = getClusterLocations("staging0", "working0");
+        cluster.setLocations(locations);
 
         Assert.assertNotNull(cluster);
         Assert.assertEquals(cluster.getName(), "testCluster");
@@ -84,7 +88,9 @@ public class ClusterEntityParserTest extends AbstractTestBase {
         Assert.assertEquals(workflow.getVersion(), "4.0");
 
         Assert.assertEquals(ClusterHelper.getLocation(cluster, ClusterLocationType.STAGING).getPath(),
-                "/projects/falcon/staging");
+                CLUSTER_LOCATIONS_BASE_DIR + "staging0");
+        Assert.assertEquals(ClusterHelper.getLocation(cluster, ClusterLocationType.WORKING).getPath(),
+                CLUSTER_LOCATIONS_BASE_DIR + "working0");
 
         StringWriter stringWriter = new StringWriter();
         Marshaller marshaller = EntityType.CLUSTER.getMarshaller();
@@ -95,8 +101,7 @@ public class ClusterEntityParserTest extends AbstractTestBase {
         Assert.assertEquals(catalog.getEndpoint(), "http://localhost:48080/templeton/v1");
         Assert.assertEquals(catalog.getVersion(), "0.11.0");
 
-        Assert.assertEquals(ClusterHelper.getLocation(cluster, ClusterLocationType.STAGING).getPath(),
-                "/projects/falcon/staging");
+
     }
 
     @Test
@@ -205,18 +210,15 @@ public class ClusterEntityParserTest extends AbstractTestBase {
      *
      * @throws ValidationException
      */
-    @Test(expectedExceptions = ValidationException.class) public void testClusterWithoutStaging()
throws Exception {
+    @Test(expectedExceptions = ValidationException.class, expectedExceptionsMessageRegExp
= ".*Unable to find.*")
+    public void testClusterWithoutStaging() throws Exception {
         ClusterEntityParser clusterEntityParser = Mockito
                 .spy((ClusterEntityParser) EntityParserFactory.getParser(EntityType.CLUSTER));
         Cluster cluster = (Cluster) this.dfsCluster.getCluster().copy();
         Mockito.doNothing().when(clusterEntityParser).validateWorkflowInterface(cluster);
         Mockito.doNothing().when(clusterEntityParser).validateMessagingInterface(cluster);
         Mockito.doNothing().when(clusterEntityParser).validateRegistryInterface(cluster);
-        Location location = new Location();
-        location.setName(ClusterLocationType.WORKING);
-        location.setPath("/apps/non/existent/path");
-        Locations locations = new Locations();
-        locations.getLocations().add(location);
+        Locations locations = getClusterLocations(null, "non/existent/path");
         cluster.setLocations(locations);
         clusterEntityParser.validate(cluster);
         Assert.fail("Should have thrown a validation exception");
@@ -228,28 +230,17 @@ public class ClusterEntityParserTest extends AbstractTestBase {
      *
      * @throws ValidationException
      */
-    @Test(expectedExceptions = ValidationException.class)
+    @Test(expectedExceptions = ValidationException.class, expectedExceptionsMessageRegExp
= ".*Location.*must exist.")
     public void testClusterWithInvalidLocationsPaths() throws Exception {
         ClusterEntityParser clusterEntityParser = Mockito
                 .spy((ClusterEntityParser) EntityParserFactory.getParser(EntityType.CLUSTER));
         Cluster cluster = (Cluster)this.dfsCluster.getCluster().copy();
-        Location location = new Location();
-        location.setName(ClusterLocationType.STAGING);
-        location.setPath("/apps/non/existent/path");
-        Locations locations = new Locations();
-        locations.getLocations().add(location);
+        Locations locations = getClusterLocations("non/existent/path", null);
         cluster.setLocations(locations);
         Mockito.doNothing().when(clusterEntityParser).validateWorkflowInterface(cluster);
         Mockito.doNothing().when(clusterEntityParser).validateMessagingInterface(cluster);
         Mockito.doNothing().when(clusterEntityParser).validateRegistryInterface(cluster);
-        try {
-            clusterEntityParser.validate(cluster);
-        } catch (ValidationException e) {
-            String errorMessage =
-                    "Location " + location.getPath() + " for cluster " + cluster.getName()
+ " must exist.";
-            Assert.assertEquals(e.getMessage(), errorMessage);
-            throw e;
-        }
+        clusterEntityParser.validate(cluster);
         Assert.fail("Should have thrown a validation exception");
     }
 
@@ -259,12 +250,15 @@ public class ClusterEntityParserTest extends AbstractTestBase {
      *
      * @throws ValidationException
      */
-    @Test(expectedExceptions = ValidationException.class)
+    @Test(expectedExceptions = ValidationException.class, expectedExceptionsMessageRegExp
= ".*same path:.*")
     public void testClusterWithSameWorkingAndStaging() throws Exception {
         ClusterEntityParser clusterEntityParser = Mockito
                 .spy((ClusterEntityParser) EntityParserFactory.getParser(EntityType.CLUSTER));
         Cluster cluster = (Cluster) this.dfsCluster.getCluster().copy();
-        cluster.getLocations().getLocations().get(1).setPath("/projects/falcon/staging");
+        Locations locations = getClusterLocations("staging1", "staging1");
+        cluster.setLocations(locations);
+        this.dfsCluster.getFileSystem().mkdirs(new Path(cluster.getLocations().getLocations().get(0).getPath()),
+                HadoopClientFactory.ALL_PERMISSION);
         Mockito.doNothing().when(clusterEntityParser).validateWorkflowInterface(cluster);
         Mockito.doNothing().when(clusterEntityParser).validateMessagingInterface(cluster);
         Mockito.doNothing().when(clusterEntityParser).validateRegistryInterface(cluster);
@@ -277,11 +271,13 @@ public class ClusterEntityParserTest extends AbstractTestBase {
      * It should automatically get generated
      * Extensive tests are found in ClusterEntityValidationIT.
      */
-    @Test public void testClusterWithOnlyStaging() throws Exception {
+    @Test
+    public void testClusterWithOnlyStaging() throws Exception {
         ClusterEntityParser clusterEntityParser = Mockito
                 .spy((ClusterEntityParser) EntityParserFactory.getParser(EntityType.CLUSTER));
         Cluster cluster = (Cluster) this.dfsCluster.getCluster().copy();
-        cluster.getLocations().getLocations().remove(1);
+        Locations locations = getClusterLocations("staging2", null);
+        cluster.setLocations(locations);
         Mockito.doNothing().when(clusterEntityParser).validateWorkflowInterface(cluster);
         Mockito.doNothing().when(clusterEntityParser).validateMessagingInterface(cluster);
         Mockito.doNothing().when(clusterEntityParser).validateRegistryInterface(cluster);
@@ -303,16 +299,21 @@ public class ClusterEntityParserTest extends AbstractTestBase {
      *
      * @throws ValidationException
      */
-    @Test(expectedExceptions = ValidationException.class)
+    @Test(expectedExceptions = ValidationException.class, expectedExceptionsMessageRegExp
= ".*rwxr-xr-x.*rwxrwxrwx")
     public void testClusterWithSubdirInStaging() throws Exception {
         ClusterEntityParser clusterEntityParser = Mockito
                 .spy((ClusterEntityParser) EntityParserFactory.getParser(EntityType.CLUSTER));
         Cluster cluster = (Cluster) this.dfsCluster.getCluster().copy();
-        cluster.getLocations().getLocations().get(1).setPath("/projects/falcon/staging");
-        cluster.getLocations().getLocations().remove(1);
+        Locations locations = getClusterLocations("staging3", null);
+        cluster.setLocations(locations);
+
+        HadoopClientFactory.mkdirs(this.dfsCluster.getFileSystem(),
+                new Path(cluster.getLocations().getLocations().get(0).getPath()),
+                HadoopClientFactory.ALL_PERMISSION);
         HadoopClientFactory.mkdirs(this.dfsCluster.getFileSystem(),
-                new Path(ClusterHelper.getLocation(cluster, ClusterLocationType.WORKING).getPath()),
+                new Path(cluster.getLocations().getLocations().get(0).getPath() + "/working"),
                 HadoopClientFactory.ALL_PERMISSION);
+
         Mockito.doNothing().when(clusterEntityParser).validateWorkflowInterface(cluster);
         Mockito.doNothing().when(clusterEntityParser).validateMessagingInterface(cluster);
         Mockito.doNothing().when(clusterEntityParser).validateRegistryInterface(cluster);
@@ -328,13 +329,13 @@ public class ClusterEntityParserTest extends AbstractTestBase {
      *
      * @throws ValidationException
      */
-    @Test(expectedExceptions = ValidationException.class)
+    @Test(expectedExceptions = ValidationException.class, expectedExceptionsMessageRegExp
= ".*rwxr-xr-x.*rwxrwxrwx")
     public void testClusterWithStagingPermission() throws Exception {
         ClusterEntityParser clusterEntityParser = Mockito
                 .spy((ClusterEntityParser) EntityParserFactory.getParser(EntityType.CLUSTER));
         Cluster cluster = (Cluster) this.dfsCluster.getCluster().copy();
-        cluster.getLocations().getLocations().get(0).setPath("/projects/falcon/staging2");
-        cluster.getLocations().getLocations().remove(1);
+        Locations locations = getClusterLocations("staging4", null);
+        cluster.setLocations(locations);
         Mockito.doNothing().when(clusterEntityParser).validateWorkflowInterface(cluster);
         Mockito.doNothing().when(clusterEntityParser).validateMessagingInterface(cluster);
         Mockito.doNothing().when(clusterEntityParser).validateRegistryInterface(cluster);
@@ -348,10 +349,32 @@ public class ClusterEntityParserTest extends AbstractTestBase {
     public void init() throws Exception {
         this.dfsCluster = EmbeddedCluster.newCluster("testCluster");
         this.conf = dfsCluster.getConf();
+        this.dfsCluster.getFileSystem().mkdirs(new Path(CLUSTER_LOCATIONS_BASE_DIR));
     }
 
     @AfterClass
-    public void tearDown() {
+    public void tearDown() throws IOException {
+        this.dfsCluster.getFileSystem().delete(new Path(CLUSTER_LOCATIONS_BASE_DIR), true);
         this.dfsCluster.shutdown();
     }
+
+    private Locations getClusterLocations(String staging, String working) {
+        Locations locations = new Locations();
+
+        Location loc = new Location();
+        loc.setName(ClusterLocationType.STAGING);
+        if (StringUtils.isNotEmpty(staging)) {
+            loc.setPath(CLUSTER_LOCATIONS_BASE_DIR + staging);
+            locations.getLocations().add(loc);
+        }
+
+        loc = new Location();
+        loc.setName(ClusterLocationType.WORKING);
+        if (StringUtils.isNotEmpty(working)) {
+            loc.setPath(CLUSTER_LOCATIONS_BASE_DIR + working);
+            locations.getLocations().add(loc);
+        }
+
+        return locations;
+    }
 }


Mime
View raw message