falcon-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ar...@apache.org
Subject git commit: FALCON-606 hcat tests should stop using root dir. Contributed by Shwetha GS Raghav Kumar Gautam.
Date Thu, 21 Aug 2014 23:01:25 GMT
Repository: incubator-falcon
Updated Branches:
  refs/heads/FALCON-585 d5fdb3294 -> e9c6a2dcf


FALCON-606 hcat tests should stop using root dir. Contributed by Shwetha GS Raghav Kumar Gautam.


Project: http://git-wip-us.apache.org/repos/asf/incubator-falcon/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-falcon/commit/e9c6a2dc
Tree: http://git-wip-us.apache.org/repos/asf/incubator-falcon/tree/e9c6a2dc
Diff: http://git-wip-us.apache.org/repos/asf/incubator-falcon/diff/e9c6a2dc

Branch: refs/heads/FALCON-585
Commit: e9c6a2dcf7b91dc669eb637141791a2e19594fac
Parents: d5fdb32
Author: arpit <arpit@apache.org>
Authored: Thu Aug 21 16:01:07 2014 -0700
Committer: arpit <arpit@apache.org>
Committed: Thu Aug 21 16:01:07 2014 -0700

----------------------------------------------------------------------
 falcon-regression/CHANGES.txt                   | 10 +++++++--
 .../falcon/regression/core/bundle/Bundle.java   | 19 +++++++++++++++++
 .../falcon/regression/core/util/BundleUtil.java | 22 ++++++++++++++++----
 .../falcon/regression/core/util/HadoopUtil.java | 20 ++++++++++++++++++
 .../regression/testHelper/BaseTestClass.java    | 13 +++++++-----
 .../falcon/regression/hcat/HCatProcessTest.java |  2 +-
 .../regression/hcat/HCatReplicationTest.java    |  2 +-
 .../regression/hcat/HCatRetentionTest.java      |  3 ++-
 .../src/test/resources/hcat/data/_SUCCESS       | 13 ------------
 9 files changed, 77 insertions(+), 27 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/e9c6a2dc/falcon-regression/CHANGES.txt
----------------------------------------------------------------------
diff --git a/falcon-regression/CHANGES.txt b/falcon-regression/CHANGES.txt
index 929a881..a0b318f 100644
--- a/falcon-regression/CHANGES.txt
+++ b/falcon-regression/CHANGES.txt
@@ -7,9 +7,15 @@ Trunk (Unreleased)
   NEW FEATURES
 
   IMPROVEMENTS
-   FALCON-619 ELExp_FutureAndLatestTest stabilization (Paul Isaychuk via Arpit Gupta)
+   FALCON-585 Remove requirement to have write access to / (Raghav Kumar Gautam)
+      FALCON-606 hcat tests should stop using root dir (Raghav Kumar Gautam 
+      via Arpit Gupta)
 
-   FALCON-610 Refactoring and documentation updates (Paul Isaychuk via Arpit Gupta)
+   FALCON-619 ELExp_FutureAndLatestTest stabilization (Paul Isaychuk via 
+   Arpit Gupta)
+
+   FALCON-610 Refactoring and documentation updates (Paul Isaychuk via 
+   Arpit Gupta)
 
    FALCON-581 Refactor code for cross product and make it a method 
    (Raghav Kumar Gautam via Arpit Gupta) 

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/e9c6a2dc/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/bundle/Bundle.java
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/bundle/Bundle.java
b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/bundle/Bundle.java
index 57c7292..ff5a6a8 100644
--- a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/bundle/Bundle.java
+++ b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/bundle/Bundle.java
@@ -44,6 +44,7 @@ import org.apache.falcon.regression.Entities.ProcessMerlin;
 import org.apache.falcon.regression.core.helpers.ColoHelper;
 import org.apache.falcon.regression.core.response.ServiceResponse;
 import org.apache.falcon.regression.core.util.AssertUtil;
+import org.apache.falcon.regression.core.util.HadoopUtil;
 import org.apache.falcon.regression.core.util.TimeUtil;
 import org.apache.falcon.regression.core.util.Util;
 import org.apache.falcon.regression.core.util.Util.URLS;
@@ -1031,4 +1032,22 @@ public class Bundle {
         return feedObject.getName();
     }
 
+    /**
+     * Update locations in cluster and base them at test path. For eg. the staging location
will be
+     * {testPath}/staging
+     * @param testAppPath  test app path of which locations will be based
+     */
+    public void updateClusterLocations(String testAppPath) {
+        List<String> newClusters = new ArrayList<String>();
+        for (String cluster : clusters) {
+            ClusterMerlin clusterMerlin = new ClusterMerlin(cluster);
+            for (org.apache.falcon.entity.v0.cluster.Location location : clusterMerlin
+                    .getLocations().getLocations()) {
+                location.setPath(HadoopUtil.stitchHdfsPath(testAppPath, location.getName()));
+            }
+            newClusters.add(clusterMerlin.toString());
+        }
+        clusters = newClusters;
+    }
+
 }

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/e9c6a2dc/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/BundleUtil.java
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/BundleUtil.java
b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/BundleUtil.java
index 1f73523..561443b 100644
--- a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/BundleUtil.java
+++ b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/BundleUtil.java
@@ -60,12 +60,12 @@ public final class BundleUtil {
         return readBundleFromFolder("ELbundle");
     }
 
-    public static Bundle readHCatBundle() throws IOException {
-        return readBundleFromFolder("hcat");
+    public static Bundle readHCatBundle(String appPath, String testName) throws IOException
{
+        return generateBundleFromTemplate("hcat", appPath, testName);
     }
 
-    public static Bundle readHCat2Bundle() throws IOException {
-        return readBundleFromFolder("hcat_2");
+    public static Bundle readHCat2Bundle(String appPath, String testName) throws IOException
{
+        return generateBundleFromTemplate("hcat_2", appPath, testName);
     }
 
     public static Bundle readLocalDCBundle() throws IOException {
@@ -80,6 +80,20 @@ public final class BundleUtil {
         return readBundleFromFolder("updateBundle");
     }
 
+    /** Generate a bundle from the template stored in a directory
+     * @param templatePath name of directory where template is stored
+     * @param appPath application path where staging/temp/working directory will live
+     * @param testName name of test
+     * @return the customized bundle
+     * @throws IOException
+     */
+    private static Bundle generateBundleFromTemplate(String templatePath, String appPath,
+            String testName) throws IOException {
+        final Bundle bundle = readBundleFromFolder(templatePath);
+        bundle.updateClusterLocations(HadoopUtil.stitchHdfsPath(appPath, testName));
+        return bundle;
+    }
+
     private static Bundle readBundleFromFolder(final String folderPath) throws IOException
{
         LOGGER.info("Loading xmls from directory: " + folderPath);
         File directory = null;

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/e9c6a2dc/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/HadoopUtil.java
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/HadoopUtil.java
b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/HadoopUtil.java
index dc0a4d4..925ee76 100644
--- a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/HadoopUtil.java
+++ b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/HadoopUtil.java
@@ -422,4 +422,24 @@ public final class HadoopUtil {
             }
         }
     }
+
+    /**
+     * Stitch two or more hadoop paths. For eg: stitchHdfsPath("/tmp/", "/test") = "/tmp/test",
+     * stitchHdfsPath("/tmp/", "test") = "/tmp/test", stitchHdfsPath("/tmp", "test") = "/tmp/test"
+     * @param path1 the first path to be stitched
+     * @param path2 the second path to be stitched
+     * @param pathParts other paths to be stitched
+     * @return final stitched path
+     */
+    public static String stitchHdfsPath(String path1, String path2, String... pathParts)
{
+        String retValue = stitchTwoHdfsPaths(path1, path2);
+        for (String pathPart : pathParts) {
+            retValue = stitchTwoHdfsPaths(retValue, pathPart);
+        }
+        return retValue;
+    }
+
+    private static String stitchTwoHdfsPaths(String path1, String path2) {
+        return path1.replaceAll("/*$", "") + "/" + path2.replaceAll("^/*", "");
+    }
 }

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/e9c6a2dc/falcon-regression/merlin/src/main/java/org/apache/falcon/regression/testHelper/BaseTestClass.java
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin/src/main/java/org/apache/falcon/regression/testHelper/BaseTestClass.java
b/falcon-regression/merlin/src/main/java/org/apache/falcon/regression/testHelper/BaseTestClass.java
index c64ddad..d892332 100644
--- a/falcon-regression/merlin/src/main/java/org/apache/falcon/regression/testHelper/BaseTestClass.java
+++ b/falcon-regression/merlin/src/main/java/org/apache/falcon/regression/testHelper/BaseTestClass.java
@@ -45,11 +45,14 @@ public class BaseTestClass {
         }
     }
 
-    public ColoHelper prism;
-    public List<ColoHelper> servers;
-    public List<FileSystem> serverFS;
-    public List<OozieClient> serverOC;
-    public String baseHDFSDir = "/tmp/falcon-regression";
+    public final ColoHelper prism;
+    public final List<ColoHelper> servers;
+    public final List<FileSystem> serverFS;
+    public final List<OozieClient> serverOC;
+    public final String baseHDFSDir = Config.getProperty("falcon.regression.test.dir",
+        "/tmp/falcon-regression");
+    public final String baseAppHDFSDir = Config.getProperty("falcon.regression.app.dir",
+        "/tmp/falcon-regression-app");
     public static final String PRISM_PREFIX = "prism";
     protected Bundle[] bundles;
 

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/e9c6a2dc/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/hcat/HCatProcessTest.java
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/hcat/HCatProcessTest.java
b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/hcat/HCatProcessTest.java
index 32062ba..d62ddde 100644
--- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/hcat/HCatProcessTest.java
+++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/hcat/HCatProcessTest.java
@@ -103,7 +103,7 @@ public class HCatProcessTest extends BaseTestClass {
     @BeforeMethod(alwaysRun = true)
     public void setUp() throws Exception {
         clusterHC = cluster.getClusterHelper().getHCatClient();
-        bundles[0] = BundleUtil.readHCatBundle();
+        bundles[0] = BundleUtil.readHCatBundle(baseAppHDFSDir, this.getClass().getSimpleName());
         bundles[0] = new Bundle(bundles[0], cluster);
         bundles[0].generateUniqueBundle();
         bundles[0].setProcessWorkflow(hiveScriptFile, EngineType.HIVE);

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/e9c6a2dc/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/hcat/HCatReplicationTest.java
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/hcat/HCatReplicationTest.java
b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/hcat/HCatReplicationTest.java
index 5452f50..c960ed6 100644
--- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/hcat/HCatReplicationTest.java
+++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/hcat/HCatReplicationTest.java
@@ -95,7 +95,7 @@ public class HCatReplicationTest extends BaseTestClass {
 
     @BeforeMethod(alwaysRun = true)
     public void setUp() throws Exception {
-        Bundle bundle = BundleUtil.readHCatBundle();
+        Bundle bundle = BundleUtil.readHCatBundle(baseAppHDFSDir, this.getClass().getSimpleName());
         bundles[0] = new Bundle(bundle, cluster.getPrefix());
         bundles[0].generateUniqueBundle();
         bundles[0].setClusterInterface(Interfacetype.REGISTRY,

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/e9c6a2dc/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/hcat/HCatRetentionTest.java
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/hcat/HCatRetentionTest.java
b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/hcat/HCatRetentionTest.java
index cce2860..557bd43 100644
--- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/hcat/HCatRetentionTest.java
+++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/hcat/HCatRetentionTest.java
@@ -78,7 +78,8 @@ public class HCatRetentionTest extends BaseTestClass {
     public void setUp() throws Exception {
         HadoopUtil.recreateDir(clusterFS, baseTestHDFSDir);
         cli = cluster.getClusterHelper().getHCatClient();
-        bundle = new Bundle(BundleUtil.readHCat2Bundle(), cluster);
+        bundle = BundleUtil.readHCat2Bundle(baseAppHDFSDir, this.getClass().getSimpleName());
+        bundle = new Bundle(bundle, cluster);
         bundle.generateUniqueBundle();
         bundle.submitClusters(prism);
     }

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/e9c6a2dc/falcon-regression/merlin/src/test/resources/hcat/data/_SUCCESS
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin/src/test/resources/hcat/data/_SUCCESS b/falcon-regression/merlin/src/test/resources/hcat/data/_SUCCESS
deleted file mode 100644
index 004c62e..0000000
--- a/falcon-regression/merlin/src/test/resources/hcat/data/_SUCCESS
+++ /dev/null
@@ -1,13 +0,0 @@
-Copyright [yyyy] [name of copyright owner]
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.


Mime
View raw message