falcon-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From rostafiyc...@apache.org
Subject incubator-falcon git commit: FALCON-884 Add option to dump xmls generated by falcon. Contributed by Raghav Kumar Gautam
Date Fri, 05 Dec 2014 16:21:28 GMT
Repository: incubator-falcon
Updated Branches:
  refs/heads/master 56900a2b8 -> bf9ed92b3


FALCON-884 Add option to dump xmls generated by falcon. Contributed by Raghav Kumar Gautam


Project: http://git-wip-us.apache.org/repos/asf/incubator-falcon/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-falcon/commit/bf9ed92b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-falcon/tree/bf9ed92b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-falcon/diff/bf9ed92b

Branch: refs/heads/master
Commit: bf9ed92b3a60a350d4589ff2a6f8485ab238d23e
Parents: 56900a2
Author: Ruslan Ostafiychuk <rostafiychuk@apache.org>
Authored: Fri Dec 5 18:16:51 2014 +0200
Committer: Ruslan Ostafiychuk <rostafiychuk@apache.org>
Committed: Fri Dec 5 18:16:51 2014 +0200

----------------------------------------------------------------------
 falcon-regression/CHANGES.txt                   |  3 ++
 falcon-regression/README.md                     |  6 +++
 .../core/enumsAndConstants/MerlinConstants.java |  8 +++
 .../falcon/regression/core/util/BundleUtil.java |  6 +--
 .../falcon/regression/core/util/Config.java     | 28 ++++++++---
 .../falcon/regression/TestngListener.java       | 52 +++++++++++++++++---
 6 files changed, 86 insertions(+), 17 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/bf9ed92b/falcon-regression/CHANGES.txt
----------------------------------------------------------------------
diff --git a/falcon-regression/CHANGES.txt b/falcon-regression/CHANGES.txt
index a0a1a7d..ea9ac36 100644
--- a/falcon-regression/CHANGES.txt
+++ b/falcon-regression/CHANGES.txt
@@ -5,6 +5,9 @@ Trunk (Unreleased)
   INCOMPATIBLE CHANGES
 
   NEW FEATURES
+   FALCON-884 Add option to dump xmls generated by falcon (Raghav Kumar Gautam via Ruslan
+   Ostafiychuk)
+
    FALCON-893 Add tests for replication to wasb filesystem
    (Raghav Kumar Gautam and Ruslan Ostafiychuk via Arpit Gupta)
 

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/bf9ed92b/falcon-regression/README.md
----------------------------------------------------------------------
diff --git a/falcon-regression/README.md b/falcon-regression/README.md
index eca1ca0..33fd7cf 100644
--- a/falcon-regression/README.md
+++ b/falcon-regression/README.md
@@ -142,3 +142,9 @@ info and logs at the end of the test. This can be done by configuring
Merlin.pro
     log.capture.oozie.skip_info = false
     log.capture.oozie.skip_log = true
     log.capture.location = ../
+
+Dumping entities generated by falcon
+------------------------------------
+Add -Dmerlin.dump.staging to the maven command. For example:
+
+    mvn clean test -Phadoop-2 -Dmerlin.dump.staging=true

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/bf9ed92b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/enumsAndConstants/MerlinConstants.java
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/enumsAndConstants/MerlinConstants.java
b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/enumsAndConstants/MerlinConstants.java
index 354137c..10cddf0 100644
--- a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/enumsAndConstants/MerlinConstants.java
+++ b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/enumsAndConstants/MerlinConstants.java
@@ -37,6 +37,14 @@ public final class MerlinConstants {
 
     public static final boolean IS_SECURE =
         "kerberos".equals(new Configuration().get("hadoop.security.authentication", "simple"));
+
+    /** Staging location to use in cluster xml. */
+    public static final String STAGING_LOCATION = Config.getProperty("merlin.staging.location",
+        "/tmp/falcon-regression-staging");
+    /** Working location to use in cluster xml. */
+    public static final String WORKING_LOCATION = Config.getProperty("merlin.working.location",
+        "/tmp/falcon-regression-working");
+
     /** the user that is going to run tests. */
     public static final String CURRENT_USER_NAME = System.getProperty("user.name");
     /** keytab of current user. */

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/bf9ed92b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/BundleUtil.java
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/BundleUtil.java
b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/BundleUtil.java
index f70b8f5..f52d563 100644
--- a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/BundleUtil.java
+++ b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/BundleUtil.java
@@ -112,13 +112,11 @@ public final class BundleUtil {
                 clusterMerlin.getLocations().getLocations().clear();
                 final Location staging = new Location();
                 staging.setName("staging");
-                staging.setPath(Config.getProperty("merlin.staging.location",
-                        "/tmp/falcon-regression-staging"));
+                staging.setPath(MerlinConstants.STAGING_LOCATION);
                 clusterMerlin.getLocations().getLocations().add(staging);
                 final Location working = new Location();
                 working.setName("working");
-                working.setPath(Config.getProperty("merlin.working.location",
-                        "/tmp/falcon-regression-working"));
+                working.setPath(MerlinConstants.WORKING_LOCATION);
                 clusterMerlin.getLocations().getLocations().add(working);
                 final String protectionPropName = "hadoop.rpc.protection";
                 final String protectionPropValue = Config.getProperty(protectionPropName);

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/bf9ed92b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/Config.java
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/Config.java
b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/Config.java
index 9172455..ba32d11 100644
--- a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/Config.java
+++ b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/Config.java
@@ -18,8 +18,11 @@
 
 package org.apache.falcon.regression.core.util;
 
+import org.apache.commons.configuration.AbstractConfiguration;
+import org.apache.commons.configuration.CompositeConfiguration;
 import org.apache.commons.configuration.ConfigurationException;
 import org.apache.commons.configuration.PropertiesConfiguration;
+import org.apache.commons.configuration.SystemConfiguration;
 import org.apache.commons.configuration.reloading.FileChangedReloadingStrategy;
 import org.apache.log4j.Logger;
 import org.testng.Assert;
@@ -31,20 +34,31 @@ public final class Config {
     private static final String MERLIN_PROPERTIES = "Merlin.properties";
     private static final Config INSTANCE = new Config(MERLIN_PROPERTIES);
 
-    private PropertiesConfiguration confObj;
+    private AbstractConfiguration confObj;
     private Config(String propFileName) {
         try {
-            LOGGER.info("Going to read properties from: " + propFileName);
-            confObj = new PropertiesConfiguration(Config.class.getResource("/" + propFileName));
-            //if changed configuration will be reloaded within 2 min
-            final FileChangedReloadingStrategy reloadingStrategy = new FileChangedReloadingStrategy();
-            reloadingStrategy.setRefreshDelay(2 * 60 * 1000);
-            confObj.setReloadingStrategy(reloadingStrategy);
+            initConfig(propFileName);
         } catch (ConfigurationException e) {
             Assert.fail("Could not read properties because of exception: " + e);
         }
     }
 
+    private void initConfig(String propFileName) throws ConfigurationException {
+        CompositeConfiguration compositeConfiguration = new CompositeConfiguration();
+        LOGGER.info("Going to add properties from system properties.");
+        compositeConfiguration.addConfiguration(new SystemConfiguration());
+
+        LOGGER.info("Going to read properties from: " + propFileName);
+        final PropertiesConfiguration merlinConfig =
+            new PropertiesConfiguration(Config.class.getResource("/" + propFileName));
+        //if changed configuration will be reloaded within 2 min
+        final FileChangedReloadingStrategy reloadingStrategy = new FileChangedReloadingStrategy();
+        reloadingStrategy.setRefreshDelay(2 * 60 * 1000);
+        merlinConfig.setReloadingStrategy(reloadingStrategy);
+        compositeConfiguration.addConfiguration(merlinConfig);
+        this.confObj = compositeConfiguration;
+    }
+
     public static String getProperty(String key) {
         return INSTANCE.confObj.getString(key);
     }

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/bf9ed92b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/TestngListener.java
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/TestngListener.java
b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/TestngListener.java
index 645d63b..e3aac1d 100644
--- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/TestngListener.java
+++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/TestngListener.java
@@ -19,13 +19,17 @@
 package org.apache.falcon.regression;
 
 import org.apache.commons.io.FileUtils;
-import org.apache.commons.lang.exception.ExceptionUtils;
 import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang.exception.ExceptionUtils;
+import org.apache.falcon.regression.core.enumsAndConstants.MerlinConstants;
 import org.apache.falcon.regression.core.helpers.ColoHelper;
 import org.apache.falcon.regression.core.util.Config;
 import org.apache.falcon.regression.core.util.LogUtil;
 import org.apache.falcon.regression.core.util.OSUtil;
+import org.apache.falcon.regression.core.util.TimeUtil;
 import org.apache.falcon.regression.testHelper.BaseUITestClass;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
 import org.apache.log4j.Logger;
 import org.apache.log4j.NDC;
 import org.openqa.selenium.OutputType;
@@ -38,6 +42,7 @@ import org.testng.ITestResult;
 import java.io.File;
 import java.io.IOException;
 import java.util.Arrays;
+import java.util.Date;
 
 /**
  * Testng listener class. This is useful for things that are applicable to all the tests
as well
@@ -47,6 +52,8 @@ public class TestngListener implements ITestListener, IExecutionListener
{
     private static final Logger LOGGER = Logger.getLogger(TestngListener.class);
     private final String hr = StringUtils.repeat("-", 100);
 
+    private enum RunResult {SUCCESS, FAILED, SKIPPED, TestFailedButWithinSuccessPercentage
}
+
     @Override
     public void onTestStart(ITestResult result) {
         LOGGER.info(hr);
@@ -56,7 +63,12 @@ public class TestngListener implements ITestListener, IExecutionListener
{
         NDC.push(result.getName());
     }
 
-    private void logEndOfTest(ITestResult result, String outcome) {
+    private void endOfTestHook(ITestResult result, RunResult outcome) {
+        try {
+            dumpFalconStore(result);
+        } catch (Exception e) {
+            LOGGER.info("Dumping of falcon store failed: " + e);
+        }
         LOGGER.info(
             String.format("Testing going to end for: %s.%s(%s) %s", result.getTestClass().getName(),
                 result.getName(), Arrays.toString(result.getParameters()), outcome));
@@ -64,14 +76,42 @@ public class TestngListener implements ITestListener, IExecutionListener
{
         LOGGER.info(hr);
     }
 
+    private void dumpFalconStore(ITestResult result) throws IOException {
+        if (Config.getBoolean("merlin.dump.staging", false)) {
+            final String[] serverNames = Config.getStringArray("servers");
+            for (final String serverName : serverNames) {
+                final ColoHelper coloHelper = new ColoHelper(serverName.trim());
+                final FileSystem clusterFs = coloHelper.getClusterHelper().getHadoopFS();
+                final String fileNameTemp = StringUtils.join(
+                    new String[]{serverName,
+                        result.getName(),
+                        Arrays.toString(result.getParameters()),
+                        TimeUtil.dateToOozieDate(new Date()),
+                    },
+                    "-");
+                final String localFileName = fileNameTemp.replaceAll(":", "-");
+                LOGGER.info("Dumping staging contents to: " + fileNameTemp);
+                clusterFs.copyToLocalFile(false, new Path(MerlinConstants.STAGING_LOCATION),
+                    new Path(localFileName));
+            }
+        }
+    }
+
     @Override
     public void onTestSuccess(ITestResult result) {
-        logEndOfTest(result, "SUCCESS");
+        endOfTestHook(result, RunResult.SUCCESS);
     }
 
     @Override
     public void onTestFailure(ITestResult result) {
-        logEndOfTest(result, "FAILED");
+        endOfTestHook(result, RunResult.FAILED);
+        takeScreenShot(result);
+
+        LOGGER.info(ExceptionUtils.getStackTrace(result.getThrowable()));
+        LOGGER.info(hr);
+    }
+
+    private void takeScreenShot(ITestResult result) {
         if (BaseUITestClass.getDriver() != null) {
             byte[] scrFile =
                 ((TakesScreenshot)BaseUITestClass.getDriver()).getScreenshotAs(OutputType.BYTES);
@@ -90,12 +130,12 @@ public class TestngListener implements ITestListener, IExecutionListener
{
 
     @Override
     public void onTestSkipped(ITestResult result) {
-        logEndOfTest(result, "SKIPPED");
+        endOfTestHook(result, RunResult.SKIPPED);
     }
 
     @Override
     public void onTestFailedButWithinSuccessPercentage(ITestResult result) {
-        logEndOfTest(result, "TestFailedButWithinSuccessPercentage");
+        endOfTestHook(result, RunResult.TestFailedButWithinSuccessPercentage);
     }
 
     @Override


Mime
View raw message