bigtop-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From j..@apache.org
Subject git commit: BIGTOP-1222. Simplify and gradleize a subset of the bigtop smokes.
Date Wed, 20 Aug 2014 04:01:29 GMT
Repository: bigtop
Updated Branches:
  refs/heads/master 4fca4573b -> da7ce0f3e


BIGTOP-1222. Simplify and gradleize a subset of the bigtop smokes.

Signed-off-by: jay@apache.org <jayunit100>


Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo
Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/da7ce0f3
Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/da7ce0f3
Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/da7ce0f3

Branch: refs/heads/master
Commit: da7ce0f3e3fa2e24fddfb73a60733ded2061c698
Parents: 4fca457
Author: Jay Vyas <jayunit100@gmail.com>
Authored: Fri Jun 20 23:48:43 2014 -0400
Committer: jay@apache.org <jayunit100>
Committed: Wed Aug 20 00:00:04 2014 -0400

----------------------------------------------------------------------
 .../org/apache/bigtop/itest/TestUtils.groovy    |  12 +-
 bigtop-tests/smoke-tests/README                 |  59 +++++++
 bigtop-tests/smoke-tests/build.gradle           |  71 ++++++++
 .../smoke-tests/flume/TestFlumeNG.groovy        | 112 +++++++++++++
 bigtop-tests/smoke-tests/flume/build.gradle     |  55 +++++++
 bigtop-tests/smoke-tests/flume/conf/flume.conf  |  20 +++
 bigtop-tests/smoke-tests/flume/log4j.properties |   9 ++
 bigtop-tests/smoke-tests/hive/build.gradle      |  58 +++++++
 bigtop-tests/smoke-tests/hive/log4j.properties  |   9 ++
 bigtop-tests/smoke-tests/mahout/build.gradle    |  56 +++++++
 .../smoke-tests/mahout/log4j.properties         |   9 ++
 bigtop-tests/smoke-tests/mapreduce/build.gradle |  59 +++++++
 .../smoke-tests/pig/TestPigSmoke.groovy         |  58 +++++++
 bigtop-tests/smoke-tests/pig/build.gradle       |  37 +++++
 bigtop-tests/smoke-tests/pig/log4j.properties   |   9 ++
 bigtop-tests/smoke-tests/pig/wordcount.pig      |   6 +
 bigtop-tests/smoke-tests/settings.gradle        |  50 ++++++
 .../smoke-tests/sqoop/TestSqoopETLHsql.groovy   | 160 +++++++++++++++++++
 bigtop-tests/smoke-tests/sqoop/build.gradle     |  57 +++++++
 bigtop-tests/smoke-tests/sqoop/log4j.properties |   9 ++
 .../hadoop/mapreduce/TestHadoopExamples.groovy  |  44 ++++-
 .../hadoop/mapreduce/TestHadoopSmoke.groovy     |   2 +-
 .../hivesmoke/HiveBulkScriptExecutor.groovy     |  26 ++-
 .../itest/hivesmoke/TestHiveSmokeBulk.groovy    |  11 ++
 .../src/main/resources/scripts/ql/basic/filter  |   2 +-
 25 files changed, 983 insertions(+), 17 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/bigtop/blob/da7ce0f3/bigtop-test-framework/src/main/groovy/org/apache/bigtop/itest/TestUtils.groovy
----------------------------------------------------------------------
diff --git a/bigtop-test-framework/src/main/groovy/org/apache/bigtop/itest/TestUtils.groovy b/bigtop-test-framework/src/main/groovy/org/apache/bigtop/itest/TestUtils.groovy
index 3020303..f4bc247 100644
--- a/bigtop-test-framework/src/main/groovy/org/apache/bigtop/itest/TestUtils.groovy
+++ b/bigtop-test-framework/src/main/groovy/org/apache/bigtop/itest/TestUtils.groovy
@@ -47,21 +47,21 @@ public class TestUtils {
       sh.exec("hadoop fs -test -e ${inputDir}");
       if (sh.getRet() == 0) {
         sh.exec("hadoop fs -rmr -skipTrash ${inputDir}");
-        assertTrue("Deletion of previous $inputDir from HDFS failed",
+        assertTrue("Deletion of previous $inputDir from the DFS failed",
             sh.getRet() == 0);
       }
       if (inputFiles != null) {
         sh.exec("hadoop fs -mkdir -p ${inputDir}");
-        assertTrue("Could not create input directory to HDFS", sh.getRet() == 0);
+        assertTrue("Could not create input directory to the DFS", sh.getRet() == 0);
         // copy additional files into HDFS input folder
         inputFiles.each {
           sh.exec("hadoop fs -put ${it} ${inputDir}");
-          assertTrue("Could not copy input files into input folder in HDFS", sh.getRet() == 0);
+          assertTrue("Could not copy input files into input folder in the DFS", sh.getRet() == 0);
         }
       } else {
         // copy the entire resource folder into HDFS
         sh.exec("hadoop fs -put ${inputDir} ${inputDir}");
-        assertTrue("Could not copy input directory to HDFS", sh.getRet() == 0);
+        assertTrue("Could not copy input directory to the DFS", sh.getRet() == 0);
       }
     }
 
@@ -70,11 +70,11 @@ public class TestUtils {
       sh.exec("hadoop fs -test -e ${outputDir}");
       if (sh.getRet() == 0) {
         sh.exec("hadoop fs -rmr -skipTrash ${outputDir}");
-        assertTrue("Deletion of previous examples output from HDFS failed",
+        assertTrue("Deletion of previous examples output from the DFS failed",
             sh.getRet() == 0);
       }
       sh.exec("hadoop fs -mkdir -p ${outputDir}");
-      assertTrue("Could not create output directory in HDFS", sh.getRet() == 0);
+      assertTrue("Could not create output directory in DFS", sh.getRet() == 0);
     }
   }
 }

http://git-wip-us.apache.org/repos/asf/bigtop/blob/da7ce0f3/bigtop-tests/smoke-tests/README
----------------------------------------------------------------------
diff --git a/bigtop-tests/smoke-tests/README b/bigtop-tests/smoke-tests/README
new file mode 100644
index 0000000..c8ac345
--- /dev/null
+++ b/bigtop-tests/smoke-tests/README
@@ -0,0 +1,59 @@
+# This is the new smoke testing module for bigtop
+
+## Goals
+
+This gradle based test framework aims at making it extremely simple to
+customize and test Apache BigTop's hadoop distribution.
+
+ - Easily add new tests to bigtop without adding new directories / xml files.
+ - Debug tests easily and make logging extremely transparent.
+ - Geneate easy to read test reports.
+ - Run bigtop tests without needing a jar file.
+ - (todo) Run bigtop tests easily, ideally without gradle/maven
+   (this can be done with gradle wrapper)
+
+So overall, this framework makes testing with apache bigtop easy as well as fun!
+
+## Running
+
+Each directory is a gradle "subproject" mapping to an ecosystem component.
+If you specify, for example, "flume", then the flume test runs.  To test hive as well,
+you would specify "flume,hive", and so on.   An example is below:
+```
+    gradle compileGroovy test -Dsmoke-tests=flume,hive --info
+```
+
+The --info option is a critical feature here:  You can easily debug your tests this way,
+just make sure you have log4j.properties in your conf/ directory locally.
+
+## Adding more tests
+
+To add a new test you will want to add a groovy file to an existing ecosystem test or create a new folder
+(if you are adding a new component to bigtop that has no tests).
+
+This can be done easily by copying an existing test subproject, and simply modifying it
+for your own needs.  The settings.gradle file will automatically see the new folder you have added,
+and run your build.gradle.
+
+## How it works
+
+Subprojects in gradle are configured at the top level by the root build.gradle file.  Each
+subproject is injected with bits of functionality from the top level (i.e. generic functions
+for confirming environment variables are set, calling of the test task, etc).  Each subproject
+then is responsible for defining its source directories and resources, and does little more.
+
+Right now, this library leverages tests both from test-artifacts, as well as locally.
+Hopefully over time all tests will be merged here, because it is easier to maintain
+this script based framework.
+
+## Typical recipes
+
+There are a lot of ways to run the bigtop tests, and not all are applicable to all clusters.
+So, below, we've accumulated some typical modifications which you can easily implement
+when running tests through this gradle interface (rather than as jars).
+
+1) Modify the TestMahoutExamples.groovy file to exclude clustering tests, and re run.
+2) Add new tests to hive , other than the "basic" hive test.
+3) Modify the Sqoop test to write more data via the embedded database.
+4) Add your own custom Groovy test in a directory which calls a shell script of your own desgn.
+And so on.

http://git-wip-us.apache.org/repos/asf/bigtop/blob/da7ce0f3/bigtop-tests/smoke-tests/build.gradle
----------------------------------------------------------------------
diff --git a/bigtop-tests/smoke-tests/build.gradle b/bigtop-tests/smoke-tests/build.gradle
new file mode 100644
index 0000000..57286e5
--- /dev/null
+++ b/bigtop-tests/smoke-tests/build.gradle
@@ -0,0 +1,71 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */apply plugin: 'groovy'
+
+repositories {
+  mavenCentral()
+}
+
+dependencies {
+}
+
+subprojects {
+
+  ext.hadoopVersion = '2.3.0'
+  //Note you need a snapshot itest to run this.
+  //if you dont have it, change to 0.7.0.
+  ext.itestVersion = '0.8.0-SNAPSHOT'
+  ext.doExclude = { String filename ->
+    //print("Exclude? ${filename} ... ")
+    def keep_this_test = true;
+    if (filename.contains(".groovy") ||
+        filename.contains(".java")) {
+      keep_this_test =
+          tests_to_include().find({
+            it.contains(filename)
+          }) != null;
+    }
+
+    //println("Keep = ${keep_this_test} "+filename);
+    return !keep_this_test;
+  }
+
+  println("Now testing...");
+  test {
+
+    systemProperties['org.apache.bigtop.itest.hivesmoke.TestHiveSmokeBulk.test_include'] = 'basic'
+       testLogging {
+      events "passed", "skipped", "failed"
+    }
+  }
+
+  /**
+   *  Utility function for tests to use to confirm EVN Variables.
+   */
+  ext.checkEnv = { env_vars ->
+    env_vars.each() {
+      def value = System.getenv("${it}")
+      if (value == null || value == "null")
+        throw new GradleException("undeclared env variable: ${it}")
+      else
+        println("ENV VARIABLE: ${it} = $value");
+    }
+  }
+
+test.dependsOn compileGroovy
+compileGroovy.dependsOn clean
+}

http://git-wip-us.apache.org/repos/asf/bigtop/blob/da7ce0f3/bigtop-tests/smoke-tests/flume/TestFlumeNG.groovy
----------------------------------------------------------------------
diff --git a/bigtop-tests/smoke-tests/flume/TestFlumeNG.groovy b/bigtop-tests/smoke-tests/flume/TestFlumeNG.groovy
new file mode 100644
index 0000000..568e44b
--- /dev/null
+++ b/bigtop-tests/smoke-tests/flume/TestFlumeNG.groovy
@@ -0,0 +1,112 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.bigtop.itest.hadoop.mapreduce
+
+import org.junit.BeforeClass
+import org.junit.AfterClass
+import static org.junit.Assert.assertNotNull
+import org.apache.bigtop.itest.shell.Shell
+import static org.junit.Assert.assertTrue
+import org.junit.Test
+import org.apache.hadoop.conf.Configuration
+import org.apache.bigtop.itest.JarContent
+import org.apache.bigtop.itest.TestUtils
+import org.apache.commons.logging.LogFactory
+import org.apache.commons.logging.Log
+
+import org.junit.runner.RunWith
+
+class TestFlumeNG {
+  static private Log LOG = LogFactory.getLog(Object.class);
+
+  static Shell sh = new Shell("/bin/bash -s");
+
+
+  @AfterClass
+  public static void tearDown() {
+    LOG.warn("Not deleting flume-test from hdfs, delete manually if necessary")
+    //sh.exec("hadoop fs -rmr -skipTrash /tmp/flume-test");
+  }
+
+  @BeforeClass
+  static void setUp() {
+    sh.exec("hadoop fs -rmr -skipTrash /tmp/flume-test");
+    /**
+     *  Start a collecter that will run for 60 seconds.
+     *  This test is HCFS Compatible, see FLUME-2140.
+     */
+    String conf = """\n
+        agent.channels.memory-channel.type = memory
+        agent.channels.memory-channel.capacity = 2000
+        agent.channels.memory-channel.transactionCapacity = 100
+        agent.sources.tail-source.type = exec
+        agent.sources.tail-source.command = tail -F /tmp/flume-smoke.source
+        agent.sources.tail-source.channels = memory-channel
+
+        agent.sinks.log-sink.channel = memory-channel
+        agent.sinks.log-sink.type = logger
+
+        # Define a sink that outputs to the DFS
+        agent.sinks.hdfs-sink.channel = memory-channel
+        agent.sinks.hdfs-sink.type = hdfs
+        agent.sinks.hdfs-sink.hdfs.path = /tmp/flume-test
+        agent.sinks.hdfs-sink.hdfs.fileType = DataStream
+
+        # activate the channels/sinks/sources
+        agent.channels = memory-channel
+        agent.sources = tail-source
+        agent.sinks = log-sink hdfs-sink"""
+    new File("./conf/flume.conf").write(conf);
+
+    //Start the listener...
+
+    Thread.start {
+      sh.exec("timeout 60 flume-ng agent " +
+          "--conf ./conf/ " +
+          "-f ./conf/flume.conf " +
+          "-Dflume.root.logger=DEBUG,console " +
+          "-n agent > /tmp/flumetest.log")
+    }
+
+    LOG.info("Started threaded listener.")
+    LOG.info("Waiting 60 seconds to finish ....")
+    LOG.info("check /tmp/flumetest.log for progress")
+  }
+
+
+  @Test
+  void test() {
+    /**
+     *    Now write to the sink.
+     */
+    File source = new File("/tmp/flume-smoke.source");
+    for (i in 1..100) {
+      Thread.sleep(1000 - i)
+      source.withWriterAppend("UTF-8") {
+        it.write("hello ${i} \n")
+      }
+    }
+
+    LOG.info("RESULTS.........")
+    sh.exec("hadoop fs -cat /tmp/flume-test/* | grep -q hello")
+    assertTrue("Did not detect the contents in the flume SINK." + sh.getOut(), sh.getRet() == 0);
+
+
+  }
+}

http://git-wip-us.apache.org/repos/asf/bigtop/blob/da7ce0f3/bigtop-tests/smoke-tests/flume/build.gradle
----------------------------------------------------------------------
diff --git a/bigtop-tests/smoke-tests/flume/build.gradle b/bigtop-tests/smoke-tests/flume/build.gradle
new file mode 100644
index 0000000..37ffc0c
--- /dev/null
+++ b/bigtop-tests/smoke-tests/flume/build.gradle
@@ -0,0 +1,55 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+apply plugin: 'groovy'
+
+repositories {
+  mavenCentral()
+}
+
+dependencies {
+  //needed to avoid groovy not on classpath error.
+  testCompile module('org.codehaus.groovy:groovy:1.8.0')
+  testCompile group: 'org.apache.bigtop.itest', name: 'itest-common', version: itestVersion, transitive: 'true'
+  testCompile group: 'org.apache.hadoop', name: 'hadoop-common', version: hadoopVersion, transitive: 'true'
+  testCompile group: 'org.apache.hadoop', name: 'hadoop-hdfs', version: hadoopVersion, transitive: 'true'
+}
+
+
+def tests_to_include() {
+  return [
+      "TestFlumeNG.groovy"
+  ];
+}
+
+sourceSets {
+  test {
+    resources {
+      srcDirs = [
+          'conf/'
+      ]
+    }
+    groovy {
+      srcDirs = ["./"]
+      exclude { FileTreeElement elem -> (doExclude(elem.getName())) }
+    }
+  }
+}
+
+test.doFirst {
+  checkEnv(["FLUME_HOME"])
+}

http://git-wip-us.apache.org/repos/asf/bigtop/blob/da7ce0f3/bigtop-tests/smoke-tests/flume/conf/flume.conf
----------------------------------------------------------------------
diff --git a/bigtop-tests/smoke-tests/flume/conf/flume.conf b/bigtop-tests/smoke-tests/flume/conf/flume.conf
new file mode 100644
index 0000000..c06c1e3
--- /dev/null
+++ b/bigtop-tests/smoke-tests/flume/conf/flume.conf
@@ -0,0 +1,20 @@
+agent.channels.memory-channel.type = memory
+agent.channels.memory-channel.capacity = 2000
+agent.channels.memory-channel.transactionCapacity = 100
+agent.sources.tail-source.type = exec
+agent.sources.tail-source.command = tail -F /tmp/flume-smoke.source
+agent.sources.tail-source.channels = memory-channel
+
+agent.sinks.log-sink.channel = memory-channel
+agent.sinks.log-sink.type = logger
+
+# Define a sink that outputs to the DFS
+agent.sinks.hdfs-sink.channel = memory-channel
+agent.sinks.hdfs-sink.type = hdfs
+agent.sinks.hdfs-sink.hdfs.path = /tmp/flume-test
+agent.sinks.hdfs-sink.hdfs.fileType = DataStream
+
+# activate the channels/sinks/sources
+agent.channels = memory-channel
+agent.sources = tail-source
+agent.sinks = log-sink hdfs-sink
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/bigtop/blob/da7ce0f3/bigtop-tests/smoke-tests/flume/log4j.properties
----------------------------------------------------------------------
diff --git a/bigtop-tests/smoke-tests/flume/log4j.properties b/bigtop-tests/smoke-tests/flume/log4j.properties
new file mode 100644
index 0000000..02d115f
--- /dev/null
+++ b/bigtop-tests/smoke-tests/flume/log4j.properties
@@ -0,0 +1,9 @@
+# Set root logger level to DEBUG and its only appender to A1.
+log4j.rootLogger=TRACE, A1
+
+# A1 is set to be a ConsoleAppender.
+log4j.appender.A1=org.apache.log4j.ConsoleAppender
+
+# A1 uses PatternLayout.
+log4j.appender.A1.layout=org.apache.log4j.PatternLayout
+log4j.appender.A1.layout.ConversionPattern=%-4r [BIGTOP-TEST-LOG %t] %-5p %c %x --- %m%n

http://git-wip-us.apache.org/repos/asf/bigtop/blob/da7ce0f3/bigtop-tests/smoke-tests/hive/build.gradle
----------------------------------------------------------------------
diff --git a/bigtop-tests/smoke-tests/hive/build.gradle b/bigtop-tests/smoke-tests/hive/build.gradle
new file mode 100644
index 0000000..447fb00
--- /dev/null
+++ b/bigtop-tests/smoke-tests/hive/build.gradle
@@ -0,0 +1,58 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+apply plugin: 'groovy'
+
+repositories {
+  mavenCentral()
+}
+
+dependencies {
+  //needed to avoid groovy not on classpath error.
+  testCompile module('org.codehaus.groovy:groovy:1.8.0')
+  testCompile group: 'org.apache.bigtop.itest', name: 'itest-common', version: itestVersion, transitive: 'true'
+  testCompile group: 'org.apache.hadoop', name: 'hadoop-common', version: hadoopVersion, transitive: 'true'
+  testCompile group: 'org.apache.hadoop', name: 'hadoop-hdfs', version: hadoopVersion, transitive: 'true'
+}
+
+
+def tests_to_include() {
+  return [
+      "TestHiveSmokeBulk.groovy",
+      "HiveBulkScriptExecutor.groovy"
+  ];
+}
+
+sourceSets {
+  test {
+    resources {
+      srcDirs = [
+          'conf/'
+      ]
+    }
+    groovy {
+      srcDirs = ["$System.env.BIGTOP_HOME"+"/bigtop-tests/test-artifacts/hive/"]
+      exclude { FileTreeElement elem -> (doExclude(elem.getName())) }
+    }
+  }
+}
+
+test.doFirst {
+  // BIGTOP_HOME is necessary here, since we use it to read in from
+  // test-artifacts.
+  checkEnv(["HIVE_HOME", "HIVE_CONF_DIR", "BIGTOP_HOME"])
+}

http://git-wip-us.apache.org/repos/asf/bigtop/blob/da7ce0f3/bigtop-tests/smoke-tests/hive/log4j.properties
----------------------------------------------------------------------
diff --git a/bigtop-tests/smoke-tests/hive/log4j.properties b/bigtop-tests/smoke-tests/hive/log4j.properties
new file mode 100644
index 0000000..02d115f
--- /dev/null
+++ b/bigtop-tests/smoke-tests/hive/log4j.properties
@@ -0,0 +1,9 @@
+# Set root logger level to DEBUG and its only appender to A1.
+log4j.rootLogger=TRACE, A1
+
+# A1 is set to be a ConsoleAppender.
+log4j.appender.A1=org.apache.log4j.ConsoleAppender
+
+# A1 uses PatternLayout.
+log4j.appender.A1.layout=org.apache.log4j.PatternLayout
+log4j.appender.A1.layout.ConversionPattern=%-4r [BIGTOP-TEST-LOG %t] %-5p %c %x --- %m%n

http://git-wip-us.apache.org/repos/asf/bigtop/blob/da7ce0f3/bigtop-tests/smoke-tests/mahout/build.gradle
----------------------------------------------------------------------
diff --git a/bigtop-tests/smoke-tests/mahout/build.gradle b/bigtop-tests/smoke-tests/mahout/build.gradle
new file mode 100644
index 0000000..5a562b4
--- /dev/null
+++ b/bigtop-tests/smoke-tests/mahout/build.gradle
@@ -0,0 +1,56 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+apply plugin: 'groovy'
+
+repositories {
+  mavenCentral()
+}
+
+dependencies {
+  //needed to avoid groovy not on classpath error.
+  testCompile module('org.codehaus.groovy:groovy:1.8.0')
+  testCompile group: 'org.apache.bigtop.itest', name: 'itest-common', version: itestVersion, transitive: 'true'
+  testCompile group: 'org.apache.hadoop', name: 'hadoop-common', version: hadoopVersion, transitive: 'true'
+  testCompile group: 'org.apache.hadoop', name: 'hadoop-hdfs', version: hadoopVersion, transitive: 'true'
+}
+
+
+def tests_to_include() {
+  return [
+      "TestMahoutExamples.groovy"
+  ];
+}
+
+sourceSets {
+  test {
+    resources {
+      srcDirs = [
+          'conf/'
+      ]
+    }
+    groovy {
+      srcDirs = ["$System.env.BIGTOP_HOME"+"/bigtop-tests/test-artifacts/mahout/"]
+      exclude { FileTreeElement elem -> (doExclude(elem.getName())) }
+    }
+  }
+}
+
+test.doFirst {
+  checkEnv(["BIGTOP_HOME", "MAHOUT_HOME"])
+}

http://git-wip-us.apache.org/repos/asf/bigtop/blob/da7ce0f3/bigtop-tests/smoke-tests/mahout/log4j.properties
----------------------------------------------------------------------
diff --git a/bigtop-tests/smoke-tests/mahout/log4j.properties b/bigtop-tests/smoke-tests/mahout/log4j.properties
new file mode 100644
index 0000000..02d115f
--- /dev/null
+++ b/bigtop-tests/smoke-tests/mahout/log4j.properties
@@ -0,0 +1,9 @@
+# Set root logger level to DEBUG and its only appender to A1.
+log4j.rootLogger=TRACE, A1
+
+# A1 is set to be a ConsoleAppender.
+log4j.appender.A1=org.apache.log4j.ConsoleAppender
+
+# A1 uses PatternLayout.
+log4j.appender.A1.layout=org.apache.log4j.PatternLayout
+log4j.appender.A1.layout.ConversionPattern=%-4r [BIGTOP-TEST-LOG %t] %-5p %c %x --- %m%n

http://git-wip-us.apache.org/repos/asf/bigtop/blob/da7ce0f3/bigtop-tests/smoke-tests/mapreduce/build.gradle
----------------------------------------------------------------------
diff --git a/bigtop-tests/smoke-tests/mapreduce/build.gradle b/bigtop-tests/smoke-tests/mapreduce/build.gradle
new file mode 100644
index 0000000..7b67b4f
--- /dev/null
+++ b/bigtop-tests/smoke-tests/mapreduce/build.gradle
@@ -0,0 +1,59 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+apply plugin: 'groovy'
+
+repositories {
+  mavenCentral()
+}
+
+dependencies {
+  //needed to avoid groovy not on classpath error.
+  testCompile module('org.codehaus.groovy:groovy:1.8.0')
+  testCompile group: 'org.apache.bigtop.itest', name: 'itest-common', version: itestVersion, transitive: 'true'
+  testCompile group: 'org.apache.hadoop', name: 'hadoop-common', version: hadoopVersion, transitive: 'true'
+  testCompile group: 'org.apache.hadoop', name: 'hadoop-hdfs', version: hadoopVersion, transitive: 'true'
+}
+
+
+def tests_to_include() {
+  return [
+      "TestHadoopExamples.groovy"
+  ];
+}
+
+sourceSets {
+  test {
+    resources {
+      srcDirs = [
+          'conf/',
+      ]
+    }
+    groovy {
+      srcDirs = [
+          "$System.env.BIGTOP_HOME"+"/bigtop-tests/test-artifacts/hadoop/src/main/groovy/org/apache/bigtop/itest/hadoop/mapreduce"]
+      exclude {
+        FileTreeElement elem -> (doExclude(elem.getName()))
+      }
+    }
+  }
+}
+
+test.doFirst {
+  checkEnv(["HADOOP_CONF_DIR"])
+}

http://git-wip-us.apache.org/repos/asf/bigtop/blob/da7ce0f3/bigtop-tests/smoke-tests/pig/TestPigSmoke.groovy
----------------------------------------------------------------------
diff --git a/bigtop-tests/smoke-tests/pig/TestPigSmoke.groovy b/bigtop-tests/smoke-tests/pig/TestPigSmoke.groovy
new file mode 100644
index 0000000..9902267
--- /dev/null
+++ b/bigtop-tests/smoke-tests/pig/TestPigSmoke.groovy
@@ -0,0 +1,58 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.bigtop.itest.hadoop.mapreduce
+
+import org.junit.BeforeClass
+import org.junit.AfterClass
+import static org.junit.Assert.assertNotNull
+import org.apache.bigtop.itest.shell.Shell
+import static org.junit.Assert.assertTrue
+import org.junit.Test
+import org.apache.hadoop.conf.Configuration
+import org.apache.bigtop.itest.JarContent
+import org.apache.bigtop.itest.TestUtils
+import org.apache.commons.logging.LogFactory
+import org.apache.commons.logging.Log
+
+import org.junit.runner.RunWith
+
+class TestPigSmoke {
+  static private Log LOG = LogFactory.getLog(Object.class);
+
+  static Shell sh = new Shell("/bin/bash -s");
+
+
+  @AfterClass
+  public static void tearDown() {
+    sh.exec("hadoop fs -rmr -skipTrash pigsmoketest");
+  }
+
+  @BeforeClass
+  static void setUp() {
+    sh.exec("cat /etc/passwd > passwd");
+    sh.exec("hadoop fs -mkdir pigsmoketest");
+    sh.exec("hadoop fs -copyFromLocal passwd pigsmoketest/input");
+  }
+
+  @Test
+  void test() {
+    sh.exec("pig wordcount.pig");
+    assertTrue("Example word count failed. " + sh.getOut() + " " + sh.getErr(), sh.getRet() == 0);
+  }
+}

http://git-wip-us.apache.org/repos/asf/bigtop/blob/da7ce0f3/bigtop-tests/smoke-tests/pig/build.gradle
----------------------------------------------------------------------
diff --git a/bigtop-tests/smoke-tests/pig/build.gradle b/bigtop-tests/smoke-tests/pig/build.gradle
new file mode 100644
index 0000000..9b3d323
--- /dev/null
+++ b/bigtop-tests/smoke-tests/pig/build.gradle
@@ -0,0 +1,37 @@
+apply plugin: 'groovy'
+
+repositories {
+  mavenCentral()
+}
+
+dependencies {
+  //needed to avoid groovy not on classpath error.
+  testCompile module('org.codehaus.groovy:groovy:1.8.0')
+  testCompile group: 'org.apache.bigtop.itest', name: 'itest-common', version: itestVersion, transitive: 'true'
+  testCompile group: 'org.apache.hadoop', name: 'hadoop-common', version: hadoopVersion, transitive: 'true'
+  testCompile group: 'org.apache.hadoop', name: 'hadoop-hdfs', version: hadoopVersion, transitive: 'true'
+  testCompile group: 'org.hsqldb', name: 'hsqldb', version: '1.8.0.10'
+}
+
+
+def tests_to_include() {
+  return ["TestPigSmoke.groovy"];
+}
+
+sourceSets {
+  test {
+    resources {
+      srcDirs = ['conf/']
+    }
+
+    groovy {
+      srcDirs = ["./"]
+      exclude 'src/main/groovy/org/apache/bigtop/itest/hadoop/hdfs/**'
+      exclude { FileTreeElement elem -> (doExclude(elem.getName())) }
+    }
+  }
+}
+
+test.doFirst {
+  checkEnv(["PIG_HOME"])
+}

http://git-wip-us.apache.org/repos/asf/bigtop/blob/da7ce0f3/bigtop-tests/smoke-tests/pig/log4j.properties
----------------------------------------------------------------------
diff --git a/bigtop-tests/smoke-tests/pig/log4j.properties b/bigtop-tests/smoke-tests/pig/log4j.properties
new file mode 100644
index 0000000..02d115f
--- /dev/null
+++ b/bigtop-tests/smoke-tests/pig/log4j.properties
@@ -0,0 +1,9 @@
+# Set root logger level to DEBUG and its only appender to A1.
+log4j.rootLogger=TRACE, A1
+
+# A1 is set to be a ConsoleAppender.
+log4j.appender.A1=org.apache.log4j.ConsoleAppender
+
+# A1 uses PatternLayout.
+log4j.appender.A1.layout=org.apache.log4j.PatternLayout
+log4j.appender.A1.layout.ConversionPattern=%-4r [BIGTOP-TEST-LOG %t] %-5p %c %x --- %m%n

http://git-wip-us.apache.org/repos/asf/bigtop/blob/da7ce0f3/bigtop-tests/smoke-tests/pig/wordcount.pig
----------------------------------------------------------------------
diff --git a/bigtop-tests/smoke-tests/pig/wordcount.pig b/bigtop-tests/smoke-tests/pig/wordcount.pig
new file mode 100644
index 0000000..2b03219
--- /dev/null
+++ b/bigtop-tests/smoke-tests/pig/wordcount.pig
@@ -0,0 +1,6 @@
+A = load 'pigsmoketest/input';
+B = foreach A generate flatten(TOKENIZE((chararray)$0)) as word;
+C = group B by word;
+D = foreach C generate COUNT(B), group;
+store D into 'pig-output-wordcount';
+dump D ;

http://git-wip-us.apache.org/repos/asf/bigtop/blob/da7ce0f3/bigtop-tests/smoke-tests/settings.gradle
----------------------------------------------------------------------
diff --git a/bigtop-tests/smoke-tests/settings.gradle b/bigtop-tests/smoke-tests/settings.gradle
new file mode 100644
index 0000000..464988c
--- /dev/null
+++ b/bigtop-tests/smoke-tests/settings.gradle
@@ -0,0 +1,50 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+def tests_comma_str = System.properties['smoke.tests']
+
+if(tests_comma_str != null) {
+  def tests = tests_comma_str.split(",")
+  tests.each() { t ->
+    println("including $t")
+    include(t)
+  }
+}
+else {
+  println("""
+
+     ******************************************************************************
+     Welcome to the bigtop smoke tests.
+
+     This is an extensible test module, with one directory for every test component.
+
+     To run it, just use the following command, adding components (directories) into
+
+     the smoke.tests system property.
+
+     For example, to run the hive/ and pig/ smoke tests:
+
+         gradle clean compileGroovy test -Dsmoke.tests=hive,pig
+
+     NOTE:  You must run clean.  Otherwise gradle might skip some of the tests.
+
+     ... Exiting now ...
+     ******************************************************************************
+  """)
+  System.exit(1)
+}

http://git-wip-us.apache.org/repos/asf/bigtop/blob/da7ce0f3/bigtop-tests/smoke-tests/sqoop/TestSqoopETLHsql.groovy
----------------------------------------------------------------------
diff --git a/bigtop-tests/smoke-tests/sqoop/TestSqoopETLHsql.groovy b/bigtop-tests/smoke-tests/sqoop/TestSqoopETLHsql.groovy
new file mode 100644
index 0000000..3dafc99
--- /dev/null
+++ b/bigtop-tests/smoke-tests/sqoop/TestSqoopETLHsql.groovy
@@ -0,0 +1,160 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.bigtop.itest.hadoop.mapreduce
+
+import org.junit.Assert
+import org.junit.BeforeClass
+import org.junit.AfterClass
+import static org.junit.Assert.assertNotNull
+import org.apache.bigtop.itest.shell.Shell
+import static org.junit.Assert.assertTrue
+import org.junit.Test
+import org.apache.hadoop.conf.Configuration
+import org.apache.bigtop.itest.JarContent
+import org.apache.bigtop.itest.TestUtils
+import org.apache.commons.logging.LogFactory
+import org.apache.commons.logging.Log
+import java.lang.reflect.Constructor;
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.SQLException;
+import org.hsqldb.Server;
+import junit.framework.TestCase;
+import junit.framework.TestResult;
+import org.junit.runner.RunWith
+import org.hsqldb.persist.HsqlDatabaseProperties;
+import org.hsqldb.persist.HsqlProperties;
+import java.sql.DriverManager;
+import java.sql.Connection;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.io.PrintWriter;
+
+class TestSqoopHSQLDBETL {
+  static private Log LOG = LogFactory.getLog(Object.class);
+
+  static Shell sh = new Shell("/bin/bash -s");
+
+
+  @AfterClass
+  public static void tearDown() {
+    //sh.exec("hadoop fs -rmr -skipTrash pigsmoketest");
+    server.stop();
+  }
+
+  static Server server;
+
+  @BeforeClass
+  static void setUp() {
+    sh.exec("rm -rf /tmp/sqooptest*")
+    HsqlProperties p = new HsqlProperties();
+    p.setProperty("server.database.0", "file:/tmp/sqooptest");
+    // just use default.
+    p.setProperty("server.dbname.0", "sqooptest");
+    p.setProperty("hsqldb.default_table_type", "TEXT");
+
+    server = new Server();
+    server.setProperties(p);
+    server.setLogWriter(new PrintWriter(System.out, true));
+    // can use custom writer
+    server.setErrWriter(new PrintWriter(System.out, true));
+    // can use custom writer
+    server.start();
+
+  }
+
+  final static String DB_DRIVER = "org.hsqldb.jdbcDriver";
+  static String DB_CONNECTION =
+      "jdbc:hsqldb:hsql://localhost:9001/sqooptest";
+  static String DB_USER = "sa";
+  static String DB_PASSWORD = "";
+
+
+  void execDB(String query) {
+    Connection dbConnection = null;
+    Statement statement = null;
+    try {
+      dbConnection = getDBConnection();
+      statement = dbConnection.createStatement();
+      statement.execute(query);
+    }
+    catch (Exception e) {
+      e.printStackTrace();
+      Assert.fail(e.getMessage() + "")
+    }
+    finally {
+      if (statement != null) statement.close();
+      if (dbConnection != null) dbConnection.close();
+    }
+
+  }
+
+
+  String hsql = "/usr/lib/sqoop/lib/hsqldb-1.8.0.10.jar"
+  String port = "9001"
+
+  @Test
+  void test() {
+    createDB()
+    sh.exec("sqoop import --libjars /usr/lib/sqoop/lib/hsqldb-1.8.0.10.jar --connect \"jdbc:hsqldb:hsql://localhost:9001/sqooptest\" --username \"SA\" --query \'select ANIMALS.* from ANIMALS WHERE \$CONDITIONS\' --split-by ANIMALS.id --target-dir /tmp/sqooptest")
+  }
+
+  void createDB() {
+    String createTableSQL =
+        """
+        CREATE TABLE ANIMALS(
+         NAME VARCHAR(20) NOT NULL,
+         COUNTRY VARCHAR(20) NOT NULL,
+         ID INTEGER NOT NULL)
+	"""
+
+    execDB(createTableSQL);
+    LOG.info("DONE CREATING ANIMALS DB")
+
+    for (i in 1..300) {
+      execDB("INSERT INTO ANIMALS (NAME,COUNTRY,ID) VALUES (\'ELEPHANT\',\'SOUTH AFRICA\',1)");
+    }
+    execDB("INSERT INTO ANIMALS (NAME,COUNTRY,ID) VALUES (\'ELEPHANT\',\'SOUTH AFRICA\',1)");
+    execDB("INSERT INTO ANIMALS (NAME,COUNTRY,ID) VALUES (\'KOALA\', \'AUSTRALIA\', 2)");
+    execDB("INSERT INTO ANIMALS (NAME,COUNTRY,ID) VALUES (\'ZEBRA\', \'USA\', 3)");
+
+    LOG.info("DONE inserting 3 animals");
+  }
+
+  private static Connection getDBConnection() {
+    Connection dbConnection = null;
+    try {
+      Class.forName(DB_DRIVER);
+    }
+    catch (ClassNotFoundException e) {
+      LOG.info(e.getMessage());
+      Assert.fail("Driver not found ${DB_DRIVER}")
+    }
+    try {
+      dbConnection = DriverManager.
+          getConnection(DB_CONNECTION, DB_USER, DB_PASSWORD);
+      return dbConnection;
+    }
+    catch (Exception e) {
+      Assert.fail("Couldnt get connection ")
+    }
+    return dbConnection;
+  }
+}

http://git-wip-us.apache.org/repos/asf/bigtop/blob/da7ce0f3/bigtop-tests/smoke-tests/sqoop/build.gradle
----------------------------------------------------------------------
diff --git a/bigtop-tests/smoke-tests/sqoop/build.gradle b/bigtop-tests/smoke-tests/sqoop/build.gradle
new file mode 100644
index 0000000..35f4abd
--- /dev/null
+++ b/bigtop-tests/smoke-tests/sqoop/build.gradle
@@ -0,0 +1,57 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+apply plugin: 'groovy'
+
+repositories {
+  mavenCentral()
+}
+
+dependencies {
+  //needed to avoid groovy not on classpath error.
+  testCompile module('org.codehaus.groovy:groovy:1.8.0')
+  testCompile group: 'org.apache.bigtop.itest', name: 'itest-common', version: itestVersion, transitive: 'true'
+  testCompile group: 'org.apache.hadoop', name: 'hadoop-common', version: hadoopVersion, transitive: 'true'
+  testCompile group: 'org.apache.hadoop', name: 'hadoop-hdfs', version: hadoopVersion, transitive: 'true'
+  testCompile group: 'org.hsqldb', name: 'hsqldb', version: '1.8.0.10'
+}
+
+
+def tests_to_include() {
+  return [
+      "TestSqoopETLHsql.groovy"
+  ];
+}
+
+sourceSets {
+  test {
+    resources {
+      srcDirs = [
+          'conf/'
+      ]
+    }
+    groovy {
+      srcDirs = ["./"]
+      exclude { FileTreeElement elem -> (doExclude(elem.getName())) }
+    }
+  }
+}
+
+test.doFirst {
+  checkEnv(["SQOOP_HOME"])
+}

http://git-wip-us.apache.org/repos/asf/bigtop/blob/da7ce0f3/bigtop-tests/smoke-tests/sqoop/log4j.properties
----------------------------------------------------------------------
diff --git a/bigtop-tests/smoke-tests/sqoop/log4j.properties b/bigtop-tests/smoke-tests/sqoop/log4j.properties
new file mode 100644
index 0000000..02d115f
--- /dev/null
+++ b/bigtop-tests/smoke-tests/sqoop/log4j.properties
@@ -0,0 +1,9 @@
+# Set root logger level to DEBUG and its only appender to A1.
+log4j.rootLogger=TRACE, A1
+
+# A1 is set to be a ConsoleAppender.
+log4j.appender.A1=org.apache.log4j.ConsoleAppender
+
+# A1 uses PatternLayout.
+log4j.appender.A1.layout=org.apache.log4j.PatternLayout
+log4j.appender.A1.layout.ConversionPattern=%-4r [BIGTOP-TEST-LOG %t] %-5p %c %x --- %m%n

http://git-wip-us.apache.org/repos/asf/bigtop/blob/da7ce0f3/bigtop-tests/test-artifacts/hadoop/src/main/groovy/org/apache/bigtop/itest/hadoop/mapreduce/TestHadoopExamples.groovy
----------------------------------------------------------------------
diff --git a/bigtop-tests/test-artifacts/hadoop/src/main/groovy/org/apache/bigtop/itest/hadoop/mapreduce/TestHadoopExamples.groovy b/bigtop-tests/test-artifacts/hadoop/src/main/groovy/org/apache/bigtop/itest/hadoop/mapreduce/TestHadoopExamples.groovy
index 1afdaf8..6d290ef 100644
--- a/bigtop-tests/test-artifacts/hadoop/src/main/groovy/org/apache/bigtop/itest/hadoop/mapreduce/TestHadoopExamples.groovy
+++ b/bigtop-tests/test-artifacts/hadoop/src/main/groovy/org/apache/bigtop/itest/hadoop/mapreduce/TestHadoopExamples.groovy
@@ -19,6 +19,7 @@
 package org.apache.bigtop.itest.hadoop.mapreduce
 
 import org.junit.BeforeClass
+import org.junit.AfterClass
 import static org.junit.Assert.assertNotNull
 import org.apache.bigtop.itest.shell.Shell
 import static org.junit.Assert.assertTrue
@@ -35,7 +36,7 @@ import org.junit.runner.RunWith
 
 @RunWith(OrderedParameterized.class)
 class TestHadoopExamples {
-  static private Log LOG = LogFactory.getLog(TestHadoopExamples.class);
+  static private Log LOG = LogFactory.getLog(Object.class);
 
   private static final String HADOOP_MAPRED_HOME = System.getenv('HADOOP_MAPRED_HOME');
   private static final String HADOOP_CONF_DIR = System.getenv('HADOOP_CONF_DIR');
@@ -52,20 +53,54 @@ class TestHadoopExamples {
     HADOOP_MAPRED_HOME + "/" + hadoopExamplesJar;
 
   static Shell sh = new Shell("/bin/bash -s");
+
+  /**
+  * Public so that we can run these tests as scripts
+  * and the scripts can manually copy resoruces into DFS
+  * See BIGTOP-1222 for example.
+  */
+  public static final String SOURCE ="bigtop-tests/test-artifacts/hadoop/src/main/resources/"
   private static final String EXAMPLES = "examples";
   private static final String EXAMPLES_OUT = "examples-output";
   private static Configuration conf;
 
   private static String mr_version = System.getProperty("mr.version", "mr2");
   
-    static final String RANDOMTEXTWRITER_TOTALBYTES = (mr_version == "mr1") ?
+  static final String RANDOMTEXTWRITER_TOTALBYTES = (mr_version == "mr1") ?
       "test.randomtextwrite.total_bytes" : "mapreduce.randomtextwriter.totalbytes";
 
+  @AfterClass
+  public static void tearDown() {
+    sh.exec("hadoop fs -rmr -skipTrash ${EXAMPLES}",
+            "hadoop fs -rmr -skipTrash ${EXAMPLES_OUT}");
+  }
+
+
   @BeforeClass
   static void setUp() {
     conf = new Configuration();
-    TestUtils.unpackTestResources(TestHadoopExamples.class, EXAMPLES, null, EXAMPLES_OUT);
-    assertTrue("Failed to copy TestHadoopExamples.class to FS", sh.getRet() == 0);
+    try{
+       //copy examples/ int /user/root/ and
+       //then create examples-output directory
+       TestUtils.unpackTestResources(TestHadoopExamples.class, EXAMPLES, null, EXAMPLES_OUT);
+    }
+    catch(java.lang.Throwable t){
+        LOG.info("Failed to unpack jar resources.  Attemting to use bigtop sources");
+        def source = System.getenv("BIGTOP_HOME")+"/"+SOURCE;
+
+        assertNotNull("Can't copy test input files from bigtop source dir,"+
+                      "and jar specific attempt failed also", examples);
+
+        LOG.info("MAKING DIRECTORIES ..................... ${EXAMPLES} ${EXAMPLES_OUT}");
+
+        //add the files in resources/
+        sh.exec("hadoop fs -put ${source}/*.* .");
+        //add the directories under resources (like examples/)
+        sh.exec("hadoop fs -put ${source}/${EXAMPLES} ${EXAMPLES}");
+        sh.exec("hadoop fs -mkdir -p ${EXAMPLES_OUT}");
+   }
+   sh.exec("hadoop fs -ls ${EXAMPLES}");
+   assertTrue("Failed asserting that 'examples' were created in the DFS", sh.getRet()==0);
   }
 
   static long terasortid = System.currentTimeMillis();
@@ -76,7 +111,6 @@ class TestHadoopExamples {
   //Number of maps and samples for pi
   public static String pi_maps = System.getProperty("pi_maps", "2");
   public static String pi_samples = System.getProperty("pi_samples", "1000");
-
   static LinkedHashMap examples =
     [
         pi                :"${pi_maps} ${pi_samples}",

http://git-wip-us.apache.org/repos/asf/bigtop/blob/da7ce0f3/bigtop-tests/test-artifacts/hadoop/src/main/groovy/org/apache/bigtop/itest/hadoop/mapreduce/TestHadoopSmoke.groovy
----------------------------------------------------------------------
diff --git a/bigtop-tests/test-artifacts/hadoop/src/main/groovy/org/apache/bigtop/itest/hadoop/mapreduce/TestHadoopSmoke.groovy b/bigtop-tests/test-artifacts/hadoop/src/main/groovy/org/apache/bigtop/itest/hadoop/mapreduce/TestHadoopSmoke.groovy
index 0b2970b..da31115 100644
--- a/bigtop-tests/test-artifacts/hadoop/src/main/groovy/org/apache/bigtop/itest/hadoop/mapreduce/TestHadoopSmoke.groovy
+++ b/bigtop-tests/test-artifacts/hadoop/src/main/groovy/org/apache/bigtop/itest/hadoop/mapreduce/TestHadoopSmoke.groovy
@@ -56,7 +56,7 @@ class TestHadoopSmoke {
   @BeforeClass
   static void  setUp() throws IOException {
     String[] inputFiles = ["cachedir.jar", "input.txt"];
-    TestUtils.unpackTestResources(TestHadoopSmoke.class, "${testDir}/cachefile", inputFiles, null);
+    //TestUtils.unpackTestResources(TestHadoopSmoke.class, "${testDir}/cachefile", inputFiles, null);
   }
 
   @AfterClass

http://git-wip-us.apache.org/repos/asf/bigtop/blob/da7ce0f3/bigtop-tests/test-artifacts/hive/src/main/groovy/org/apache/bigtop/itest/hivesmoke/HiveBulkScriptExecutor.groovy
----------------------------------------------------------------------
diff --git a/bigtop-tests/test-artifacts/hive/src/main/groovy/org/apache/bigtop/itest/hivesmoke/HiveBulkScriptExecutor.groovy b/bigtop-tests/test-artifacts/hive/src/main/groovy/org/apache/bigtop/itest/hivesmoke/HiveBulkScriptExecutor.groovy
index 23444b4..a25214a 100644
--- a/bigtop-tests/test-artifacts/hive/src/main/groovy/org/apache/bigtop/itest/hivesmoke/HiveBulkScriptExecutor.groovy
+++ b/bigtop-tests/test-artifacts/hive/src/main/groovy/org/apache/bigtop/itest/hivesmoke/HiveBulkScriptExecutor.groovy
@@ -17,23 +17,39 @@
  */
 package org.apache.bigtop.itest.hivesmoke
 
+import org.apache.commons.logging.LogFactory
+import org.apache.commons.logging.Log
 import org.apache.bigtop.itest.JarContent
 import org.apache.bigtop.itest.shell.Shell
 import static junit.framework.Assert.assertEquals
 
 public class HiveBulkScriptExecutor {
+  static private Log LOG = LogFactory.getLog(Object.class);
+
   static Shell sh = new Shell("/bin/bash -s");
 
   private File scripts;
   private String location;
 
+  public static final String RESOURCES ="bigtop-tests/test-artifacts/hive/src/main/resources/"
+
   public HiveBulkScriptExecutor(String l) {
     location = l;
     scripts = new File(location);
 
     if (!scripts.exists()) {
-      JarContent.unpackJarContainer(HiveBulkScriptExecutor.class, '.' , null);
-    }
+      try{
+        JarContent.unpackJarContainer(HiveBulkScriptExecutor.class, '.' , null);
+      }
+      //BIGTOP-1222 : Support script execution.
+      catch(Throwable t){
+        LOG.info("Didnt find jar resource, copying resources locally...");
+        def resources = System.getenv("BIGTOP_HOME")+"/"+RESOURCES ;
+        sh.exec("cp -r ${resources}/* .");
+        sh.exec("ls ${l}");
+
+      }
+   }
   }
 
   public List<String> getScripts() {
@@ -47,14 +63,16 @@ public class HiveBulkScriptExecutor {
 
   public void runScript(String test, String extraArgs) {
     String l = "${location}/${test}";
+    String test_command="""diff -u <(\$F < ${l}/actual) <(\$F < ${l}/out)""" ;
     sh.exec("""
     F=cat
     if [ -f ${l}/filter ]; then
       chmod 777 ${l}/filter
       F=${l}/filter
     fi
-    hive ${extraArgs} -v -f ${l}/in > ${l}/actual && diff -u <(\$F < ${l}/actual) <(\$F < ${l}/out)
-    """);
+    hive ${extraArgs} -v -f ${l}/in > ${l}/actual && ${test_command}"""
+    ) ;
+
     assertEquals("Got unexpected output from test script ${test}",
                   0, sh.ret);
   }

http://git-wip-us.apache.org/repos/asf/bigtop/blob/da7ce0f3/bigtop-tests/test-artifacts/hive/src/main/groovy/org/apache/bigtop/itest/hivesmoke/TestHiveSmokeBulk.groovy
----------------------------------------------------------------------
diff --git a/bigtop-tests/test-artifacts/hive/src/main/groovy/org/apache/bigtop/itest/hivesmoke/TestHiveSmokeBulk.groovy b/bigtop-tests/test-artifacts/hive/src/main/groovy/org/apache/bigtop/itest/hivesmoke/TestHiveSmokeBulk.groovy
index 6a0bbff..60ca698 100644
--- a/bigtop-tests/test-artifacts/hive/src/main/groovy/org/apache/bigtop/itest/hivesmoke/TestHiveSmokeBulk.groovy
+++ b/bigtop-tests/test-artifacts/hive/src/main/groovy/org/apache/bigtop/itest/hivesmoke/TestHiveSmokeBulk.groovy
@@ -25,9 +25,15 @@ import org.apache.bigtop.itest.shell.Shell
 import org.apache.bigtop.itest.junit.OrderedParameterized
 import org.junit.runner.RunWith
 import org.junit.runners.Parameterized.Parameters
+import org.apache.commons.logging.LogFactory
+import org.apache.commons.logging.Log
+import static org.junit.Assert.assertTrue
 
 @RunWith(OrderedParameterized.class)
 public class TestHiveSmokeBulk {
+
+  static private Log LOG = LogFactory.getLog(Object.class);
+
   private static String test_include =
     System.getProperty("org.apache.bigtop.itest.hivesmoke.TestHiveSmokeBulk.test_include");
   private static String test_exclude =
@@ -61,6 +67,9 @@ public class TestHiveSmokeBulk {
   @BeforeClass
   public static void setUp() {
     sh.exec("hive -f ./seed.hql");
+    LOG.info(sh.getOut())
+    LOG.info(sh.getErr())
+    assertTrue("FAILED.. "+sh.getOut()+" "+sh.getErr(),sh.getRet()==0);
   }
 
   @AfterClass
@@ -71,6 +80,7 @@ public class TestHiveSmokeBulk {
 
   @Parameters
   public static Map<String, Object[]> readTestCases() {
+    LOG.info("hive includes = " + test_include);
     List<String> tests;
     if (test_include != null) {
       tests = scripts.getScripts().intersect(Arrays.asList(test_include.split(",")));
@@ -79,6 +89,7 @@ public class TestHiveSmokeBulk {
     } else {
       tests = scripts.getScripts();
     }
+    LOG.info("HIVE TESTS = " + tests);
     Map res = [:];
     tests.each {
       res[it] = ([it] as String[]);

http://git-wip-us.apache.org/repos/asf/bigtop/blob/da7ce0f3/bigtop-tests/test-artifacts/hive/src/main/resources/scripts/ql/basic/filter
----------------------------------------------------------------------
diff --git a/bigtop-tests/test-artifacts/hive/src/main/resources/scripts/ql/basic/filter b/bigtop-tests/test-artifacts/hive/src/main/resources/scripts/ql/basic/filter
index 0bba73c..e8e6135 100644
--- a/bigtop-tests/test-artifacts/hive/src/main/resources/scripts/ql/basic/filter
+++ b/bigtop-tests/test-artifacts/hive/src/main/resources/scripts/ql/basic/filter
@@ -1,4 +1,4 @@
 sed -e 's#hdfs://[^/]*/#hdfs://HADOOP/#' \
-    -e 's#Copying file:.*u.data#Copying file: u.data#' \
+    -e '/.*Copying .*u.data$/d' \
     -e '/^Deleted.*u_data$/d' \
     -e '/.*jobconf.xml:an attempt to override final parameter: mapreduce.job.end-notification.*;  Ignoring\./ d'


Mime
View raw message