incubator-bigtop-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From aba...@apache.org
Subject svn commit: r1148501 [7/47] - in /incubator/bigtop: branches/ tags/ trunk/ trunk/docs/ trunk/src/ trunk/src/pkg/ trunk/src/pkg/common/ trunk/src/pkg/common/flume/ trunk/src/pkg/common/hadoop/ trunk/src/pkg/common/hadoop/conf.pseudo/ trunk/src/pkg/commo...
Date Tue, 19 Jul 2011 19:45:54 GMT
Added: incubator/bigtop/trunk/test/site/src/site/apt/userguide.apt
URL: http://svn.apache.org/viewvc/incubator/bigtop/trunk/test/site/src/site/apt/userguide.apt?rev=1148501&view=auto
==============================================================================
--- incubator/bigtop/trunk/test/site/src/site/apt/userguide.apt (added)
+++ incubator/bigtop/trunk/test/site/src/site/apt/userguide.apt Tue Jul 19 19:44:48 2011
@@ -0,0 +1,84 @@
+Develop with iTest
+~~~~~~~~~~~~~~~~~~
+
+* iTest provided functionality
+
+** Shell support
+
+  iTest implements transparent shell-outs from test written in Java and Groovy.
+  This support is implemented in <<<com.cloudera.itest.shell>>> package.
+
++------------------------+
+...
+  Shell sh = new Shell("/bin/bash -s");
+  sh.exec("hadoop jar $HADOOP_HOME/hadoop-examples.jar sleep -mt 1 -rt 1 -m 10 -r 10");
+  assertTrue ("Non-zero return code", sh.ret == 0);
+...
++------------------------+
+
+  Of cource the same can be done using pure Java (Groovy) code
+
++------------------------+
+...
+  Configuration conf = new Configuration();
+  SleepJob sljob = new SleepJob();
+  sljob.setConf(conf);
+  conf = sljob.setupJobConf(10, 10, 100, 100, 100, 100);
+  RunningJob rJob = JobClient.runJob(conf);
+...
++------------------------+
+
+** Package management
+
+  In order to manipulate native packages (e.g. for the purpose of testing
+  their behavior) iTest provides high-level packaging abstraction with concrete
+  implementations for .deb, .rpm, packages. More is in the pipeline.
+
+  A typical use of packaged software involves installing/updating/removing
+  packages; manipulating statuses of the services, etc. Here how this can be done
+  with iTest:
+
++------------------------+
+  static public PackageManager pm = PackageManager.getPackageManager();
+...
+  PackageInstance pkg = PackageInstance.getPackageInstance(pm, "hadoop-0.20");
+  pm.install(pkg);
++------------------------+
+
+  This code snippet takes care about getting an instance of the host OS' package
+  manager, creating an instance of "hadoop-0.20" package, and instructing
+  management layer to install said package.
+
+  Because package management is implemented on top of JVM platform one can easily
+  run the following scenario of package testing:
+
+    * install package
+
+    * run existing system tests
+
+    * check results
+
+    * remove the package
+
+  iTest's package management subsystem also allows certain manipulations with
+  repositories for apt-get, yum, and zypper.
+
+** Jar helpers
+
+  These helpers are a special kind of functionality to facilitate test execution
+  contained in appopriate artifacts.
+
+  A typical use case is to unpack a data which has been archived along with the
+  test code from within the test itself. Normally this might be non-so-easy task.
+  Not so with iTest help.
+
++------------------------+
+  JarContent.unpackJarContainer(MyTestClass.class, '.', DATA_DIR);
++------------------------+
+
+  The above static call will look in the classpath for the jar containing
+  MyTestClass class file and unpack everything matching the pattern ('.' everything
+  in this case) to the specified destination DATA_DIR.
+
+  Other manipulations are enabled. Please refer to project's documentation for
+  more information about current framework APIs.

Added: incubator/bigtop/trunk/test/site/src/site/fml/faq.fml
URL: http://svn.apache.org/viewvc/incubator/bigtop/trunk/test/site/src/site/fml/faq.fml?rev=1148501&view=auto
==============================================================================
--- incubator/bigtop/trunk/test/site/src/site/fml/faq.fml (added)
+++ incubator/bigtop/trunk/test/site/src/site/fml/faq.fml Tue Jul 19 19:44:48 2011
@@ -0,0 +1,70 @@
+<?xml version="1.0"?>
+<!--
+  ~ Copyright (c) 2011, Cloudera, Inc. All Rights Reserved.
+  ~
+  ~ Cloudera, Inc. licenses this file to you under the Apache License,
+  ~ Version 2.0 (the "License"). You may not use this file except in
+  ~ compliance with the License. You may obtain a copy of the License at
+  ~
+  ~     http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ This software is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+  ~ CONDITIONS OF ANY KIND, either express or implied. See the License for
+  ~ the specific language governing permissions and limitations under the
+  ~ License.
+  -->
+
+<faqs xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns="http://maven.apache.org/FML/1.0"
+  xsi:schemaLocation="http://maven.apache.org/FML/1.0 http://maven.apache.org/xsd/fml-1.0.xsd"
+  id="General FAQ">
+  <part id="General">
+    <faq id="where">
+      <question>What is iTest?</question>
+      <answer>
+        <p>
+          iTest is a library providing a set of highly useful primitives to enable
+          seamless deployment, execution, dependency tracking, and reports
+          gathering for software validation.
+        </p>
+        <p>
+          iTest works on top of JVM platform, using Groovy for most of its
+          implementation.
+        </p>
+        <p>
+          iTest bridges certain gaps between Maven and available test frameworks
+          such as JUnit and TestNG to allow better system and integration testing
+          experience.
+        </p>
+      </answer>
+    </faq>
+    <faq id="why">
+      <question>Why do I need iTest?</question>
+       <answer>
+         <p>
+          if you need to have a robust environment for your software validation
+          needs where you can run different kind of tests and user applications
+          in a repeatable and controllable fashion then iTest is for you.
+         </p>
+         <p>
+          If you are looking for help with deployment of your software into
+          clusters or cloud infrastructure, iTest might still be at help, but we
+          suggest you look for a more specialized tools.
+         </p>
+       </answer>
+    </faq>
+    <faq id="where">
+      <question>Where I can download iTest?</question>
+      <answer>
+        <p>
+          iTest comes in two forms: binary Maven artifact (basically a jar file)
+          and as a source code. The latter can be used for development and or as
+          a reference material. Whether the former is intended for use in the
+          field i.e. for testing/deployment automation.
+        </p>
+        <p>
+          See "Getting iTest" section for more information
+        </p>
+      </answer>
+    </faq>
+  </part>
+</faqs>

Added: incubator/bigtop/trunk/test/site/src/site/resources/images/banner.png
URL: http://svn.apache.org/viewvc/incubator/bigtop/trunk/test/site/src/site/resources/images/banner.png?rev=1148501&view=auto
==============================================================================
Binary file - no diff available.

Propchange: incubator/bigtop/trunk/test/site/src/site/resources/images/banner.png
------------------------------------------------------------------------------
    svn:mime-type = application/octet-stream

Added: incubator/bigtop/trunk/test/site/src/site/resources/images/itest.png
URL: http://svn.apache.org/viewvc/incubator/bigtop/trunk/test/site/src/site/resources/images/itest.png?rev=1148501&view=auto
==============================================================================
Binary file - no diff available.

Propchange: incubator/bigtop/trunk/test/site/src/site/resources/images/itest.png
------------------------------------------------------------------------------
    svn:mime-type = application/octet-stream

Added: incubator/bigtop/trunk/test/site/src/site/site.xml
URL: http://svn.apache.org/viewvc/incubator/bigtop/trunk/test/site/src/site/site.xml?rev=1148501&view=auto
==============================================================================
--- incubator/bigtop/trunk/test/site/src/site/site.xml (added)
+++ incubator/bigtop/trunk/test/site/src/site/site.xml Tue Jul 19 19:44:48 2011
@@ -0,0 +1,59 @@
+<?xml version="1.0" encoding="ISO-8859-1"?>
+
+<!--
+  ~ Copyright (c) 2011, Cloudera, Inc. All Rights Reserved.
+  ~
+  ~ Cloudera, Inc. licenses this file to you under the Apache License,
+  ~ Version 2.0 (the "License"). You may not use this file except in
+  ~ compliance with the License. You may obtain a copy of the License at
+  ~
+  ~     http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ This software is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+  ~ CONDITIONS OF ANY KIND, either express or implied. See the License for
+  ~ the specific language governing permissions and limitations under the
+  ~ License.
+  -->
+
+<project name="iTest" xmlns="http://maven.apache.org/DECORATION/1.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+  xsi:schemaLocation="http://maven.apache.org/DECORATION/1.0.0 http://maven.apache.org/xsd/decoration-1.0.0.xsd">
+  <bannerLeft>
+    <name>iTest</name>
+    <src>http://cloudera.github.com/iTest/images/itest.png</src>
+    <href>http://cloudera.github.com/iTest/</href>
+  </bannerLeft>
+  <bannerRight>
+    <src>http://cloudera.github.com/iTest/images/banner.png</src>
+  </bannerRight>
+
+  <body>
+    <links>
+      <item name="Apache" href="http://www.apache.org/" />
+      <item name="iTest 1.0" href="http://github.com/cloudera/iTest"/>
+    </links>
+
+    <menu name="iTest">
+      <item name="Welcome" href="index.html"/>
+      <item name="License" href="license.html"/>
+
+    </menu>
+    <menu name="About iTest">
+      <item name="iTest" href="itest.html"/>
+      <item name="Contributing" href="devguide.html"/>
+      <item name="Sources" href="source-repository.html"/>
+      <item name="API docs" href="apidocs/index.html"/>
+      <item name="Issues" href="issue-tracking.html"/>
+      <item name="FAQ" href="faq.html"/>
+      <item name="Team" href="team-list.html"/>
+    </menu>
+    <menu name="Using iTest">
+      <item name="User guide" href="userguide.html"/>
+      <item name="Examples" href="examples.html"/>
+    </menu>
+
+    <menu name="Getting iTest">
+      <item name="Binary" href="downloads.html#binary"/>
+      <item name="Sources" href="downloads.html#sources"/>
+    </menu>
+  </body>
+</project>

Added: incubator/bigtop/trunk/test/src/integration/sqoop/pom.xml
URL: http://svn.apache.org/viewvc/incubator/bigtop/trunk/test/src/integration/sqoop/pom.xml?rev=1148501&view=auto
==============================================================================
--- incubator/bigtop/trunk/test/src/integration/sqoop/pom.xml (added)
+++ incubator/bigtop/trunk/test/src/integration/sqoop/pom.xml Tue Jul 19 19:44:48 2011
@@ -0,0 +1,16 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+  <modelVersion>4.0.0</modelVersion>
+
+  <parent>
+    <groupId>com.cloudera.itest</groupId>
+    <artifactId>cdh-smokes</artifactId>
+    <version>1.0-cdh3u1-SNAPSHOT</version>
+  </parent>
+  <groupId>com.cloudera.itest</groupId>
+  <artifactId>sqoop-integration</artifactId>
+  <version>1.2.0-cdh3u1-SNAPSHOT</version>
+  <name>sqoop-integration</name>
+</project>

Added: incubator/bigtop/trunk/test/src/integration/sqoop/src/test/groovy/com/cloudera/itest/integration/sqoop/IntegrationTestSqoopHBase.groovy
URL: http://svn.apache.org/viewvc/incubator/bigtop/trunk/test/src/integration/sqoop/src/test/groovy/com/cloudera/itest/integration/sqoop/IntegrationTestSqoopHBase.groovy?rev=1148501&view=auto
==============================================================================
--- incubator/bigtop/trunk/test/src/integration/sqoop/src/test/groovy/com/cloudera/itest/integration/sqoop/IntegrationTestSqoopHBase.groovy (added)
+++ incubator/bigtop/trunk/test/src/integration/sqoop/src/test/groovy/com/cloudera/itest/integration/sqoop/IntegrationTestSqoopHBase.groovy Tue Jul 19 19:44:48 2011
@@ -0,0 +1,115 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.cloudera.itest.integration.sqoop;
+
+import com.cloudera.itest.shell.Shell
+import org.junit.Test
+import org.junit.Before
+import com.cloudera.itest.JarContent
+import static org.junit.Assert.assertEquals
+import static org.junit.Assert.assertNotNull
+import org.junit.After
+import static org.junit.Assert.assertFalse;
+
+public class IntegrationTestSqoopHBase {
+  private static final String MYSQL_ROOTPW =
+    System.getProperty("MYSQL_ROOTPW", "");
+  private static final String HADOOP_HOME =
+    System.getenv("HADOOP_HOME");
+  private static final String SQOOP_HOME =
+    System.getenv("SQOOP_HOME");
+  private static final String HBASE_HOME =
+    System.getenv("HBASE_HOME");
+  private static final String ZOOKEEPER_HOME =
+    System.getenv("ZOOKEEPER_HOME");
+
+  static {
+    // make all validations of the above here
+    // check for mysql connector library
+    assertNotNull("SQOOP_HOME is unset", SQOOP_HOME);
+    assertNotNull("HBASE_HOME is unset", HBASE_HOME);
+    assertNotNull("ZOOKEEPER_HOME is unset", ZOOKEEPER_HOME);
+    assertNotNull("mysql connector jars are required to be present in $SQOOP_HOME/lib",
+      JarContent.getJarName("$SQOOP_HOME/lib", "mysql-connector-java.*.jar"));
+  }
+  private static final String DATA_DIR = 'hbase-sqoop';
+//  private static final String MYSQL_COMMAND = "mysql -u root -p${MYSQL_ROOTPW}";
+  private static final String MYSQL_COMMAND = "mysql";
+
+  private static Shell shell = new Shell("/bin/bash -s");
+
+  public static int mkdir(String path) {
+    Shell test = new Shell("/bin/bash -s");
+    test.exec("hadoop fs -mkdir $path");
+    return test.getRet();
+  }
+  public static int rmr(String path) {
+    Shell rmr = new Shell("/bin/bash -s");
+    rmr.exec("hadoop fs -rmr -skipTrash $path");
+    return rmr.getRet();
+  }
+
+  public static String mktemps() {
+     mkdir("IntegrationTestSqoopHBase-${(new Date().getTime())}");
+  }
+
+  @Before
+  public void setUp () {
+    JarContent.unpackJarContainer(IntegrationTestSqoopHBase.class, '.', DATA_DIR);
+
+    if (!"".equals(MYSQL_ROOTPW))
+      MYSQL_ROOTPW = " -p$MYSQL_ROOTPW";
+    rmr('test_table');
+    shell.exec("cat $DATA_DIR/mysql-create-db.sql | $MYSQL_COMMAND -u root $MYSQL_ROOTPW");
+    assertEquals('Unable to run mysql-create-db.sql script', 0, shell.getRet());
+    shell.exec("cat $DATA_DIR/mysql-load-db.sql | $MYSQL_COMMAND -u root testhbase");
+    assertEquals('Unable to run mysql-load-db.sql script', 0, shell.getRet());
+    println "MySQL database prepared for test";
+
+    shell.exec("cat $DATA_DIR/drop-table.hxt | $HBASE_HOME/bin/hbase shell");
+    shell.exec("cat $DATA_DIR/create-table.hxt | $HBASE_HOME/bin/hbase shell")
+    def out = shell.out.join('\n');
+
+    assertFalse("Unable to create HBase table by script create-table.hxt",
+        (out =~ /ERROR/).find());
+  }
+
+  @After
+  public void tearDown() {
+    shell.exec("cat $DATA_DIR/drop-table.hxt | $HBASE_HOME/bin/hbase shell");
+    rmr('test_table');
+  }
+
+  @Test
+  public void hBaseSqoop () {
+    def hostname = shell.exec('hostname').out.get(0);
+    def dbURL = "jdbc:mysql://$hostname/testhbase";
+    def OUTFILE = 'outfile.txt';
+    //Run Sqoop HBase import now
+    shell.exec("$SQOOP_HOME/bin/sqoop import --connect $dbURL --username root --table test_table --hbase-table test_table --column-family data --verbose");
+    assertEquals("Failed to run Sqoop import with HBase", 0, shell.getRet());
+    // Verify if it was imported correctly
+    shell.exec("cat $DATA_DIR/select-table.hxt| ${HBASE_HOME}/bin/hbase shell | awk '/^ [0-9]+/ { print \$1\"  \"\$4 }' | sort > $OUTFILE");
+    shell.exec("sort $DATA_DIR/expected-hbase-output.txt > expected-hbase-output.txt.resorted");
+    // TODO need to conver shell callouts to power tools with Java parsing of
+    // ' 10                                    column=data:b, timestamp=1301075559859, value=ten'
+    assertEquals("HBase scan output did not match expected output. File: $OUTFILE",
+        0, shell.exec("diff -u $OUTFILE expected-hbase-output.txt.resorted").getRet() );
+  }
+}

Added: incubator/bigtop/trunk/test/src/integration/sqoop/src/test/groovy/com/cloudera/itest/integration/sqoop/IntegrationTestSqoopHive.groovy
URL: http://svn.apache.org/viewvc/incubator/bigtop/trunk/test/src/integration/sqoop/src/test/groovy/com/cloudera/itest/integration/sqoop/IntegrationTestSqoopHive.groovy?rev=1148501&view=auto
==============================================================================
--- incubator/bigtop/trunk/test/src/integration/sqoop/src/test/groovy/com/cloudera/itest/integration/sqoop/IntegrationTestSqoopHive.groovy (added)
+++ incubator/bigtop/trunk/test/src/integration/sqoop/src/test/groovy/com/cloudera/itest/integration/sqoop/IntegrationTestSqoopHive.groovy Tue Jul 19 19:44:48 2011
@@ -0,0 +1,100 @@
+package com.cloudera.itest.integration.sqoop
+
+import org.junit.Test
+import org.junit.After
+import static org.junit.Assert.assertEquals
+import static org.junit.Assert.assertNotNull
+import com.cloudera.itest.JarContent
+import org.junit.Before
+import com.cloudera.itest.shell.Shell
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+class IntegrationTestSqoopHive {
+  private static final String MYSQL_ROOTPW =
+    System.getProperty("MYSQL_ROOTPW", "");
+  private static final String HADOOP_HOME =
+    System.getenv("HADOOP_HOME");
+  private static final String SQOOP_HOME =
+    System.getenv("SQOOP_HOME");
+  private static final String HIVE_HOME =
+    System.getenv("HIVE_HOME");
+  private static final String ZOOKEEPER_HOME =
+    System.getenv("ZOOKEEPER_HOME");
+
+  static {
+    // make all validations of the above here
+    // check for mysql connector library
+    assertNotNull("SQOOP_HOME is unset", SQOOP_HOME);
+    assertNotNull("HIVE_HOME is unset", HIVE_HOME);
+    assertNotNull("ZOOKEEPER_HOME is unset", ZOOKEEPER_HOME);
+    assertNotNull("mysql connector jars are required to be present in $SQOOP_HOME/lib",
+      JarContent.getJarName("$SQOOP_HOME/lib", "mysql-connector-java.*.jar"));
+  }
+  private static final String DATA_DIR = 'hive-sqoop';
+  private static final String MYSQL_COMMAND = "mysql";
+
+  private static Shell shell = new Shell("/bin/bash -s");
+  def final OUTFILE = 'outfile.txt';
+
+  public static int rmr(String path) {
+    Shell rmr = new Shell("/bin/bash -s");
+    rmr.exec("hadoop fs -rmr -skipTrash $path");
+    return rmr.getRet();
+  }
+
+  @Before
+  public void setUp () {
+    JarContent.unpackJarContainer(IntegrationTestSqoopHive.class, '.', DATA_DIR);
+    // MySQL preparations
+    if (!"".equals(MYSQL_ROOTPW))
+      MYSQL_ROOTPW = " -p$MYSQL_ROOTPW";
+    rmr('test_table');
+    shell.exec("cat $DATA_DIR/mysql-create-db.sql | $MYSQL_COMMAND -u root $MYSQL_ROOTPW");
+    assertEquals('Unable to run mysql-create-db.sql script', 0, shell.getRet());
+    shell.exec("cat $DATA_DIR/mysql-load-db.sql | $MYSQL_COMMAND -u testhiveuser testhive");
+    assertEquals('Unable to run mysql-load-db.sql script', 0, shell.getRet());
+    println "MySQL database prepared for test";
+    // Hive preparations
+    shell.exec("$HIVE_HOME/bin/hive -f $DATA_DIR/hive-drop-table.hql");
+    assertEquals("Unable to run hive-drop-table.hql script",
+      0, shell.ret);
+  }
+
+  @After
+  public void tearDown() {
+    File outfile = new File(OUTFILE);
+    outfile.deleteOnExit();
+    shell.exec("$HIVE_HOME/bin/hive -f $DATA_DIR/hive-drop-table.hql");
+    rmr('test_table');
+  }
+
+  @Test
+  public void hiveSqoop () {
+    def hostname = shell.exec('hostname').out.get(0);
+    def dbURL = "jdbc:mysql://$hostname/testhive";
+    //Run Sqoop Hive import now
+    shell.exec("$SQOOP_HOME/bin/sqoop import --connect $dbURL --username root --table test_table --hive-import --verbose");
+    assertEquals("Failed to run Sqoop import with Hive", 0, shell.getRet());
+    // Verify if it was imported correctly
+    shell.exec("${HIVE_HOME}/bin/hive -f $DATA_DIR/hive-select-table.hql > $OUTFILE");
+    assertEquals("Unable to run hive-select-table.hql script", 0, shell.ret);
+    assertEquals("Hive output did not match expected output. File: $OUTFILE",
+        0, shell.exec("diff -u $OUTFILE $DATA_DIR/expected-hive-output.txt").getRet() );
+  }
+}

Added: incubator/bigtop/trunk/test/src/integration/sqoop/src/test/resources/hbase-sqoop/create-table.hxt
URL: http://svn.apache.org/viewvc/incubator/bigtop/trunk/test/src/integration/sqoop/src/test/resources/hbase-sqoop/create-table.hxt?rev=1148501&view=auto
==============================================================================
--- incubator/bigtop/trunk/test/src/integration/sqoop/src/test/resources/hbase-sqoop/create-table.hxt (added)
+++ incubator/bigtop/trunk/test/src/integration/sqoop/src/test/resources/hbase-sqoop/create-table.hxt Tue Jul 19 19:44:48 2011
@@ -0,0 +1 @@
+create 'test_table', 'data'

Added: incubator/bigtop/trunk/test/src/integration/sqoop/src/test/resources/hbase-sqoop/drop-table.hxt
URL: http://svn.apache.org/viewvc/incubator/bigtop/trunk/test/src/integration/sqoop/src/test/resources/hbase-sqoop/drop-table.hxt?rev=1148501&view=auto
==============================================================================
--- incubator/bigtop/trunk/test/src/integration/sqoop/src/test/resources/hbase-sqoop/drop-table.hxt (added)
+++ incubator/bigtop/trunk/test/src/integration/sqoop/src/test/resources/hbase-sqoop/drop-table.hxt Tue Jul 19 19:44:48 2011
@@ -0,0 +1,2 @@
+disable 'test_table'
+drop 'test_table'

Added: incubator/bigtop/trunk/test/src/integration/sqoop/src/test/resources/hbase-sqoop/expected-hbase-output.txt
URL: http://svn.apache.org/viewvc/incubator/bigtop/trunk/test/src/integration/sqoop/src/test/resources/hbase-sqoop/expected-hbase-output.txt?rev=1148501&view=auto
==============================================================================
--- incubator/bigtop/trunk/test/src/integration/sqoop/src/test/resources/hbase-sqoop/expected-hbase-output.txt (added)
+++ incubator/bigtop/trunk/test/src/integration/sqoop/src/test/resources/hbase-sqoop/expected-hbase-output.txt Tue Jul 19 19:44:48 2011
@@ -0,0 +1,12 @@
+10  value=ten
+11  value=eleven
+12  value=twelve
+1  value=one
+2  value=two
+3  value=three
+4  value=four
+5  value=five
+6  value=six
+7  value=seven
+8  value=eight
+9  value=nine

Added: incubator/bigtop/trunk/test/src/integration/sqoop/src/test/resources/hbase-sqoop/mysql-create-db.sql
URL: http://svn.apache.org/viewvc/incubator/bigtop/trunk/test/src/integration/sqoop/src/test/resources/hbase-sqoop/mysql-create-db.sql?rev=1148501&view=auto
==============================================================================
--- incubator/bigtop/trunk/test/src/integration/sqoop/src/test/resources/hbase-sqoop/mysql-create-db.sql (added)
+++ incubator/bigtop/trunk/test/src/integration/sqoop/src/test/resources/hbase-sqoop/mysql-create-db.sql Tue Jul 19 19:44:48 2011
@@ -0,0 +1,23 @@
+#
+# Run this script once as root user before the test run. For example:
+# mysql -u root -p < /path/to/this/script.sql
+#
+
+#
+# Drop old databases
+#
+drop database if exists testhbase;
+
+#
+# Create new database
+#
+create database testhbase;
+
+#
+# Grant permissions to the testhbaseuser
+#
+use mysql;
+grant all privileges on testhbase.* to 'testhbaseuser'@'localhost';
+grant all privileges on testhbase.* to 'testhbaseuser'@'%';
+grant all privileges on testhbase.* to 'root'@'%';
+flush privileges;

Added: incubator/bigtop/trunk/test/src/integration/sqoop/src/test/resources/hbase-sqoop/mysql-load-db.sql
URL: http://svn.apache.org/viewvc/incubator/bigtop/trunk/test/src/integration/sqoop/src/test/resources/hbase-sqoop/mysql-load-db.sql?rev=1148501&view=auto
==============================================================================
--- incubator/bigtop/trunk/test/src/integration/sqoop/src/test/resources/hbase-sqoop/mysql-load-db.sql (added)
+++ incubator/bigtop/trunk/test/src/integration/sqoop/src/test/resources/hbase-sqoop/mysql-load-db.sql Tue Jul 19 19:44:48 2011
@@ -0,0 +1,23 @@
+#
+# Run this script as testhbaseuser, specifically for the testhbase database.
+# This script must be run after the mysql-create-db.sql has been run as root.
+# Example of command to run this script:
+# mysql testhbase -u testhbaseuser < /path/to/this/script.sql
+#
+
+#
+# Drop test_table
+#
+drop table if exists test_table;
+
+#
+# Create test_table
+#
+create table test_table (a integer primary key, b varchar(32));
+
+#
+# Load table data
+#
+insert into test_table values (1, 'one'), (2, 'two'), (3, 'three'), (4, 'four'),
+   (5, 'five'), (6, 'six'), (7, 'seven'), (8, 'eight'), (9, 'nine'), (10, 'ten'),
+   (11, 'eleven'), (12, 'twelve');

Added: incubator/bigtop/trunk/test/src/integration/sqoop/src/test/resources/hbase-sqoop/select-table.hxt
URL: http://svn.apache.org/viewvc/incubator/bigtop/trunk/test/src/integration/sqoop/src/test/resources/hbase-sqoop/select-table.hxt?rev=1148501&view=auto
==============================================================================
--- incubator/bigtop/trunk/test/src/integration/sqoop/src/test/resources/hbase-sqoop/select-table.hxt (added)
+++ incubator/bigtop/trunk/test/src/integration/sqoop/src/test/resources/hbase-sqoop/select-table.hxt Tue Jul 19 19:44:48 2011
@@ -0,0 +1 @@
+scan 'test_table'

Added: incubator/bigtop/trunk/test/src/integration/sqoop/src/test/resources/hive-sqoop/expected-hive-output.txt
URL: http://svn.apache.org/viewvc/incubator/bigtop/trunk/test/src/integration/sqoop/src/test/resources/hive-sqoop/expected-hive-output.txt?rev=1148501&view=auto
==============================================================================
--- incubator/bigtop/trunk/test/src/integration/sqoop/src/test/resources/hive-sqoop/expected-hive-output.txt (added)
+++ incubator/bigtop/trunk/test/src/integration/sqoop/src/test/resources/hive-sqoop/expected-hive-output.txt Tue Jul 19 19:44:48 2011
@@ -0,0 +1,12 @@
+1	one
+2	two
+3	three
+4	four
+5	five
+6	six
+7	seven
+8	eight
+9	nine
+10	ten
+11	eleven
+12	twelve

Added: incubator/bigtop/trunk/test/src/integration/sqoop/src/test/resources/hive-sqoop/hive-drop-table.hql
URL: http://svn.apache.org/viewvc/incubator/bigtop/trunk/test/src/integration/sqoop/src/test/resources/hive-sqoop/hive-drop-table.hql?rev=1148501&view=auto
==============================================================================
--- incubator/bigtop/trunk/test/src/integration/sqoop/src/test/resources/hive-sqoop/hive-drop-table.hql (added)
+++ incubator/bigtop/trunk/test/src/integration/sqoop/src/test/resources/hive-sqoop/hive-drop-table.hql Tue Jul 19 19:44:48 2011
@@ -0,0 +1 @@
+drop table if exists test_table

Added: incubator/bigtop/trunk/test/src/integration/sqoop/src/test/resources/hive-sqoop/hive-select-table.hql
URL: http://svn.apache.org/viewvc/incubator/bigtop/trunk/test/src/integration/sqoop/src/test/resources/hive-sqoop/hive-select-table.hql?rev=1148501&view=auto
==============================================================================
--- incubator/bigtop/trunk/test/src/integration/sqoop/src/test/resources/hive-sqoop/hive-select-table.hql (added)
+++ incubator/bigtop/trunk/test/src/integration/sqoop/src/test/resources/hive-sqoop/hive-select-table.hql Tue Jul 19 19:44:48 2011
@@ -0,0 +1 @@
+select * from test_table order by a

Added: incubator/bigtop/trunk/test/src/integration/sqoop/src/test/resources/hive-sqoop/mysql-create-db.sql
URL: http://svn.apache.org/viewvc/incubator/bigtop/trunk/test/src/integration/sqoop/src/test/resources/hive-sqoop/mysql-create-db.sql?rev=1148501&view=auto
==============================================================================
--- incubator/bigtop/trunk/test/src/integration/sqoop/src/test/resources/hive-sqoop/mysql-create-db.sql (added)
+++ incubator/bigtop/trunk/test/src/integration/sqoop/src/test/resources/hive-sqoop/mysql-create-db.sql Tue Jul 19 19:44:48 2011
@@ -0,0 +1,23 @@
+#
+# Run this script once as root user before the test run. For example:
+# mysql -u root -p < /path/to/this/script.sql
+#
+
+#
+# Drop old databases
+#
+drop database if exists testhive;
+
+#
+# Create new database
+#
+create database testhive;
+
+#
+# Grant permissions to the testhiveuser
+#
+use mysql;
+grant all privileges on testhive.* to 'testhiveuser'@'localhost';
+grant all privileges on testhive.* to 'testhiveuser'@'%';
+grant all privileges on testhive.* to 'root'@'%';
+flush privileges;

Added: incubator/bigtop/trunk/test/src/integration/sqoop/src/test/resources/hive-sqoop/mysql-load-db.sql
URL: http://svn.apache.org/viewvc/incubator/bigtop/trunk/test/src/integration/sqoop/src/test/resources/hive-sqoop/mysql-load-db.sql?rev=1148501&view=auto
==============================================================================
--- incubator/bigtop/trunk/test/src/integration/sqoop/src/test/resources/hive-sqoop/mysql-load-db.sql (added)
+++ incubator/bigtop/trunk/test/src/integration/sqoop/src/test/resources/hive-sqoop/mysql-load-db.sql Tue Jul 19 19:44:48 2011
@@ -0,0 +1,23 @@
+#
+# Run this script as testhiveuser, specifically for the testhive database.
+# This script must be run after the mysql-create-db.sql has been run as root.
+# Example of command to run this script:
+# mysql testhive -u testhiveuser < /path/to/this/script.sql
+#
+
+#
+# Drop test_table
+#
+drop table if exists test_table;
+
+#
+# Create test_table
+#
+create table test_table (a integer primary key, b varchar(32));
+
+#
+# Load table data
+#
+insert into test_table values (1, 'one'), (2, 'two'), (3, 'three'), (4, 'four'),
+   (5, 'five'), (6, 'six'), (7, 'seven'), (8, 'eight'), (9, 'nine'), (10, 'ten'),
+   (11, 'eleven'), (12, 'twelve');

Added: incubator/bigtop/trunk/test/src/itest-common/README
URL: http://svn.apache.org/viewvc/incubator/bigtop/trunk/test/src/itest-common/README?rev=1148501&view=auto
==============================================================================
--- incubator/bigtop/trunk/test/src/itest-common/README (added)
+++ incubator/bigtop/trunk/test/src/itest-common/README Tue Jul 19 19:44:48 2011
@@ -0,0 +1,12 @@
+This project will develop a set of JarRunner adapters to perform the following
+tasks via specific adapters:
+  - provide a list of files within a jar file (artifact) which can be passed to a test
+    framework such as JUnit for further execution (generic adapter)
+  - request a list of tests from an artifact to be execute by an external
+    framework (specific adapter)
+  - get a list of tests within an artifact, supply them with certain arguments
+    and run as POJO applications (concrete adapter: an extension of generic)
+
+This project contains the experimental foundation of JarRunner and a
+meta-service utilizing it to perform tests execution. These two components
+might be joined together later.

Added: incubator/bigtop/trunk/test/src/itest-common/pom.xml
URL: http://svn.apache.org/viewvc/incubator/bigtop/trunk/test/src/itest-common/pom.xml?rev=1148501&view=auto
==============================================================================
--- incubator/bigtop/trunk/test/src/itest-common/pom.xml (added)
+++ incubator/bigtop/trunk/test/src/itest-common/pom.xml Tue Jul 19 19:44:48 2011
@@ -0,0 +1,136 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  ~ Copyright (c) 2011, Cloudera, Inc. All Rights Reserved.
+  ~
+  ~ Cloudera, Inc. licenses this file to you under the Apache License,
+  ~ Version 2.0 (the "License"). You may not use this file except in
+  ~ compliance with the License. You may obtain a copy of the License at
+  ~
+  ~     http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ This software is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+  ~ CONDITIONS OF ANY KIND, either express or implied. See the License for
+  ~ the specific language governing permissions and limitations under the
+  ~ License.
+  -->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+
+  <modelVersion>4.0.0</modelVersion>
+
+  <groupId>com.cloudera.itest</groupId>
+  <artifactId>itest-common</artifactId>
+  <name>iTest: system and integration testing in the cloud</name>
+  <version>1.1-SNAPSHOT</version>
+
+  <dependencies>
+    <dependency>
+      <groupId>org.codehaus.groovy.maven.runtime</groupId>
+      <artifactId>gmaven-runtime-1.6</artifactId>
+      <version>1.0</version>
+    </dependency>
+
+    <dependency>
+      <groupId>junit</groupId>
+      <artifactId>junit</artifactId>
+      <version>4.8.1</version>
+    </dependency>
+
+    <dependency>
+      <groupId>commons-logging</groupId>
+      <artifactId>commons-logging</artifactId>
+      <version>1.1</version>
+    </dependency>
+
+    <dependency>
+      <groupId>org.apache.ant</groupId>
+      <artifactId>ant-junit</artifactId>
+      <version>1.8.2</version>
+    </dependency>
+  </dependencies>
+
+  <build>
+    <plugins>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-compiler-plugin</artifactId>
+        <configuration>
+          <source>1.5</source>
+          <target>1.5</target>
+        </configuration>
+      </plugin>
+      <plugin>
+        <groupId>org.codehaus.groovy.maven</groupId>
+        <artifactId>gmaven-plugin</artifactId>
+        <version>1.0</version>
+        <executions>
+          <execution>
+            <goals>
+              <goal>generateStubs</goal>
+              <goal>compile</goal>
+              <goal>generateTestStubs</goal>
+              <goal>testCompile</goal>
+            </goals>
+          </execution>
+        </executions>
+      </plugin>
+
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-jar-plugin</artifactId>
+        <version>2.2</version>
+        <executions>
+          <execution>
+            <goals>
+              <!-- Make sure that test artifacts is being deployed as well -->
+              <goal>test-jar</goal>
+            </goals>
+          </execution>
+        </executions>
+      </plugin>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-javadoc-plugin</artifactId>
+        <version>2.7</version>
+      </plugin>
+    </plugins>
+
+    <extensions>
+      <extension>
+        <groupId>org.apache.maven.scm</groupId>
+        <artifactId>maven-scm-provider-gitexe</artifactId>
+        <version>1.3</version>
+      </extension>
+      <extension>
+        <groupId>org.apache.maven.scm</groupId>
+        <artifactId>maven-scm-manager-plexus</artifactId>
+        <version>1.3</version>
+      </extension>
+      <extension>
+        <groupId>org.kathrynhuxtable.maven.wagon</groupId>
+        <artifactId>wagon-gitsite</artifactId>
+        <version>0.3.1</version>
+      </extension>
+    </extensions>
+  </build>
+
+  <scm>
+    <connection>scm:git:git://github.com/cloudera/iTest.git</connection>
+  </scm>
+
+  <distributionManagement>
+    <snapshotRepository>
+      <id>cloudera-snapshots</id>
+      <name>Internal Snapshots</name>
+      <url>https://repository.cloudera.com/content/repositories/testsnaps/</url>
+    </snapshotRepository>
+    <repository>
+      <id>azov01.sf.cloudera.com</id>
+      <name>Internal Releases</name>
+      <url>http://azov01.sf.cloudera.com:8081/artifactory/libs-release-local</url>
+    </repository>
+    <site>
+      <id>github-project-site</id>
+      <url>gitsite:git@github.com/cloudera/iTest.git</url>
+    </site>
+  </distributionManagement>
+</project>

Added: incubator/bigtop/trunk/test/src/itest-common/src/main/groovy/com/cloudera/itest/JUnitUtils.groovy
URL: http://svn.apache.org/viewvc/incubator/bigtop/trunk/test/src/itest-common/src/main/groovy/com/cloudera/itest/JUnitUtils.groovy?rev=1148501&view=auto
==============================================================================
--- incubator/bigtop/trunk/test/src/itest-common/src/main/groovy/com/cloudera/itest/JUnitUtils.groovy (added)
+++ incubator/bigtop/trunk/test/src/itest-common/src/main/groovy/com/cloudera/itest/JUnitUtils.groovy Tue Jul 19 19:44:48 2011
@@ -0,0 +1,50 @@
+/*
+ * Copyright (c) 2011, Cloudera, Inc. All Rights Reserved.
+ *
+ * Cloudera, Inc. licenses this file to you under the Apache License,
+ * Version 2.0 (the "License"). You may not use this file except in
+ * compliance with the License. You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * This software is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+ * CONDITIONS OF ANY KIND, either express or implied. See the License for
+ * the specific language governing permissions and limitations under the
+ * License.
+ */
+package com.cloudera.itest
+
+public class JUnitUtils {
+  final private static String DETECT_ERRORS = "totalErrorsDetected";
+  final private static String DETECT_FAILS = "totalFailsDetected";
+  final private static String RESULTS_DIR = "com.cloudera.itest.JUnitUtils.results.dir";
+
+  /**
+   * A bulk executor of JUnit test clases capable of producing XML testresults
+   * and forking a VM.
+   *
+   * @param testClasses an array of JUnit test classes to be executed in a forked VM
+   * @return true if ALL the tests passed and false in any other case 
+   */
+  static boolean executeTests(Class... testClasses) {
+    def ant = new AntBuilder()
+    def res = ant.junit (printsummary:'yes', fork:'yes', forkmode:'once', 
+                         errorproperty: DETECT_ERRORS, failureproperty: DETECT_FAILS) {
+          System.getProperties().each { k, v ->
+            sysproperty(key: k, value: v)
+          }
+          classpath {
+             System.getProperty('java.class.path').
+                    split(System.getProperty('path.separator',':')).each {
+               pathelement(location: it);
+             }
+          }
+          testClasses.each {
+            test(name: it.getName(), todir: System.getProperty(RESULTS_DIR, '.'));
+          }
+          formatter(type:'xml');
+    }
+    return !(ant.project.getProperty(DETECT_FAILS) == "true" ||
+             ant.project.getProperty(DETECT_ERRORS) == "true");
+  }
+}

Added: incubator/bigtop/trunk/test/src/itest-common/src/main/groovy/com/cloudera/itest/JarContent.groovy
URL: http://svn.apache.org/viewvc/incubator/bigtop/trunk/test/src/itest-common/src/main/groovy/com/cloudera/itest/JarContent.groovy?rev=1148501&view=auto
==============================================================================
--- incubator/bigtop/trunk/test/src/itest-common/src/main/groovy/com/cloudera/itest/JarContent.groovy (added)
+++ incubator/bigtop/trunk/test/src/itest-common/src/main/groovy/com/cloudera/itest/JarContent.groovy Tue Jul 19 19:44:48 2011
@@ -0,0 +1,235 @@
+/*
+ * Copyright (c) 2011, Cloudera, Inc. All Rights Reserved.
+ *
+ * Cloudera, Inc. licenses this file to you under the Apache License,
+ * Version 2.0 (the "License"). You may not use this file except in
+ * compliance with the License. You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * This software is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+ * CONDITIONS OF ANY KIND, either express or implied. See the License for
+ * the specific language governing permissions and limitations under the
+ * License.
+ */
+
+
+package com.cloudera.itest
+
+import java.util.jar.JarEntry
+import java.util.jar.JarFile
+import java.util.zip.ZipInputStream
+import java.util.zip.ZipException
+import org.apache.commons.logging.LogFactory
+import org.apache.commons.logging.Log
+
+public abstract class JarContent {
+  static private Log LOG = LogFactory.getLog(JarContent.class);
+
+  static {
+    // Calling MOP hooks
+    bootstrapPlugins();
+  }
+
+  private static List<String> patterns = null;
+  private static List<String> content = null;
+  // Exclude META* and inner classes
+  def public static defaultExclPattern = ['.*META.*', '.*\\$.*.class']
+
+  /**
+   * Lists content of a given jar file excluding defaultExclPattern and any extra
+   * patterns set via {@link JarContent#setPatterns} call
+   * @param jarFileName file for a jar
+   * @return list of Strings representing jar's entries
+   * @throws IOException if file isn't found and anything else goes wrong
+   */
+  def static List<String> listContent(String jarFileName) throws IOException {
+    content = new ArrayList<String>();
+    if (jarFileName == null) {
+      throw new IOException("Specify a jar file name");
+    }
+    JarFile jarFile;
+    try {
+      jarFile = new JarFile(jarFileName)
+    } catch (ZipException ze) {
+      throw new IOException("Could not open " + jarFileName + " file.", ze);
+    };
+
+    Enumeration<JarEntry> entries = jarFile.entries();
+    while (entries.hasMoreElements()) {
+      def en = process(entries.nextElement());
+      en != null ? content.add(en) : null;
+    }
+    setPatterns(defaultExclPattern);
+
+    content = applyExcludeFilter(content, patterns);
+    return content;
+  }
+
+  private static String process(JarEntry jarEntry) throws IOException {
+    JarEntry entry = jarEntry;
+    String name = entry.getName();
+    if (!entry.isDirectory())
+      return name;
+    return null;
+  }
+
+  /**
+   * Set a list of new patterns to be applied in the processing of a jar file
+   * @param filterPatters list of pattern strings
+   * @return list of currently set patterns. Next call of this method will
+   * reset the content of patterns' list.
+   */
+  def static List<String> setPatterns(List<String> filterPatters) {
+    patterns = new ArrayList<String>();
+    filterPatters.each {
+      patterns.add(it);
+    }
+    return patterns;
+  }
+
+  /**
+   * Filter out any entries which match given patterns
+   * @param list of entries
+   * @param filters list of patterns
+   * @return filtered-out list of entries
+   */
+  def static List<String> applyExcludeFilter(final List<String> list, final List<String> filters) {
+    List<String> filtered = list.asList();
+    ArrayList<String> toRemove = new ArrayList<String>();
+
+    filters.each {
+      def pattern = ~/${it}/
+      for (l in list) {
+        if (pattern.matcher(l).matches())
+          toRemove.add(l);
+      }
+    }
+    filtered.removeAll(toRemove);
+    return filtered;
+  }
+
+  /**
+   * Finds JAR URL of an object's class belongs to
+   * @param ref Class reference of a class belongs to a jar file
+   * @return JAR URL or <code>null</code> if class doesn't belong
+   * to a JAR in the classpath
+   */
+  public static URL getJarURL(Class ref) {
+    URL clsUrl = ref.getResource(ref.getSimpleName() + ".class");
+    if (clsUrl != null) {
+      try {
+        URLConnection conn = clsUrl.openConnection();
+        if (conn instanceof JarURLConnection) {
+          JarURLConnection connection = (JarURLConnection) conn;
+          return connection.getJarFileURL();
+        }
+      }
+      catch (IOException e) {
+        throw new RuntimeException(e);
+      }
+    }
+    return null;
+  }
+
+  /**
+   * Finds JAR URL of an object's class belongs to
+   * @param className is full name of the class e.g. java.lang.String
+   * @return JAR URL or <code>null</code> if class doesn't belong
+   * to a JAR in the classpath
+   * @throws ClassNotFoundException if class specified by className wasn't found
+   */
+  public static URL getJarURL(String className) throws ClassNotFoundException {
+    Class cl = Class.forName(className)
+    return getJarURL(cl)
+  }
+
+  /**
+   * Returns full name of a jar file by a pattern
+   * @param baseDir to look for a jar under
+   * @param namePattern to look for a jar by
+   * @return name of the jar file if found; <code>null</code> otherwise
+   */
+  public static String getJarName(String baseDir, String namePattern) {
+    try {
+      return new File(baseDir).list(
+          [accept: {d, f -> f ==~ /$namePattern/ }] as FilenameFilter
+      ).toList().get(0);
+    } catch (java.lang.IndexOutOfBoundsException ioob) {
+      LOG.error ("No $namePattern has been found under $baseDir. Check your installation.");
+    } catch (java.lang.NullPointerException npe) {
+      LOG.error("No $baseDir exists. Check your installation.");
+    }
+    return null;
+  }
+
+  /**
+   * Finds and unpack a jar file by locating to what jar file a given class belongs
+   * and unpacking jar content to desalination according to given includes
+   * @param ref
+   * @param destination
+   * @param includes
+   * @throws IOException if can't find class' jar file in the classpath
+   */
+  public static void unpackJarContainer (Class ref,
+    String destination, String includes) throws IOException {
+    URL connection = JarContent.getJarURL(ref);
+    if (connection == null) {
+      throw new IOException("Class " + ref.getSimpleName() +
+          " doesn't belong to any jar file in the classpath");
+    }
+    ZipInputStream fis =
+      new ZipInputStream(connection.openConnection().getInputStream());
+    fis.unzip (destination, includes);
+
+  }
+
+  public static unpackJarContainer (String className,
+    String destination, String includes) throws IOException {
+    Class cl = Class.forName(className)
+    unpackJarContainer (cl, destination, includes)
+  }
+
+  private static void bootstrapPlugins() {
+    /**
+     * Adding an ability to unpack a content of an given ZipInputStream
+     * to specified destination with given pattern
+     * @param dest directory where the content will be unpacked
+     * @param includes regexps to include resources to be unpacked
+     */
+    ZipInputStream.metaClass.unzip = { String dest, String includes ->
+      //in metaclass added methods, 'delegate' is the object on which
+      //the method is called. Here it's the file to unzip
+      if (includes == null) includes = "";
+      def result = delegate
+      def destFile = new File(dest)
+      if (!destFile.exists()) {
+        destFile.mkdir();
+      }
+      result.withStream {
+        def entry
+        while (entry = result.nextEntry) {
+          if (!entry.name.contains(includes)) {
+            continue
+          };
+          if (!entry.isDirectory()) {
+            new File(dest + File.separator + entry.name).parentFile?.mkdirs()
+            def output = new FileOutputStream(dest + File.separator
+                + entry.name)
+            output.withStream {
+              int len;
+              byte[] buffer = new byte[4096]
+              while ((len = result.read(buffer)) > 0) {
+                output.write(buffer, 0, len);
+              }
+            }
+          }
+          else {
+            new File(dest + File.separator + entry.name).mkdir()
+          }
+        }
+      }
+    }
+  }
+}
+

Added: incubator/bigtop/trunk/test/src/itest-common/src/main/groovy/com/cloudera/itest/TestListUtils.groovy
URL: http://svn.apache.org/viewvc/incubator/bigtop/trunk/test/src/itest-common/src/main/groovy/com/cloudera/itest/TestListUtils.groovy?rev=1148501&view=auto
==============================================================================
--- incubator/bigtop/trunk/test/src/itest-common/src/main/groovy/com/cloudera/itest/TestListUtils.groovy (added)
+++ incubator/bigtop/trunk/test/src/itest-common/src/main/groovy/com/cloudera/itest/TestListUtils.groovy Tue Jul 19 19:44:48 2011
@@ -0,0 +1,47 @@
+/*
+ * Copyright (c) 2011, Cloudera, Inc. All Rights Reserved.
+ *
+ * Cloudera, Inc. licenses this file to you under the Apache License,
+ * Version 2.0 (the "License"). You may not use this file except in
+ * compliance with the License. You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * This software is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+ * CONDITIONS OF ANY KIND, either express or implied. See the License for
+ * the specific language governing permissions and limitations under the
+ * License.
+ */
+package com.cloudera.itest
+
+public class TestListUtils {
+
+  /**
+   * Touching files to force Surefire plugin to pick them up
+   * @param pathName
+   * @throws IOException
+   */
+  static final def FS = System.getProperty('file.separator', '/');
+
+  static void touchTestFiles(String dirPrefix, String pathName) throws IOException {
+    if (!pathName.endsWith('.class')) {
+      return;
+    }
+
+    List<String> pathArray = pathName.split(FS).toList();
+    def prefix =  "";
+    if (dirPrefix != null)
+      prefix = dirPrefix;
+
+    String fileName =
+      pathArray.remove(pathArray.size() - 1).replaceAll('.class', '.touched')
+    String dirName = prefix + FS + pathArray.join(FS)
+
+    File dir = new File(dirName);
+    dir.mkdirs();
+    File file = new File(dirName, fileName);
+    file.createNewFile();
+
+    assert file.exists();
+  }
+}

Added: incubator/bigtop/trunk/test/src/itest-common/src/main/groovy/com/cloudera/itest/junit/OrderedParameterized.java
URL: http://svn.apache.org/viewvc/incubator/bigtop/trunk/test/src/itest-common/src/main/groovy/com/cloudera/itest/junit/OrderedParameterized.java?rev=1148501&view=auto
==============================================================================
--- incubator/bigtop/trunk/test/src/itest-common/src/main/groovy/com/cloudera/itest/junit/OrderedParameterized.java (added)
+++ incubator/bigtop/trunk/test/src/itest-common/src/main/groovy/com/cloudera/itest/junit/OrderedParameterized.java Tue Jul 19 19:44:48 2011
@@ -0,0 +1,389 @@
+/*
+ * Copyright (c) 2011, Cloudera, Inc. All Rights Reserved.
+*/
+/*
+JUnit
+
+Common Public License - v 1.0
+
+THE ACCOMPANYING PROGRAM IS PROVIDED UNDER THE TERMS OF THIS COMMON PUBLIC
+LICENSE ("AGREEMENT"). ANY USE, REPRODUCTION OR DISTRIBUTION OF THE PROGRAM
+CONSTITUTES RECIPIENT'S ACCEPTANCE OF THIS AGREEMENT.
+
+1. DEFINITIONS
+
+"Contribution" means:
+
+      a) in the case of the initial Contributor, the initial code and
+         documentation distributed under this Agreement, and
+      b) in the case of each subsequent Contributor:
+
+      i) changes to the Program, and
+
+      ii) additions to the Program;
+
+      where such changes and/or additions to the Program originate from and are
+distributed by that particular Contributor. A Contribution 'originates' from a
+Contributor if it was added to the Program by such Contributor itself or anyone
+acting on such Contributor's behalf. Contributions do not include additions to
+the Program which: (i) are separate modules of software distributed in
+conjunction with the Program under their own license agreement, and (ii) are
+not derivative works of the Program.
+
+"Contributor" means any person or entity that distributes the Program.
+
+"Licensed Patents " mean patent claims licensable by a Contributor which are
+necessarily infringed by the use or sale of its Contribution alone or when
+combined with the Program.
+
+"Program" means the Contributions distributed in accordance with this Agreement.
+
+"Recipient" means anyone who receives the Program under this Agreement,
+including all Contributors.
+
+2. GRANT OF RIGHTS
+
+      a) Subject to the terms of this Agreement, each Contributor hereby grants
+Recipient a non-exclusive, worldwide, royalty-free copyright license to
+reproduce, prepare derivative works of, publicly display, publicly perform,
+distribute and sublicense the Contribution of such Contributor, if any, and
+such derivative works, in source code and object code form.
+
+      b) Subject to the terms of this Agreement, each Contributor hereby grants
+Recipient a non-exclusive, worldwide, royalty-free patent license under
+Licensed Patents to make, use, sell, offer to sell, import and otherwise
+transfer the Contribution of such Contributor, if any, in source code and
+object code form. This patent license shall apply to the combination of the
+Contribution and the Program if, at the time the Contribution is added by the
+Contributor, such addition of the Contribution causes such combination to be
+covered by the Licensed Patents. The patent license shall not apply to any
+other combinations which include the Contribution. No hardware per se is
+licensed hereunder.
+
+      c) Recipient understands that although each Contributor grants the
+licenses to its Contributions set forth herein, no assurances are provided by
+any Contributor that the Program does not infringe the patent or other
+intellectual property rights of any other entity. Each Contributor disclaims
+any liability to Recipient for claims brought by any other entity based on
+infringement of intellectual property rights or otherwise. As a condition to
+exercising the rights and licenses granted hereunder, each Recipient hereby
+assumes sole responsibility to secure any other intellectual property rights
+needed, if any. For example, if a third party patent license is required to
+allow Recipient to distribute the Program, it is Recipient's responsibility to
+acquire that license before distributing the Program.
+
+      d) Each Contributor represents that to its knowledge it has sufficient
+copyright rights in its Contribution, if any, to grant the copyright license
+set forth in this Agreement.
+
+3. REQUIREMENTS
+
+A Contributor may choose to distribute the Program in object code form under
+its own license agreement, provided that:
+
+      a) it complies with the terms and conditions of this Agreement; and
+
+      b) its license agreement:
+
+      i) effectively disclaims on behalf of all Contributors all warranties and
+conditions, express and implied, including warranties or conditions of title
+and non-infringement, and implied warranties or conditions of merchantability
+and fitness for a particular purpose;
+
+      ii) effectively excludes on behalf of all Contributors all liability for
+damages, including direct, indirect, special, incidental and consequential
+damages, such as lost profits;
+
+      iii) states that any provisions which differ from this Agreement are
+offered by that Contributor alone and not by any other party; and
+
+      iv) states that source code for the Program is available from such
+Contributor, and informs licensees how to obtain it in a reasonable manner on
+or through a medium customarily used for software exchange.
+
+When the Program is made available in source code form:
+
+      a) it must be made available under this Agreement; and
+
+      b) a copy of this Agreement must be included with each copy of the
+Program.
+
+Contributors may not remove or alter any copyright notices contained within the
+Program.
+
+Each Contributor must identify itself as the originator of its Contribution, if
+any, in a manner that reasonably allows subsequent Recipients to identify the
+originator of the Contribution.
+
+4. COMMERCIAL DISTRIBUTION
+
+Commercial distributors of software may accept certain responsibilities with
+respect to end users, business partners and the like. While this license is
+intended to facilitate the commercial use of the Program, the Contributor who
+includes the Program in a commercial product offering should do so in a manner
+which does not create potential liability for other Contributors. Therefore, if
+a Contributor includes the Program in a commercial product offering, such
+Contributor ("Commercial Contributor") hereby agrees to defend and indemnify
+every other Contributor ("Indemnified Contributor") against any losses, damages
+and costs (collectively "Losses") arising from claims, lawsuits and other legal
+actions brought by a third party against the Indemnified Contributor to the
+extent caused by the acts or omissions of such Commercial Contributor in
+connection with its distribution of the Program in a commercial product
+offering. The obligations in this section do not apply to any claims or Losses
+relating to any actual or alleged intellectual property infringement. In order
+to qualify, an Indemnified Contributor must: a) promptly notify the Commercial
+Contributor in writing of such claim, and b) allow the Commercial Contributor
+to control, and cooperate with the Commercial Contributor in, the defense and
+any related settlement negotiations. The Indemnified Contributor may
+participate in any such claim at its own expense.
+
+For example, a Contributor might include the Program in a commercial product
+offering, Product X. That Contributor is then a Commercial Contributor. If that
+Commercial Contributor then makes performance claims, or offers warranties
+related to Product X, those performance claims and warranties are such
+Commercial Contributor's responsibility alone. Under this section, the
+Commercial Contributor would have to defend claims against the other
+Contributors related to those performance claims and warranties, and if a court
+requires any other Contributor to pay any damages as a result, the Commercial
+Contributor must pay those damages.
+
+5. NO WARRANTY
+
+EXCEPT AS EXPRESSLY SET FORTH IN THIS AGREEMENT, THE PROGRAM IS PROVIDED ON AN
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER EXPRESS OR
+IMPLIED INCLUDING, WITHOUT LIMITATION, ANY WARRANTIES OR CONDITIONS OF TITLE,
+NON-INFRINGEMENT, MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE. Each
+Recipient is solely responsible for determining the appropriateness of using
+and distributing the Program and assumes all risks associated with its exercise
+of rights under this Agreement, including but not limited to the risks and
+costs of program errors, compliance with applicable laws, damage to or loss of
+data, programs or equipment, and unavailability or interruption of operations.
+
+6. DISCLAIMER OF LIABILITY
+
+EXCEPT AS EXPRESSLY SET FORTH IN THIS AGREEMENT, NEITHER RECIPIENT NOR ANY
+CONTRIBUTORS SHALL HAVE ANY LIABILITY FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING WITHOUT LIMITATION LOST
+PROFITS), HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY
+WAY OUT OF THE USE OR DISTRIBUTION OF THE PROGRAM OR THE EXERCISE OF ANY RIGHTS
+GRANTED HEREUNDER, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGES.
+
+7. GENERAL
+
+If any provision of this Agreement is invalid or unenforceable under applicable
+law, it shall not affect the validity or enforceability of the remainder of the
+terms of this Agreement, and without further action by the parties hereto, such
+provision shall be reformed to the minimum extent necessary to make such
+provision valid and enforceable.
+
+If Recipient institutes patent litigation against a Contributor with respect to
+a patent applicable to software (including a cross-claim or counterclaim in a
+lawsuit), then any patent licenses granted by that Contributor to such
+Recipient under this Agreement shall terminate as of the date such litigation
+is filed. In addition, if Recipient institutes patent litigation against any
+entity (including a cross-claim or counterclaim in a lawsuit) alleging that the
+Program itself (excluding combinations of the Program with other software or
+hardware) infringes such Recipient's patent(s), then such Recipient's rights
+granted under Section 2(b) shall terminate as of the date such litigation is
+filed.
+
+All Recipient's rights under this Agreement shall terminate if it fails to
+comply with any of the material terms or conditions of this Agreement and does
+not cure such failure in a reasonable period of time after becoming aware of
+such noncompliance. If all Recipient's rights under this Agreement terminate,
+Recipient agrees to cease use and distribution of the Program as soon as
+reasonably practicable. However, Recipient's obligations under this Agreement
+and any licenses granted by Recipient relating to the Program shall continue
+and survive.
+
+Everyone is permitted to copy and distribute copies of this Agreement, but in
+order to avoid inconsistency the Agreement is copyrighted and may only be
+modified in the following manner. The Agreement Steward reserves the right to
+publish new versions (including revisions) of this Agreement from time to time.
+No one other than the Agreement Steward has the right to modify this Agreement.
+IBM is the initial Agreement Steward. IBM may assign the responsibility to
+serve as the Agreement Steward to a suitable separate entity. Each new version
+of the Agreement will be given a distinguishing version number. The Program
+(including Contributions) may always be distributed subject to the version of
+the Agreement under which it was received. In addition, after a new version of
+the Agreement is published, Contributor may elect to distribute the Program
+(including its Contributions) under the new version. Except as expressly stated
+in Sections 2(a) and 2(b) above, Recipient receives no rights or licenses to
+the intellectual property of any Contributor under this Agreement, whether
+expressly, by implication, estoppel or otherwise. All rights in the Program not
+expressly granted under this Agreement are reserved.
+
+This Agreement is governed by the laws of the State of New York and the
+intellectual property laws of the United States of America. No party to this
+Agreement will bring a legal action under this Agreement more than one year
+after the cause of action arose. Each party waives its rights to a jury trial
+in any resulting litigation.
+ */
+package com.cloudera.itest.junit;
+
+import org.junit.runner.Runner;
+import org.junit.runner.notification.RunNotifier;
+import org.junit.runners.BlockJUnit4ClassRunner;
+import org.junit.runners.Parameterized;
+import org.junit.runners.Suite;
+import org.junit.runners.model.FrameworkMethod;
+import org.junit.runners.model.InitializationError;
+import org.junit.runners.model.Statement;
+import org.junit.runners.model.TestClass;
+
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+import java.lang.reflect.Modifier;
+import java.util.*;
+
+/**
+ * This is a modification of a Parameterized JUnit runner that take care of two things:
+ *   1. it lets arrange individual tests into ordered sequence of run stages via adding a
+ *      a &#064;RunStage(level=X) annotation to the desired testcases (default run stage
+ *      is 0). Later on run stages are executed according to the order of their levels
+ *      and testcases within the same run stage have no guaranteed order of execution.
+ *   2. it lets give names to the parameterized testcases via making the factory method
+ *      &#064;Parameters return a Map (mapping names to testcases) instead of a List.
+ *
+ * Here's how to use it:
+ * <pre>
+ * public class Example {
+ *    <b>&#064;RunStage(level=-1)</b>
+ *    <b>&#064;Test</b>
+ *    public void earlyTest() {
+ *    }
+ *
+ *    <b>&#064;RunStage(level=1)</b>
+ *    <b>&#064;Test</b>
+ *    public void lateTest() {
+ *    }
+ *
+ *    <b>&#064;Test</b>
+ *    public void defaultTest() {
+ *      // this test will be executed at run stage 0 (default of level)
+ *    }
+ *
+ *    &#064;Parameters
+ *    public static Map<String, Object[]> generateTests() {
+ *      HashMap<String, Object[]> res = new HashMap();
+ *      res.put("test name", new Object[] {1, 2});
+        return res;
+ *    }
+ * }
+ * </pre>
+ *
+ */
+public class OrderedParameterized extends Suite {
+	/**
+	 * Annotation for a method which provides parameters to be injected into the
+	 * test class constructor by <code>Parameterized</code>
+	 */
+    @Retention(RetentionPolicy.RUNTIME)
+	@Target(ElementType.METHOD)
+    public @interface RunStage {
+      int level() default 0;
+    };
+
+    private class TestClassRunnerForParameters extends
+            BlockJUnit4ClassRunner {
+        private final String fParameterSetNumber;
+
+        private final Map<String, Object[]> fParameterList;
+
+        TestClassRunnerForParameters(Class<?> type,
+                Map<String, Object[]> parameterList, String i) throws InitializationError {
+            super(type);
+            fParameterList= parameterList;
+            fParameterSetNumber= i;
+        }
+
+        @Override
+        public Object createTest() throws Exception {
+            return getTestClass().getOnlyConstructor().newInstance(
+                    computeParams());
+        }
+
+        private Object[] computeParams() throws Exception {
+            try {
+                return fParameterList.get(fParameterSetNumber);
+            } catch (ClassCastException e) {
+                throw new Exception(String.format(
+                        "%s.%s() must return a Map from Strings to arrays.",
+                        getTestClass().getName(), getParametersMethod(
+                                getTestClass()).getName()));
+            }
+        }
+
+        @Override
+        protected List<FrameworkMethod> getChildren() {
+            List<FrameworkMethod> c = super.getChildren();
+            Collections.sort(c, new Comparator<FrameworkMethod>() {
+              public int compare(FrameworkMethod m1, FrameworkMethod m2) {
+                RunStage r1 = m1.getAnnotation(RunStage.class);
+                RunStage r2 = m2.getAnnotation(RunStage.class);
+                return ((r1 != null) ? r1.level() : 0) -
+                       ((r2 != null) ? r2.level() : 0);
+              }
+            });
+            return c;
+        }
+
+        @Override
+        protected String getName() {
+            return String.format("[%s]", fParameterSetNumber);
+        }
+
+        @Override
+        protected String testName(final FrameworkMethod method) {
+            return String.format("%s[%s]", method.getName(),
+                    fParameterSetNumber);
+        }
+
+        @Override
+        protected void validateConstructor(List<Throwable> errors) {
+            validateOnlyOneConstructor(errors);
+        }
+
+        @Override
+        protected Statement classBlock(RunNotifier notifier) {
+            return childrenInvoker(notifier);
+        }
+    }
+
+    private FrameworkMethod getParametersMethod(TestClass testClass)
+            throws Exception {
+        List<FrameworkMethod> methods= testClass
+                .getAnnotatedMethods(Parameterized.Parameters.class);
+        for (FrameworkMethod each : methods) {
+            int modifiers= each.getMethod().getModifiers();
+            if (Modifier.isStatic(modifiers) && Modifier.isPublic(modifiers))
+                return each;
+        }
+
+        throw new Exception("No public static parameters method on class "
+                + testClass.getName());
+    }
+
+    private final ArrayList<Runner> runners= new ArrayList<Runner>();
+
+    @Override
+    protected List<Runner> getChildren() {
+        return runners;
+    }
+
+    @SuppressWarnings("unchecked")
+    private Map<String, Object[]> getParametersList(TestClass klass) throws Throwable {
+        return (Map<String, Object[]>) getParametersMethod(klass).invokeExplosively(null);
+    }
+
+    public OrderedParameterized(Class<?> klass) throws Throwable {
+        super(klass, Collections.<Runner>emptyList());
+        Map<String, Object[]> parametersMap= getParametersList(getTestClass());
+        for (Map.Entry<String, Object[]> entry : parametersMap.entrySet())
+            runners.add(new TestClassRunnerForParameters(getTestClass().getJavaClass(),
+                    parametersMap, entry.getKey()));
+    }
+}

Added: incubator/bigtop/trunk/test/src/itest-common/src/main/groovy/com/cloudera/itest/pmanager/AptCmdLinePackageManager.groovy
URL: http://svn.apache.org/viewvc/incubator/bigtop/trunk/test/src/itest-common/src/main/groovy/com/cloudera/itest/pmanager/AptCmdLinePackageManager.groovy?rev=1148501&view=auto
==============================================================================
--- incubator/bigtop/trunk/test/src/itest-common/src/main/groovy/com/cloudera/itest/pmanager/AptCmdLinePackageManager.groovy (added)
+++ incubator/bigtop/trunk/test/src/itest-common/src/main/groovy/com/cloudera/itest/pmanager/AptCmdLinePackageManager.groovy Tue Jul 19 19:44:48 2011
@@ -0,0 +1,82 @@
+/*
+ * Copyright (c) 2011, Cloudera, Inc. All Rights Reserved.
+ *
+ * Cloudera, Inc. licenses this file to you under the Apache License,
+ * Version 2.0 (the "License"). You may not use this file except in
+ * compliance with the License. You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * This software is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+ * CONDITIONS OF ANY KIND, either express or implied. See the License for
+ * the specific language governing permissions and limitations under the
+ * License.
+ */
+package com.cloudera.itest.pmanager
+
+class AptCmdLinePackageManager extends PackageManager {
+  // FIXME: NB export DEBIAN_FRONTEND=noninteractive
+  static String type  = "apt";
+  static String repository_registry = "/etc/apt/sources.list.d/%s.list";
+
+  private static final ROOT_URL = "http://us.archive.ubuntu.com/ubuntu/";
+
+  public void setDefaults(String defaults) {
+     shRoot.exec("debconf-set-selections <<__EOT__\n${defaults}\n__EOT__");
+  }
+
+  public int addBinRepo(String record, String url, String key, String cookie) {
+    if (!url)
+      url = ROOT_URL;
+
+    if (key) {
+        def text = key.toURL().text;
+        shRoot.exec("apt-key add - <<__EOT__\n${text}\n__EOT__");
+        if (shRoot.getRet()) {
+          return shRoot.getRet();
+        }
+    }
+
+    return addBinRepo(record, "deb ${url} ${cookie}\ndeb-src ${url} ${cookie}");
+  }
+
+  public int refresh() {
+    shRoot.exec("apt-get update");
+    return shRoot.getRet();
+  }
+
+  public int cleanup() {
+    shRoot.exec("apt-get -y clean", "apt-get -y autoremove");
+    return shRoot.getRet();
+  }
+
+  public List<PackageInstance> search(String name) {
+    def packages = new ArrayList<PackageInstance>();
+    shUser.exec("apt-cache search --names-only $name").out.each {
+      packages.add(PackageInstance.getPackageInstance (this, ((it =~ /^(.*)( - .*)$/)[0][1])))
+    }
+    return packages
+  }
+
+  public List<PackageInstance> lookup(String name) {
+    shUser.exec("apt-cache show $name");
+    return (shUser.getRet() == 0) ? DEBPackage.parseMetaOutput(null, shUser.out, this) : [];
+  }
+
+  public int install(PackageInstance pkg) {
+    shRoot.exec("env DEBIAN_FRONTEND=noninteractive apt-get -y install ${pkg.name}");
+    pkg.installMessages = shRoot.getOut().join('\n');
+    return shRoot.getRet();
+  }
+
+  public int remove(PackageInstance pkg) {
+    // All config files need to be removed as well.
+    shRoot.exec("env DEBIAN_FRONTEND=noninteractive apt-get -y purge ${pkg.name}");
+    return shRoot.getRet();
+  }
+
+  public boolean isInstalled(PackageInstance pkg) {
+    def text = shUser.exec("env DEBIAN_FRONTEND=noninteractive apt-get -s remove ${pkg.name}").out.join('\n')
+    return (text =~ /\nRemv ${pkg.name} /).find()
+  }
+}

Added: incubator/bigtop/trunk/test/src/itest-common/src/main/groovy/com/cloudera/itest/pmanager/DEBPackage.groovy
URL: http://svn.apache.org/viewvc/incubator/bigtop/trunk/test/src/itest-common/src/main/groovy/com/cloudera/itest/pmanager/DEBPackage.groovy?rev=1148501&view=auto
==============================================================================
--- incubator/bigtop/trunk/test/src/itest-common/src/main/groovy/com/cloudera/itest/pmanager/DEBPackage.groovy (added)
+++ incubator/bigtop/trunk/test/src/itest-common/src/main/groovy/com/cloudera/itest/pmanager/DEBPackage.groovy Tue Jul 19 19:44:48 2011
@@ -0,0 +1,115 @@
+/*
+ * Copyright (c) 2011, Cloudera, Inc. All Rights Reserved.
+ *
+ * Cloudera, Inc. licenses this file to you under the Apache License,
+ * Version 2.0 (the "License"). You may not use this file except in
+ * compliance with the License. You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * This software is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+ * CONDITIONS OF ANY KIND, either express or implied. See the License for
+ * the specific language governing permissions and limitations under the
+ * License.
+ */
+
+package com.cloudera.itest.pmanager
+
+import com.cloudera.itest.shell.Shell
+import com.cloudera.itest.posix.Service
+
+class DEBPackage extends ManagedPackage {
+  Shell shRoot = new Shell("/bin/bash -s", "root");
+  Shell shUser = new Shell();
+
+  private static String translateMeta(String key) {
+    def mapping = [Homepage: "url", Origin: "vendor", Section: "group"];
+    return mapping[key] ?: key.toLowerCase();
+  }
+
+  /**
+   * A helper method that parses a stream of metadata describing one or more packages.
+   *
+   * NOTE: the design of this is a bit ugly, we need to rethink how it is done.
+   *
+   * @param givenPkg a package to which the beginning of the metadata stream applies
+   * @param text stream of metadata as a list of Strings
+   * @param pm a package manager that this package belong to
+   * @return list of EXTRA packages that were detected (and created!) during metadata parsing
+   */
+  public static List parseMetaOutput(PackageInstance givenPkg, List<String> text, PackageManager pm) {
+    def packages = new ArrayList<PackageInstance>();
+    PackageInstance pkg = givenPkg;
+    String curMetaKey = "";
+    text.each {
+      if ((it =~ /^ /).find()) {
+        pkg.meta[curMetaKey] <<= "\n$it";
+      } else {
+        def m = (it =~ /(\S+):(.*)/);
+        if (m.size()) {
+          String[] matcher = m[0];
+          if ("Package" == matcher[1] && !givenPkg) {
+            pkg = PackageInstance.getPackageInstance(pm, matcher[2]);
+            packages.add(pkg);
+          } else {
+            curMetaKey = translateMeta(matcher[1]);
+            pkg.meta[curMetaKey] = matcher[2].trim();
+          }
+        }
+      }
+    }
+
+    (packages.size() == 0 ? [givenPkg] : packages).each {
+      it.version = it.meta["version"] ?: it.version;
+      it.arch = it.meta["architecture"] ?: it.arch;
+      if (it.meta["conffiles"]) {
+        it.configs = it.meta["conffiles"].toString().split("\n").collect { lines ->
+          lines.replaceAll(/ [^ ]*$/, "").trim();
+        }.findAll { l -> (l != ""); };
+      }
+    };
+    return packages;
+  }
+
+  public void refresh() {
+    parseMetaOutput(this, shUser.exec("env DEBIAN_FRONTEND=noninteractive dpkg -s $name").out, mgr);
+  }
+
+  public Map<String, Service> getServices() {
+    Map res = [:];
+    shUser.exec("env DEBIAN_FRONTEND=noninteractive dpkg -L $name | sed -ne '/^.etc.init.d./s#^.etc.init.d.##p'");
+    shUser.out.collect {
+      res[it] = new Service(it);
+    }
+    return res;
+  }
+
+  public List<String> getFiles() {
+    shUser.exec("env DEBIAN_FRONTEND=noninteractive dpkg -L $name");
+    return shUser.out.collect({"$it"});
+  }
+
+  public List<String> getConfigs() {
+    shUser.exec("""
+    env DEBIAN_FRONTEND=noninteractive dpkg -s $name | sed -ne '/Conffiles:/,/^[^ ]*:/{/:/!s# [^ ]*\$##p}'
+    """);
+    return shUser.out.collect({ it.toString().trim() });
+  }
+
+  public List<String> getDocs() {
+    return [];
+  }
+
+  Map<String, String> getDeps() {
+    Map<String, String> res = [:];
+    // sample input: sun-java6-jre | sun-java6-sdk, hadoop-zookeeper (= 3.3.1+8), hadoop-0.20 (>= 0.20.2+700)
+    // NOTE: we currently do NOT support alternative specification i.e. sun-java6-jre | sun-java6-sdk
+    shUser.exec("dpkg -s $name | sed -ne 's#^Depends: ##p'").getOut().join(' ').split(',').each {
+      def matcher = (it =~ /(\S+)( \(.+\))*/);
+      if (matcher.size() == 1) {
+        res[matcher[0][1]] = (matcher[0][2] ?: "").replaceAll(/( |\(|\))/, '');
+      }
+    }
+    return res;
+  }
+}

Added: incubator/bigtop/trunk/test/src/itest-common/src/main/groovy/com/cloudera/itest/pmanager/ManagedPackage.groovy
URL: http://svn.apache.org/viewvc/incubator/bigtop/trunk/test/src/itest-common/src/main/groovy/com/cloudera/itest/pmanager/ManagedPackage.groovy?rev=1148501&view=auto
==============================================================================
--- incubator/bigtop/trunk/test/src/itest-common/src/main/groovy/com/cloudera/itest/pmanager/ManagedPackage.groovy (added)
+++ incubator/bigtop/trunk/test/src/itest-common/src/main/groovy/com/cloudera/itest/pmanager/ManagedPackage.groovy Tue Jul 19 19:44:48 2011
@@ -0,0 +1,31 @@
+/*
+ * Copyright (c) 2011, Cloudera, Inc. All Rights Reserved.
+ *
+ * Cloudera, Inc. licenses this file to you under the Apache License,
+ * Version 2.0 (the "License"). You may not use this file except in
+ * compliance with the License. You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * This software is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+ * CONDITIONS OF ANY KIND, either express or implied. See the License for
+ * the specific language governing permissions and limitations under the
+ * License.
+ */
+package com.cloudera.itest.pmanager
+
+import com.cloudera.itest.posix.Service
+
+abstract class ManagedPackage extends PackageInstance {
+  public boolean isInstalled() {
+    return mgr.isInstalled(this);
+  }
+
+  public int install() {
+    return mgr.install(this);
+  }
+
+  public int remove() {
+    return mgr.remove(this);
+  }
+}

Added: incubator/bigtop/trunk/test/src/itest-common/src/main/groovy/com/cloudera/itest/pmanager/PackageInstance.groovy
URL: http://svn.apache.org/viewvc/incubator/bigtop/trunk/test/src/itest-common/src/main/groovy/com/cloudera/itest/pmanager/PackageInstance.groovy?rev=1148501&view=auto
==============================================================================
--- incubator/bigtop/trunk/test/src/itest-common/src/main/groovy/com/cloudera/itest/pmanager/PackageInstance.groovy (added)
+++ incubator/bigtop/trunk/test/src/itest-common/src/main/groovy/com/cloudera/itest/pmanager/PackageInstance.groovy Tue Jul 19 19:44:48 2011
@@ -0,0 +1,115 @@
+/*
+ * Copyright (c) 2011, Cloudera, Inc. All Rights Reserved.
+ *
+ * Cloudera, Inc. licenses this file to you under the Apache License,
+ * Version 2.0 (the "License"). You may not use this file except in
+ * compliance with the License. You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * This software is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+ * CONDITIONS OF ANY KIND, either express or implied. See the License for
+ * the specific language governing permissions and limitations under the
+ * License.
+ */
+package com.cloudera.itest.pmanager
+
+import com.cloudera.itest.posix.Service
+
+abstract class PackageInstance {
+  PackageManager mgr;
+
+  String name;
+  String version;
+  String release;
+  String arch;
+
+  Map meta = [:];
+  String installMessages;
+
+  protected List<String> files;
+  protected List<String> docs;
+  protected List<String> configs;
+  protected Map<String, Service> services;
+  protected Map<String, String> deps;
+
+  /**
+   * Check if this package is installed on the system
+   *
+   * @return true if the package is installed and can be used, false otherwise
+   */
+  abstract public boolean isInstalled();
+  /**
+   * Install this package (from collection of all the packages available in all the repositories)
+   *
+   * @return int return code of the operation: o in case of success, non-zero otherwise
+   */
+  abstract public int install();
+  /**
+   * Remove this package from a system
+   *
+   * @return int return code of the operation: o in case of success, non-zero otherwise
+   */
+  abstract public int remove();
+  /**
+   * Re-sync metadata associated with this package with the underlying package management system
+   *
+   * @return int return code of the operation: o in case of success, non-zero otherwise
+   */
+  abstract public void refresh();
+  /**
+   * Get a list of services (System V init scripts) provided by this package
+   *
+   * @return list of Service instances
+   */
+  abstract public Map<String, Service> getServices();
+  /**
+   * Get a list of files provided by this package. This list will include ALL the files regardless
+   * of whether they are also marked as configs or documentation.
+   *
+   * @return list file and directory names belong to the package.
+   */
+  abstract public List<String> getFiles();
+  /**
+   * Get a list of documentation files provided by this package (if the underlying package
+   * management system doesn't support a notion of a documentation file -- empty list is expected
+   * to be returned.
+   *
+   * @return list config file names that belong to the package.
+   */
+  abstract public List<String> getDocs();
+  /**
+   * Get a list of configuration files provided by this package (if the underlying package
+   * management system doesn't support a notion of a configuration file -- empty list is expected
+   * to be returned.
+   *
+   * @return list config file names that belong to the package.
+   */
+  abstract public List<String> getConfigs();
+  /**
+   * Get a list of dependencies for this package. Dependency information is stored in a map with
+   * keys being names of the objects and values being version information in the following format:
+   *     [version operator] version
+   * E.g. >= 3.3.1+10
+   *
+   * @return map of objects (mostly packaged) this package depends on
+   */
+  abstract public Map<String, String> getDeps();
+
+  /**
+   * Factory method for creating an instance of a Package that can reside in
+   * a particular instance of a PackageManager.
+   * NOTE: For now only 'natural' pairing is supported (e.g. deb with apt, rpm
+   * with yum/zypper)
+   *
+   * @param mgr package manager that is expected to manage this type of package
+   * @param name package manager dependent name of a package
+   */
+  static public PackageInstance getPackageInstance(PackageManager mgr, String name) {
+    PackageInstance pkg = (mgr.type == "apt" ) ? new DEBPackage() :
+                                                 new RPMPackage();
+    pkg.mgr = mgr;
+    pkg.name = name;
+    return pkg;
+  }
+}



Mime
View raw message