incubator-bigtop-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From r..@apache.org
Subject svn commit: r1196868 [2/4] - in /incubator/bigtop/branches/hadoop-0.22: ./ bigtop-deploy/puppet/manifests/ bigtop-deploy/puppet/modules/hadoop/manifests/ bigtop-deploy/puppet/modules/kerberos/manifests/ bigtop-packages/src/common/hive/ bigtop-packages/...
Date Wed, 02 Nov 2011 23:40:26 GMT
Modified: incubator/bigtop/branches/hadoop-0.22/bigtop-tests/test-artifacts/package/pom.xml
URL: http://svn.apache.org/viewvc/incubator/bigtop/branches/hadoop-0.22/bigtop-tests/test-artifacts/package/pom.xml?rev=1196868&r1=1196867&r2=1196868&view=diff
==============================================================================
--- incubator/bigtop/branches/hadoop-0.22/bigtop-tests/test-artifacts/package/pom.xml (original)
+++ incubator/bigtop/branches/hadoop-0.22/bigtop-tests/test-artifacts/package/pom.xml Wed
Nov  2 23:40:24 2011
@@ -49,11 +49,33 @@
       <artifactId>ant-junit</artifactId>
       <version>1.8.2</version>
     </dependency>
-
-    <dependency>
-      <groupId>org.apache.bigtop.itest</groupId>
-      <artifactId>hadoop-smoke</artifactId>
-      <version>0.2.0-incubating-SNAPSHOT</version>
-    </dependency>
   </dependencies>
+
+  <build>
+    <plugins>
+        <plugin>
+          <groupId>org.codehaus.groovy.maven</groupId>
+          <artifactId>gmaven-plugin</artifactId>
+          <version>1.0</version>
+          <executions>
+            <execution>
+              <id>check-manifest</id>
+              <phase>process-test-classes</phase>
+              <goals>
+                <goal>execute</goal>
+              </goals>
+              <configuration>
+                <source><![CDATA[
+                   ["yum", "zypper", "apt", "urpmi"].each { 
+                     def a = new XmlSlurper().parseText(new File("${project.build.directory}/classes/package_data_${it}.xml").text);
+                   }
+                   println "Manifest XML validated";
+                  ]]>
+                </source>
+              </configuration>
+            </execution>
+          </executions>
+        </plugin>
+     </plugins>
+  </build>
 </project>

Modified: incubator/bigtop/branches/hadoop-0.22/bigtop-tests/test-artifacts/package/src/main/groovy/org/apache/bigtop/itest/packagesmoke/CDHServices.groovy
URL: http://svn.apache.org/viewvc/incubator/bigtop/branches/hadoop-0.22/bigtop-tests/test-artifacts/package/src/main/groovy/org/apache/bigtop/itest/packagesmoke/CDHServices.groovy?rev=1196868&r1=1196867&r2=1196868&view=diff
==============================================================================
--- incubator/bigtop/branches/hadoop-0.22/bigtop-tests/test-artifacts/package/src/main/groovy/org/apache/bigtop/itest/packagesmoke/CDHServices.groovy
(original)
+++ incubator/bigtop/branches/hadoop-0.22/bigtop-tests/test-artifacts/package/src/main/groovy/org/apache/bigtop/itest/packagesmoke/CDHServices.groovy
Wed Nov  2 23:40:24 2011
@@ -21,47 +21,65 @@ package org.apache.bigtop.itest.packages
 import org.apache.bigtop.itest.pmanager.PackageManager
 
 class CDHServices {
+  static final List serviceDaemonUserNames = [ "flume", "hbase" , "hdfs" , "hue" , "mapred"
, "oozie" , "sqoop" ,
+                                               "zookeeper", "hadoop"];
+
   static final Map components = [
-                     HDFS           : [ services : [ "hadoop-0.20-namenode", "hadoop-0.20-datanode",
-                                                     "hadoop-0.20-secondarynamenode" ],
+                     HDFS           : [ services : [ "hadoop-namenode", "hadoop-datanode",
+                                                     "hadoop-secondarynamenode" ],
                                         verifier : new StateVerifierHDFS(),
+                                        killIDs  : [ "hdfs" ],
                                       ],
-                     mapreduce      : [ services : [ "hadoop-0.20-namenode", "hadoop-0.20-datanode",
-                                                     "hadoop-0.20-jobtracker", "hadoop-0.20-tasktracker"
],
+                     mapreduce      : [ services : [ "hadoop-namenode", "hadoop-datanode",
+                                                     "hadoop-jobtracker", "hadoop-tasktracker"
],
+                                        killIDs  : [ "hdfs", "mapred" ],
                                         verifier : new StateVerifierMapreduce(),
                                       ],
-                     HBase          : [ services : [ "hadoop-0.20-namenode", "hadoop-0.20-datanode",
+                     hive           : [ services : [ "hadoop-namenode", "hadoop-datanode",
+                                                     "hadoop-jobtracker", "hadoop-tasktracker"
],
+                                        killIDs  : [ "hdfs", "mapred" ],
+                                        verifier : new StateVerifierHive(),
+                                      ],
+                     HBase          : [ services : [ "hadoop-namenode", "hadoop-datanode",
                                                      "hadoop-hbase-master" ],
+                                        killIDs  : [ "hdfs", "hbase" ],
                                         verifier : new StateVerifierHBase(),
                                       ],
-                     zookeeper      : [ services : [ "${PackageManager.getPackageManager().type
== 'apt' ? 'hadoop-zookeeper-server' : 'hadoop-zookeeper'}" ],
+                     zookeeper      : [ services : [ "hadoop-zookeeper" ],
                                         verifier : new StateVerifierZookeeper(),
+                                        killIDs  : [ "zookeeper" ],
                                       ],
-                     oozie          : [ services : [ "hadoop-0.20-namenode", "hadoop-0.20-datanode",
"hadoop-0.20-jobtracker", "hadoop-0.20-tasktracker",
+                     oozie          : [ services : [ "hadoop-namenode", "hadoop-datanode",
"hadoop-jobtracker", "hadoop-tasktracker",
                                                      "oozie" ],
+                                        killIDs  : [ "hdfs", "mapred", "oozie" ],
                                         verifier : new StateVerifierOozie(),
                                       ],
-                     flume          : [ services : [ "hadoop-0.20-namenode", "hadoop-0.20-datanode",
+                     flume          : [ services : [ "hadoop-namenode", "hadoop-datanode",
                                                      "flume-master", "flume-node" ],
+                                        killIDs  : [ "hdfs", "flume" ],
                                         verifier : new StateVerifierFlume(),
                                       ],
-                     sqoop          : [ services : [ "hadoop-0.20-namenode", "hadoop-0.20-datanode",
+                     sqoop          : [ services : [ "hadoop-namenode", "hadoop-datanode",
                                                      "sqoop-metastore" ],
+                                        killIDs  : [ "hdfs", "sqoop" ],
                                         verifier : new StateVerifierSqoop(),
                                       ],
-                     hue            : [ services : [ "hadoop-0.20-namenode", "hadoop-0.20-datanode",
"hadoop-0.20-jobtracker", "hadoop-0.20-tasktracker",
+                     hue            : [ services : [ "hadoop-namenode", "hadoop-datanode",
"hadoop-jobtracker", "hadoop-tasktracker",
                                                      "hue" ],
+                                        killIDs  : [ "hdfs", "mapred", "hue" ],
                                         verifier : new StateVerifierHue(),
                                       ],
                    ];
 
   static final Map<String, List<String>> release2services = [
-                     "2"            : [ "HDFS", "mapreduce" ],
-                     "3b2"          : [ "HDFS", "mapreduce", "HBase", "zookeeper", "oozie",
"flume",          "hue" ],
-                     "3b3"          : [ "HDFS", "mapreduce", "HBase", "zookeeper", "oozie",
"flume", "sqoop", "hue" ],
-                     "3b4"          : [ "HDFS", "mapreduce", "HBase", "zookeeper", "oozie",
"flume", "sqoop", "hue" ],
-                     "3u0"          : [ "HDFS", "mapreduce", "HBase", "zookeeper", "oozie",
"flume", "sqoop", "hue" ],
-                     "3"            : [ "HDFS", "mapreduce", "HBase", "zookeeper", "oozie",
"flume", "sqoop", "hue" ],
+                     "bigtop"       : [ "HDFS", "mapreduce", "hive", "HBase", "zookeeper",
"oozie", "flume" ],
+                     "2"            : [ "HDFS", "mapreduce", "hive" ],
+                     "3b2"          : [ "HDFS", "mapreduce", "hive", "HBase", "zookeeper",
"oozie", "flume",          "hue" ],
+                     "3b3"          : [ "HDFS", "mapreduce", "hive", "HBase", "zookeeper",
"oozie", "flume", "sqoop", "hue" ],
+                     "3b4"          : [ "HDFS", "mapreduce", "hive", "HBase", "zookeeper",
"oozie", "flume", "sqoop", "hue" ],
+                     "3u0"          : [ "HDFS", "mapreduce", "hive", "HBase", "zookeeper",
"oozie", "flume", "sqoop", "hue" ],
+                     "3u1"          : [ "HDFS", "mapreduce", "hive", "HBase", "zookeeper",
"oozie", "flume", "sqoop", "hue" ],
+                     "3"            : [ "HDFS", "mapreduce", "hive", "HBase", "zookeeper",
"oozie", "flume" ],
                    ];
 
   public static Map getServices(String release) {
@@ -69,8 +87,10 @@ class CDHServices {
     List<String> services;
 
     if ((release =~ /\.\./).find()) {
-      services = release2services[release.replaceAll(/^.*\.\./, "")].clone();
-      release2services[release.replaceAll(/\.\..*$/, "")].each {
+      String release_from = release.replaceAll(/\.\..*$/, "");
+      release = release.replaceAll(/^.*\.\./, "");
+      services = release2services[release].clone();
+      release2services[release_from].each {
         services.remove(it);
       }
     } else {
@@ -78,7 +98,16 @@ class CDHServices {
     }
 
     services.each {
-        res[it] = components[it];
+        // zookeeper is a very messy case of naming :-(
+        if (it == "zookeeper" &&
+            (PackageManager.getPackageManager().type == 'apt' ||
+             release == "3" || release == "3u1" || release == "bigtop")) {
+          res[it] = [ services : [ "hadoop-zookeeper-server" ],
+                      verifier : new StateVerifierZookeeper(),
+                    ];
+        } else {
+          res[it] = components[it];
+        }
     }
     return res;
   }

Modified: incubator/bigtop/branches/hadoop-0.22/bigtop-tests/test-artifacts/package/src/main/groovy/org/apache/bigtop/itest/packagesmoke/DeployCDH.groovy
URL: http://svn.apache.org/viewvc/incubator/bigtop/branches/hadoop-0.22/bigtop-tests/test-artifacts/package/src/main/groovy/org/apache/bigtop/itest/packagesmoke/DeployCDH.groovy?rev=1196868&r1=1196867&r2=1196868&view=diff
==============================================================================
--- incubator/bigtop/branches/hadoop-0.22/bigtop-tests/test-artifacts/package/src/main/groovy/org/apache/bigtop/itest/packagesmoke/DeployCDH.groovy
(original)
+++ incubator/bigtop/branches/hadoop-0.22/bigtop-tests/test-artifacts/package/src/main/groovy/org/apache/bigtop/itest/packagesmoke/DeployCDH.groovy
Wed Nov  2 23:40:24 2011
@@ -58,13 +58,16 @@ class DeployCDH {
     Map<String, LinkedHashMap<String, Collection>> distPackages = [
       "2" : [ "apt" : cdh2 + aptPkg,
               "yum" : cdh2 + yumPkg + [ "hadoop-0.20-docs", "hadoop-hive-webinterface" ],
+               "zypper" : [],
               // "cloudera-desktop", "cloudera-desktop-plugins",
             ],
     "3b2" : [ "apt" : cdh3b2 + aptPkg,
               "yum" : cdh3b2 + yumPkg + [ "hadoop-0.20-docs", "hadoop-hive-webinterface"
],
+              "zypper" : [],
             ],
     "3b3" : [ "apt" : cdh3b3 + aptPkg + [ "hadoop-hbase-doc",  ],
               "yum" : cdh3b3 + yumPkg + [ "hadoop-0.20-docs", "hadoop-hive-webinterface"
],
+              "zypper" : [],
             ],
     "3b4" : [ "apt" : cdh3b3 + aptPkg + [ "hadoop-hbase-doc",  ],
               "yum" : cdh3b3 + yumPkg + [ "hadoop-0.20-doc",  "hadoop-hive-webinterface"
],
@@ -78,6 +81,10 @@ class DeployCDH {
               "yum" : cdh3b3 + yumPkg + [ "hadoop-0.20-doc" ],
               "zypper" : cdh3b3 + zypperPkg,
             ],
+    "3u1" : [ "apt" : cdh3b3 + aptPkg + [ "hadoop-hbase-doc",  ],
+              "yum" : cdh3b3 + yumPkg + [ "hadoop-0.20-doc" ],
+              "zypper" : cdh3b3 + zypperPkg,
+            ],
     ];
 
   @Rule
@@ -94,10 +101,22 @@ class DeployCDH {
     checkThat("failed to add repository for pre-upgrade CDH deployment",
               oldRepo.getPm().refresh(), equalTo(0));
 
+    // Lets try to remove existing packages -- just in case
+    List stalePkgs = [];
+    distPackages.each { key, value -> stalePkgs.addAll(value[pm.type]); }
+    (stalePkgs as Set).each {
+      PackageInstance pkg = PackageInstance.getPackageInstance(pm, it);
+      pkg.remove();
+    }
+
     packages.each {
       PackageInstance pkg = PackageInstance.getPackageInstance(pm, it);
-      checkThat("failed to install required package ${pkg.getName()}",
-                pkg.install(), equalTo(0));
+      // Some packages get installed as requirement for others that we install.
+      // We don't want to install them for a second time.
+      if (!pkg.isInstalled()) {
+        checkThat("failed to install required package ${pkg.getName()}",
+                  pkg.install(), equalTo(0));
+      }
       pkg.getServices().each { it.value.stop(); }
     }
 

Modified: incubator/bigtop/branches/hadoop-0.22/bigtop-tests/test-artifacts/package/src/main/groovy/org/apache/bigtop/itest/packagesmoke/PackageTestCommon.groovy
URL: http://svn.apache.org/viewvc/incubator/bigtop/branches/hadoop-0.22/bigtop-tests/test-artifacts/package/src/main/groovy/org/apache/bigtop/itest/packagesmoke/PackageTestCommon.groovy?rev=1196868&r1=1196867&r2=1196868&view=diff
==============================================================================
--- incubator/bigtop/branches/hadoop-0.22/bigtop-tests/test-artifacts/package/src/main/groovy/org/apache/bigtop/itest/packagesmoke/PackageTestCommon.groovy
(original)
+++ incubator/bigtop/branches/hadoop-0.22/bigtop-tests/test-artifacts/package/src/main/groovy/org/apache/bigtop/itest/packagesmoke/PackageTestCommon.groovy
Wed Nov  2 23:40:24 2011
@@ -30,6 +30,7 @@ import org.hamcrest.Matcher
 
 import static org.apache.bigtop.itest.packagesmoke.PackageTestMatchers.MapHasKeys.hasSameKeys
 import org.apache.bigtop.itest.shell.Shell
+import groovy.xml.MarkupBuilder
 
 class PackageTestCommon {
   static public PackageManager pm;
@@ -48,7 +49,7 @@ class PackageTestCommon {
   }
 
   String formatDescription(String description, String summary) {
-    return ((summary ?: "") + ' ' + description).replaceAll(/\s+/,' ').trim();
+    return ((summary ?: "") + ' ' + description).replaceAll(/\s+/,' ').replaceAll(/\s\.\s/,'
').replaceAll(/\s\.$/,' ').trim();
   }
 
   private void checkMetadata(PackageInstance pkg, Map expected_metadata) {
@@ -96,7 +97,13 @@ class PackageTestCommon {
   }
 
   public void checkPulledDeps(Map expected_deps) {
-    Map pkgDeps = pkg.getDeps();
+    Map pkgDeps = [:];
+
+    pkg.getDeps().each { k, v ->
+      if (!(k =~ /\.so\.[0-9]/).find()) {
+        pkgDeps[k] = v;
+      }
+    }
 
     checkThat("a set of dependencies of package $name is different from what was expected",
               pkgDeps, hasSameKeys(expected_deps));
@@ -367,14 +374,23 @@ class PackageTestCommon {
                 problemFiles, equalTo([]));
 
     // a bit of debug output
-    pkg.getFiles().each {
-      Map meta = fileMeta[it] ?: [:];
-      String target = meta.target ? " target=\"${meta.target}\"" : "";
-      String tip = configs[it] ? "config" : (docs[it] ? "doc " : "file");
-      println "\n::: ${name}    <${tip} name=\"${it}\" owners=\"${meta.owners}\" perm=\"${meta.perm}\""
+
-                                  " user=\"${meta.user}\" group=\"${meta.group}\"${target}/>";
+    def newManifest = new MarkupBuilder(new FileWriter("${pkg.name}_manifest.xml"));
+
+    newManifest.content() {
+      fileMeta = getLsMetadata(pkg.getFiles());
+
+      pkg.getFiles().each {
+        Map meta = fileMeta[it] ?: [:];
+        String node = configs[it] ? "config" : (docs[it] ? "doc " : "file");
+        int owners = meta.owners ?: -1;
+
+        if (meta.target) {
+          "$node"(name : it, owners : owners, perm : meta.perm, user : meta.user, group :
meta.group, target : meta.target);
+        } else {
+          "$node"(name : it, owners : owners, perm : meta.perm, user : meta.user, group :
meta.group);
+        }
+      }
     }
-    // println "";
   }
 
   public void checkComplimentary32bitInstall() {

Modified: incubator/bigtop/branches/hadoop-0.22/bigtop-tests/test-artifacts/package/src/main/groovy/org/apache/bigtop/itest/packagesmoke/PackageTestRepoMgr.groovy
URL: http://svn.apache.org/viewvc/incubator/bigtop/branches/hadoop-0.22/bigtop-tests/test-artifacts/package/src/main/groovy/org/apache/bigtop/itest/packagesmoke/PackageTestRepoMgr.groovy?rev=1196868&r1=1196867&r2=1196868&view=diff
==============================================================================
--- incubator/bigtop/branches/hadoop-0.22/bigtop-tests/test-artifacts/package/src/main/groovy/org/apache/bigtop/itest/packagesmoke/PackageTestRepoMgr.groovy
(original)
+++ incubator/bigtop/branches/hadoop-0.22/bigtop-tests/test-artifacts/package/src/main/groovy/org/apache/bigtop/itest/packagesmoke/PackageTestRepoMgr.groovy
Wed Nov  2 23:40:24 2011
@@ -50,7 +50,7 @@ class PackageTestRepoMgr {
 
   public parseRepoSpec(String prefix) {
     cdhRepoHost = System.getProperty("${prefix}.host", "nightly.cloudera.com");
-    cdhRepoVersion = System.getProperty("${prefix}.version", "3");
+    cdhRepoVersion = System.getProperty("${prefix}.version", "bigtop");
 
     Map cdhKeys  = [ yum    : "http://${cdhRepoHost}/redhat/cdh/RPM-GPG-KEY-cloudera",
                      zypper : null,
@@ -69,6 +69,7 @@ class PackageTestRepoMgr {
 
   public boolean addRepo() {
     repoName = "cloudera-cdh${cdhRepoVersion}";
+    pm.cleanup();
     try {
       String repoText = cdhRepoFileURL.toURL().text;
       if (pm.addBinRepo(repoName, repoText)) {
@@ -80,10 +81,11 @@ class PackageTestRepoMgr {
         return false;
       }
     }
+    pm.refresh();
     return true;
   }
 
   public boolean removeRepo() {
     return (pm.removeBinRepo(repoName) == 0);
   }
-}
\ No newline at end of file
+}

Modified: incubator/bigtop/branches/hadoop-0.22/bigtop-tests/test-artifacts/package/src/main/groovy/org/apache/bigtop/itest/packagesmoke/StateVerifierFlume.groovy
URL: http://svn.apache.org/viewvc/incubator/bigtop/branches/hadoop-0.22/bigtop-tests/test-artifacts/package/src/main/groovy/org/apache/bigtop/itest/packagesmoke/StateVerifierFlume.groovy?rev=1196868&r1=1196867&r2=1196868&view=diff
==============================================================================
--- incubator/bigtop/branches/hadoop-0.22/bigtop-tests/test-artifacts/package/src/main/groovy/org/apache/bigtop/itest/packagesmoke/StateVerifierFlume.groovy
(original)
+++ incubator/bigtop/branches/hadoop-0.22/bigtop-tests/test-artifacts/package/src/main/groovy/org/apache/bigtop/itest/packagesmoke/StateVerifierFlume.groovy
Wed Nov  2 23:40:24 2011
@@ -28,6 +28,7 @@ class StateVerifierFlume extends StateVe
 
   public void createState() {
     String node;
+    sleep(120001);
     shFlume.exec("connect localhost",
                  "getnodestatus",
                  "quit\n");
@@ -35,14 +36,15 @@ class StateVerifierFlume extends StateVe
                                      .replaceAll(/^.*Master knows about [0-9]* nodes /,'')
                                      .trim();
     shFlume.exec("connect localhost",
-                 "exec config $node 'text(\"/etc/group\")' 'dfs(\"hdfs://localhost/flume.test\")'",
+                 "exec config $node 'text(\"/etc/group\")' 'collectorSink(\"hdfs://localhost/flume\",\"test\")'",
                  "quit\n");
     sleep(5001);
-    (new Shell()).exec("hadoop fs -rm /flume.test");
   }
 
   public boolean verifyState() {
     sleep(5001);
-    return ((new Shell()).exec("hadoop fs -ls /flume.test >/dev/null 2>&1").getRet()
== 0);
+    boolean ret = ((new Shell()).exec("hadoop fs -ls /flume >/dev/null 2>&1").getRet()
== 0);
+    (new Shell()).exec("hadoop fs -rmr /flume");
+    return ret;
   }
 }
\ No newline at end of file

Modified: incubator/bigtop/branches/hadoop-0.22/bigtop-tests/test-artifacts/package/src/main/groovy/org/apache/bigtop/itest/packagesmoke/StateVerifierHBase.groovy
URL: http://svn.apache.org/viewvc/incubator/bigtop/branches/hadoop-0.22/bigtop-tests/test-artifacts/package/src/main/groovy/org/apache/bigtop/itest/packagesmoke/StateVerifierHBase.groovy?rev=1196868&r1=1196867&r2=1196868&view=diff
==============================================================================
--- incubator/bigtop/branches/hadoop-0.22/bigtop-tests/test-artifacts/package/src/main/groovy/org/apache/bigtop/itest/packagesmoke/StateVerifierHBase.groovy
(original)
+++ incubator/bigtop/branches/hadoop-0.22/bigtop-tests/test-artifacts/package/src/main/groovy/org/apache/bigtop/itest/packagesmoke/StateVerifierHBase.groovy
Wed Nov  2 23:40:24 2011
@@ -36,6 +36,7 @@ class StateVerifierHBase extends StateVe
   public static void createStaticState() {
     shHBase.exec("create 't1', 'f1'",
                  "put 't1', 'r1', 'f1:q', 'val'",
+                 "flush 't1'",
                  "quit\n");
   }
 

Modified: incubator/bigtop/branches/hadoop-0.22/bigtop-tests/test-artifacts/package/src/main/groovy/org/apache/bigtop/itest/packagesmoke/StateVerifierHDFS.groovy
URL: http://svn.apache.org/viewvc/incubator/bigtop/branches/hadoop-0.22/bigtop-tests/test-artifacts/package/src/main/groovy/org/apache/bigtop/itest/packagesmoke/StateVerifierHDFS.groovy?rev=1196868&r1=1196867&r2=1196868&view=diff
==============================================================================
--- incubator/bigtop/branches/hadoop-0.22/bigtop-tests/test-artifacts/package/src/main/groovy/org/apache/bigtop/itest/packagesmoke/StateVerifierHDFS.groovy
(original)
+++ incubator/bigtop/branches/hadoop-0.22/bigtop-tests/test-artifacts/package/src/main/groovy/org/apache/bigtop/itest/packagesmoke/StateVerifierHDFS.groovy
Wed Nov  2 23:40:24 2011
@@ -55,6 +55,11 @@ class StateVerifierHDFS extends StateVer
     return (Arrays.equals(digest, MD5.digest()));
   } */
 
+  public boolean config() {
+    Shell shRoot = new Shell("/bin/bash", "root");
+    return 0 == shRoot.exec("sed -i -e 's#<configuration>\$#<configuration><property><name>dfs.safemode.min.datanodes</name><value>1</value></property><property><name>dfs.safemode.extension</name><value>0</value></property>#'
/etc/hadoop/conf/hdfs-site.xml").getRet();
+  }
+
   public static void createStaticState() {
     sh.exec("hadoop fs -put <(echo StateHDFSVErifier) /StateHDFSVErifier");
   }

Copied: incubator/bigtop/branches/hadoop-0.22/bigtop-tests/test-artifacts/package/src/main/groovy/org/apache/bigtop/itest/packagesmoke/StateVerifierHive.groovy
(from r1196225, incubator/bigtop/branches/hadoop-0.22/bigtop-tests/test-artifacts/package/src/main/groovy/org/apache/bigtop/itest/packagesmoke/StateVerifierFlume.groovy)
URL: http://svn.apache.org/viewvc/incubator/bigtop/branches/hadoop-0.22/bigtop-tests/test-artifacts/package/src/main/groovy/org/apache/bigtop/itest/packagesmoke/StateVerifierHive.groovy?p2=incubator/bigtop/branches/hadoop-0.22/bigtop-tests/test-artifacts/package/src/main/groovy/org/apache/bigtop/itest/packagesmoke/StateVerifierHive.groovy&p1=incubator/bigtop/branches/hadoop-0.22/bigtop-tests/test-artifacts/package/src/main/groovy/org/apache/bigtop/itest/packagesmoke/StateVerifierFlume.groovy&r1=1196225&r2=1196868&rev=1196868&view=diff
==============================================================================
--- incubator/bigtop/branches/hadoop-0.22/bigtop-tests/test-artifacts/package/src/main/groovy/org/apache/bigtop/itest/packagesmoke/StateVerifierFlume.groovy
(original)
+++ incubator/bigtop/branches/hadoop-0.22/bigtop-tests/test-artifacts/package/src/main/groovy/org/apache/bigtop/itest/packagesmoke/StateVerifierHive.groovy
Wed Nov  2 23:40:24 2011
@@ -15,34 +15,29 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 package org.apache.bigtop.itest.packagesmoke
 
 import org.apache.bigtop.itest.shell.Shell
-import java.security.MessageDigest
-import org.junit.Before
-import org.junit.After
-import org.junit.Test
 
-class StateVerifierFlume extends StateVerifier {
-  Shell shFlume = new Shell("flume shell");
+class StateVerifierHive extends StateVerifier {
+  final static String schema = "CREATE TABLE state_table(A INT) PARTITIONED BY (dt STRING)
row format delimited fields terminated by ','  escaped by '\\\\\\\\' stored as textfile;";
+
+  Shell sh = new Shell();
 
   public void createState() {
-    String node;
-    shFlume.exec("connect localhost",
-                 "getnodestatus",
-                 "quit\n");
-    node = shFlume.getOut().join(' ').replaceAll(/ --> IDLE.*$/,'')
-                                     .replaceAll(/^.*Master knows about [0-9]* nodes /,'')
-                                     .trim();
-    shFlume.exec("connect localhost",
-                 "exec config $node 'text(\"/etc/group\")' 'dfs(\"hdfs://localhost/flume.test\")'",
-                 "quit\n");
-    sleep(5001);
-    (new Shell()).exec("hadoop fs -rm /flume.test");
+    File tmpFile = File.createTempFile("StateVerifierHiveData", ".txt");
+    tmpFile.withWriter { (1..15).each { num -> it.write("$num\n"); } };
+
+    sh.exec("hive -e \"${schema}\"");
+    ["2008-08-08", "2008-08-09"].each {
+      sh.exec("hive -e \"LOAD DATA LOCAL INPATH '${tmpFile.getCanonicalPath()}' OVERWRITE
INTO TABLE state_table partition (dt='$it');\"");
+    }
+    tmpFile.delete();
   }
 
   public boolean verifyState() {
-    sleep(5001);
-    return ((new Shell()).exec("hadoop fs -ls /flume.test >/dev/null 2>&1").getRet()
== 0);
+    sh.exec("hive -e 'SELECT COUNT(*) from state_table;'")
+    return (sh.getOut() =~ /30/).find();
   }
-}
\ No newline at end of file
+}

Modified: incubator/bigtop/branches/hadoop-0.22/bigtop-tests/test-artifacts/package/src/main/groovy/org/apache/bigtop/itest/packagesmoke/StateVerifierHue.groovy
URL: http://svn.apache.org/viewvc/incubator/bigtop/branches/hadoop-0.22/bigtop-tests/test-artifacts/package/src/main/groovy/org/apache/bigtop/itest/packagesmoke/StateVerifierHue.groovy?rev=1196868&r1=1196867&r2=1196868&view=diff
==============================================================================
--- incubator/bigtop/branches/hadoop-0.22/bigtop-tests/test-artifacts/package/src/main/groovy/org/apache/bigtop/itest/packagesmoke/StateVerifierHue.groovy
(original)
+++ incubator/bigtop/branches/hadoop-0.22/bigtop-tests/test-artifacts/package/src/main/groovy/org/apache/bigtop/itest/packagesmoke/StateVerifierHue.groovy
Wed Nov  2 23:40:24 2011
@@ -36,20 +36,22 @@ class StateVerifierHue extends StateVeri
 
   public void createState() {
     // first call creates admin/admin username/keypair
-    sh.exec("curl --data '${creds}' ${loginURL}");
+    sh.exec("curl -m 60 --data '${creds}' ${loginURL}");
   }
 
   public boolean verifyState() {
     String sessionId;
     boolean res;
 
-    sh.exec("curl -i --data '${creds}' ${loginURL} | sed -e 's#Set-Cookie: *##' -e 's#;.*\$##'
| grep '^sessionid'");
+    sh.exec("curl -m 60 -i --data '${creds}' ${loginURL} | sed -e 's#Set-Cookie: *##' -e
's#;.*\$##' | grep '^sessionid'");
     sessionId = sh.getOut().join('');
 
-    res = (sh.exec("curl -b '${sessionId}' ${checkURL} | grep -q 'All ok. Configuration check
passed'").getRet() == 0);
+    sh.exec("curl -m 60 -b '${sessionId}' ${checkURL}");
+    res = (sh.getOut().grep( ~/.*All ok. Configuration check passed.*/ ).size() != 0)
     checkApps.each {
-      res = res && (sh.exec("curl -b '${sessionId}' ${hueServer}/${it}/ | grep -q
'Page Not Found'").getRet() != 0);
+      sh.exec("curl -m 60 -b '${sessionId}' ${hueServer}/${it}/");
+      res = res && (sh.getOut().grep( ~/.*Page Not Found.*/ ).size() == 0);
     }
     return res;
   }
-}
\ No newline at end of file
+}

Modified: incubator/bigtop/branches/hadoop-0.22/bigtop-tests/test-artifacts/package/src/main/groovy/org/apache/bigtop/itest/packagesmoke/StateVerifierMapreduce.groovy
URL: http://svn.apache.org/viewvc/incubator/bigtop/branches/hadoop-0.22/bigtop-tests/test-artifacts/package/src/main/groovy/org/apache/bigtop/itest/packagesmoke/StateVerifierMapreduce.groovy?rev=1196868&r1=1196867&r2=1196868&view=diff
==============================================================================
--- incubator/bigtop/branches/hadoop-0.22/bigtop-tests/test-artifacts/package/src/main/groovy/org/apache/bigtop/itest/packagesmoke/StateVerifierMapreduce.groovy
(original)
+++ incubator/bigtop/branches/hadoop-0.22/bigtop-tests/test-artifacts/package/src/main/groovy/org/apache/bigtop/itest/packagesmoke/StateVerifierMapreduce.groovy
Wed Nov  2 23:40:24 2011
@@ -28,7 +28,7 @@ class StateVerifierMapreduce extends Sta
   }
 
   public static boolean verifyStaticState() {
-    return sh.exec("hadoop jar `ls /usr/lib/hadoop-0.20/hadoop*examples*jar | head -1` pi
10 100").getRet() == 0;
+    return sh.exec("hadoop jar `ls /usr/lib/hadoop*/hadoop*examples*jar | head -1` pi 10
100").getRet() == 0;
   }
 
   void createState() {

Modified: incubator/bigtop/branches/hadoop-0.22/bigtop-tests/test-artifacts/package/src/main/groovy/org/apache/bigtop/itest/packagesmoke/TestPackagesBasics.groovy
URL: http://svn.apache.org/viewvc/incubator/bigtop/branches/hadoop-0.22/bigtop-tests/test-artifacts/package/src/main/groovy/org/apache/bigtop/itest/packagesmoke/TestPackagesBasics.groovy?rev=1196868&r1=1196867&r2=1196868&view=diff
==============================================================================
--- incubator/bigtop/branches/hadoop-0.22/bigtop-tests/test-artifacts/package/src/main/groovy/org/apache/bigtop/itest/packagesmoke/TestPackagesBasics.groovy
(original)
+++ incubator/bigtop/branches/hadoop-0.22/bigtop-tests/test-artifacts/package/src/main/groovy/org/apache/bigtop/itest/packagesmoke/TestPackagesBasics.groovy
Wed Nov  2 23:40:24 2011
@@ -125,10 +125,9 @@ class TestPackagesBasics extends Package
   @Test
   synchronized void testPackageInstall() {
     // WARNING: sometimes packages do not install because the server is busy
-    int i;
-    for (i=3; pkg.install() && i>0; i--) {};
-    checkThat("could only install package $name on the ${3-i} try",
-              i, equalTo(3));
+    for (int i=3; pkg.install() && i>0; i--) {
+      recordFailure("can not install pacakge $name will retry $i times");
+    }
 
     // TODO: we need to come up with a way to abort any further execution to avoid spurious
failures
 
@@ -163,12 +162,6 @@ class TestPackagesBasics extends Package
     checkAlternatives(getMap(golden.alternatives));
   }
 
-  @RunStage(level=1)
-  @Test
-  void testPackageRemove() {
-    checkRemoval();
-  }
-
   static void tryOrFail(Closure cl, int retries, String fail) {
     while (!cl.call()) {
       retries--;

Modified: incubator/bigtop/branches/hadoop-0.22/bigtop-tests/test-artifacts/package/src/main/groovy/org/apache/bigtop/itest/packagesmoke/TestPackagesPseudoDistributed.groovy
URL: http://svn.apache.org/viewvc/incubator/bigtop/branches/hadoop-0.22/bigtop-tests/test-artifacts/package/src/main/groovy/org/apache/bigtop/itest/packagesmoke/TestPackagesPseudoDistributed.groovy?rev=1196868&r1=1196867&r2=1196868&view=diff
==============================================================================
--- incubator/bigtop/branches/hadoop-0.22/bigtop-tests/test-artifacts/package/src/main/groovy/org/apache/bigtop/itest/packagesmoke/TestPackagesPseudoDistributed.groovy
(original)
+++ incubator/bigtop/branches/hadoop-0.22/bigtop-tests/test-artifacts/package/src/main/groovy/org/apache/bigtop/itest/packagesmoke/TestPackagesPseudoDistributed.groovy
Wed Nov  2 23:40:24 2011
@@ -23,7 +23,7 @@ import org.apache.bigtop.itest.junit.Ord
 import org.junit.Test
 
 @RunWith(OrderedParameterized.class)
-class TestPackagesPseudoDistributed extends TestPackagesSingleNode {
+class TestPackagesPseudoDistributed extends TestPackagesBasics {
   public TestPackagesPseudoDistributed(String pkgName, Node pkgGolden) {
     super(pkgName, pkgGolden);
   }
@@ -33,8 +33,28 @@ class TestPackagesPseudoDistributed exte
   synchronized void testRemoteMetadata() {
   }
 
-  @RunStage(level=1)
+  @RunStage(level=-1)
   @Test
-  void testPackageRemove() {
+  void testPackageUpgrade() {
+    if (isUpgrade()) {
+      checkThat("upgrade sequence on a package $name failed to be executed",
+                CDHUpgradeSequence.execute(name, System.getProperty("cdh.prev.repo.version"),
"3"), equalTo(0));
+    }
+  }
+
+  @Test
+  void testPulledDeps() {
+    checkPulledDeps(getMap(golden.deps));
+  }
+
+  @Test
+  void testPackageContent() {
+    Map files = getMap(golden.content);
+    checkFiles(files.config, files.doc, files.file);
+  }
+
+  @Test
+  void testPackageServices() {
+    checkServices(getMap(golden.services));
   }
 }

Copied: incubator/bigtop/branches/hadoop-0.22/bigtop-tests/test-artifacts/package/src/main/groovy/org/apache/bigtop/itest/packagesmoke/TestPackagesPseudoDistributedWithRM.groovy
(from r1196225, incubator/bigtop/branches/hadoop-0.22/bigtop-tests/test-artifacts/package/src/main/groovy/org/apache/bigtop/itest/packagesmoke/TestPackagesReadiness.groovy)
URL: http://svn.apache.org/viewvc/incubator/bigtop/branches/hadoop-0.22/bigtop-tests/test-artifacts/package/src/main/groovy/org/apache/bigtop/itest/packagesmoke/TestPackagesPseudoDistributedWithRM.groovy?p2=incubator/bigtop/branches/hadoop-0.22/bigtop-tests/test-artifacts/package/src/main/groovy/org/apache/bigtop/itest/packagesmoke/TestPackagesPseudoDistributedWithRM.groovy&p1=incubator/bigtop/branches/hadoop-0.22/bigtop-tests/test-artifacts/package/src/main/groovy/org/apache/bigtop/itest/packagesmoke/TestPackagesReadiness.groovy&r1=1196225&r2=1196868&rev=1196868&view=diff
==============================================================================
--- incubator/bigtop/branches/hadoop-0.22/bigtop-tests/test-artifacts/package/src/main/groovy/org/apache/bigtop/itest/packagesmoke/TestPackagesReadiness.groovy
(original)
+++ incubator/bigtop/branches/hadoop-0.22/bigtop-tests/test-artifacts/package/src/main/groovy/org/apache/bigtop/itest/packagesmoke/TestPackagesPseudoDistributedWithRM.groovy
Wed Nov  2 23:40:24 2011
@@ -15,58 +15,30 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 package org.apache.bigtop.itest.packagesmoke
 
-import org.apache.bigtop.itest.junit.OrderedParameterized
-import org.junit.runner.RunWith
-import org.apache.bigtop.itest.junit.OrderedParameterized.RunStage
 import org.junit.Test
 
-@RunWith(OrderedParameterized.class)
-class TestPackagesReadiness extends TestPackagesSingleNode {
-  public TestPackagesReadiness(String pkgName, Node pkgGolden) {
-    super(pkgName, pkgGolden);
-  }
-  
-  @RunStage(level=-1)
-  @Test
-  void testPackageUpgrade() {
-  }
-
-  @Test
-  void testRepoFile() {
-  }
-
-  @Test
-  void testPulledDeps() {
-  }
-
-  @Test
-  void testPackageMetadata() {
-  }
-
-  @Test
-  void testPackageContent() {
-  }
-
-  @Test
-  void testPackageServices() {
-  }
+import static org.hamcrest.CoreMatchers.equalTo
+import org.junit.runner.RunWith
 
-  @Test
-  void testUsers() {
-  }
+import org.apache.bigtop.itest.junit.OrderedParameterized
+import org.apache.bigtop.itest.junit.OrderedParameterized.RunStage
 
-  @Test
-  void testGroups() {
-  }
+@RunWith(OrderedParameterized.class)
+class TestPackagesPseudoDistributedWithRM extends TestPackagesPseudoDistributed {
 
-  @Test
-  void testAlternatives() {
+  public TestPackagesPseudoDistributedWithRM(String pkgName, Node pkgGolden) {
+    super(pkgName, pkgGolden);
   }
 
   @RunStage(level=1)
   @Test
   void testPackageRemove() {
+    checkComplimentary32bitInstall();
+    checkDaemonStart();
+    sleep(3001); // TODO FIXME: CDH-2816 should address the timing of daemons start.
+    checkRemoval();
   }
 }

Modified: incubator/bigtop/branches/hadoop-0.22/bigtop-tests/test-artifacts/package/src/main/groovy/org/apache/bigtop/itest/packagesmoke/TestServices.groovy
URL: http://svn.apache.org/viewvc/incubator/bigtop/branches/hadoop-0.22/bigtop-tests/test-artifacts/package/src/main/groovy/org/apache/bigtop/itest/packagesmoke/TestServices.groovy?rev=1196868&r1=1196867&r2=1196868&view=diff
==============================================================================
--- incubator/bigtop/branches/hadoop-0.22/bigtop-tests/test-artifacts/package/src/main/groovy/org/apache/bigtop/itest/packagesmoke/TestServices.groovy
(original)
+++ incubator/bigtop/branches/hadoop-0.22/bigtop-tests/test-artifacts/package/src/main/groovy/org/apache/bigtop/itest/packagesmoke/TestServices.groovy
Wed Nov  2 23:40:24 2011
@@ -28,12 +28,16 @@ import org.apache.bigtop.itest.posix.Ser
 import org.apache.bigtop.itest.junit.OrderedParameterized.RunStage
 import org.hamcrest.Matcher
 import static org.hamcrest.core.IsEqual.equalTo
+import org.apache.bigtop.itest.shell.Shell
 
 @RunWith(OrderedParameterized.class)
 class TestServices {
   Map.Entry svcDescr;
   List<Service> svcs;
   StateVerifier verifier;
+  List<String> killIDs;
+
+  static Shell shRoot = new Shell("/bin/bash", "root");
 
   @Rule
   public ErrorCollector errors = new ErrorCollector();
@@ -49,6 +53,7 @@ class TestServices {
     svcDescr = svc;
     svcs = svcDescr.value.services.collect { new Service(it); };
     verifier = svcDescr.value.verifier;
+    killIDs = svcDescr.value.killIDs;
   }
 
   static Map<String, Object[]> selectServices(String CDHrelease) {
@@ -60,6 +65,14 @@ class TestServices {
     return res;
   }
 
+  @AfterClass
+  static void tearDown() {
+    // TODO: this is pretty silly, but it'll do for now
+    CDHServices.serviceDaemonUserNames.each {
+      shRoot.exec("kill -9 `ps -U${it} -opid=`");
+    }
+  }
+
   @RunStage(level=-1)
   @Test
   void createState() {
@@ -79,16 +92,28 @@ class TestServices {
               verifier.verifyState(), equalTo(true));
 
     svcs.reverseEach {
-      checkThat("service ${it.getName()} failed to stop",
-                it.stop(), equalTo(0));
+      // TODO: we're only trying the best we can here
+      // there's a total eradication of  services happening at @BeforeClass
+      it.stop();
+      sleep(5001);
     }
     sleep(5001);
+
+    // TODO: this is pretty silly, but it'll do for now
+    killIDs.each {
+      shRoot.exec("kill -9 `ps -U${it} -opid=`");
+    }
   }
 
   @RunStage(level=1)
   @Test
   void verifyState() {
     svcs.each {
+      checkThat("failed to configure service ${it.getName()}",
+                verifier.config(), equalTo(true));
+    }
+
+    svcs.each {
       checkThat("service ${it.getName()} failed to start",
                 it.start(), equalTo(0));
     }
@@ -97,6 +122,16 @@ class TestServices {
               verifier.verifyState(), equalTo(true));
 
     svcs.reverseEach { it.stop(); }
+    sleep(5001);
+    // lets check if they are really stopped (if not -- we'll complain and kill them)
+    killIDs.each {
+      shRoot.exec("kill -0 `ps -U${it} -opid=`");
+      if (!shRoot.getRet()) {
+        shRoot.exec("kill -9 `ps -U${it} -opid=`");
+        checkThat("service running under the name of $it is supposed to be stopped, but it
is not",
+                  true, equalTo(false));
+      }
+    }
   }
 
   public void checkThat(String msg, Object value, Matcher<Object> matcher) {



Mime
View raw message