bigtop-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From evan...@apache.org
Subject [bigtop] branch master updated: BIGTOP-3137: Giraph smoke tests implementation
Date Fri, 15 Feb 2019 17:42:53 GMT
This is an automated email from the ASF dual-hosted git repository.

evansye pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/bigtop.git


The following commit(s) were added to refs/heads/master by this push:
     new f1500c1  BIGTOP-3137: Giraph smoke tests implementation
f1500c1 is described below

commit f1500c13124759897105ceb5270de5d242b2b5eb
Author: Yuqi Gu <yuqi.gu@arm.com>
AuthorDate: Thu Jan 17 02:45:36 2019 +0000

    BIGTOP-3137: Giraph smoke tests implementation
    
    Add smoke tests for Giraph
    
    Change-Id: I80ec1fbc916d5f616e92a16a15db777c4f4d3979
    Signed-off-by: Yuqi Gu <yuqi.gu@arm.com>
    Signed-off-by: Evans Ye <evansye@apache.org>
---
 .../modules/hadoop/templates/mapred-site.xml       |  5 ++
 .../src/common/giraph/install_giraph.sh            |  4 +-
 bigtop-tests/smoke-tests/giraph/build.gradle       | 37 ++++++++++++
 .../itest/giraphsmoke/TestGiraphSmoke.groovy       | 68 +++++-----------------
 provisioner/utils/smoke-tests.sh                   |  1 +
 5 files changed, 61 insertions(+), 54 deletions(-)

diff --git a/bigtop-deploy/puppet/modules/hadoop/templates/mapred-site.xml b/bigtop-deploy/puppet/modules/hadoop/templates/mapred-site.xml
index a25f66b..b7f2bba 100644
--- a/bigtop-deploy/puppet/modules/hadoop/templates/mapred-site.xml
+++ b/bigtop-deploy/puppet/modules/hadoop/templates/mapred-site.xml
@@ -77,6 +77,11 @@
 <% end %>
 <% if @mapreduce_jobhistory_host %>
   <property>
+    <name>mapred.job.tracker</name>
+    <value>BIGTOP-3137: Required by Giraph checkLocalJobRunnerConfiguration</value>
+  </property>
+
+  <property>
     <name>mapreduce.jobhistory.address</name>
     <value><%= @mapreduce_jobhistory_host %>:<%= @mapreduce_jobhistory_port
%></value>
   </property>
diff --git a/bigtop-packages/src/common/giraph/install_giraph.sh b/bigtop-packages/src/common/giraph/install_giraph.sh
index 0cc9d94..79b7c37 100644
--- a/bigtop-packages/src/common/giraph/install_giraph.sh
+++ b/bigtop-packages/src/common/giraph/install_giraph.sh
@@ -110,11 +110,12 @@ install -d -m 0755 ${PREFIX}/${LIB_DIR}
 # Installing Giraph core
 install -d -m 0755 ${PREFIX}/${LIB_DIR}
 cp -r $BUILD_DIR/giraph-dist/target/giraph*-bin/*/* ${PREFIX}/${LIB_DIR}
+cp  $BUILD_DIR/giraph-examples/target/giraph-examples*.jar ${PREFIX}/${LIB_DIR}
 
 # Installing docs and examples
 install -d -m 0755 $PREFIX/${DOC_DIR}
 # cp -r $BUILD_DIR/target/staging/*  $PREFIX/${DOC_DIR}
-mv ${PREFIX}/${LIB_DIR}/giraph-examples*.jar $PREFIX/${DOC_DIR}
+cp ${PREFIX}/${LIB_DIR}/giraph-examples*.jar $PREFIX/${DOC_DIR}
 
 # Install executable wrappers
 install -d -m 0755 $PREFIX/usr/bin
@@ -134,6 +135,7 @@ for i in giraph ; do
 # Workaround for GIRAPH-199
 export HADOOP_HOME=\${HADOOP_HOME:-/usr/lib/hadoop}
 export HADOOP_CONF_DIR=\${HADOOP_CONF_DIR:-/etc/hadoop/conf}
+export HADOOP_CLASSPATH=\${HADOOP_CLASSPATH:-/etc/giraph/conf}
 
 export GIRAPH_HOME=$LIB_DIR
 exec $BIN_DIR/$i "\$@"
diff --git a/bigtop-tests/smoke-tests/giraph/build.gradle b/bigtop-tests/smoke-tests/giraph/build.gradle
new file mode 100644
index 0000000..06d47cf
--- /dev/null
+++ b/bigtop-tests/smoke-tests/giraph/build.gradle
@@ -0,0 +1,37 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+def tests_to_include() {
+  return [
+    "TestGiraphSmoke.groovy"
+  ];
+}
+
+sourceSets {
+  test {
+    groovy {
+      srcDirs = [
+        "${BIGTOP_HOME}/bigtop-tests/test-artifacts/giraph/src/main/groovy/org/apache/bigtop/itest/giraphsmoke/"
+      ]
+      exclude { FileTreeElement elem -> (doExclude(elem.getName())) }
+    }
+  }
+}
+
+test.doFirst {
+  checkEnv(["GIRAPH_HOME"])
+}
diff --git a/bigtop-tests/test-artifacts/giraph/src/main/groovy/org/apache/bigtop/itest/giraphsmoke/TestGiraphSmoke.groovy
b/bigtop-tests/test-artifacts/giraph/src/main/groovy/org/apache/bigtop/itest/giraphsmoke/TestGiraphSmoke.groovy
index 39a150e..7afbee1 100644
--- a/bigtop-tests/test-artifacts/giraph/src/main/groovy/org/apache/bigtop/itest/giraphsmoke/TestGiraphSmoke.groovy
+++ b/bigtop-tests/test-artifacts/giraph/src/main/groovy/org/apache/bigtop/itest/giraphsmoke/TestGiraphSmoke.groovy
@@ -24,17 +24,12 @@ import org.apache.bigtop.itest.shell.Shell
 import org.junit.Test
 
 public class TestGiraphSmoke {
-  static String runnerScript = "HADOOP_CLASSPATH=/etc/giraph/conf hadoop jar"
   static String testDir = "test.giraphsmoke." + (new Date().getTime())
-
-  static String giraphHome = System.getProperty('GIRAPH_HOME', '/usr/lib/giraph')
-  static String giraphJar = "${giraphHome}/giraph-jar-with-dependencies.jar"
-
   static Shell sh = new Shell("/bin/bash -s")
 
   @Test(timeout = 300000L)
   public void testPageRankBenchmark() {
-    sh.exec("${runnerScript} ${giraphJar}"
+    sh.exec("hadoop jar /usr/lib/giraph/giraph-examples*with-dependencies.jar"
       + " org.apache.giraph.benchmark.PageRankBenchmark"
       + " -v  "        // verbose
       + " -e 1"        // edges per vertex
@@ -47,7 +42,7 @@ public class TestGiraphSmoke {
 
   @Test(timeout = 300000L)
   public void testRandomMessageBenchmark() {
-    sh.exec("${runnerScript} ${giraphJar}"
+    sh.exec("hadoop jar /usr/lib/giraph/giraph-examples*with-dependencies.jar"
       + " org.apache.giraph.benchmark.RandomMessageBenchmark"
       + " -v  "        // verbose
       + " -e 1"        // edges per vertex
@@ -61,55 +56,22 @@ public class TestGiraphSmoke {
   }
 
   @Test(timeout = 300000L)
-  public void testSimpleCheckpointVertex() {
-    sh.exec("hadoop fs -rmr ${testDir}");
-    sh.exec("${runnerScript} ${giraphJar}"
-      + " org.apache.giraph.examples.SimpleCheckpointVertex"
-      + " -v  "        // verbose
-      + " -s 3"        // number of supersteps
-      + " -w 3"        // workers
-      + " -o ${testDir}"
-    )
-    assertEquals("running SimpleCheckpointVertex failed", 0, sh.getRet());
-  }
-
-  @Test(timeout = 300000L)
-  public void testSimpleVertexWithWorkerContext() {
-    sh.exec("hadoop fs -rmr ${testDir}");
-    sh.exec("${runnerScript} ${giraphJar}"
-      + " org.apache.giraph.examples.SimpleVertexWithWorkerContext"
-      + " ${testDir} 3"
-    )
-    assertEquals("running SimpleCheckpointVertex failed", 0, sh.getRet());
-  }
-
-  @Test(timeout = 300000L)
-  public void testSimpleShortestPathsVertex() {
-    // A graph definition: 
-    //   [vertex id, vertex value, [[edge1, value1], .. [edgeN, valueN]]] 
-    List graphDescription = [[0, 0, [[1, 1], [2, 2]]],
-      [1, 1, [[2, 2], [3, 3]]],
-      [2, 2, [[3, 3], [4, 4]]],
-      [3, 3, [[4, 4], [5, 5]]],
-      [4, 4, [[5, 5], [0, 0]]],
-      [5, 5, [[0, 0], [1, 1]]]];
-    int partitionSize = 2;
+  public void testSimpleShortestPathsComputation() {
+    sh.exec("echo -e '[0,0,[[1,1],[3,3]]]\n[1,0,[[0,1],[2,2],[3,1]]]\n[2,0,[[1,2],[4,4]]]\n[3,0,[[0,3],[1,1],[4,4]]]\n[4,0,[[3,4],[2,4]]]'
> ./giraphTest.txt");
 
-    sh.exec("hadoop fs -rmr ${testDir}",
-      "hadoop fs -mkdir ${testDir}/input");
+    sh.exec("hadoop fs -mkdir ${testDir}");
 
-    for (int i = 0; i < graphDescription.size(); i += partitionSize) {
-      String part = graphDescription[i..(i + partitionSize - 1)].join("\n");
-      int partId = i / partitionSize;
-      sh.exec("hadoop fs -put <(echo '${part}') ${testDir}/input/part-m-${partId}");
-    }
+    sh.exec("hadoop fs -copyFromLocal ./giraphTest.txt ${testDir}/giraphTest.txt");
 
-    sh.exec("${runnerScript} ${giraphJar}"
-      + " org.apache.giraph.examples.SimpleShortestPathsVertex"
-      + " ${testDir}/input"
-      + " ${testDir}/output"
-      + " 0 ${graphDescription.size() / partitionSize}"
+    sh.exec("hadoop jar /usr/lib/giraph/giraph-examples*with-dependencies.jar"
+      + " org.apache.giraph.GiraphRunner"
+      + " org.apache.giraph.examples.SimpleShortestPathsComputation"
+      + " -vif org.apache.giraph.io.formats.JsonLongDoubleFloatDoubleVertexInputFormat"
+      + " -vip ${testDir}/giraphTest.txt"
+      + " -vof org.apache.giraph.io.formats.IdWithValueTextOutputFormat"
+      + " -op ${testDir}/shortestpaths"
+      + " -w 3"
     )
-    assertEquals("running SimpleShortestPathsVertex failed", 0, sh.getRet());
+    assertEquals("running SimpleShortestPathsComputation failed", 0, sh.getRet());
   }
 }
diff --git a/provisioner/utils/smoke-tests.sh b/provisioner/utils/smoke-tests.sh
index 6250cba..a28f799 100755
--- a/provisioner/utils/smoke-tests.sh
+++ b/provisioner/utils/smoke-tests.sh
@@ -47,6 +47,7 @@ export MAHOUT_HOME=${MAHOUT_HOME:-/usr/lib/mahout}
 export SPARK_HOME=${SPARK_HOME:-/usr/lib/spark}
 export SQOOP_HOME=${SQOOP_HOME:-/usr/lib/sqoop}
 export ZOOKEEPER_HOME=${ZOOKEEPER_HOME:-/usr/lib/zookeeper}
+export GIRAPH_HOME=${GIRAPH_HOME:-/usr/lib/giraph}
 
 echo -e "\n===== START TO RUN SMOKE TESTS: $SMOKE_TESTS =====\n"
 


Mime
View raw message