bigtop-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ju...@apache.org
Subject [bigtop] branch master updated: BIGTOP-3151: Add flink smoke test
Date Wed, 27 Feb 2019 05:10:47 GMT
This is an automated email from the ASF dual-hosted git repository.

junhe pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/bigtop.git


The following commit(s) were added to refs/heads/master by this push:
     new 3134c0c  BIGTOP-3151: Add flink smoke test
3134c0c is described below

commit 3134c0c0b883fbdf7d9254545f72722fe356c8f7
Author: Jun He <jun.he@linaro.org>
AuthorDate: Thu Feb 14 15:04:47 2019 +0800

    BIGTOP-3151: Add flink smoke test
    
    Add flink to smoke tests.
    
    Change-Id: Ib6a538a97347ec0e77b51c81b09a9d23a1a25a1a
    Signed-off-by: Jun He <jun.he@linaro.org>
---
 bigtop-tests/smoke-tests/flink/TestFlink.groovy | 87 +++++++++++++++++++++++++
 bigtop-tests/smoke-tests/flink/build.gradle     | 34 ++++++++++
 bigtop-tests/smoke-tests/flink/test.data        | 11 ++++
 build.gradle                                    |  1 +
 provisioner/utils/smoke-tests.sh                |  1 +
 5 files changed, 134 insertions(+)

diff --git a/bigtop-tests/smoke-tests/flink/TestFlink.groovy b/bigtop-tests/smoke-tests/flink/TestFlink.groovy
new file mode 100644
index 0000000..d569f53
--- /dev/null
+++ b/bigtop-tests/smoke-tests/flink/TestFlink.groovy
@@ -0,0 +1,87 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.bigtop.itest.hadoop.flink
+
+import org.apache.bigtop.itest.shell.Shell
+import org.apache.commons.logging.Log
+import org.apache.commons.logging.LogFactory
+import org.junit.BeforeClass
+import org.junit.AfterClass
+import org.junit.Test
+
+import static org.junit.Assert.assertEquals
+
+class TestFlink {
+  static private Log LOG = LogFactory.getLog(Object.class)
+
+  static Shell sh = new Shell("/bin/bash -s")
+
+  static String cmdPrefix = "export HADOOP_CONF_DIR=/etc/hadoop/conf;HADOOP_USER_NAME=hdfs"
+  static private final String config_file = "/etc/flink/conf/flink-conf.yaml";
+
+
+  private static void execCommand(String cmd) {
+    LOG.info(cmd)
+
+    sh.exec("$cmdPrefix $cmd")
+  }
+
+  @BeforeClass
+  static void setupTest() {
+    execCommand("hadoop fs -rm -r /flink");
+    execCommand("hadoop fs -mkdir /flink")
+    execCommand("hadoop fs -put test.data /flink/")
+  }
+
+  @AfterClass
+  static void tearDown() {
+    execCommand("hadoop fs -rm -r /flink")
+  }
+
+  @Test
+  void testCheckRestfulAPI() {
+    // read JM address and port from conf
+    execCommand("awk '{if(/jobmanager.rpc.address:/) print \$2}' < "+ config_file);
+    final String jmhost = sh.out.join('\n');
+    execCommand("awk '{if(/jobmanager.web.port:/) print \$2}' < "+config_file);
+    final String webPort = sh.out.join('\n');
+    // check web API
+    execCommand("curl http://"+jmhost+":"+webPort+"/config");
+    final String result = sh.out.join('\n');
+    assert(result.contains("flink-version"));
+  }
+
+  @Test
+  void testWordCountBatch() {
+    execCommand("flink run \$FLINK_HOME/examples/batch/WordCount.jar --input hdfs:///flink/test.data
--output hdfs:///tmp/result.txt")
+
+    execCommand("hadoop fs -cat /tmp/result.txt")
+
+    String expected =
+        "black 5\n" +
+        "blue 11\n" +
+        "green 11\n" +
+        "white 5\n" +
+        "yellow 11";
+
+    String actual = sh.out.join('\n')
+
+    assertEquals("Incorrect output", expected, actual)
+  }
+}
diff --git a/bigtop-tests/smoke-tests/flink/build.gradle b/bigtop-tests/smoke-tests/flink/build.gradle
new file mode 100644
index 0000000..05ccdab
--- /dev/null
+++ b/bigtop-tests/smoke-tests/flink/build.gradle
@@ -0,0 +1,34 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+def tests_to_include() {
+  return [
+      "TestFlink.groovy"
+  ];
+}
+
+sourceSets {
+  test {
+    groovy {
+        srcDirs = ["${BIGTOP_HOME}/bigtop-tests/smoke-tests/flink/"]
+    }
+  }
+}
+
+test.doFirst {
+  checkEnv(["FLINK_HOME"])
+}
diff --git a/bigtop-tests/smoke-tests/flink/test.data b/bigtop-tests/smoke-tests/flink/test.data
new file mode 100644
index 0000000..6599669
--- /dev/null
+++ b/bigtop-tests/smoke-tests/flink/test.data
@@ -0,0 +1,11 @@
+green blue yellow
+yellow green blue black white
+yellow green blue
+white yellow green blue black
+blue yellow green
+black white yellow green blue
+green blue yellow
+blue black white yellow green
+yellow green blue
+green blue black white yellow
+blue yellow green
diff --git a/build.gradle b/build.gradle
index facdcd5..d7e1045 100644
--- a/build.gradle
+++ b/build.gradle
@@ -74,6 +74,7 @@ rat {
        /* Test data with rigid structure and/or binary */
        "bigtop-tests/test-artifacts/**/resources/**",
        "bigtop-tests/smoke-tests/alluxio/datafile",
+       "bigtop-tests/smoke-tests/flink/test.data",
        "bigtop-tests/smoke-tests/hive/passwd.ql",
        "bigtop-tests/smoke-tests/phoenix/*.csv",
        "bigtop-tests/smoke-tests/phoenix/*.sql",
diff --git a/provisioner/utils/smoke-tests.sh b/provisioner/utils/smoke-tests.sh
index a28f799..4f2ef84 100755
--- a/provisioner/utils/smoke-tests.sh
+++ b/provisioner/utils/smoke-tests.sh
@@ -48,6 +48,7 @@ export SPARK_HOME=${SPARK_HOME:-/usr/lib/spark}
 export SQOOP_HOME=${SQOOP_HOME:-/usr/lib/sqoop}
 export ZOOKEEPER_HOME=${ZOOKEEPER_HOME:-/usr/lib/zookeeper}
 export GIRAPH_HOME=${GIRAPH_HOME:-/usr/lib/giraph}
+export FLINK_HOME=${FLINK_HOME:-/usr/lib/flink}
 
 echo -e "\n===== START TO RUN SMOKE TESTS: $SMOKE_TESTS =====\n"
 


Mime
View raw message