bigtop-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From c..@apache.org
Subject bigtop git commit: 1821: added ignite smoke tests ported from grid gain tests.
Date Fri, 07 Aug 2015 21:36:52 GMT
Repository: bigtop
Updated Branches:
  refs/heads/master f6eac001a -> 6f2a5fc9f


1821: added ignite smoke tests ported from grid gain tests.

Signed-off-by: Konstantin Boudnik <cos@apache.org>


Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo
Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/6f2a5fc9
Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/6f2a5fc9
Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/6f2a5fc9

Branch: refs/heads/master
Commit: 6f2a5fc9f6a61dc1163acbfe9387d784f96e313b
Parents: f6eac00
Author: iveselovskiy <iveselovskiy@gridgain.com>
Authored: Tue Aug 4 15:12:22 2015 +0300
Committer: Konstantin Boudnik <cos@apache.org>
Committed: Fri Aug 7 14:33:39 2015 -0700

----------------------------------------------------------------------
 .../ignite-hadoop/TestIgniteHadoop.groovy       | 72 ++++++++++++++++
 .../smoke-tests/ignite-hadoop/build.gradle      | 53 ++++++++++++
 .../ignite-hadoop/conf/core-site.xml            | 90 ++++++++++++++++++++
 .../ignite-hadoop/conf/log4j.properties         | 24 ++++++
 .../ignite-hadoop/conf/mapred-site.xml          | 66 ++++++++++++++
 .../smoke-tests/ignite-hadoop/test.data         | 11 +++
 pom.xml                                         |  1 +
 7 files changed, 317 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/bigtop/blob/6f2a5fc9/bigtop-tests/smoke-tests/ignite-hadoop/TestIgniteHadoop.groovy
----------------------------------------------------------------------
diff --git a/bigtop-tests/smoke-tests/ignite-hadoop/TestIgniteHadoop.groovy b/bigtop-tests/smoke-tests/ignite-hadoop/TestIgniteHadoop.groovy
new file mode 100644
index 0000000..6358b7f
--- /dev/null
+++ b/bigtop-tests/smoke-tests/ignite-hadoop/TestIgniteHadoop.groovy
@@ -0,0 +1,72 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.bigtop.itest.hadoop.ignite
+
+import org.apache.bigtop.itest.shell.Shell
+import org.apache.commons.logging.Log
+import org.apache.commons.logging.LogFactory
+import org.junit.Before
+import org.junit.Test
+
+import static org.junit.Assert.assertEquals
+
+class TestIgniteHadoop {
+  static private Log LOG = LogFactory.getLog(Object.class)
+
+  static Shell sh = new Shell("/bin/bash -s")
+
+  static String hadoopClassPath = "/usr/lib/ignite-hadoop/libs/*:/usr/lib/ignite-hadoop/libs/ignite-hadoop/*"
+
+  static String cmdPrefix = "export HADOOP_CLASSPATH=$hadoopClassPath:\$HADOOP_CLASSPATH;
" +
+    "export HADOOP_CONF_DIR=\$(pwd)/conf;"
+
+
+  private static void execCommand(String cmd) {
+    LOG.info(cmd)
+
+    sh.exec("$cmdPrefix $cmd")
+  }
+
+  @Before
+  void cleanFileSystem() {
+    execCommand("hadoop fs -rm -r /gh-input")
+    execCommand("hadoop fs -rm -r /gh-output")
+  }
+
+  @Test
+  void test() {
+    execCommand("hadoop fs -mkdir /gh-input")
+    execCommand("hadoop fs -put test.data /gh-input/")
+
+    execCommand("hadoop jar \$HADOOP_MAPRED_HOME/hadoop-mapreduce-examples.jar wordcount
/gh-input /gh-output")
+
+    execCommand("hadoop fs -cat /gh-output/part-r-00000")
+
+    String expected =
+        "black\t5\n" +
+        "blue\t11\n" +
+        "green\t11\n" +
+        "white\t5\n" +
+        "yellow\t11";
+
+    String actual = sh.out.join('\n')
+
+    assertEquals("Incorrect output", expected, actual)
+  }
+}

http://git-wip-us.apache.org/repos/asf/bigtop/blob/6f2a5fc9/bigtop-tests/smoke-tests/ignite-hadoop/build.gradle
----------------------------------------------------------------------
diff --git a/bigtop-tests/smoke-tests/ignite-hadoop/build.gradle b/bigtop-tests/smoke-tests/ignite-hadoop/build.gradle
new file mode 100644
index 0000000..f0973ff
--- /dev/null
+++ b/bigtop-tests/smoke-tests/ignite-hadoop/build.gradle
@@ -0,0 +1,53 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+apply plugin: 'groovy'
+
+repositories {
+    mavenCentral()
+}
+
+dependencies {
+    //needed to avoid groovy not on classpath error.
+    testCompile module('org.codehaus.groovy:groovy:1.8.0')
+    testCompile group: 'org.apache.bigtop.itest', name: 'itest-common', version: itestVersion,
transitive: 'true'
+    testCompile group: 'org.apache.hadoop', name: 'hadoop-common', version: hadoopVersion,
transitive: 'true'
+    testCompile group: 'org.apache.hadoop', name: 'hadoop-hdfs', version: hadoopVersion,
transitive: 'true'
+}
+
+def tests_to_include() {
+  return [
+      "TestIgniteHadoop.groovy"
+  ];
+}
+
+sourceSets {
+  test {
+    resources {
+      srcDirs = [
+          'conf/'
+      ]
+    }
+    groovy {
+        srcDirs = ["$System.env.BIGTOP_HOME"+"/bigtop-tests/smoke-tests/ignite-hadoop/"]
+    }
+  }
+}
+
+test.doFirst {
+  checkEnv(["BIGTOP_HOME", "HADOOP_MAPRED_HOME"])
+}

http://git-wip-us.apache.org/repos/asf/bigtop/blob/6f2a5fc9/bigtop-tests/smoke-tests/ignite-hadoop/conf/core-site.xml
----------------------------------------------------------------------
diff --git a/bigtop-tests/smoke-tests/ignite-hadoop/conf/core-site.xml b/bigtop-tests/smoke-tests/ignite-hadoop/conf/core-site.xml
new file mode 100644
index 0000000..8b8e634
--- /dev/null
+++ b/bigtop-tests/smoke-tests/ignite-hadoop/conf/core-site.xml
@@ -0,0 +1,90 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+
+<!--
+  Licensed to the Apache Software Foundation (ASF) under one or more
+  contributor license agreements.  See the NOTICE file distributed with
+  this work for additional information regarding copyright ownership.
+  The ASF licenses this file to You under the Apache License, Version 2.0
+  (the "License"); you may not use this file except in compliance with
+  the License.  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+-->
+
+<!--
+    This template file contains settings needed to run Apache Hadoop jobs
+    with Apache Ignite's distributed in-memory file system IGFS.
+
+    You can replace '$HADOOP_HOME/etc/hadoop/core-site.xml' file with this one
+    to work with IGFS nodes running on localhost (these local nodes can be
+    a part of distributed cluster though). To work with file system on remote
+    hosts you need to change the host of file system URI to any host running
+    IGFS node.
+
+    Note that Ignite jars must be in Apache Hadoop client classpath to work
+    with this configuration.
+
+    Run script '$IGNITE_HOME/bin/setup-hadoop.{sh|bat}' for Apache Hadoop client setup.
+-->
+
+<configuration>
+    <!--
+        Set default file system to IGFS instance named "igfs" configured in Ignite.
+    -->
+    <property>
+        <name>fs.default.name</name>
+        <value>igfs://igfs@localhost</value>
+    </property>
+
+    <!--
+        Set Hadoop 1.* file system implementation class for IGFS.
+    -->
+    <property>
+        <name>fs.igfs.impl</name>
+        <value>org.apache.ignite.hadoop.fs.v1.IgniteHadoopFileSystem</value>
+    </property>
+
+    <!--
+        Set Hadoop 2.* file system implementation class for IGFS.
+    -->
+    <property>
+        <name>fs.AbstractFileSystem.igfs.impl</name>
+        <value>org.apache.ignite.hadoop.fs.v2.IgniteHadoopFileSystem</value>
+    </property>
+
+    <!--
+        Disallow data node replacement since it does not make sense for IGFS nodes.
+    -->
+    <property>
+        <name>dfs.client.block.write.replace-datanode-on-failure.policy</name>
+        <value>NEVER</value>
+    </property>
+
+    <!--
+        Allow to write the job statistics into IGFS.
+    -->
+    <!--
+    <property>
+        <name>ignite.counters.writer</name>
+        <value>org.apache.ignite.hadoop.fs.IgniteHadoopFileSystemCounterWriter</value>
+    </property>
+    -->
+
+    <!--
+        By default data is placed into the file /user/<user_name>/<job_id>/performance
+        You can override this path with using macro ${USER} that is to injection of submitter
user name.
+    -->
+    <!--
+    <property>
+        <name>ignite.counters.fswriter.directory</name>
+        <value>/user/${USER}</value>
+    </property>
+    -->
+</configuration>

http://git-wip-us.apache.org/repos/asf/bigtop/blob/6f2a5fc9/bigtop-tests/smoke-tests/ignite-hadoop/conf/log4j.properties
----------------------------------------------------------------------
diff --git a/bigtop-tests/smoke-tests/ignite-hadoop/conf/log4j.properties b/bigtop-tests/smoke-tests/ignite-hadoop/conf/log4j.properties
new file mode 100644
index 0000000..0723326
--- /dev/null
+++ b/bigtop-tests/smoke-tests/ignite-hadoop/conf/log4j.properties
@@ -0,0 +1,24 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Set root logger level to INFO and its only appender to A1.
+log4j.rootLogger=INFO, A1
+
+# A1 is set to be a ConsoleAppender.
+log4j.appender.A1=org.apache.log4j.ConsoleAppender
+
+# A1 uses PatternLayout.
+log4j.appender.A1.layout=org.apache.log4j.PatternLayout
+log4j.appender.A1.layout.ConversionPattern=%-4r [BIGTOP-TEST-LOG %t] %-5p %c %x --- %m%n

http://git-wip-us.apache.org/repos/asf/bigtop/blob/6f2a5fc9/bigtop-tests/smoke-tests/ignite-hadoop/conf/mapred-site.xml
----------------------------------------------------------------------
diff --git a/bigtop-tests/smoke-tests/ignite-hadoop/conf/mapred-site.xml b/bigtop-tests/smoke-tests/ignite-hadoop/conf/mapred-site.xml
new file mode 100644
index 0000000..a2ed437
--- /dev/null
+++ b/bigtop-tests/smoke-tests/ignite-hadoop/conf/mapred-site.xml
@@ -0,0 +1,66 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+
+<!--
+  Licensed to the Apache Software Foundation (ASF) under one or more
+  contributor license agreements.  See the NOTICE file distributed with
+  this work for additional information regarding copyright ownership.
+  The ASF licenses this file to You under the Apache License, Version 2.0
+  (the "License"); you may not use this file except in compliance with
+  the License.  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+-->
+
+<!--
+    This template file contains settings needed to run Apache Hadoop jobs
+    with Apache Ignite In-Memory Accelerator.
+
+    You can replace '$HADOOP_HOME/etc/hadoop/mapred-site.xml' file with this one
+    to run jobs on localhost (local node can be a part of distributed cluster though).
+    To run jobs on remote host you have to change jobtracker address to the REST address
+    of any running Ignite node.
+
+    Note that Ignite jars must be in Apache Hadoop client classpath to work
+    with this configuration.
+
+    Run script '$IGNITE_HOME/bin/setup-hadoop.{sh|bat}' for Apache Hadoop client setup.
+-->
+
+<configuration>
+    <!--
+        Framework name must be set to 'ignite'.
+    -->
+    <property>
+        <name>mapreduce.framework.name</name>
+        <value>ignite</value>
+    </property>
+
+    <!--
+        Job tracker address must be set to the REST address of any running Ignite node.
+    -->
+    <property>
+        <name>mapreduce.jobtracker.address</name>
+        <value>localhost:11211</value>
+    </property>
+
+    <!-- Parameters for job tuning. -->
+    <!--
+    <property>
+        <name>mapreduce.job.reduces</name>
+        <value>1</value>
+    </property>
+
+    <property>
+        <name>mapreduce.job.maps</name>
+        <value>4</value>
+    </property>
+    -->
+
+</configuration>

http://git-wip-us.apache.org/repos/asf/bigtop/blob/6f2a5fc9/bigtop-tests/smoke-tests/ignite-hadoop/test.data
----------------------------------------------------------------------
diff --git a/bigtop-tests/smoke-tests/ignite-hadoop/test.data b/bigtop-tests/smoke-tests/ignite-hadoop/test.data
new file mode 100644
index 0000000..6599669
--- /dev/null
+++ b/bigtop-tests/smoke-tests/ignite-hadoop/test.data
@@ -0,0 +1,11 @@
+green blue yellow
+yellow green blue black white
+yellow green blue
+white yellow green blue black
+blue yellow green
+black white yellow green blue
+green blue yellow
+blue black white yellow green
+yellow green blue
+green blue black white yellow
+blue yellow green

http://git-wip-us.apache.org/repos/asf/bigtop/blob/6f2a5fc9/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 455f35a..49e9e1d 100644
--- a/pom.xml
+++ b/pom.xml
@@ -304,6 +304,7 @@
               <exclude>bigtop-tests/smoke-tests/kite/sandwiches.csv</exclude>
               <exclude>bigtop-tests/smoke-tests/phoenix/*.csv</exclude>
               <exclude>bigtop-tests/smoke-tests/phoenix/*.sql</exclude>
+              <exclude>bigtop-tests/smoke-tests/ignite-hadoop/*.data</exclude>
               <exclude>**/target/**</exclude>
               <exclude>**/build/**</exclude>
               <exclude>**/.gradle/**</exclude>


Mime
View raw message