bigtop-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From c..@apache.org
Subject git commit: BIGTOP-1208. Add DFSIO test into longevity suite
Date Thu, 20 Feb 2014 03:21:50 GMT
Repository: bigtop
Updated Branches:
  refs/heads/master 158e74608 -> 54d45491f


BIGTOP-1208. Add DFSIO test into longevity suite


Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo
Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/54d45491
Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/54d45491
Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/54d45491

Branch: refs/heads/master
Commit: 54d45491fb56ae29df0fee8507cb96582727f70c
Parents: 158e746
Author: Dasha Boudnik <dasha@wandisco.com>
Authored: Wed Feb 19 19:21:12 2014 -0800
Committer: Konstantin Boudnik <cos@apache.org>
Committed: Wed Feb 19 19:21:12 2014 -0800

----------------------------------------------------------------------
 .../bigtop/itest/iolongevity/TestDFSIO.groovy   | 97 ++++++++++++++++++++
 bigtop-tests/test-execution/README              |  4 +
 2 files changed, 101 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/bigtop/blob/54d45491/bigtop-tests/test-artifacts/longevity/src/main/groovy/org/apache/bigtop/itest/iolongevity/TestDFSIO.groovy
----------------------------------------------------------------------
diff --git a/bigtop-tests/test-artifacts/longevity/src/main/groovy/org/apache/bigtop/itest/iolongevity/TestDFSIO.groovy
b/bigtop-tests/test-artifacts/longevity/src/main/groovy/org/apache/bigtop/itest/iolongevity/TestDFSIO.groovy
new file mode 100644
index 0000000..e32d56e
--- /dev/null
+++ b/bigtop-tests/test-artifacts/longevity/src/main/groovy/org/apache/bigtop/itest/iolongevity/TestDFSIO.groovy
@@ -0,0 +1,97 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.bigtop.itest.iolongevity
+
+import org.apache.bigtop.itest.JarContent
+import org.apache.bigtop.itest.shell.Shell
+import static org.apache.bigtop.itest.LogErrorsUtils.logError
+import static org.junit.Assert.assertNotNull
+import static org.junit.Assert.assertTrue
+import org.junit.BeforeClass
+import org.junit.Before
+import org.junit.Test
+
+public class TestDFSIO {
+  static Shell sh = new Shell("/bin/bash -s");
+
+  private static final String WRITE_CMD = "-write";
+  private static final String APPEND_CMD = "-append";
+  private static final String READ_CMD = "-read";
+
+  private static final String BENCHMARKS_DIR = "/benchmarks/TestDFSIO";
+  private static final String HADOOP_MAPRED_HOME = System.getenv('HADOOP_MAPRED_HOME');
+
+  private final String numFiles = System.getProperty("numFiles", "10");
+  private final String sizeFile = System.getProperty("sizeFile", "10MB");
+  private final int numOfIterations = Integer.getInteger("numOfIterations", 1);
+  private final int TIMEOUT = 5000;
+
+  private String DFSIO_TEMPLATE;
+  private String result = "TestDFSIO_Results.log";
+
+  final String hadoopExamplesJar =
+    JarContent.getJarName(HADOOP_MAPRED_HOME, 'hadoop.*jobclient.*tests.*.jar');
+
+  final String hadoopTestJar = HADOOP_MAPRED_HOME + '/' + hadoopExamplesJar;
+
+  @BeforeClass
+  static void setUp() throws IOException {
+    assertNotNull("HADOOP_MAPRED_HOME has to be set to run this test", HADOOP_MAPRED_HOME)
+  }
+
+  @Before
+  public void cleanup() {
+    sh.exec("hadoop fs -rm -r " + BENCHMARKS_DIR);
+    Thread.sleep(TIMEOUT);
+    sh.exec("hadoop fs -mkdir -p " + BENCHMARKS_DIR);
+    assertTrue("Failed to create " + BENCHMARKS_DIR, sh.getRet() == 0);
+  }
+
+  @Test
+  public void testDFSIO() {
+    DFSIO_TEMPLATE = "hadoop jar " + hadoopTestJar + " TestDFSIO %s ";
+    final String DFSIO_ARGS_TEMPLATE = "-nrFiles %s -fileSize %s -resFile %s";
+
+    final String writeSuccess = BENCHMARKS_DIR + "/io_write/_SUCCESS";
+    final String appendSuccess = BENCHMARKS_DIR + "/io_append/_SUCCESS";
+    final String readSuccess = BENCHMARKS_DIR + "/io_read/_SUCCESS";
+    String argStr = String.format(DFSIO_ARGS_TEMPLATE, numFiles, sizeFile, result);
+
+    for (int counter = 0; counter < numOfIterations; counter++) {
+      executeCmd(String.format(DFSIO_TEMPLATE + argStr, WRITE_CMD), writeSuccess);
+      executeCmd(String.format(DFSIO_TEMPLATE + argStr, APPEND_CMD), appendSuccess);
+      executeCmd(String.format(DFSIO_TEMPLATE + argStr, READ_CMD), readSuccess);
+      argStr = String.format(DFSIO_ARGS_TEMPLATE, numFiles, sizeFile, result + "." + counter);
+    }
+  }
+
+  private void executeCmd(String cmd, String expectedFile){
+    sh.exec(cmd);
+    logError(sh);
+    assertTrue("Command " + cmd + " is unsuccessful", sh.getRet() == 0);
+    sh.exec("hadoop fs -ls " + expectedFile);
+    boolean success = false;
+    sh.getOut().each { str ->
+        if (str.contains(expectedFile)) {
+            success = true;
+        }
+    }
+    assertTrue("File " + expectedFile + " was not found", success);
+  }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/bigtop/blob/54d45491/bigtop-tests/test-execution/README
----------------------------------------------------------------------
diff --git a/bigtop-tests/test-execution/README b/bigtop-tests/test-execution/README
index c3e5bba..136cbbf 100644
--- a/bigtop-tests/test-execution/README
+++ b/bigtop-tests/test-execution/README
@@ -16,3 +16,7 @@
 Repository contains the collection of starter test suites for different kinds
 of Hadoop stack validations using iTest framework
 
+To run the DFSIO longevity test, you can parametrize the number of files (-nrFiles),
+size of files (-fileSize), and number of cycles using system properties
+numFiles, sizeFile, and numOfIterations, respectively.
+User must make sure that the user home directory exists prior to running the test.
\ No newline at end of file


Mime
View raw message