hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From a..@apache.org
Subject [hadoop] branch trunk updated: HDDS-1388. Add a shell script to run MiniOzoneChaosCluster using mvn exec. Contributed by Mukul Kumar Singh. (#709)
Date Tue, 09 Apr 2019 20:08:32 GMT
This is an automated email from the ASF dual-hosted git repository.

arp pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git


The following commit(s) were added to refs/heads/trunk by this push:
     new 312d4d4  HDDS-1388. Add a shell script to run MiniOzoneChaosCluster using mvn exec.
Contributed by Mukul Kumar Singh. (#709)
312d4d4 is described below

commit 312d4d4cc5d6f022a839f7ae390ee0f4520898d4
Author: Mukul Kumar Singh <mksingh1987@gmail.com>
AuthorDate: Wed Apr 10 01:38:25 2019 +0530

    HDDS-1388. Add a shell script to run MiniOzoneChaosCluster using mvn exec. Contributed
by Mukul Kumar Singh. (#709)
---
 .../hadoop/hdds/scm/block/BlockManagerImpl.java    |  2 ++
 .../integration-test/src/test/bin/start-chaos.sh   | 35 ++++++++++++++++++++++
 .../apache/hadoop/ozone/MiniOzoneChaosCluster.java |  9 ++++--
 .../hadoop/ozone/MiniOzoneLoadGenerator.java       | 15 +++++++---
 .../hadoop/ozone/TestMiniChaosOzoneCluster.java    |  6 ++--
 5 files changed, 58 insertions(+), 9 deletions(-)

diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/BlockManagerImpl.java
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/BlockManagerImpl.java
index 5ae4115..d15f07b 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/BlockManagerImpl.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/BlockManagerImpl.java
@@ -189,6 +189,8 @@ public class BlockManagerImpl implements BlockManager, BlockmanagerMXBean
{
           // factors are handled by pipeline creator
           pipeline = pipelineManager.createPipeline(type, factor);
         } catch (IOException e) {
+          LOG.error("pipeline creation failed type:{} factor:{}", type,
+              factor, e);
           break;
         }
       } else {
diff --git a/hadoop-ozone/integration-test/src/test/bin/start-chaos.sh b/hadoop-ozone/integration-test/src/test/bin/start-chaos.sh
new file mode 100755
index 0000000..dcec909
--- /dev/null
+++ b/hadoop-ozone/integration-test/src/test/bin/start-chaos.sh
@@ -0,0 +1,35 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#!/usr/bin/env bash
+
+date=`date +"%m-%d-%y-%T"`
+fileformat=".MiniOzoneChaosCluster.log"
+heapformat=".dump"
+current="/tmp/"
+filename=$current$date$fileformat
+heapdumpfile=$current$date$heapformat
+
+export MAVEN_OPTS="-XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath={$heapdumpfile}"
+
+echo "logging to" ${filename}
+echo "heapdump to" ${heapdumpfile}
+
+echo "Starting MiniOzoneChaosCluster"
+mvn clean install -DskipTests > ${filename} 2>&1
+mvn exec:java \
+  -Dexec.mainClass="org.apache.hadoop.ozone.TestMiniChaosOzoneCluster" \
+  -Dexec.classpathScope=test \
+  -Dexec.args="$*" >> ${filename} 2>&1
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneChaosCluster.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneChaosCluster.java
index 8e25d48..52a2d40 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneChaosCluster.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneChaosCluster.java
@@ -66,6 +66,7 @@ public class MiniOzoneChaosCluster extends MiniOzoneClusterImpl {
 
     this.executorService =  Executors.newSingleThreadScheduledExecutor();
     this.numDatanodes = getHddsDatanodes().size();
+    LOG.info("Starting MiniOzoneChaosCluster with:{} datanodes" + numDatanodes);
     LogUtils.setLogLevel(GrpcClientProtocolClient.LOG, Level.WARN);
   }
 
@@ -117,13 +118,16 @@ public class MiniOzoneChaosCluster extends MiniOzoneClusterImpl {
   }
 
   void startChaos(long initialDelay, long period, TimeUnit timeUnit) {
+    LOG.info("Starting Chaos with failure period:{} unit:{}", period, timeUnit);
     scheduledFuture = executorService.scheduleAtFixedRate(this::fail,
         initialDelay, period, timeUnit);
   }
 
   void stopChaos() throws Exception {
-    scheduledFuture.cancel(false);
-    scheduledFuture.get();
+    if (scheduledFuture != null) {
+      scheduledFuture.cancel(false);
+      scheduledFuture.get();
+    }
   }
 
   public void shutdown() {
@@ -192,6 +196,7 @@ public class MiniOzoneChaosCluster extends MiniOzoneClusterImpl {
           1, TimeUnit.SECONDS);
       conf.setTimeDuration(HddsConfigKeys.HDDS_HEARTBEAT_INTERVAL, 1,
           TimeUnit.SECONDS);
+      conf.setInt(OzoneConfigKeys.OZONE_CONTAINER_CACHE_SIZE, 8);
     }
 
     @Override
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneLoadGenerator.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneLoadGenerator.java
index efb3b66..005a528 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneLoadGenerator.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneLoadGenerator.java
@@ -81,7 +81,8 @@ public class MiniOzoneLoadGenerator {
 
   // Start IO load on an Ozone bucket.
   private void load(long runTimeMillis) {
-    LOG.info("Started IO Thread" + Thread.currentThread().getId());
+    long threadID = Thread.currentThread().getId();
+    LOG.info("Started IO Thread:{}.", threadID);
     String threadName = Thread.currentThread().getName();
     long startTime = Time.monotonicNow();
 
@@ -98,8 +99,11 @@ public class MiniOzoneLoadGenerator {
           new HashMap<>())) {
         stream.write(buffer.array());
       } catch (Exception e) {
-        LOG.error("LOADGEN: Create key:{} failed with exception", keyName, e);
-        break;
+        LOG.error("LOADGEN: Create key:{} failed with exception, skipping",
+            keyName, e);
+        continue;
+        // TODO: HDDS-1403.A key write can fail after multiple block writes
+        //  to closed container. add a break here once that is fixed.
       }
 
       try (OzoneInputStream stream = ozoneBucket.readKey(keyName)) {
@@ -119,17 +123,20 @@ public class MiniOzoneLoadGenerator {
         }
 
       } catch (Exception e) {
-        LOG.error("Read key:{} failed with exception", keyName, e);
+        LOG.error("LOADGEN: Read key:{} failed with exception", keyName, e);
         break;
       }
 
     }
     // This will terminate other threads too.
     isWriteThreadRunning.set(false);
+    LOG.info("Terminating IO thread:{}.", threadID);
   }
 
   public void startIO(long time, TimeUnit timeUnit) {
     List<CompletableFuture<Void>> writeFutures = new ArrayList<>();
+    LOG.info("Starting MiniOzoneLoadGenerator for time {}:{} with {} buffers " +
+            "and {} threads", time, timeUnit, numBuffers, numWriteThreads);
     if (isWriteThreadRunning.compareAndSet(false, true)) {
       // Start the IO thread
       for (int i = 0; i < numWriteThreads; i++) {
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestMiniChaosOzoneCluster.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestMiniChaosOzoneCluster.java
index 0438351..a979c40 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestMiniChaosOzoneCluster.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestMiniChaosOzoneCluster.java
@@ -96,7 +96,7 @@ public class TestMiniChaosOzoneCluster implements Runnable {
   public void run() {
     try {
       init();
-      cluster.startChaos(5, failureInterval, TimeUnit.SECONDS);
+      cluster.startChaos(failureInterval, failureInterval, TimeUnit.SECONDS);
       loadGenerator.startIO(numMinutes, TimeUnit.MINUTES);
     } catch (Exception e) {
     } finally {
@@ -109,8 +109,8 @@ public class TestMiniChaosOzoneCluster implements Runnable {
   }
 
   @Test
-  public void testReadWriteWithChaosCluster() throws Exception {
-    cluster.startChaos(5, 1, TimeUnit.SECONDS);
+  public void testReadWriteWithChaosCluster() {
+    cluster.startChaos(5, 10, TimeUnit.SECONDS);
     loadGenerator.startIO(1, TimeUnit.MINUTES);
   }
 }


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message