hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From aengin...@apache.org
Subject hadoop git commit: HDFS-12216. Ozone: TestKeys is failing consistently. Contributed by Mukul Kumar Singh.
Date Sat, 19 Aug 2017 08:02:26 GMT
Repository: hadoop
Updated Branches:
  refs/heads/HDFS-7240 a245c60bb -> 7d132596d


HDFS-12216. Ozone: TestKeys is failing consistently. Contributed by Mukul Kumar Singh.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/7d132596
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/7d132596
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/7d132596

Branch: refs/heads/HDFS-7240
Commit: 7d132596da65fcdb5aa52eaff7865b661edb2097
Parents: a245c60
Author: Anu Engineer <aengineer@apache.org>
Authored: Sat Aug 19 00:55:43 2017 -0700
Committer: Anu Engineer <aengineer@apache.org>
Committed: Sat Aug 19 00:55:43 2017 -0700

----------------------------------------------------------------------
 .../common/helpers/ContainerUtils.java          |  6 ++-
 .../container/common/helpers/KeyUtils.java      | 16 +------
 .../common/transport/server/XceiverServer.java  |  6 ++-
 .../container/common/utils/ContainerCache.java  | 49 ++++++++++++++++----
 .../org/apache/hadoop/hdfs/MiniDFSCluster.java  | 10 +++-
 .../apache/hadoop/ozone/MiniOzoneCluster.java   | 24 ++++++++++
 .../hadoop/ozone/web/client/TestKeys.java       |  6 +--
 7 files changed, 85 insertions(+), 32 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/7d132596/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ContainerUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ContainerUtils.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ContainerUtils.java
index 0a360e2..d16b908 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ContainerUtils.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ContainerUtils.java
@@ -126,7 +126,8 @@ public final class ContainerUtils {
   public static ContainerProtos.ContainerCommandResponseProto logAndReturnError(
       Logger log, StorageContainerException ex,
       ContainerProtos.ContainerCommandRequestProto msg) {
-    log.info("Trace ID: {} : Message: {} : Result: {}", msg.getTraceID(),
+    log.info("Operation: {} : Trace ID: {} : Message: {} : Result: {}",
+        msg.getCmdType().name(), msg.getTraceID(),
         ex.getMessage(), ex.getResult().getValueDescriptor().getName());
     return getContainerResponse(msg, ex.getResult(), ex.getMessage()).build();
   }
@@ -142,7 +143,8 @@ public final class ContainerUtils {
   public static ContainerProtos.ContainerCommandResponseProto logAndReturnError(
       Logger log, RuntimeException ex,
       ContainerProtos.ContainerCommandRequestProto msg) {
-    log.info("Trace ID: {} : Message: {} ", msg.getTraceID(), ex.getMessage());
+    log.info("Operation: {} : Trace ID: {} : Message: {} ",
+        msg.getCmdType().name(), msg.getTraceID(), ex.getMessage());
     return getContainerResponse(msg, INVALID_ARGUMENT, ex.getMessage()).build();
   }
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/7d132596/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/helpers/KeyUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/helpers/KeyUtils.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/helpers/KeyUtils.java
index 1a36b71..bced2e2 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/helpers/KeyUtils.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/helpers/KeyUtils.java
@@ -21,12 +21,9 @@ import com.google.common.base.Preconditions;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos;
 import org.apache.hadoop.scm.container.common.helpers.StorageContainerException;
-import org.apache.hadoop.ozone.container.common.impl.KeyManagerImpl;
 import org.apache.hadoop.ozone.container.common.utils.ContainerCache;
 import org.apache.hadoop.utils.MetadataStore;
 import org.apache.hadoop.utils.MetadataStoreBuilder;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
 
 import java.io.File;
 import java.io.IOException;
@@ -100,21 +97,10 @@ public final class KeyUtils {
    * Shutdown all DB Handles.
    *
    * @param cache - Cache for DB Handles.
-   * @throws IOException
    */
   @SuppressWarnings("unchecked")
   public static void shutdownCache(ContainerCache cache)  {
-    Logger log = LoggerFactory.getLogger(KeyManagerImpl.class);
-    MetadataStore[] handles = new MetadataStore[cache.values().size()];
-    cache.values().toArray(handles);
-    Preconditions.checkState(handles.length == cache.values().size());
-    for (MetadataStore db : handles) {
-      try {
-        db.close();
-      } catch (IOException ex) {
-        log.error("error closing db. error {}", ex);
-      }
-    }
+    cache.shutdownCache();
   }
 
   /**

http://git-wip-us.apache.org/repos/asf/hadoop/blob/7d132596/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/XceiverServer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/XceiverServer.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/XceiverServer.java
index 7271cb2..3a5ce18 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/XceiverServer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/XceiverServer.java
@@ -34,7 +34,9 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import java.io.IOException;
+import java.net.InetSocketAddress;
 import java.net.ServerSocket;
+import java.net.SocketAddress;
 
 /**
  * Creates a netty server endpoint that acts as the communication layer for
@@ -65,8 +67,10 @@ public final class XceiverServer implements XceiverServerSpi {
     // use that as the container port
     if (conf.getBoolean(OzoneConfigKeys.DFS_CONTAINER_IPC_RANDOM_PORT,
         OzoneConfigKeys.DFS_CONTAINER_IPC_RANDOM_PORT_DEFAULT)) {
-      try (ServerSocket socket = new ServerSocket(0)) {
+      try (ServerSocket socket = new ServerSocket()) {
         socket.setReuseAddress(true);
+        SocketAddress address = new InetSocketAddress(0);
+        socket.bind(address);
         this.port = socket.getLocalPort();
         LOG.info("Found a free port for the server : {}", this.port);
       } catch (IOException e) {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/7d132596/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/utils/ContainerCache.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/utils/ContainerCache.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/utils/ContainerCache.java
index f4caad0..0dc08fb 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/utils/ContainerCache.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/utils/ContainerCache.java
@@ -19,6 +19,7 @@
 package org.apache.hadoop.ozone.container.common.utils;
 
 import com.google.common.base.Preconditions;
+import org.apache.commons.collections.MapIterator;
 import org.apache.commons.collections.map.LRUMap;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
@@ -63,6 +64,42 @@ public final class ContainerCache extends LRUMap {
   }
 
   /**
+   * Closes a db instance.
+   *
+   * @param container - name of the container to be closed.
+   * @param db - db instance to close.
+   */
+  private void closeDB(String container, MetadataStore db) {
+    if (db != null) {
+      try {
+        db.close();
+      } catch (IOException e) {
+        LOG.error("Error closing DB. Container: " + container, e);
+      }
+    }
+  }
+
+  /**
+   * Closes all the db instances and resets the cache.
+   */
+  public void shutdownCache() {
+    lock.lock();
+    try {
+      // iterate the cache and close each db
+      MapIterator iterator = cache.mapIterator();
+      while (iterator.hasNext()) {
+        iterator.next();
+        MetadataStore db = (MetadataStore) iterator.getValue();
+        closeDB(iterator.getKey().toString(), db);
+      }
+      // reset the cache
+      cache.clear();
+    } finally {
+      lock.unlock();
+    }
+  }
+
+  /**
    * {@inheritDoc}
    */
   @Override
@@ -70,9 +107,7 @@ public final class ContainerCache extends LRUMap {
     lock.lock();
     try {
       MetadataStore db = (MetadataStore) entry.getValue();
-      db.close();
-    } catch (IOException e) {
-      LOG.error("Error closing DB. Container: " + entry.getKey().toString(), e);
+      closeDB(entry.getKey().toString(), db);
     } finally {
       lock.unlock();
     }
@@ -107,13 +142,7 @@ public final class ContainerCache extends LRUMap {
     lock.lock();
     try {
       MetadataStore db = this.getDB(containerName);
-      if (db != null) {
-        try {
-          db.close();
-        } catch (IOException e) {
-          LOG.warn("There is some issue to stop an unused DB handler.", e);
-        }
-      }
+      closeDB(containerName, db);
       this.remove(containerName);
     } finally {
       lock.unlock();

http://git-wip-us.apache.org/repos/asf/hadoop/blob/7d132596/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
index a113cdb..820917f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
@@ -530,6 +530,14 @@ public class MiniDFSCluster implements AutoCloseable {
       this.ipcPort = ipcPort;
     }
 
+    public Configuration getConf() {
+      return conf;
+    }
+
+    public DataNode getDatanode() {
+      return datanode;
+    }
+
     public void setDnArgs(String ... args) {
       dnArgs = args;
     }
@@ -2312,7 +2320,7 @@ public class MiniDFSCluster implements AutoCloseable {
       conf.set(DFS_DATANODE_ADDRESS_KEY, 
           addr.getAddress().getHostAddress() + ":" + addr.getPort());
       conf.set(DFS_DATANODE_IPC_ADDRESS_KEY,
-          addr.getAddress().getHostAddress() + ":" + dnprop.ipcPort); 
+          addr.getAddress().getHostAddress() + ":" + dnprop.ipcPort);
     }
     final DataNode newDn = DataNode.createDataNode(args, conf, secureResources);
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/7d132596/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/MiniOzoneCluster.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/MiniOzoneCluster.java
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/MiniOzoneCluster.java
index 61de1e2..e91c523 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/MiniOzoneCluster.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/MiniOzoneCluster.java
@@ -53,6 +53,10 @@ import java.util.Random;
 import java.util.UUID;
 import java.util.concurrent.TimeoutException;
 
+import static org.apache.hadoop.ozone.OzoneConfigKeys
+    .DFS_CONTAINER_IPC_PORT;
+import static org.apache.hadoop.ozone.OzoneConfigKeys
+    .DFS_CONTAINER_IPC_RANDOM_PORT;
 import static org.apache.hadoop.ozone.protocol.proto.OzoneProtos.NodeState
     .HEALTHY;
 import static org.junit.Assert.assertFalse;
@@ -135,6 +139,26 @@ public final class MiniOzoneCluster extends MiniDFSCluster
     }
   }
 
+  public boolean restartDataNode(int i) throws IOException {
+    return restartDataNode(i, true);
+  }
+  /*
+   * Restart a particular datanode, wait for it to become active
+   */
+  public boolean restartDataNode(int i, boolean keepPort) throws IOException {
+    if (keepPort) {
+      DataNodeProperties dnProp = dataNodes.get(i);
+      int currentPort = dnProp.getDatanode().getOzoneContainerManager()
+          .getContainerServerPort();
+      Configuration config = dnProp.getConf();
+      config.setInt(DFS_CONTAINER_IPC_PORT, currentPort);
+      config.setBoolean(DFS_CONTAINER_IPC_RANDOM_PORT, false);
+    }
+    boolean status =  super.restartDataNode(i, true);
+    this.waitActive();
+    return status;
+  }
+
   @Override
   public void shutdown() {
     super.shutdown();

http://git-wip-us.apache.org/repos/asf/hadoop/blob/7d132596/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/web/client/TestKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/web/client/TestKeys.java
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/web/client/TestKeys.java
index 456b1689..af871e5 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/web/client/TestKeys.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/web/client/TestKeys.java
@@ -65,8 +65,9 @@ public class TestKeys {
   public Timeout testTimeout = new Timeout(300000);
 
   private static MiniOzoneCluster ozoneCluster = null;
-  static private String path;
+  private static String path;
   private static OzoneRestClient ozoneRestClient = null;
+  private static long currentTime;
 
   /**
    * Create a MiniDFSCluster for testing.
@@ -93,6 +94,7 @@ public class TestKeys {
     final int port = dataNode.getInfoPort();
     ozoneRestClient = new OzoneRestClient(
         String.format("http://localhost:%d", port));
+    currentTime = Time.now();
   }
 
   /**
@@ -405,7 +407,6 @@ public class TestKeys {
     assertNotNull(helper.getBucket());
     assertNotNull(helper.getFile());
 
-    long currentTime = Time.now();
     // add keys [list-key0, list-key1, ..., list-key9]
     for (int x = 0; x < 10; x++) {
       String newkeyName = "list-key" + x;
@@ -490,7 +491,6 @@ public class TestKeys {
 
   static void runTestGetKeyInfo(PutHelper helper)
       throws OzoneException, ParseException {
-    long currentTime = Time.now();
     String keyName = helper.putKey();
     assertNotNull(helper.getBucket());
     assertNotNull(helper.getFile());


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message