hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From aengin...@apache.org
Subject hadoop git commit: HDFS-11594. Ozone: close container should call compactDB. Contributed by Anu Engineer.
Date Thu, 30 Mar 2017 23:54:31 GMT
Repository: hadoop
Updated Branches:
  refs/heads/HDFS-7240 1590b9f7e -> fbfdf6990


HDFS-11594. Ozone: close container should call compactDB. Contributed by Anu Engineer.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/fbfdf699
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/fbfdf699
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/fbfdf699

Branch: refs/heads/HDFS-7240
Commit: fbfdf699048c53f72b802ddcebb5b8cb852916b1
Parents: 1590b9f
Author: Anu Engineer <aengineer@apache.org>
Authored: Thu Mar 30 16:53:28 2017 -0700
Committer: Anu Engineer <aengineer@apache.org>
Committed: Thu Mar 30 16:53:28 2017 -0700

----------------------------------------------------------------------
 .../main/proto/DatanodeContainerProtocol.proto  |  1 +
 .../container/common/helpers/ChunkUtils.java    | 11 +++++---
 .../common/impl/ContainerManagerImpl.java       | 28 ++++++++++++++++----
 .../org/apache/hadoop/utils/LevelDBStore.java   | 12 +++++++++
 .../container/ozoneimpl/TestOzoneContainer.java | 22 +++++++++------
 .../hadoop/ozone/scm/node/TestNodeManager.java  |  4 +--
 6 files changed, 59 insertions(+), 19 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/fbfdf699/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/DatanodeContainerProtocol.proto
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/DatanodeContainerProtocol.proto
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/DatanodeContainerProtocol.proto
index 522a716..09b7602 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/DatanodeContainerProtocol.proto
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/DatanodeContainerProtocol.proto
@@ -125,6 +125,7 @@ enum Result {
   GET_SMALL_FILE_ERROR = 21;
   CLOSED_CONTAINER_IO = 22;
   ERROR_CONTAINER_NOT_EMPTY = 23;
+  ERROR_IN_COMPACT_DB = 24;
 }
 
 message ContainerCommandRequestProto {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/fbfdf699/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ChunkUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ChunkUtils.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ChunkUtils.java
index 277fa18..4eb211b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ChunkUtils.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ChunkUtils.java
@@ -24,9 +24,9 @@ import org.apache.commons.codec.digest.DigestUtils;
 import org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos;
 import org.apache.hadoop.io.IOUtils;
 import org.apache.hadoop.ozone.OzoneConsts;
-import org.apache.hadoop.scm.container.common.helpers.StorageContainerException;
 import org.apache.hadoop.ozone.container.common.impl.ChunkManagerImpl;
 import org.apache.hadoop.scm.container.common.helpers.Pipeline;
+import org.apache.hadoop.scm.container.common.helpers.StorageContainerException;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -217,7 +217,12 @@ public final class ChunkUtils {
         }
       }
       if (file != null) {
-        IOUtils.closeStream(file);
+        try {
+          file.close();
+        } catch (IOException e) {
+          throw new StorageContainerException("Error closing chunk file",
+              e, CONTAINER_INTERNAL_ERROR);
+        }
       }
     }
   }
@@ -250,9 +255,7 @@ public final class ChunkUtils {
    *
    * @param chunkFile - file where data lives.
    * @param data - chunk definition.
-   *
    * @return ByteBuffer
-   *
    * @throws StorageContainerException
    * @throws ExecutionException
    * @throws InterruptedException

http://git-wip-us.apache.org/repos/asf/hadoop/blob/fbfdf699/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerManagerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerManagerImpl.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerManagerImpl.java
index 532266f..3a9eb11 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerManagerImpl.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerManagerImpl.java
@@ -24,6 +24,7 @@ import org.apache.commons.codec.digest.DigestUtils;
 import org.apache.commons.io.FileUtils;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos;
+import org.apache.hadoop.ozone.container.common.helpers.KeyUtils;
 import org.apache.hadoop.scm.container.common.helpers.StorageContainerException;
 import org.apache.hadoop.ozone.protocol.proto
     .StorageContainerDatanodeProtocolProtos.SCMNodeReport;
@@ -40,6 +41,7 @@ import org.apache.hadoop.ozone.container.common.interfaces
 import org.apache.hadoop.ozone.container.common.interfaces.ContainerManager;
 import org.apache.hadoop.ozone.container.common.interfaces.KeyManager;
 import org.apache.hadoop.scm.container.common.helpers.Pipeline;
+import org.apache.hadoop.utils.LevelDBStore;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -78,6 +80,8 @@ import static org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos
     .Result.UNABLE_TO_READ_METADATA_DB;
 import static org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos
     .Result.UNSUPPORTED_REQUEST;
+import static org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos
+   .Result.ERROR_IN_COMPACT_DB;
 import static org.apache.hadoop.ozone.OzoneConsts.CONTAINER_EXTENSION;
 import static org.apache.hadoop.ozone.OzoneConsts.CONTAINER_META;
 
@@ -301,19 +305,23 @@ public class ContainerManagerImpl implements ContainerManager {
     FileOutputStream metaStream = null;
 
     try {
-      Path location = locationManager.getContainerPath();
+      Path metadataPath = null;
+      Path location = (!overwrite) ? locationManager.getContainerPath():
+          Paths.get(containerData.getContainerPath()).getParent();
       File containerFile = ContainerUtils.getContainerFile(containerData,
           location);
       File metadataFile = ContainerUtils.getMetadataFile(containerData,
           location);
+
       if(!overwrite) {
         ContainerUtils.verifyIsNewContainer(containerFile, metadataFile);
+        metadataPath = this.locationManager.getDataPath(
+            containerData.getContainerName());
+        metadataPath = ContainerUtils.createMetadata(metadataPath);
+      }  else {
+        metadataPath = ContainerUtils.getMetadataDirectory(containerData);
       }
 
-      Path metadataPath = this.locationManager.getDataPath(
-          containerData.getContainerName());
-      metadataPath = ContainerUtils.createMetadata(metadataPath);
-
       containerStream = new FileOutputStream(containerFile);
       metaStream = new FileOutputStream(metadataFile);
       MessageDigest sha = MessageDigest.getInstance(OzoneConsts.FILE_HASH);
@@ -470,6 +478,16 @@ public class ContainerManagerImpl implements ContainerManager {
     ContainerData containerData = readContainer(containerName);
     containerData.closeContainer();
     writeContainerInfo(containerData, true);
+    LevelDBStore db = KeyUtils.getDB(containerData, conf);
+
+    // It is ok if this operation takes a bit of time.
+    // Close container is not expected to be instantaneous.
+    try {
+      db.compactDB();
+    } catch (IOException e) {
+      LOG.error("Error in DB compaction while closing container", e);
+      throw new StorageContainerException(e, ERROR_IN_COMPACT_DB);
+    }
 
     // Active is different from closed. Closed means it is immutable, active
     // false means we have some internal error that is happening to this

http://git-wip-us.apache.org/repos/asf/hadoop/blob/fbfdf699/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/utils/LevelDBStore.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/utils/LevelDBStore.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/utils/LevelDBStore.java
index 46a29b4..247610b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/utils/LevelDBStore.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/utils/LevelDBStore.java
@@ -156,4 +156,16 @@ public class LevelDBStore implements Closeable {
   public void destroy() throws IOException {
     JniDBFactory.factory.destroy(dbFile, dbOptions);
   }
+
+
+  /**
+   * Compacts the DB by removing deleted keys etc.
+   * @throws IOException if there is an error.
+   */
+  public void compactDB() throws IOException {
+    if(db != null) {
+      // From LevelDB docs : begin == null and end == null means the whole DB.
+      db.compactRange(null, null);
+    }
+  }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/fbfdf699/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOzoneContainer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOzoneContainer.java
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOzoneContainer.java
index 6c28388..09d1a81 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOzoneContainer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOzoneContainer.java
@@ -276,7 +276,8 @@ public class TestOzoneContainer {
     }
   }
 
-  private void testCloseContainer() throws Exception {
+  @Test
+  public void testCloseContainer() throws Exception {
     MiniOzoneCluster cluster = null;
     XceiverClient client = null;
     try {
@@ -307,17 +308,19 @@ public class TestOzoneContainer {
       client.connect();
 
 
+
+      // Create container
+      ContainerProtos.ContainerCommandRequestProto request =
+          ContainerTestHelper.getCreateContainerRequest(containerName);
+      ContainerProtos.ContainerCommandResponseProto response =
+          client.sendCommand(request);
+      Assert.assertNotNull(response);
+      Assert.assertTrue(request.getTraceID().equals(response.getTraceID()));
+
       ContainerProtos.ContainerCommandRequestProto writeChunkRequest =
           ContainerTestHelper.getWriteChunkRequest(pipeline, containerName,
               keyName, 1024);
 
-      ContainerProtos.ContainerCommandRequestProto request;
-      ContainerProtos.ContainerCommandResponseProto response;
-
-      ContainerProtos.ContainerCommandRequestProto putKeyRequest =
-          ContainerTestHelper.getPutKeyRequest(writeChunkRequest
-              .getWriteChunk());
-
       // Write Chunk before closing
       response = client.sendCommand(writeChunkRequest);
       Assert.assertNotNull(response);
@@ -327,6 +330,9 @@ public class TestOzoneContainer {
           .getTraceID()));
 
 
+      ContainerProtos.ContainerCommandRequestProto putKeyRequest =
+          ContainerTestHelper.getPutKeyRequest(writeChunkRequest
+              .getWriteChunk());
       // Put key before closing.
       response = client.sendCommand(putKeyRequest);
       Assert.assertNotNull(response);

http://git-wip-us.apache.org/repos/asf/hadoop/blob/fbfdf699/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/scm/node/TestNodeManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/scm/node/TestNodeManager.java
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/scm/node/TestNodeManager.java
index 2eeb889..da567cb 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/scm/node/TestNodeManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/scm/node/TestNodeManager.java
@@ -866,7 +866,7 @@ public class TestNodeManager {
   }
 
   @Test
-  public void testScmEnterAndExistChillMode() throws IOException,
+  public void testScmEnterAndExitChillMode() throws IOException,
       InterruptedException {
     OzoneConfiguration conf = getConf();
     conf.setInt(OZONE_SCM_HEARTBEAT_PROCESS_INTERVAL_MS, 100);
@@ -879,7 +879,7 @@ public class TestNodeManager {
       Assert.assertThat(status, CoreMatchers.containsString("Still in chill " +
           "mode. Waiting on nodes to report in."));
 
-      // Should not exist chill mode since 10 nodes have not heartbeat yet.
+      // Should not exit chill mode since 10 nodes have not heartbeat yet.
       assertFalse(nodeManager.isOutOfNodeChillMode());
       assertFalse((nodeManager.isInManualChillMode()));
 


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message