hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From na...@apache.org
Subject [hadoop] branch trunk updated: HDDS-1898. GrpcReplicationService#download cannot replicate the container. (#1326)
Date Thu, 05 Sep 2019 12:42:47 GMT
This is an automated email from the ASF dual-hosted git repository.

nanda pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git


The following commit(s) were added to refs/heads/trunk by this push:
     new 2b16d53  HDDS-1898. GrpcReplicationService#download cannot replicate the container.
(#1326)
2b16d53 is described below

commit 2b16d5377c39518ed0140fd27f5b000d464c2f43
Author: Nanda kumar <nanda@apache.org>
AuthorDate: Thu Sep 5 18:12:36 2019 +0530

    HDDS-1898. GrpcReplicationService#download cannot replicate the container. (#1326)
---
 .../ozone/container/common/interfaces/Handler.java | 16 ++++++++++---
 .../commandhandler/DeleteBlocksCommandHandler.java |  7 +++++-
 .../container/keyvalue/KeyValueContainer.java      |  9 +++----
 .../ozone/container/keyvalue/KeyValueHandler.java  | 28 +++++++++++++++++-----
 .../background/BlockDeletingService.java           |  6 +++++
 .../container/ozoneimpl/ContainerController.java   | 12 ++++++++--
 .../OnDemandContainerReplicationSource.java        | 18 ++++----------
 .../ozone/container/TestContainerReplication.java  |  8 +++++++
 8 files changed, 75 insertions(+), 29 deletions(-)

diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/Handler.java
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/Handler.java
index 97601ec..8c3b981 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/Handler.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/Handler.java
@@ -19,8 +19,9 @@
 package org.apache.hadoop.ozone.container.common.interfaces;
 
 
-import java.io.FileInputStream;
 import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hdds.protocol.DatanodeDetails;
@@ -109,14 +110,23 @@ public abstract class Handler {
       DispatcherContext dispatcherContext);
 
   /**
-   * Import container data from a raw input stream.
+   * Imports container from a raw input stream.
    */
   public abstract Container importContainer(
       long containerID,
       long maxSize,
       String originPipelineId,
       String originNodeId,
-      FileInputStream rawContainerStream,
+      InputStream rawContainerStream,
+      TarContainerPacker packer)
+      throws IOException;
+
+  /**
+   * Exports container to the output stream.
+   */
+  public abstract void exportContainer(
+      Container container,
+      OutputStream outputStream,
       TarContainerPacker packer)
       throws IOException;
 
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/DeleteBlocksCommandHandler.java
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/DeleteBlocksCommandHandler.java
index a5d4760..a4849f2 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/DeleteBlocksCommandHandler.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/DeleteBlocksCommandHandler.java
@@ -127,7 +127,12 @@ public class DeleteBlocksCommandHandler implements CommandHandler {
           case KeyValueContainer:
             KeyValueContainerData containerData = (KeyValueContainerData)
                 cont.getContainerData();
-            deleteKeyValueContainerBlocks(containerData, entry);
+            cont.writeLock();
+            try {
+              deleteKeyValueContainerBlocks(containerData, entry);
+            } finally {
+              cont.writeUnlock();
+            }
             txResultBuilder.setContainerID(containerId)
                 .setSuccess(true);
             break;
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueContainer.java
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueContainer.java
index 53065cc..b7f46c9 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueContainer.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueContainer.java
@@ -330,6 +330,9 @@ public class KeyValueContainer implements Container<KeyValueContainerData>
{
     } finally {
       writeUnlock();
     }
+    LOG.info("Container {} is closed with bcsId {}.",
+        containerData.getContainerID(),
+        containerData.getBlockCommitSequenceId());
   }
 
   /**
@@ -361,13 +364,10 @@ public class KeyValueContainer implements Container<KeyValueContainerData>
{
     }
   }
 
-  void compactDB() throws StorageContainerException {
+  private void compactDB() throws StorageContainerException {
     try {
       try(ReferenceCountedDB db = BlockUtils.getDB(containerData, config)) {
         db.getStore().compactDB();
-        LOG.info("Container {} is closed with bcsId {}.",
-            containerData.getContainerID(),
-            containerData.getBlockCommitSequenceId());
       }
     } catch (StorageContainerException ex) {
       throw ex;
@@ -524,6 +524,7 @@ public class KeyValueContainer implements Container<KeyValueContainerData>
{
           "Only closed containers could be exported: ContainerId="
               + getContainerData().getContainerID());
     }
+    compactDB();
     packer.pack(this, destination);
   }
 
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java
index 50e3706..ab1d124 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java
@@ -18,8 +18,9 @@
 
 package org.apache.hadoop.ozone.container.keyvalue;
 
-import java.io.FileInputStream;
 import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
 import java.nio.ByteBuffer;
 import java.util.HashMap;
 import java.util.LinkedList;
@@ -841,13 +842,14 @@ public class KeyValueHandler extends Handler {
     throw new StorageContainerException(msg, result);
   }
 
-  public Container importContainer(long containerID, long maxSize,
-      String originPipelineId,
-      String originNodeId,
-      FileInputStream rawContainerStream,
-      TarContainerPacker packer)
+  @Override
+  public Container importContainer(final long containerID,
+      final long maxSize, final String originPipelineId,
+      final String originNodeId, final InputStream rawContainerStream,
+      final TarContainerPacker packer)
       throws IOException {
 
+    // TODO: Add layout version!
     KeyValueContainerData containerData =
         new KeyValueContainerData(containerID,
             maxSize, originPipelineId, originNodeId);
@@ -863,6 +865,20 @@ public class KeyValueHandler extends Handler {
   }
 
   @Override
+  public void exportContainer(final Container container,
+      final OutputStream outputStream,
+      final TarContainerPacker packer)
+      throws IOException{
+    container.readLock();
+    try {
+      final KeyValueContainer kvc = (KeyValueContainer) container;
+      kvc.exportContainerData(outputStream, packer);
+    } finally {
+      container.readUnlock();
+    }
+  }
+
+  @Override
   public void markContainerForClose(Container container)
       throws IOException {
     // Move the container to CLOSING state only if it's OPEN
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/statemachine/background/BlockDeletingService.java
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/statemachine/background/BlockDeletingService.java
index 25c00c3..fd048d7 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/statemachine/background/BlockDeletingService.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/statemachine/background/BlockDeletingService.java
@@ -23,6 +23,7 @@ import org.apache.hadoop.hdds.scm.ScmConfigKeys;
 import org.apache.hadoop.hdds.scm.pipeline.PipelineID;
 import org.apache.hadoop.ozone.container.common.impl.ContainerData;
 import org.apache.hadoop.ozone.container.common.impl.TopNOrderedContainerDeletionChoosingPolicy;
+import org.apache.hadoop.ozone.container.common.interfaces.Container;
 import org.apache.hadoop.ozone.container.common.interfaces.ContainerDeletionChoosingPolicy;
 import org.apache.hadoop.ozone.container.common.transport.server.ratis.XceiverServerRatis;
 import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainerData;
@@ -247,6 +248,9 @@ public class BlockDeletingService extends BackgroundService {
     @Override
     public BackgroundTaskResult call() throws Exception {
       ContainerBackgroundTaskResult crr = new ContainerBackgroundTaskResult();
+      final Container container = ozoneContainer.getContainerSet()
+          .getContainer(containerData.getContainerID());
+      container.writeLock();
       long startTime = Time.monotonicNow();
       // Scan container's db and get list of under deletion blocks
       try (ReferenceCountedDB meta = BlockUtils.getDB(containerData, conf)) {
@@ -313,6 +317,8 @@ public class BlockDeletingService extends BackgroundService {
         }
         crr.addAll(succeedBlocks);
         return crr;
+      } finally {
+        container.writeUnlock();
       }
     }
 
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/ContainerController.java
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/ContainerController.java
index 523e63f..eb672a7 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/ContainerController.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/ContainerController.java
@@ -29,8 +29,9 @@ import org.apache.hadoop.ozone.container.common.interfaces.Handler;
 import org.apache.hadoop.ozone.container.common.volume.HddsVolume;
 import org.apache.hadoop.ozone.container.keyvalue.TarContainerPacker;
 
-import java.io.FileInputStream;
 import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
 import java.util.Iterator;
 import java.util.Map;
 
@@ -120,13 +121,20 @@ public class ContainerController {
 
   public Container importContainer(final ContainerType type,
       final long containerId, final long maxSize, final String originPipelineId,
-      final String originNodeId, final FileInputStream rawContainerStream,
+      final String originNodeId, final InputStream rawContainerStream,
       final TarContainerPacker packer)
       throws IOException {
     return handlers.get(type).importContainer(containerId, maxSize,
         originPipelineId, originNodeId, rawContainerStream, packer);
   }
 
+  public void exportContainer(final ContainerType type,
+      final long containerId, final OutputStream outputStream,
+      final TarContainerPacker packer) throws IOException {
+    handlers.get(type).exportContainer(
+        containerSet.getContainer(containerId), outputStream, packer);
+  }
+
   /**
    * Deletes a container given its Id.
    * @param containerId Id of the container to be deleted
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/OnDemandContainerReplicationSource.java
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/OnDemandContainerReplicationSource.java
index 28b8713..d318ffa 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/OnDemandContainerReplicationSource.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/OnDemandContainerReplicationSource.java
@@ -21,7 +21,6 @@ import java.io.IOException;
 import java.io.OutputStream;
 
 import org.apache.hadoop.ozone.container.common.interfaces.Container;
-import org.apache.hadoop.ozone.container.common.interfaces.ContainerPacker;
 import org.apache.hadoop.ozone.container.keyvalue.TarContainerPacker;
 
 import com.google.common.base.Preconditions;
@@ -41,7 +40,7 @@ public class OnDemandContainerReplicationSource
 
   private ContainerController controller;
 
-  private ContainerPacker packer = new TarContainerPacker();
+  private TarContainerPacker packer = new TarContainerPacker();
 
   public OnDemandContainerReplicationSource(
       ContainerController controller) {
@@ -59,18 +58,11 @@ public class OnDemandContainerReplicationSource
 
     Container container = controller.getContainer(containerId);
 
-    Preconditions
-        .checkNotNull(container, "Container is not found " + containerId);
+    Preconditions.checkNotNull(
+        container, "Container is not found " + containerId);
 
-    switch (container.getContainerType()) {
-    case KeyValueContainer:
-      packer.pack(container,
-          destination);
-      break;
-    default:
-      LOG.warn("Container type " + container.getContainerType()
-          + " is not replicable as no compression algorithm for that.");
-    }
+    controller.exportContainer(
+        container.getContainerType(), containerId, destination, packer);
 
   }
 }
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/TestContainerReplication.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/TestContainerReplication.java
index ab78705..7e8ff3c 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/TestContainerReplication.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/TestContainerReplication.java
@@ -46,6 +46,7 @@ import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainerData;
 import org.apache.hadoop.ozone.container.keyvalue.KeyValueHandler;
 import org.apache.hadoop.ozone.container.ozoneimpl.OzoneContainer;
 import org.apache.hadoop.ozone.container.ozoneimpl.TestOzoneContainer;
+import org.apache.hadoop.ozone.protocol.commands.CloseContainerCommand;
 import org.apache.hadoop.ozone.protocol.commands.ReplicateContainerCommand;
 import org.apache.hadoop.test.GenericTestUtils;
 
@@ -119,6 +120,13 @@ public class TestContainerReplication {
         chooseDatanodeWithoutContainer(sourcePipelines,
             cluster.getHddsDatanodes());
 
+    // Close the container
+    cluster.getStorageContainerManager().getScmNodeManager()
+        .addDatanodeCommand(
+            sourceDatanodes.get(0).getUuid(),
+            new CloseContainerCommand(containerId,
+                sourcePipelines.getId(), true));
+
     //WHEN: send the order to replicate the container
     cluster.getStorageContainerManager().getScmNodeManager()
         .addDatanodeCommand(destinationDatanode.getDatanodeDetails().getUuid(),


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message