hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From aengin...@apache.org
Subject hadoop git commit: HDFS-11796. Ozone: MiniOzoneCluster should set "ozone.handler.type" key correctly. Contributed by Mukul Kumar Singh.
Date Fri, 02 Jun 2017 23:28:39 GMT
Repository: hadoop
Updated Branches:
  refs/heads/HDFS-7240 5cdd88007 -> 776d2d4de


HDFS-11796. Ozone: MiniOzoneCluster should set "ozone.handler.type" key correctly. Contributed
by Mukul Kumar Singh.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/776d2d4d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/776d2d4d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/776d2d4d

Branch: refs/heads/HDFS-7240
Commit: 776d2d4de79bc3f37bb0fa510442841225a87af4
Parents: 5cdd880
Author: Anu Engineer <aengineer@apache.org>
Authored: Fri Jun 2 16:24:16 2017 -0700
Committer: Anu Engineer <aengineer@apache.org>
Committed: Fri Jun 2 16:24:16 2017 -0700

----------------------------------------------------------------------
 .../apache/hadoop/ozone/ksm/KeyManagerImpl.java | 27 ++++++++++++--------
 .../apache/hadoop/ozone/MiniOzoneCluster.java   |  2 ++
 .../hadoop/ozone/ksm/TestKeySpaceManager.java   | 10 +++++---
 .../ozone/web/TestOzoneRestWithMiniCluster.java |  1 +
 4 files changed, 25 insertions(+), 15 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/776d2d4d/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/KeyManagerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/KeyManagerImpl.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/KeyManagerImpl.java
index 0a1b727..555ca9d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/KeyManagerImpl.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/KeyManagerImpl.java
@@ -73,18 +73,23 @@ public class KeyManagerImpl implements KeyManager {
         throw new KSMException("Bucket not found",
             KSMException.ResultCodes.FAILED_BUCKET_NOT_FOUND);
       }
-      // TODO throw exception if key exists, may change to support key
-      // overwrite in the future
-      //Check if key already exists.
-      if(metadataManager.get(keyKey) != null) {
-        LOG.error("key already exist: {}/{}/{} ", volumeName, bucketName,
-            keyName);
-        throw new KSMException("Key already exist",
-            KSMException.ResultCodes.FAILED_KEY_ALREADY_EXISTS);
-      }
 
+      // TODO: Garbage collect deleted blocks due to overwrite of a key.
+      // FIXME: BUG: Please see HDFS-11922.
+      // If user overwrites a key, then we are letting it pass without
+      // corresponding process.
+      // In reality we need to garbage collect those blocks by telling SCM to
+      // clean up those blocks when it can. Right now making this change
+      // allows us to pass tests that expect ozone can overwrite a key.
+
+
+      // When we talk to SCM make sure that we ask for at least a byte in the
+      // block. This way even if the call is for a zero length key, we back it
+      // with a actual SCM block.
+      // TODO : Review this decision later. We can get away with only a
+      // metadata entry in case of 0 length key.
       AllocatedBlock allocatedBlock =
-          scmBlockClient.allocateBlock(args.getDataSize());
+          scmBlockClient.allocateBlock(Math.max(args.getDataSize(), 1));
       KsmKeyInfo keyBlock = new KsmKeyInfo.Builder()
           .setVolumeName(args.getVolumeName())
           .setBucketName(args.getBucketName())
@@ -98,7 +103,7 @@ public class KeyManagerImpl implements KeyManager {
       LOG.debug("Key {} allocated in volume {} bucket {}",
           keyName, volumeName, bucketName);
       return keyBlock;
-    } catch (DBException ex) {
+    } catch (Exception ex) {
       LOG.error("Key allocation failed for volume:{} bucket:{} key:{}",
           volumeName, bucketName, keyName, ex);
       throw new KSMException(ex.getMessage(),

http://git-wip-us.apache.org/repos/asf/hadoop/blob/776d2d4d/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/MiniOzoneCluster.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/MiniOzoneCluster.java
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/MiniOzoneCluster.java
index bc1ade9..51b3ee1 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/MiniOzoneCluster.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/MiniOzoneCluster.java
@@ -423,6 +423,8 @@ public final class MiniOzoneCluster extends MiniDFSCluster
       if (!ozoneHandlerType.isPresent()) {
         throw new IllegalArgumentException(
             "The Ozone handler type must be specified.");
+      } else {
+        conf.set(OzoneConfigKeys.OZONE_HANDLER_TYPE_KEY, ozoneHandlerType.get());
       }
     }
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/776d2d4d/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/ksm/TestKeySpaceManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/ksm/TestKeySpaceManager.java
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/ksm/TestKeySpaceManager.java
index 0acebcb..8ef2b0e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/ksm/TestKeySpaceManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/ksm/TestKeySpaceManager.java
@@ -360,12 +360,14 @@ public class TestKeySpaceManager {
     try (OutputStream stream = storageHandler.newKeyWriter(keyArgs)) {
       stream.write(dataString.getBytes());
     }
-    // try to put the same keyArg, should raise KEY_ALREADY_EXISTS exception
-    exception.expect(IOException.class);
-    exception.expectMessage("KEY_ALREADY_EXISTS");
+
+    // We allow the key overwrite to be successful. Please note : Till HDFS-11922
+    // is fixed this causes a data block leak on the data node side. That is
+    // this overwrite only overwrites the keys on KSM. We need to garbage
+    // collect those blocks from datanode.
     KeyArgs keyArgs2 = new KeyArgs(volumeName, bucketName, keyName, userArgs);
     storageHandler.newKeyWriter(keyArgs2);
-    Assert.assertEquals(1 + numKeyAllocateFails,
+    Assert.assertEquals(numKeyAllocateFails,
         ksmMetrics.getNumKeyAllocateFails());
   }
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/776d2d4d/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/web/TestOzoneRestWithMiniCluster.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/web/TestOzoneRestWithMiniCluster.java
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/web/TestOzoneRestWithMiniCluster.java
index a3e5e2e..8e1c9ac 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/web/TestOzoneRestWithMiniCluster.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/web/TestOzoneRestWithMiniCluster.java
@@ -62,6 +62,7 @@ public class TestOzoneRestWithMiniCluster {
     conf = new OzoneConfiguration();
     cluster = new MiniOzoneCluster.Builder(conf).numDataNodes(1)
         .setHandlerType(OzoneConsts.OZONE_HANDLER_DISTRIBUTED).build();
+    cluster.waitOzoneReady();
     ozoneClient = cluster.createOzoneClient();
   }
 


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message