hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From yq...@apache.org
Subject hadoop git commit: HDFS-12170. Ozone: OzoneFileSystem: KSM should maintain key creation time and modification time. Contributed by Mukul Kumar Singh.
Date Thu, 27 Jul 2017 03:44:21 GMT
Repository: hadoop
Updated Branches:
  refs/heads/HDFS-7240 04b39066d -> f6ed18633


HDFS-12170. Ozone: OzoneFileSystem: KSM should maintain key creation time and modification
time. Contributed by Mukul Kumar Singh.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f6ed1863
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f6ed1863
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f6ed1863

Branch: refs/heads/HDFS-7240
Commit: f6ed186330fbc0d58374e1f167e73c0de870fdde
Parents: 04b3906
Author: Yiqun Lin <yqlin@apache.org>
Authored: Thu Jul 27 11:43:01 2017 +0800
Committer: Yiqun Lin <yqlin@apache.org>
Committed: Thu Jul 27 11:43:01 2017 +0800

----------------------------------------------------------------------
 .../main/java/org/apache/hadoop/util/Time.java  |  9 +++++
 .../apache/hadoop/ksm/helpers/KsmKeyInfo.java   | 35 ++++++++++++++++++--
 .../main/proto/KeySpaceManagerProtocol.proto    |  2 ++
 .../apache/hadoop/ozone/ksm/KeyManagerImpl.java |  4 +++
 .../hadoop/ozone/web/handlers/KeyHandler.java   |  2 +-
 .../hadoop/ozone/web/response/KeyInfo.java      | 19 +++++++++++
 .../web/storage/DistributedStorageHandler.java  |  4 ++-
 .../hadoop/ozone/ksm/TestKeySpaceManager.java   |  8 ++++-
 8 files changed, 77 insertions(+), 6 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/f6ed1863/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Time.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Time.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Time.java
index db5a567..e099c8f 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Time.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Time.java
@@ -17,6 +17,7 @@
  */
 package org.apache.hadoop.util;
 
+import java.text.ParseException;
 import java.text.SimpleDateFormat;
 
 import org.apache.hadoop.classification.InterfaceAudience;
@@ -82,4 +83,12 @@ public final class Time {
   public static String formatTime(long millis) {
     return DATE_FORMAT.get().format(millis);
   }
+
+  /**
+   * Convert time in human readable format to millisecond.
+   * @return time in milliseconds
+   */
+  public static long formatDate(String date) throws ParseException {
+    return DATE_FORMAT.get().parse(date).getTime();
+  }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/f6ed1863/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ksm/helpers/KsmKeyInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ksm/helpers/KsmKeyInfo.java
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ksm/helpers/KsmKeyInfo.java
index 76d22be..05748e8 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ksm/helpers/KsmKeyInfo.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ksm/helpers/KsmKeyInfo.java
@@ -35,10 +35,13 @@ public final class KsmKeyInfo {
   private final String blockID;
   private final long dataSize;
   private final boolean shouldCreateContainer;
+  private final long creationTime;
+  private final long modificationTime;
 
   private KsmKeyInfo(String volumeName, String bucketName, String keyName,
       long dataSize, String blockID, String containerName,
-      boolean shouldCreateContainer) {
+      boolean shouldCreateContainer, long creationTime,
+      long modificationTime) {
     this.volumeName = volumeName;
     this.bucketName = bucketName;
     this.keyName = keyName;
@@ -46,6 +49,8 @@ public final class KsmKeyInfo {
     this.blockID = blockID;
     this.dataSize = dataSize;
     this.shouldCreateContainer = shouldCreateContainer;
+    this.creationTime = creationTime;
+    this.modificationTime = modificationTime;
   }
 
   public String getVolumeName() {
@@ -76,6 +81,14 @@ public final class KsmKeyInfo {
     return shouldCreateContainer;
   }
 
+  public long getCreationTime() {
+    return creationTime;
+  }
+
+  public long getModificationTime() {
+    return modificationTime;
+  }
+
   /**
    * Builder of KsmKeyInfo.
    */
@@ -87,6 +100,8 @@ public final class KsmKeyInfo {
     private String blockID;
     private long dataSize;
     private boolean shouldCreateContainer;
+    private long creationTime;
+    private long modificationTime;
 
     public Builder setVolumeName(String volume) {
       this.volumeName = volume;
@@ -123,10 +138,20 @@ public final class KsmKeyInfo {
       return this;
     }
 
+    public Builder setCreationTime(long creationTime) {
+      this.creationTime = creationTime;
+      return this;
+    }
+
+    public Builder setModificationTime(long modificationTime) {
+      this.modificationTime = modificationTime;
+      return this;
+    }
+
     public KsmKeyInfo build() {
       return new KsmKeyInfo(
           volumeName, bucketName, keyName, dataSize, blockID, containerName,
-          shouldCreateContainer);
+          shouldCreateContainer, creationTime, modificationTime);
     }
   }
 
@@ -139,6 +164,8 @@ public final class KsmKeyInfo {
         .setBlockKey(blockID)
         .setContainerName(containerName)
         .setShouldCreateContainer(shouldCreateContainer)
+        .setCreationTime(creationTime)
+        .setModificationTime(modificationTime)
         .build();
   }
 
@@ -150,7 +177,9 @@ public final class KsmKeyInfo {
         keyInfo.getDataSize(),
         keyInfo.getBlockKey(),
         keyInfo.getContainerName(),
-        keyInfo.getShouldCreateContainer());
+        keyInfo.getShouldCreateContainer(),
+        keyInfo.getCreationTime(),
+        keyInfo.getModificationTime());
   }
 
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/f6ed1863/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/KeySpaceManagerProtocol.proto
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/KeySpaceManagerProtocol.proto
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/KeySpaceManagerProtocol.proto
index b8dbafc..f641898 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/KeySpaceManagerProtocol.proto
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/KeySpaceManagerProtocol.proto
@@ -233,6 +233,8 @@ message KeyInfo {
     required string blockKey = 5;
     required string containerName = 6;
     required bool shouldCreateContainer = 7;
+    required uint64 creationTime = 8;
+    required uint64 modificationTime = 9;
 }
 
 message LocateKeyRequest {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/f6ed1863/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/KeyManagerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/KeyManagerImpl.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/KeyManagerImpl.java
index aee3f7c..70e687b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/KeyManagerImpl.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/KeyManagerImpl.java
@@ -26,6 +26,7 @@ import org.apache.hadoop.ozone.protocol.proto.ScmBlockLocationProtocolProtos.Del
 import org.apache.hadoop.scm.container.common.helpers.AllocatedBlock;
 import org.apache.hadoop.scm.container.common.helpers.DeleteBlockResult;
 import org.apache.hadoop.scm.protocol.ScmBlockLocationProtocol;
+import org.apache.hadoop.util.Time;
 import org.iq80.leveldb.DBException;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -95,6 +96,7 @@ public class KeyManagerImpl implements KeyManager {
       // metadata entry in case of 0 length key.
       AllocatedBlock allocatedBlock =
           scmBlockClient.allocateBlock(Math.max(args.getDataSize(), 1));
+      long currentTime = Time.now();
       KsmKeyInfo keyBlock = new KsmKeyInfo.Builder()
           .setVolumeName(args.getVolumeName())
           .setBucketName(args.getBucketName())
@@ -103,6 +105,8 @@ public class KeyManagerImpl implements KeyManager {
           .setBlockID(allocatedBlock.getKey())
           .setContainerName(allocatedBlock.getPipeline().getContainerName())
           .setShouldCreateContainer(allocatedBlock.getCreateContainer())
+          .setCreationTime(currentTime)
+          .setModificationTime(currentTime)
           .build();
       metadataManager.put(keyKey, keyBlock.getProtobuf().toByteArray());
       LOG.debug("Key {} allocated in volume {} bucket {}",

http://git-wip-us.apache.org/repos/asf/hadoop/blob/f6ed1863/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/handlers/KeyHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/handlers/KeyHandler.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/handlers/KeyHandler.java
index 10528e0..0655a0f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/handlers/KeyHandler.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/handlers/KeyHandler.java
@@ -73,7 +73,7 @@ public class KeyHandler implements Keys {
        * @param input - The body as an Input Stream
        * @param request - Http request
        * @param headers - Parsed http Headers.
-       * @param info - UriInfo
+       * @param uriInfo - UriInfo
        *
        * @return Response
        *

http://git-wip-us.apache.org/repos/asf/hadoop/blob/f6ed1863/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/response/KeyInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/response/KeyInfo.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/response/KeyInfo.java
index e5cfd21..34885f6 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/response/KeyInfo.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/response/KeyInfo.java
@@ -69,6 +69,7 @@ public class KeyInfo implements Comparable<KeyInfo> {
   private long version;
   private String md5hash;
   private String createdOn;
+  private String modifiedOn;
   private long size;
   private String keyName;
 
@@ -84,6 +85,15 @@ public class KeyInfo implements Comparable<KeyInfo> {
   }
 
   /**
+   * When this key was modified.
+   *
+   * @return Date String
+   */
+  public String getModifiedOn() {
+    return modifiedOn;
+  }
+
+  /**
    * When this key was created.
    *
    * @param createdOn - Date String
@@ -93,6 +103,15 @@ public class KeyInfo implements Comparable<KeyInfo> {
   }
 
   /**
+   * When this key was modified.
+   *
+   * @param modifiedOn - Date String
+   */
+  public void setModifiedOn(String modifiedOn) {
+    this.modifiedOn = modifiedOn;
+  }
+
+  /**
    * Full path to where the actual data for this key is stored.
    *
    * @return String

http://git-wip-us.apache.org/repos/asf/hadoop/blob/f6ed1863/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/storage/DistributedStorageHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/storage/DistributedStorageHandler.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/storage/DistributedStorageHandler.java
index 57dcc32..1bf5b11 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/storage/DistributedStorageHandler.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/storage/DistributedStorageHandler.java
@@ -66,6 +66,7 @@ import org.apache.hadoop.scm.storage.ChunkInputStream;
 import org.apache.hadoop.scm.storage.ChunkOutputStream;
 import org.apache.hadoop.scm.storage.ContainerProtocolCalls;
 import org.apache.hadoop.util.StringUtils;
+import org.apache.hadoop.util.Time;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -482,7 +483,8 @@ public final class DistributedStorageHandler implements StorageHandler
{
     keyInfo.setVersion(0);
     keyInfo.setKeyName(ksmKeyInfo.getKeyName());
     keyInfo.setSize(ksmKeyInfo.getDataSize());
-
+    keyInfo.setCreatedOn(Time.formatTime(ksmKeyInfo.getCreationTime()));
+    keyInfo.setModifiedOn(Time.formatTime(ksmKeyInfo.getModificationTime()));
     return keyInfo;
   }
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/f6ed1863/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/ksm/TestKeySpaceManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/ksm/TestKeySpaceManager.java
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/ksm/TestKeySpaceManager.java
index 18a5114..5936ccd 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/ksm/TestKeySpaceManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/ksm/TestKeySpaceManager.java
@@ -44,6 +44,7 @@ import org.apache.hadoop.ozone.web.handlers.ListArgs;
 import org.apache.hadoop.ozone.web.response.ListBuckets;
 import org.apache.hadoop.ozone.web.response.ListKeys;
 import org.apache.hadoop.ozone.web.response.ListVolumes;
+import org.apache.hadoop.util.Time;
 import org.junit.AfterClass;
 import org.junit.Assert;
 import org.junit.BeforeClass;
@@ -55,6 +56,7 @@ import org.junit.rules.ExpectedException;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.OutputStream;
+import java.text.ParseException;
 import java.util.HashSet;
 import java.util.LinkedList;
 import java.util.Random;
@@ -941,11 +943,13 @@ public class TestKeySpaceManager {
    * @throws OzoneException
    */
   @Test
-  public void testGetKeyInfo() throws IOException, OzoneException {
+  public void testGetKeyInfo() throws IOException,
+                          OzoneException, ParseException {
     String userName = "user" + RandomStringUtils.randomNumeric(5);
     String adminName = "admin" + RandomStringUtils.randomNumeric(5);
     String volumeName = "volume" + RandomStringUtils.randomNumeric(5);
     String bucketName = "bucket" + RandomStringUtils.randomNumeric(5);
+    long currentTime = Time.monotonicNow();
 
     VolumeArgs createVolumeArgs = new VolumeArgs(volumeName, userArgs);
     createVolumeArgs.setUserName(userName);
@@ -966,6 +970,8 @@ public class TestKeySpaceManager {
     stream.close();
 
     KeyInfo keyInfo = storageHandler.getKeyInfo(keyArgs);
+    Assert.assertTrue(Time.formatDate(keyInfo.getCreatedOn()) >= currentTime);
+    Assert.assertTrue(Time.formatDate(keyInfo.getModifiedOn()) >= currentTime);
     Assert.assertEquals(keyName, keyInfo.getKeyName());
     Assert.assertEquals(4096, keyInfo.getSize());
   }


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message