hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From msi...@apache.org
Subject hadoop git commit: HDDS-20. Ozone: Add support for rename key within a bucket for rpc client. Contributed by Lokesh Jain.
Date Wed, 09 May 2018 13:37:01 GMT
Repository: hadoop
Updated Branches:
  refs/heads/trunk cd68c7cc6 -> 208b97e96


HDDS-20. Ozone: Add support for rename key within a bucket for rpc client. Contributed by
Lokesh Jain.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/208b97e9
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/208b97e9
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/208b97e9

Branch: refs/heads/trunk
Commit: 208b97e969cda59827890dfd26f53e3b7f20d7de
Parents: cd68c7c
Author: Mukul Kumar Singh <msingh@apache.org>
Authored: Wed May 9 19:06:07 2018 +0530
Committer: Mukul Kumar Singh <msingh@apache.org>
Committed: Wed May 9 19:06:07 2018 +0530

----------------------------------------------------------------------
 .../apache/hadoop/ozone/client/OzoneBucket.java |   8 ++
 .../ozone/client/protocol/ClientProtocol.java   |  10 ++
 .../hadoop/ozone/client/rest/RestClient.java    |   6 +
 .../hadoop/ozone/client/rpc/RpcClient.java      |  15 +++
 .../hadoop/ozone/ksm/helpers/KsmKeyInfo.java    |   6 +-
 .../ksm/protocol/KeySpaceManagerProtocol.java   |   9 +-
 ...ceManagerProtocolClientSideTranslatorPB.java |  27 +++++
 .../main/proto/KeySpaceManagerProtocol.proto    |  20 +++-
 .../ozone/client/rpc/TestOzoneRpcClient.java    |  44 +++++++
 .../hadoop/ozone/ksm/TestKeySpaceManager.java   | 115 +++++++++++++++++++
 .../ozone/web/interfaces/StorageHandler.java    |   9 ++
 .../web/localstorage/LocalStorageHandler.java   |   6 +
 .../web/storage/DistributedStorageHandler.java  |  11 ++
 .../org/apache/hadoop/ozone/ksm/KSMMetrics.java |  22 ++++
 .../org/apache/hadoop/ozone/ksm/KeyManager.java |  10 ++
 .../apache/hadoop/ozone/ksm/KeyManagerImpl.java |  65 +++++++++++
 .../hadoop/ozone/ksm/KeySpaceManager.java       |  11 ++
 .../ozone/ksm/exceptions/KSMException.java      |   2 +
 ...ceManagerProtocolServerSideTranslatorPB.java |  26 +++++
 .../apache/hadoop/fs/ozone/OzoneFileSystem.java |  29 ++---
 20 files changed, 427 insertions(+), 24 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/208b97e9/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java
b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java
index ba6286b..1712979 100644
--- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java
+++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java
@@ -305,6 +305,14 @@ public class OzoneBucket {
     proxy.deleteKey(volumeName, name, key);
   }
 
+  public void renameKey(String fromKeyName, String toKeyName)
+      throws IOException {
+    Preconditions.checkNotNull(proxy, "Client proxy is not set.");
+    Preconditions.checkNotNull(fromKeyName);
+    Preconditions.checkNotNull(toKeyName);
+    proxy.renameKey(volumeName, name, fromKeyName, toKeyName);
+  }
+
   /**
    * An Iterator to iterate over {@link OzoneKey} list.
    */

http://git-wip-us.apache.org/repos/asf/hadoop/blob/208b97e9/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java
b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java
index 816c185..94cc257 100644
--- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java
+++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java
@@ -285,6 +285,16 @@ public interface ClientProtocol {
   void deleteKey(String volumeName, String bucketName, String keyName)
       throws IOException;
 
+  /**
+   * Renames an existing key within a bucket.
+   * @param volumeName Name of the Volume
+   * @param bucketName Name of the Bucket
+   * @param fromKeyName Name of the Key to be renamed
+   * @param toKeyName New name to be used for the Key
+   * @throws IOException
+   */
+  void renameKey(String volumeName, String bucketName, String fromKeyName,
+      String toKeyName) throws IOException;
 
   /**
    * Returns list of Keys in {Volume/Bucket} that matches the keyPrefix,

http://git-wip-us.apache.org/repos/asf/hadoop/blob/208b97e9/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rest/RestClient.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rest/RestClient.java
b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rest/RestClient.java
index b8b4610..e9885d1 100644
--- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rest/RestClient.java
+++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rest/RestClient.java
@@ -676,6 +676,12 @@ public class RestClient implements ClientProtocol {
   }
 
   @Override
+  public void renameKey(String volumeName, String bucketName,
+      String fromKeyName, String toKeyName) throws IOException {
+    throw new UnsupportedOperationException("Not yet implemented.");
+  }
+
+  @Override
   public List<OzoneKey> listKeys(String volumeName, String bucketName,
                                  String keyPrefix, String prevKey,
                                  int maxListResult)

http://git-wip-us.apache.org/repos/asf/hadoop/blob/208b97e9/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
index 2464fe3..ffe93dd 100644
--- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
+++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
@@ -520,6 +520,21 @@ public class RpcClient implements ClientProtocol {
   }
 
   @Override
+  public void renameKey(String volumeName, String bucketName,
+      String fromKeyName, String toKeyName) throws IOException {
+    Preconditions.checkNotNull(volumeName);
+    Preconditions.checkNotNull(bucketName);
+    Preconditions.checkNotNull(fromKeyName);
+    Preconditions.checkNotNull(toKeyName);
+    KsmKeyArgs keyArgs = new KsmKeyArgs.Builder()
+        .setVolumeName(volumeName)
+        .setBucketName(bucketName)
+        .setKeyName(fromKeyName)
+        .build();
+    keySpaceManagerClient.renameKey(keyArgs, toKeyName);
+  }
+
+  @Override
   public List<OzoneKey> listKeys(String volumeName, String bucketName,
                                  String keyPrefix, String prevKey,
                                  int maxListResult)

http://git-wip-us.apache.org/repos/asf/hadoop/blob/208b97e9/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/ksm/helpers/KsmKeyInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/ksm/helpers/KsmKeyInfo.java
b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/ksm/helpers/KsmKeyInfo.java
index 41d523c..678ce92 100644
--- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/ksm/helpers/KsmKeyInfo.java
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/ksm/helpers/KsmKeyInfo.java
@@ -34,7 +34,7 @@ public final class KsmKeyInfo {
   private final String volumeName;
   private final String bucketName;
   // name of key client specified
-  private final String keyName;
+  private String keyName;
   private long dataSize;
   private List<KsmKeyLocationInfoGroup> keyLocationVersions;
   private final long creationTime;
@@ -75,6 +75,10 @@ public final class KsmKeyInfo {
     return keyName;
   }
 
+  public void setKeyName(String keyName) {
+    this.keyName = keyName;
+  }
+
   public long getDataSize() {
     return dataSize;
   }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/208b97e9/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/ksm/protocol/KeySpaceManagerProtocol.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/ksm/protocol/KeySpaceManagerProtocol.java
b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/ksm/protocol/KeySpaceManagerProtocol.java
index 5da5a27..54862d3 100644
--- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/ksm/protocol/KeySpaceManagerProtocol.java
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/ksm/protocol/KeySpaceManagerProtocol.java
@@ -166,12 +166,19 @@ public interface KeySpaceManagerProtocol {
    * Look up for the container of an existing key.
    *
    * @param args the args of the key.
-   * @return KsmKeyInfo isntacne that client uses to talk to container.
+   * @return KsmKeyInfo instance that client uses to talk to container.
    * @throws IOException
    */
   KsmKeyInfo lookupKey(KsmKeyArgs args) throws IOException;
 
   /**
+   * Rename an existing key within a bucket
+   * @param args the args of the key.
+   * @param toKeyName New name to be used for the Key
+   */
+  void renameKey(KsmKeyArgs args, String toKeyName) throws IOException;
+
+  /**
    * Deletes an existing key.
    *
    * @param args the args of the key.

http://git-wip-us.apache.org/repos/asf/hadoop/blob/208b97e9/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/ksm/protocolPB/KeySpaceManagerProtocolClientSideTranslatorPB.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/ksm/protocolPB/KeySpaceManagerProtocolClientSideTranslatorPB.java
b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/ksm/protocolPB/KeySpaceManagerProtocolClientSideTranslatorPB.java
index cc215cf..854c688 100644
--- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/ksm/protocolPB/KeySpaceManagerProtocolClientSideTranslatorPB.java
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/ksm/protocolPB/KeySpaceManagerProtocolClientSideTranslatorPB.java
@@ -70,6 +70,10 @@ import org.apache.hadoop.ozone.protocol.proto
 import org.apache.hadoop.ozone.protocol.proto
     .KeySpaceManagerProtocolProtos.LocateKeyResponse;
 import org.apache.hadoop.ozone.protocol.proto
+    .KeySpaceManagerProtocolProtos.RenameKeyRequest;
+import org.apache.hadoop.ozone.protocol.proto
+    .KeySpaceManagerProtocolProtos.RenameKeyResponse;
+import org.apache.hadoop.ozone.protocol.proto
     .KeySpaceManagerProtocolProtos.KeyArgs;
 import org.apache.hadoop.ozone.protocol.proto
     .KeySpaceManagerProtocolProtos.SetVolumePropertyRequest;
@@ -623,6 +627,29 @@ public final class KeySpaceManagerProtocolClientSideTranslatorPB
     return KsmKeyInfo.getFromProtobuf(resp.getKeyInfo());
   }
 
+  @Override
+  public void renameKey(KsmKeyArgs args, String toKeyName) throws IOException {
+    RenameKeyRequest.Builder req = RenameKeyRequest.newBuilder();
+    KeyArgs keyArgs = KeyArgs.newBuilder()
+        .setVolumeName(args.getVolumeName())
+        .setBucketName(args.getBucketName())
+        .setKeyName(args.getKeyName())
+        .setDataSize(args.getDataSize()).build();
+    req.setKeyArgs(keyArgs);
+    req.setToKeyName(toKeyName);
+
+    final RenameKeyResponse resp;
+    try {
+      resp = rpcProxy.renameKey(NULL_RPC_CONTROLLER, req.build());
+    } catch (ServiceException e) {
+      throw ProtobufHelper.getRemoteException(e);
+    }
+    if (resp.getStatus() != Status.OK) {
+      throw new IOException("Rename key failed, error:" +
+          resp.getStatus());
+    }
+  }
+
   /**
    * Deletes an existing key.
    *

http://git-wip-us.apache.org/repos/asf/hadoop/blob/208b97e9/hadoop-ozone/common/src/main/proto/KeySpaceManagerProtocol.proto
----------------------------------------------------------------------
diff --git a/hadoop-ozone/common/src/main/proto/KeySpaceManagerProtocol.proto b/hadoop-ozone/common/src/main/proto/KeySpaceManagerProtocol.proto
index 405c5b0..7b70330 100644
--- a/hadoop-ozone/common/src/main/proto/KeySpaceManagerProtocol.proto
+++ b/hadoop-ozone/common/src/main/proto/KeySpaceManagerProtocol.proto
@@ -50,8 +50,9 @@ enum Status {
     BUCKET_ALREADY_EXISTS = 10;
     KEY_ALREADY_EXISTS = 11;
     KEY_NOT_FOUND = 12;
-    ACCESS_DENIED = 13;
-    INTERNAL_ERROR = 14;
+    INVALID_KEY_NAME = 13;
+    ACCESS_DENIED = 14;
+    INTERNAL_ERROR = 15;
 }
 
 
@@ -276,6 +277,15 @@ message SetBucketPropertyResponse {
     required Status status = 1;
 }
 
+message RenameKeyRequest{
+    required KeyArgs keyArgs = 1;
+    required string toKeyName = 2;
+}
+
+message RenameKeyResponse{
+    required Status status = 1;
+}
+
 message DeleteBucketRequest {
     required string volumeName = 1;
     required string bucketName = 2;
@@ -413,6 +423,12 @@ service KeySpaceManagerService {
         returns(LocateKeyResponse);
 
     /**
+       Rename an existing key within a bucket.
+    */
+    rpc renameKey(RenameKeyRequest)
+        returns(RenameKeyResponse);
+
+    /**
        Delete an existing key.
     */
     rpc deleteKey(LocateKeyRequest)

http://git-wip-us.apache.org/repos/asf/hadoop/blob/208b97e9/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClient.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClient.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClient.java
index c9a25e5..e3823b3 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClient.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClient.java
@@ -528,6 +528,50 @@ public class TestOzoneRpcClient {
   }
 
   @Test
+  public void testRenameKey()
+      throws IOException, OzoneException {
+    String volumeName = UUID.randomUUID().toString();
+    String bucketName = UUID.randomUUID().toString();
+    String fromKeyName = UUID.randomUUID().toString();
+    String value = "sample value";
+    store.createVolume(volumeName);
+    OzoneVolume volume = store.getVolume(volumeName);
+    volume.createBucket(bucketName);
+    OzoneBucket bucket = volume.getBucket(bucketName);
+    OzoneOutputStream out = bucket.createKey(fromKeyName,
+        value.getBytes().length, ReplicationType.STAND_ALONE,
+        ReplicationFactor.ONE);
+    out.write(value.getBytes());
+    out.close();
+    OzoneKey key = bucket.getKey(fromKeyName);
+    Assert.assertEquals(fromKeyName, key.getName());
+
+    // Rename to empty string should fail.
+    IOException ioe = null;
+    String toKeyName = "";
+    try {
+      bucket.renameKey(fromKeyName, toKeyName);
+    } catch (IOException e) {
+      ioe = e;
+    }
+    Assert.assertTrue(ioe.getMessage().contains("Rename key failed, error"));
+
+    toKeyName = UUID.randomUUID().toString();
+    bucket.renameKey(fromKeyName, toKeyName);
+
+    // Lookup for old key should fail.
+    try {
+      bucket.getKey(fromKeyName);
+    } catch (IOException e) {
+      ioe = e;
+    }
+    Assert.assertTrue(ioe.getMessage().contains("Lookup key failed, error"));
+
+    key = bucket.getKey(toKeyName);
+    Assert.assertEquals(toKeyName, key.getName());
+  }
+
+  @Test
   public void testListVolume() throws IOException, OzoneException {
     String volBase = "vol-" + RandomStringUtils.randomNumeric(3);
     //Create 10 volume vol-<random>-a-0-<random> to vol-<random>-a-9-<random>

http://git-wip-us.apache.org/repos/asf/hadoop/blob/208b97e9/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/ksm/TestKeySpaceManager.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/ksm/TestKeySpaceManager.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/ksm/TestKeySpaceManager.java
index 9a116b7..9c0a488 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/ksm/TestKeySpaceManager.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/ksm/TestKeySpaceManager.java
@@ -652,6 +652,121 @@ public class TestKeySpaceManager {
         ksmMetrics.getNumKeyDeletesFails());
   }
 
+  /**
+   * Test rename key for ksm.
+   *
+   * @throws IOException
+   * @throws OzoneException
+   */
+  @Test
+  public void testRenameKey() throws IOException, OzoneException {
+    String userName = "user" + RandomStringUtils.randomNumeric(5);
+    String adminName = "admin" + RandomStringUtils.randomNumeric(5);
+    String volumeName = "volume" + RandomStringUtils.randomNumeric(5);
+    String bucketName = "bucket" + RandomStringUtils.randomNumeric(5);
+    String keyName = "key" + RandomStringUtils.randomNumeric(5);
+    long numKeyRenames = ksmMetrics.getNumKeyRenames();
+    long numKeyRenameFails = ksmMetrics.getNumKeyRenameFails();
+    int testRenameFails = 0;
+    int testRenames = 0;
+    IOException ioe = null;
+
+    VolumeArgs createVolumeArgs = new VolumeArgs(volumeName, userArgs);
+    createVolumeArgs.setUserName(userName);
+    createVolumeArgs.setAdminName(adminName);
+    storageHandler.createVolume(createVolumeArgs);
+
+    BucketArgs bucketArgs = new BucketArgs(bucketName, createVolumeArgs);
+    storageHandler.createBucket(bucketArgs);
+
+    KeyArgs keyArgs = new KeyArgs(keyName, bucketArgs);
+    keyArgs.setSize(100);
+    String toKeyName = "key" + RandomStringUtils.randomNumeric(5);
+
+    // Rename from non-existent key should fail
+    try {
+      testRenames++;
+      storageHandler.renameKey(keyArgs, toKeyName);
+    } catch (IOException e) {
+      testRenameFails++;
+      ioe = e;
+    }
+    Assert.assertTrue(ioe.getMessage().contains("Rename key failed, error"));
+
+    // Write the contents of the key to be renamed
+    String dataString = RandomStringUtils.randomAscii(100);
+    try (OutputStream stream = storageHandler.newKeyWriter(keyArgs)) {
+      stream.write(dataString.getBytes());
+    }
+
+    // Rename the key
+    toKeyName = "key" + RandomStringUtils.randomNumeric(5);
+    testRenames++;
+    storageHandler.renameKey(keyArgs, toKeyName);
+    Assert.assertEquals(numKeyRenames + testRenames,
+        ksmMetrics.getNumKeyRenames());
+    Assert.assertEquals(numKeyRenameFails + testRenameFails,
+        ksmMetrics.getNumKeyRenameFails());
+
+    // Try to get the key, should fail as it has been renamed
+    try {
+      storageHandler.newKeyReader(keyArgs);
+    } catch (IOException e) {
+      ioe = e;
+    }
+    Assert.assertTrue(ioe.getMessage().contains("KEY_NOT_FOUND"));
+
+    // Verify the contents of the renamed key
+    keyArgs = new KeyArgs(toKeyName, bucketArgs);
+    InputStream in = storageHandler.newKeyReader(keyArgs);
+    byte[] b = new byte[dataString.getBytes().length];
+    in.read(b);
+    Assert.assertEquals(new String(b), dataString);
+
+    // Rewrite the renamed key. Rename to key which already exists should fail.
+    keyArgs = new KeyArgs(keyName, bucketArgs);
+    keyArgs.setSize(100);
+    dataString = RandomStringUtils.randomAscii(100);
+    try (OutputStream stream = storageHandler.newKeyWriter(keyArgs)) {
+      stream.write(dataString.getBytes());
+      stream.close();
+      testRenames++;
+      storageHandler.renameKey(keyArgs, toKeyName);
+    } catch (IOException e) {
+      testRenameFails++;
+      ioe = e;
+    }
+    Assert.assertTrue(ioe.getMessage().contains("Rename key failed, error"));
+
+    // Rename to empty string should fail
+    toKeyName = "";
+    try {
+      testRenames++;
+      storageHandler.renameKey(keyArgs, toKeyName);
+    } catch (IOException e) {
+      testRenameFails++;
+      ioe = e;
+    }
+    Assert.assertTrue(ioe.getMessage().contains("Rename key failed, error"));
+
+    // Rename from empty string should fail
+    keyArgs = new KeyArgs("", bucketArgs);
+    toKeyName = "key" + RandomStringUtils.randomNumeric(5);
+    try {
+      testRenames++;
+      storageHandler.renameKey(keyArgs, toKeyName);
+    } catch (IOException e) {
+      testRenameFails++;
+      ioe = e;
+    }
+    Assert.assertTrue(ioe.getMessage().contains("Rename key failed, error"));
+
+    Assert.assertEquals(numKeyRenames + testRenames,
+        ksmMetrics.getNumKeyRenames());
+    Assert.assertEquals(numKeyRenameFails + testRenameFails,
+        ksmMetrics.getNumKeyRenameFails());
+  }
+
   @Test(timeout = 60000)
   public void testListBuckets() throws IOException, OzoneException {
     ListBuckets result = null;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/208b97e9/hadoop-ozone/objectstore-service/src/main/java/org/apache/hadoop/ozone/web/interfaces/StorageHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/objectstore-service/src/main/java/org/apache/hadoop/ozone/web/interfaces/StorageHandler.java
b/hadoop-ozone/objectstore-service/src/main/java/org/apache/hadoop/ozone/web/interfaces/StorageHandler.java
index 6336c90..338ff63 100644
--- a/hadoop-ozone/objectstore-service/src/main/java/org/apache/hadoop/ozone/web/interfaces/StorageHandler.java
+++ b/hadoop-ozone/objectstore-service/src/main/java/org/apache/hadoop/ozone/web/interfaces/StorageHandler.java
@@ -264,6 +264,15 @@ public interface StorageHandler extends Closeable{
    */
   void deleteKey(KeyArgs args) throws IOException, OzoneException;
 
+  /**
+   * Renames an existing key within a bucket.
+   *
+   * @param args KeyArgs
+   * @param toKeyName New name to be used for the key
+   * @throws OzoneException
+   */
+  void renameKey(KeyArgs args, String toKeyName)
+      throws IOException, OzoneException;
 
   /**
    * Returns a list of Key.

http://git-wip-us.apache.org/repos/asf/hadoop/blob/208b97e9/hadoop-ozone/objectstore-service/src/main/java/org/apache/hadoop/ozone/web/localstorage/LocalStorageHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/objectstore-service/src/main/java/org/apache/hadoop/ozone/web/localstorage/LocalStorageHandler.java
b/hadoop-ozone/objectstore-service/src/main/java/org/apache/hadoop/ozone/web/localstorage/LocalStorageHandler.java
index 9747eac..89158cb 100644
--- a/hadoop-ozone/objectstore-service/src/main/java/org/apache/hadoop/ozone/web/localstorage/LocalStorageHandler.java
+++ b/hadoop-ozone/objectstore-service/src/main/java/org/apache/hadoop/ozone/web/localstorage/LocalStorageHandler.java
@@ -339,6 +339,12 @@ public class LocalStorageHandler implements StorageHandler {
     oz.deleteKey(args);
   }
 
+  @Override
+  public void renameKey(KeyArgs args, String toKeyName)
+      throws IOException, OzoneException {
+    throw new UnsupportedOperationException("Not yet implemented");
+  }
+
   /**
    * Returns a list of Key.
    *

http://git-wip-us.apache.org/repos/asf/hadoop/blob/208b97e9/hadoop-ozone/objectstore-service/src/main/java/org/apache/hadoop/ozone/web/storage/DistributedStorageHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/objectstore-service/src/main/java/org/apache/hadoop/ozone/web/storage/DistributedStorageHandler.java
b/hadoop-ozone/objectstore-service/src/main/java/org/apache/hadoop/ozone/web/storage/DistributedStorageHandler.java
index 0f4a856..45270ea 100644
--- a/hadoop-ozone/objectstore-service/src/main/java/org/apache/hadoop/ozone/web/storage/DistributedStorageHandler.java
+++ b/hadoop-ozone/objectstore-service/src/main/java/org/apache/hadoop/ozone/web/storage/DistributedStorageHandler.java
@@ -457,6 +457,17 @@ public final class DistributedStorageHandler implements StorageHandler
{
   }
 
   @Override
+  public void renameKey(KeyArgs args, String toKeyName)
+      throws IOException, OzoneException {
+    KsmKeyArgs keyArgs = new KsmKeyArgs.Builder()
+        .setVolumeName(args.getVolumeName())
+        .setBucketName(args.getBucketName())
+        .setKeyName(args.getKeyName())
+        .build();
+    keySpaceManagerClient.renameKey(keyArgs, toKeyName);
+  }
+
+  @Override
   public KeyInfo getKeyInfo(KeyArgs args) throws IOException, OzoneException {
     KsmKeyArgs keyArgs = new KsmKeyArgs.Builder()
         .setVolumeName(args.getVolumeName())

http://git-wip-us.apache.org/repos/asf/hadoop/blob/208b97e9/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/ksm/KSMMetrics.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/ksm/KSMMetrics.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/ksm/KSMMetrics.java
index bd29012..8ee67c3 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/ksm/KSMMetrics.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/ksm/KSMMetrics.java
@@ -52,6 +52,7 @@ public class KSMMetrics {
   private @Metric MutableCounterLong numBucketDeletes;
   private @Metric MutableCounterLong numKeyAllocate;
   private @Metric MutableCounterLong numKeyLookup;
+  private @Metric MutableCounterLong numKeyRenames;
   private @Metric MutableCounterLong numKeyDeletes;
   private @Metric MutableCounterLong numBucketLists;
   private @Metric MutableCounterLong numKeyLists;
@@ -72,6 +73,7 @@ public class KSMMetrics {
   private @Metric MutableCounterLong numBucketDeleteFails;
   private @Metric MutableCounterLong numKeyAllocateFails;
   private @Metric MutableCounterLong numKeyLookupFails;
+  private @Metric MutableCounterLong numKeyRenameFails;
   private @Metric MutableCounterLong numKeyDeleteFails;
   private @Metric MutableCounterLong numBucketListFails;
   private @Metric MutableCounterLong numKeyListFails;
@@ -208,6 +210,16 @@ public class KSMMetrics {
     numKeyLookupFails.incr();
   }
 
+  public void incNumKeyRenames() {
+    numKeyOps.incr();
+    numKeyRenames.incr();
+  }
+
+  public void incNumKeyRenameFails() {
+    numKeyOps.incr();
+    numKeyRenameFails.incr();
+  }
+
   public void incNumKeyDeleteFails() {
     numKeyDeleteFails.incr();
   }
@@ -381,6 +393,16 @@ public class KSMMetrics {
   }
 
   @VisibleForTesting
+  public long getNumKeyRenames() {
+    return numKeyRenames.value();
+  }
+
+  @VisibleForTesting
+  public long getNumKeyRenameFails() {
+    return numKeyRenameFails.value();
+  }
+
+  @VisibleForTesting
   public long getNumKeyDeletes() {
     return numKeyDeletes.value();
   }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/208b97e9/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/ksm/KeyManager.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/ksm/KeyManager.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/ksm/KeyManager.java
index e71ce5f..5ec1db8 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/ksm/KeyManager.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/ksm/KeyManager.java
@@ -86,6 +86,16 @@ public interface KeyManager {
   KsmKeyInfo lookupKey(KsmKeyArgs args) throws IOException;
 
   /**
+   * Renames an existing key within a bucket.
+   *
+   * @param args the args of the key provided by client.
+   * @param toKeyName New name to be used for the key
+   * @throws IOException if specified key doesn't exist or
+   * some other I/O errors while renaming the key.
+   */
+  void renameKey(KsmKeyArgs args, String toKeyName) throws IOException;
+
+  /**
    * Deletes an object by an object key. The key will be immediately removed
    * from KSM namespace and become invisible to clients. The object data
    * will be removed in async manner that might retain for some time.

http://git-wip-us.apache.org/repos/asf/hadoop/blob/208b97e9/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/ksm/KeyManagerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/ksm/KeyManagerImpl.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/ksm/KeyManagerImpl.java
index 8ee7d25..6409a73 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/ksm/KeyManagerImpl.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/ksm/KeyManagerImpl.java
@@ -396,6 +396,71 @@ public class KeyManagerImpl implements KeyManager {
   }
 
   @Override
+  public void renameKey(KsmKeyArgs args, String toKeyName) throws IOException {
+    Preconditions.checkNotNull(args);
+    Preconditions.checkNotNull(toKeyName);
+    String volumeName = args.getVolumeName();
+    String bucketName = args.getBucketName();
+    String fromKeyName = args.getKeyName();
+    if (toKeyName.length() == 0 || fromKeyName.length() == 0) {
+      LOG.error("Rename key failed for volume:{} bucket:{} fromKey:{} toKey:{}.",
+          volumeName, bucketName, fromKeyName, toKeyName);
+      throw new KSMException("Key name is empty",
+          ResultCodes.FAILED_INVALID_KEY_NAME);
+    }
+
+    metadataManager.writeLock().lock();
+    try {
+      // fromKeyName should exist
+      byte[] fromKey = metadataManager.getDBKeyBytes(
+          volumeName, bucketName, fromKeyName);
+      byte[] fromKeyValue = metadataManager.get(fromKey);
+      if (fromKeyValue == null) {
+        // TODO: Add support for renaming open key
+        LOG.error(
+            "Rename key failed for volume:{} bucket:{} fromKey:{} toKey:{}. "
+                + "Key: {} not found.", volumeName, bucketName, fromKeyName,
+            toKeyName, fromKeyName);
+        throw new KSMException("Key not found",
+            KSMException.ResultCodes.FAILED_KEY_NOT_FOUND);
+      }
+
+      // toKeyName should not exist
+      byte[] toKey =
+          metadataManager.getDBKeyBytes(volumeName, bucketName, toKeyName);
+      byte[] toKeyValue = metadataManager.get(toKey);
+      if (toKeyValue != null) {
+        LOG.error(
+            "Rename key failed for volume:{} bucket:{} fromKey:{} toKey:{}. "
+                + "Key: {} already exists.", volumeName, bucketName,
+            fromKeyName, toKeyName, toKeyName);
+        throw new KSMException("Key not found",
+            KSMException.ResultCodes.FAILED_KEY_ALREADY_EXISTS);
+      }
+
+      if (fromKeyName.equals(toKeyName)) {
+        return;
+      }
+
+      KsmKeyInfo newKeyInfo =
+          KsmKeyInfo.getFromProtobuf(KeyInfo.parseFrom(fromKeyValue));
+      newKeyInfo.setKeyName(toKeyName);
+      newKeyInfo.updateModifcationTime();
+      BatchOperation batch = new BatchOperation();
+      batch.delete(fromKey);
+      batch.put(toKey, newKeyInfo.getProtobuf().toByteArray());
+      metadataManager.writeBatch(batch);
+    } catch (DBException ex) {
+      LOG.error("Rename key failed for volume:{} bucket:{} fromKey:{} toKey:{}.",
+          volumeName, bucketName, fromKeyName, toKeyName, ex);
+      throw new KSMException(ex.getMessage(),
+          ResultCodes.FAILED_KEY_RENAME);
+    } finally {
+      metadataManager.writeLock().unlock();
+    }
+  }
+
+  @Override
   public void deleteKey(KsmKeyArgs args) throws IOException {
     Preconditions.checkNotNull(args);
     metadataManager.writeLock().lock();

http://git-wip-us.apache.org/repos/asf/hadoop/blob/208b97e9/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/ksm/KeySpaceManager.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/ksm/KeySpaceManager.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/ksm/KeySpaceManager.java
index 120eb06..d0f0c9b 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/ksm/KeySpaceManager.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/ksm/KeySpaceManager.java
@@ -744,6 +744,17 @@ public final class KeySpaceManager extends ServiceRuntimeInfoImpl
     }
   }
 
+  @Override
+  public void renameKey(KsmKeyArgs args, String toKeyName) throws IOException {
+    try {
+      metrics.incNumKeyRenames();
+      keyManager.renameKey(args, toKeyName);
+    } catch (IOException e) {
+      metrics.incNumKeyRenameFails();
+      throw e;
+    }
+  }
+
   /**
    * Deletes an existing key.
    *

http://git-wip-us.apache.org/repos/asf/hadoop/blob/208b97e9/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/ksm/exceptions/KSMException.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/ksm/exceptions/KSMException.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/ksm/exceptions/KSMException.java
index e2f3580..b902eab 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/ksm/exceptions/KSMException.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/ksm/exceptions/KSMException.java
@@ -108,6 +108,8 @@ public class KSMException extends IOException {
     FAILED_KEY_NOT_FOUND,
     FAILED_KEY_ALLOCATION,
     FAILED_KEY_DELETION,
+    FAILED_KEY_RENAME,
+    FAILED_INVALID_KEY_NAME,
     FAILED_METADATA_ERROR,
     FAILED_INTERNAL_ERROR,
     KSM_NOT_INITIALIZED,

http://git-wip-us.apache.org/repos/asf/hadoop/blob/208b97e9/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/KeySpaceManagerProtocolServerSideTranslatorPB.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/KeySpaceManagerProtocolServerSideTranslatorPB.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/KeySpaceManagerProtocolServerSideTranslatorPB.java
index 02a4120..536f95a 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/KeySpaceManagerProtocolServerSideTranslatorPB.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/KeySpaceManagerProtocolServerSideTranslatorPB.java
@@ -63,6 +63,10 @@ import org.apache.hadoop.ozone.protocol.proto
 import org.apache.hadoop.ozone.protocol.proto
     .KeySpaceManagerProtocolProtos.LocateKeyResponse;
 import org.apache.hadoop.ozone.protocol.proto
+    .KeySpaceManagerProtocolProtos.RenameKeyRequest;
+import org.apache.hadoop.ozone.protocol.proto
+    .KeySpaceManagerProtocolProtos.RenameKeyResponse;
+import org.apache.hadoop.ozone.protocol.proto
     .KeySpaceManagerProtocolProtos.KeyArgs;
 import org.apache.hadoop.ozone.protocol.proto
     .KeySpaceManagerProtocolProtos.SetVolumePropertyRequest;
@@ -152,6 +156,8 @@ public class KeySpaceManagerProtocolServerSideTranslatorPB implements
         return Status.KEY_ALREADY_EXISTS;
       case FAILED_KEY_NOT_FOUND:
         return Status.KEY_NOT_FOUND;
+      case FAILED_INVALID_KEY_NAME:
+        return Status.INVALID_KEY_NAME;
       default:
         return Status.INTERNAL_ERROR;
       }
@@ -373,6 +379,26 @@ public class KeySpaceManagerProtocolServerSideTranslatorPB implements
   }
 
   @Override
+  public RenameKeyResponse renameKey(
+      RpcController controller, RenameKeyRequest request)
+      throws ServiceException {
+    RenameKeyResponse.Builder resp = RenameKeyResponse.newBuilder();
+    try {
+      KeyArgs keyArgs = request.getKeyArgs();
+      KsmKeyArgs ksmKeyArgs = new KsmKeyArgs.Builder()
+          .setVolumeName(keyArgs.getVolumeName())
+          .setBucketName(keyArgs.getBucketName())
+          .setKeyName(keyArgs.getKeyName())
+          .build();
+      impl.renameKey(ksmKeyArgs, request.getToKeyName());
+      resp.setStatus(Status.OK);
+    } catch (IOException e){
+      resp.setStatus(exceptionToResponseStatus(e));
+    }
+    return resp.build();
+  }
+
+  @Override
   public SetBucketPropertyResponse setBucketProperty(
       RpcController controller, SetBucketPropertyRequest request)
       throws ServiceException {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/208b97e9/hadoop-tools/hadoop-ozone/src/main/java/org/apache/hadoop/fs/ozone/OzoneFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-ozone/src/main/java/org/apache/hadoop/fs/ozone/OzoneFileSystem.java
b/hadoop-tools/hadoop-ozone/src/main/java/org/apache/hadoop/fs/ozone/OzoneFileSystem.java
index c2a2fe2..ef0d3ab 100644
--- a/hadoop-tools/hadoop-ozone/src/main/java/org/apache/hadoop/fs/ozone/OzoneFileSystem.java
+++ b/hadoop-tools/hadoop-ozone/src/main/java/org/apache/hadoop/fs/ozone/OzoneFileSystem.java
@@ -257,30 +257,17 @@ public class OzoneFileSystem extends FileSystem {
 
     boolean processKey(String key) throws IOException {
       String newKeyName = dstKey.concat(key.substring(srcKey.length()));
-      rename(key, newKeyName);
+      bucket.renameKey(key, newKeyName);
       return true;
     }
-
-    // TODO: currently rename work by copying the streams, with changes in KSM,
-    // this operation can be improved by renaming the keys in KSM directly.
-    private void rename(String src, String dst) throws IOException {
-      try (OzoneInputStream inputStream = bucket.readKey(src);
-          OzoneOutputStream outputStream = bucket
-              .createKey(dst, 0, replicationType, replicationFactor)) {
-        IOUtils.copyBytes(inputStream, outputStream, getConf());
-      }
-    }
   }
 
   /**
    * Check whether the source and destination path are valid and then perform
-   * rename by copying the data from source path to destination path.
+   * rename from source path to destination path.
    *
-   * The rename operation is performed by copying data from source key
-   * to destination key. This is done by reading the source key data into a
-   * temporary file and then writing this temporary file to destination key.
-   * The temporary file is deleted after the rename operation.
-   * TODO: Optimize the operation by renaming keys in KSM.
+   * The rename operation is performed by renaming the keys with src as prefix.
+   * For such keys the prefix is changed from src to dst.
    *
    * @param src source path for rename
    * @param dst destination path for rename
@@ -290,8 +277,11 @@ public class OzoneFileSystem extends FileSystem {
    */
   @Override
   public boolean rename(Path src, Path dst) throws IOException {
-    LOG.trace("rename() from:{} to:{}", src, dst);
+    if (src.equals(dst)) {
+      return true;
+    }
 
+    LOG.trace("rename() from:{} to:{}", src, dst);
     if (src.isRoot()) {
       // Cannot rename root of file system
       LOG.trace("Cannot rename the root of a filesystem");
@@ -367,8 +357,7 @@ public class OzoneFileSystem extends FileSystem {
       }
     }
     RenameIterator iterator = new RenameIterator(src, dst);
-    iterator.iterate();
-    return src.equals(dst) || delete(src, true);
+    return iterator.iterate();
   }
 
   private class DeleteIterator extends OzoneListingIterator {


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message