hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From e...@apache.org
Subject hadoop git commit: HDDS-798. Storage-class is showing incorrectly. Contributed by Bharat Viswanadham.
Date Tue, 06 Nov 2018 17:29:45 GMT
Repository: hadoop
Updated Branches:
  refs/heads/trunk 349168c4b -> 3d5cc1138


HDDS-798. Storage-class is showing incorrectly. Contributed by Bharat Viswanadham.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3d5cc113
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3d5cc113
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3d5cc113

Branch: refs/heads/trunk
Commit: 3d5cc1138a07d3c0964d18250966a188ab1cf0b1
Parents: 349168c
Author: Márton Elek <elek@apache.org>
Authored: Tue Nov 6 18:19:40 2018 +0100
Committer: Márton Elek <elek@apache.org>
Committed: Tue Nov 6 18:28:54 2018 +0100

----------------------------------------------------------------------
 .../apache/hadoop/ozone/client/OzoneKey.java    | 16 ++++++++++++++-
 .../hadoop/ozone/client/OzoneKeyDetails.java    |  9 ++++++---
 .../hadoop/ozone/client/rest/RestClient.java    |  6 ++++--
 .../hadoop/ozone/client/rpc/RpcClient.java      | 14 ++++++-------
 .../ozone/client/rest/response/KeyInfo.java     | 20 +++++++++++++++++++
 .../hadoop/ozone/web/response/KeyInfo.java      | 21 ++++++++++++++++++++
 .../main/smoketest/s3/objectmultidelete.robot   |  4 ++++
 .../ozone/s3/endpoint/BucketEndpoint.java       |  9 ++++++++-
 .../hadoop/ozone/client/OzoneBucketStub.java    |  2 +-
 9 files changed, 85 insertions(+), 16 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/3d5cc113/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneKey.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneKey.java
b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneKey.java
index 7c93146..bc8135f 100644
--- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneKey.java
+++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneKey.java
@@ -18,6 +18,8 @@
 
 package org.apache.hadoop.ozone.client;
 
+import org.apache.hadoop.hdds.client.ReplicationType;
+
 /**
  * A class that encapsulates OzoneKey.
  */
@@ -48,19 +50,22 @@ public class OzoneKey {
    */
   private long modificationTime;
 
+  private ReplicationType replicationType;
+
   /**
    * Constructs OzoneKey from OmKeyInfo.
    *
    */
   public OzoneKey(String volumeName, String bucketName,
                   String keyName, long size, long creationTime,
-                  long modificationTime) {
+                  long modificationTime, ReplicationType type) {
     this.volumeName = volumeName;
     this.bucketName = bucketName;
     this.name = keyName;
     this.dataSize = size;
     this.creationTime = creationTime;
     this.modificationTime = modificationTime;
+    this.replicationType = type;
   }
 
   /**
@@ -116,4 +121,13 @@ public class OzoneKey {
   public long getModificationTime() {
     return modificationTime;
   }
+
+  /**
+   * Returns the replication type of the key.
+   *
+   * @return replicationType
+   */
+  public ReplicationType getReplicationType() {
+    return replicationType;
+  }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/3d5cc113/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneKeyDetails.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneKeyDetails.java
b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneKeyDetails.java
index e7709dd..7c22f1d 100644
--- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneKeyDetails.java
+++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneKeyDetails.java
@@ -18,6 +18,8 @@
 
 package org.apache.hadoop.ozone.client;
 
+import org.apache.hadoop.hdds.client.ReplicationType;
+
 import java.util.List;
 
 /**
@@ -34,10 +36,11 @@ public class OzoneKeyDetails extends OzoneKey {
    * Constructs OzoneKeyDetails from OmKeyInfo.
    */
   public OzoneKeyDetails(String volumeName, String bucketName, String keyName,
-                  long size, long creationTime, long modificationTime,
-                         List<OzoneKeyLocation> ozoneKeyLocations) {
+                         long size, long creationTime, long modificationTime,
+                         List<OzoneKeyLocation> ozoneKeyLocations,
+                         ReplicationType type) {
     super(volumeName, bucketName, keyName, size, creationTime,
-        modificationTime);
+        modificationTime, type);
     this.ozoneKeyLocations = ozoneKeyLocations;
   }
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/3d5cc113/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rest/RestClient.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rest/RestClient.java
b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rest/RestClient.java
index 4293c08..8e106a5 100644
--- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rest/RestClient.java
+++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rest/RestClient.java
@@ -777,7 +777,8 @@ public class RestClient implements ClientProtocol {
           LOG.warn("Parse exception in getting creation time for volume", e);
         }
         return new OzoneKey(volumeName, bucketName, keyInfo.getKeyName(),
-            keyInfo.getSize(), creationTime, modificationTime);
+            keyInfo.getSize(), creationTime, modificationTime,
+            ReplicationType.valueOf(keyInfo.getType().toString()));
       }).collect(Collectors.toList());
     } catch (URISyntaxException e) {
       throw new IOException(e);
@@ -812,7 +813,8 @@ public class RestClient implements ClientProtocol {
           keyInfo.getSize(),
           HddsClientUtils.formatDateTime(keyInfo.getCreatedOn()),
           HddsClientUtils.formatDateTime(keyInfo.getModifiedOn()),
-          ozoneKeyLocations);
+          ozoneKeyLocations, ReplicationType.valueOf(
+              keyInfo.getType().toString()));
       EntityUtils.consume(response);
       return key;
     } catch (URISyntaxException | ParseException e) {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/3d5cc113/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
index 5efc086..cbb2e49 100644
--- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
+++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
@@ -537,7 +537,8 @@ public class RpcClient implements ClientProtocol {
         key.getKeyName(),
         key.getDataSize(),
         key.getCreationTime(),
-        key.getModificationTime()))
+        key.getModificationTime(),
+        ReplicationType.valueOf(key.getType().toString())))
         .collect(Collectors.toList());
   }
 
@@ -559,13 +560,10 @@ public class RpcClient implements ClientProtocol {
     keyInfo.getLatestVersionLocations().getBlocksLatestVersionOnly().forEach(
         (a) -> ozoneKeyLocations.add(new OzoneKeyLocation(a.getContainerID(),
             a.getLocalID(), a.getLength(), a.getOffset())));
-    return new OzoneKeyDetails(keyInfo.getVolumeName(),
-                        keyInfo.getBucketName(),
-                        keyInfo.getKeyName(),
-                        keyInfo.getDataSize(),
-                        keyInfo.getCreationTime(),
-                        keyInfo.getModificationTime(),
-                        ozoneKeyLocations);
+    return new OzoneKeyDetails(keyInfo.getVolumeName(), keyInfo.getBucketName(),
+        keyInfo.getKeyName(), keyInfo.getDataSize(), keyInfo.getCreationTime(),
+        keyInfo.getModificationTime(), ozoneKeyLocations, ReplicationType
+        .valueOf(keyInfo.getType().toString()));
   }
 
   @Override

http://git-wip-us.apache.org/repos/asf/hadoop/blob/3d5cc113/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/client/rest/response/KeyInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/client/rest/response/KeyInfo.java
b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/client/rest/response/KeyInfo.java
index 61c2abb..299826f 100644
--- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/client/rest/response/KeyInfo.java
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/client/rest/response/KeyInfo.java
@@ -24,6 +24,7 @@ import com.fasterxml.jackson.databind.ObjectMapper;
 import com.fasterxml.jackson.databind.ObjectReader;
 import org.apache.commons.lang3.builder.EqualsBuilder;
 import org.apache.commons.lang3.builder.HashCodeBuilder;
+import org.apache.hadoop.hdds.client.ReplicationType;
 
 /**
  * KeyInfo class is used used for parsing json response
@@ -40,6 +41,25 @@ public class KeyInfo implements Comparable<KeyInfo> {
   private String modifiedOn;
   private long size;
   private String keyName;
+  private ReplicationType type;
+
+  /**
+   * Return replication type of the key.
+   *
+   * @return replication type
+   */
+  public ReplicationType getType() {
+    return type;
+  }
+
+  /**
+   * Set replication type of the key.
+   *
+   * @param replicationType
+   */
+  public void setType(ReplicationType replicationType) {
+    this.type = replicationType;
+  }
 
   /**
    * When this key was created.

http://git-wip-us.apache.org/repos/asf/hadoop/blob/3d5cc113/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/web/response/KeyInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/web/response/KeyInfo.java
b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/web/response/KeyInfo.java
index ba47bee..f480194 100644
--- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/web/response/KeyInfo.java
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/web/response/KeyInfo.java
@@ -21,6 +21,7 @@ import java.io.IOException;
 
 import org.apache.commons.lang3.builder.EqualsBuilder;
 import org.apache.commons.lang3.builder.HashCodeBuilder;
+import org.apache.hadoop.hdds.client.ReplicationType;
 import org.apache.hadoop.ozone.web.utils.JsonUtils;
 
 import com.fasterxml.jackson.annotation.JsonAutoDetect;
@@ -75,6 +76,26 @@ public class KeyInfo implements Comparable<KeyInfo> {
 
   private String dataFileName;
 
+  private ReplicationType type;
+
+  /**
+   * Return replication type of the key.
+   *
+   * @return replication type
+   */
+  public ReplicationType getType() {
+    return type;
+  }
+
+  /**
+   * Set replication type of the key.
+   *
+   * @param replicationType
+   */
+  public void setType(ReplicationType replicationType) {
+    this.type = replicationType;
+  }
+
   /**
    * When this key was created.
    *

http://git-wip-us.apache.org/repos/asf/hadoop/blob/3d5cc113/hadoop-ozone/dist/src/main/smoketest/s3/objectmultidelete.robot
----------------------------------------------------------------------
diff --git a/hadoop-ozone/dist/src/main/smoketest/s3/objectmultidelete.robot b/hadoop-ozone/dist/src/main/smoketest/s3/objectmultidelete.robot
index e8b7bd3..dae3a8a 100644
--- a/hadoop-ozone/dist/src/main/smoketest/s3/objectmultidelete.robot
+++ b/hadoop-ozone/dist/src/main/smoketest/s3/objectmultidelete.robot
@@ -36,9 +36,13 @@ Delete file with multi delete
                         Should contain             ${result}         multidelete/f1
                         Should contain             ${result}         multidelete/f2
                         Should contain             ${result}         multidelete/f3
+                        Should contain             ${result}         REDUCED_REDUNDANCY
+                        Should not contain         ${result}         STANDARD
     ${result} =         Execute AWSS3APICli        delete-objects --bucket ${BUCKET} --delete
'Objects=[{Key=multidelete/f1},{Key=multidelete/f2},{Key=multidelete/f4}]'
                         Should not contain         ${result}         Error
     ${result} =         Execute AWSS3ApiCli        list-objects --bucket ${BUCKET} --prefix
multidelete/
                         Should not contain         ${result}         multidelete/f1
                         Should not contain         ${result}         multidelete/f2
                         Should contain             ${result}         multidelete/f3
+                        Should contain             ${result}         REDUCED_REDUNDANCY
+                        Should not contain         ${result}         STANDARD

http://git-wip-us.apache.org/repos/asf/hadoop/blob/3d5cc113/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java
b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java
index bfbbb33..1f0764b 100644
--- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java
+++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java
@@ -37,6 +37,7 @@ import java.io.InputStream;
 import java.time.Instant;
 import java.util.Iterator;
 
+import org.apache.hadoop.hdds.client.ReplicationType;
 import org.apache.hadoop.ozone.client.OzoneBucket;
 import org.apache.hadoop.ozone.client.OzoneKey;
 import org.apache.hadoop.ozone.s3.commontypes.KeyMetadata;
@@ -48,6 +49,7 @@ import org.apache.hadoop.ozone.s3.exception.S3ErrorTable;
 
 import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
 import org.apache.commons.lang3.StringUtils;
+import org.apache.hadoop.ozone.s3.util.S3StorageType;
 import org.apache.hadoop.ozone.s3.util.S3utils;
 import org.apache.http.HttpStatus;
 import org.slf4j.Logger;
@@ -296,7 +298,12 @@ public class BucketEndpoint extends EndpointBase {
     keyMetadata.setKey(next.getName());
     keyMetadata.setSize(next.getDataSize());
     keyMetadata.setETag("" + next.getModificationTime());
-    keyMetadata.setStorageClass("STANDARD");
+    if (next.getReplicationType().toString().equals(ReplicationType
+        .STAND_ALONE.toString())) {
+      keyMetadata.setStorageClass(S3StorageType.REDUCED_REDUNDANCY.toString());
+    } else {
+      keyMetadata.setStorageClass(S3StorageType.STANDARD.toString());
+    }
     keyMetadata.setLastModified(Instant.ofEpochMilli(
         next.getModificationTime()));
     response.addKey(keyMetadata);

http://git-wip-us.apache.org/repos/asf/hadoop/blob/3d5cc113/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/OzoneBucketStub.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/OzoneBucketStub.java
b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/OzoneBucketStub.java
index 918e9d0..040d6df 100644
--- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/OzoneBucketStub.java
+++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/OzoneBucketStub.java
@@ -93,7 +93,7 @@ public class OzoneBucketStub extends OzoneBucket {
                 size,
                 System.currentTimeMillis(),
                 System.currentTimeMillis(),
-                new ArrayList<>()
+                new ArrayList<>(), type
             ));
             super.close();
           }


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message