hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From l..@apache.org
Subject [02/21] hadoop git commit: Change MetadataStore.get() to return a new CachedListStatus.
Date Fri, 08 Jul 2016 21:30:09 GMT
Change MetadataStore.get() to return a new CachedListStatus.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d7bf6c82
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d7bf6c82
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d7bf6c82

Branch: refs/heads/s3_create
Commit: d7bf6c82d16b4636e108ca16e36200b750a91f66
Parents: 5e1370e
Author: Aaron Fabbri <fabbri@cloudera.com>
Authored: Fri Jun 24 15:26:44 2016 -0700
Committer: Aaron Fabbri <fabbri@cloudera.com>
Committed: Fri Jun 24 15:26:44 2016 -0700

----------------------------------------------------------------------
 .../apache/hadoop/fs/s3a/CachedFileStatus.java  | 24 ++++++++++++++++++++
 .../hadoop/fs/s3a/DynamoDBMetadataStore.java    | 24 +++++++++++++++++++-
 .../org/apache/hadoop/fs/s3a/MetadataStore.java |  4 +++-
 3 files changed, 50 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/d7bf6c82/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/CachedFileStatus.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/CachedFileStatus.java
b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/CachedFileStatus.java
new file mode 100644
index 0000000..f6e7a85
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/CachedFileStatus.java
@@ -0,0 +1,24 @@
+package org.apache.hadoop.fs.s3a;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.fs.FileStatus;
+
+/**
+ * FileStatus plus extra state for caching.
+ */
+@InterfaceAudience.LimitedPrivate("HDFS")
+@InterfaceStability.Evolving
+public class CachedFileStatus {
+
+  // TODO this may change to inheritance, not sure yet.
+  protected FileStatus fileStatus;
+
+  public CachedFileStatus(FileStatus fileStatus) {
+    this.fileStatus = fileStatus;
+  }
+
+  public FileStatus getFileStatus() {
+    return fileStatus;
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d7bf6c82/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/DynamoDBMetadataStore.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/DynamoDBMetadataStore.java
b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/DynamoDBMetadataStore.java
index 16b6a9b..214952c 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/DynamoDBMetadataStore.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/DynamoDBMetadataStore.java
@@ -42,6 +42,7 @@ import org.apache.hadoop.fs.Path;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import javax.annotation.Nullable;
 import java.io.Closeable;
 import java.io.FileNotFoundException;
 import java.io.IOException;
@@ -125,6 +126,16 @@ public class DynamoDBMetadataStore implements MetadataStore, Closeable
{
           "DynamoDBMetadata(parent=%s, file=%s, dir=%b, etag=%s)",
           parent, fileName, 0, etag);
     }
+
+    private CachedFileStatus toCachedFileStatus() {
+
+      // XXX TODO this object needs to map cleanly to/from FileStatus
+      //  hacking broken values for now
+      FileStatus fs = new FileStatus(0, isdir, 0, 0, 0, getPath());
+      CachedFileStatus cfs = new CachedFileStatus(fs);
+      return cfs;
+
+    }
   }
 
   private final AmazonDynamoDBClient client;
@@ -157,6 +168,8 @@ public class DynamoDBMetadataStore implements MetadataStore, Closeable
{
 
     tableName = conf.get(S3_CONSISTENT_METADATA_TABLE_KEY,
         S3_CONSISTENT_METADATA_TABLE_DEFAULT);
+
+    // TODO consistent read setting here?
     mapper = new DynamoDBMapper(client,
         new DynamoDBMapperConfig(new DynamoDBMapperConfig.TableNameOverride(tableName)));
 
@@ -235,7 +248,16 @@ public class DynamoDBMetadataStore implements MetadataStore, Closeable
{
   }
 
   @Override
-  public DynamoDBFileMetadata get(Path f) throws IOException {
+  public CachedFileStatus get(Path f) throws IOException {
+    DynamoDBFileMetadata dfm = dynamoGet(f);
+    if (dfm == null)
+      return null;
+
+    return dfm.toCachedFileStatus();
+  }
+
+  @Nullable
+  private DynamoDBFileMetadata dynamoGet(Path f) throws IOException {
     final String hashKey = f.getParent().toString();
     final String rangeKey = f.getName();
     DynamoDBFileMetadata status = mapper.load(DynamoDBFileMetadata.class,

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d7bf6c82/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/MetadataStore.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/MetadataStore.java
b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/MetadataStore.java
index 29138c0..bfee0d2 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/MetadataStore.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/MetadataStore.java
@@ -22,6 +22,7 @@ import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.Path;
 
+import javax.annotation.Nullable;
 import java.io.FileNotFoundException;
 import java.io.IOException;
 
@@ -30,7 +31,8 @@ import java.io.IOException;
 public interface MetadataStore {
   boolean delete(Path f, boolean recursive) throws IOException;
 
-  DynamoDBMetadataStore.DynamoDBFileMetadata get(Path f) throws IOException;
+  @Nullable
+  CachedFileStatus get(Path f) throws IOException;
 
   FileStatus[] listStatus(Path f) throws FileNotFoundException, IOException;
 }


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message