hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From l..@apache.org
Subject [01/21] hadoop git commit: MetadataStore Interface intial commit
Date Fri, 08 Jul 2016 21:30:08 GMT
Repository: hadoop
Updated Branches:
  refs/heads/s3_create [created] a8bbfa93e


MetadataStore Interface intial commit


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5e1370e3
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5e1370e3
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5e1370e3

Branch: refs/heads/s3_create
Commit: 5e1370e38451033621aca1313dec59c604979aa4
Parents: 8c1f81d
Author: Lei Xu <lei@cloudera.com>
Authored: Mon Jun 20 16:52:43 2016 -0700
Committer: Lei Xu <lei@cloudera.com>
Committed: Mon Jun 20 16:52:43 2016 -0700

----------------------------------------------------------------------
 hadoop-project/pom.xml                          |   5 +
 hadoop-tools/hadoop-aws/pom.xml                 |   5 +
 .../hadoop/fs/s3a/DynamoDBMetadataStore.java    | 250 +++++++++++++++++++
 .../org/apache/hadoop/fs/s3a/MetadataStore.java |  36 +++
 .../org/apache/hadoop/fs/s3a/S3AFileSystem.java |   2 +-
 .../fs/s3a/TestDynamoDBMetadataStore.java       |  86 +++++++
 6 files changed, 383 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/5e1370e3/hadoop-project/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
index 7558b04..2432e51 100644
--- a/hadoop-project/pom.xml
+++ b/hadoop-project/pom.xml
@@ -695,6 +695,11 @@
         <version>${aws-java-sdk.version}</version>
       </dependency>
       <dependency>
+        <groupId>com.amazonaws</groupId>
+        <artifactId>aws-java-sdk-dynamodb</artifactId>
+        <version>1.10.6</version>
+      </dependency>
+      <dependency>
         <groupId>org.apache.mina</groupId>
         <artifactId>mina-core</artifactId>
         <version>2.0.0-M5</version>

http://git-wip-us.apache.org/repos/asf/hadoop/blob/5e1370e3/hadoop-tools/hadoop-aws/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/pom.xml b/hadoop-tools/hadoop-aws/pom.xml
index 7c25e60..c413210 100644
--- a/hadoop-tools/hadoop-aws/pom.xml
+++ b/hadoop-tools/hadoop-aws/pom.xml
@@ -236,6 +236,11 @@
       <scope>test</scope>
     </dependency>
     <dependency>
+      <groupId>com.amazonaws</groupId>
+      <artifactId>aws-java-sdk-dynamodb</artifactId>
+      <scope>compile</scope>
+    </dependency>
+    <dependency>
       <groupId>junit</groupId>
       <artifactId>junit</artifactId>
       <scope>test</scope>

http://git-wip-us.apache.org/repos/asf/hadoop/blob/5e1370e3/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/DynamoDBMetadataStore.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/DynamoDBMetadataStore.java
b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/DynamoDBMetadataStore.java
new file mode 100644
index 0000000..16b6a9b
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/DynamoDBMetadataStore.java
@@ -0,0 +1,250 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.s3a;
+
+import com.amazonaws.services.dynamodbv2.AmazonDynamoDBClient;
+import com.amazonaws.services.dynamodbv2.datamodeling.DynamoDBAttribute;
+import com.amazonaws.services.dynamodbv2.datamodeling.DynamoDBHashKey;
+import com.amazonaws.services.dynamodbv2.datamodeling.DynamoDBIgnore;
+import com.amazonaws.services.dynamodbv2.datamodeling.DynamoDBMapper;
+import com.amazonaws.services.dynamodbv2.datamodeling.DynamoDBMapperConfig;
+import com.amazonaws.services.dynamodbv2.datamodeling.DynamoDBRangeKey;
+import com.amazonaws.services.dynamodbv2.datamodeling.DynamoDBTable;
+import com.amazonaws.services.dynamodbv2.document.DynamoDB;
+import com.amazonaws.services.dynamodbv2.document.Table;
+import com.amazonaws.services.dynamodbv2.model.AttributeDefinition;
+import com.amazonaws.services.dynamodbv2.model.CreateTableRequest;
+import com.amazonaws.services.dynamodbv2.model.KeySchemaElement;
+import com.amazonaws.services.dynamodbv2.model.KeyType;
+import com.amazonaws.services.dynamodbv2.model.ProvisionedThroughput;
+import com.amazonaws.services.dynamodbv2.model.ResourceNotFoundException;
+import com.amazonaws.services.dynamodbv2.model.ScalarAttributeType;
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.Path;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.Closeable;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Objects;
+
+
+/**
+ * Metadata store on DynamoDB for consistent views of S3 data.
+ */
+public class DynamoDBMetadataStore implements MetadataStore, Closeable {
+  public static final Logger LOG =
+      LoggerFactory.getLogger(DynamoDBMetadataStore.class);
+
+  static final String S3_CONSISTENT_METADATA_TABLE_KEY =
+      "fs.s3a.consistent.metadata.table";
+  static final String S3_CONSISTENT_METADATA_TABLE_DEFAULT =
+      "s3c_metadata";
+
+
+  @DynamoDBTable(tableName = S3_CONSISTENT_METADATA_TABLE_DEFAULT)
+  public static class DynamoDBFileMetadata {
+    public static final String HASH_KEY_NAME = "parent";
+    public static final String RANGE_KEY_NAME = "fileName";
+
+    private String parent;
+    private String fileName;
+    private boolean isdir = false;
+    private String etag;
+
+    @DynamoDBHashKey(attributeName = HASH_KEY_NAME)
+    public String getParent() {
+      return parent;
+    }
+
+    public void setParent(String parent) {
+      this.parent = parent;
+    }
+
+    @DynamoDBRangeKey(attributeName = RANGE_KEY_NAME)
+    public String getFileName() {
+      return fileName;
+    }
+
+    public void setFileName(String fileName) {
+      this.fileName = fileName;
+    }
+
+    @DynamoDBAttribute(attributeName = "isdir")
+    public boolean getIsDir() {
+      return isdir;
+    }
+
+    public void setIsDir(boolean isDir) {
+      this.isdir = isDir;
+    }
+
+    @DynamoDBAttribute(attributeName = "etag")
+    public String getETag() {
+      return etag;
+    }
+
+    public void setETag(String etag) {
+      this.etag = etag;
+    }
+
+    @DynamoDBIgnore
+    public Path getPath() {
+      return new Path(parent, fileName);
+    }
+
+    @Override
+    public int hashCode() {
+      return Objects.hash(parent, fileName, etag);
+    }
+
+    @Override
+    public String toString() {
+      return String.format(
+          "DynamoDBMetadata(parent=%s, file=%s, dir=%b, etag=%s)",
+          parent, fileName, 0, etag);
+    }
+  }
+
+  private final AmazonDynamoDBClient client;
+  private final DynamoDB dynamoDB;
+  private final DynamoDBMapper mapper;
+  private final String tableName;
+
+  @VisibleForTesting
+  static boolean hasTable(AmazonDynamoDBClient client, String tableName) {
+    try {
+      client.describeTable(tableName);
+      return true;
+    } catch (ResourceNotFoundException e) {
+      return false;
+    }
+  }
+
+  @VisibleForTesting
+  static void deleteTable(AmazonDynamoDBClient client, String tableName) {
+    if (hasTable(client, tableName)) {
+      client.deleteTable(tableName);
+    }
+  }
+
+  DynamoDBMetadataStore(Configuration conf) {
+    client = new AmazonDynamoDBClient();
+    // TODO: for local dev only, delete later.
+    client.setEndpoint("http://localhost:8000");
+    dynamoDB = new DynamoDB(client);
+
+    tableName = conf.get(S3_CONSISTENT_METADATA_TABLE_KEY,
+        S3_CONSISTENT_METADATA_TABLE_DEFAULT);
+    mapper = new DynamoDBMapper(client,
+        new DynamoDBMapperConfig(new DynamoDBMapperConfig.TableNameOverride(tableName)));
+
+    if (hasTable(tableName)) {
+      Table table = dynamoDB.getTable(tableName);
+      table.delete();
+    }
+    createTable(tableName);
+  }
+
+
+  @Override
+  public void close() throws IOException {
+    dynamoDB.shutdown();
+    client.shutdown();
+  }
+
+  private boolean hasTable(String tableName) {
+    try {
+      client.describeTable(tableName).getTable();
+      return true;
+    } catch (ResourceNotFoundException e) {
+      return false;
+    }
+  }
+
+  private void createTable(String tableName) {
+    LOG.info("Creating metadata store in DynamoDB table: {}", tableName);
+
+    List<AttributeDefinition> attributes = new ArrayList<>();
+    attributes.add(new AttributeDefinition()
+      .withAttributeName(DynamoDBFileMetadata.HASH_KEY_NAME)
+      .withAttributeType(ScalarAttributeType.S));
+    attributes.add(new AttributeDefinition(
+        DynamoDBFileMetadata.RANGE_KEY_NAME,
+        ScalarAttributeType.S));
+
+    List<KeySchemaElement> keySchema = new ArrayList<>();
+    keySchema.add(new KeySchemaElement(
+        DynamoDBFileMetadata.HASH_KEY_NAME, KeyType.HASH));
+    keySchema.add(new KeySchemaElement(
+        DynamoDBFileMetadata.RANGE_KEY_NAME, KeyType.RANGE));
+    System.out.println(keySchema);
+
+    ProvisionedThroughput pt = new ProvisionedThroughput(100L, 100L);
+    CreateTableRequest request = new CreateTableRequest()
+        .withTableName(tableName)
+        .withKeySchema(keySchema)
+        .withAttributeDefinitions(attributes)
+        .withProvisionedThroughput(pt);
+
+    Table table = dynamoDB.createTable(request);
+    try {
+      table.waitForActive();
+    } catch (InterruptedException e) {
+      e.printStackTrace();
+    }
+    LOG.info("Created DynamoDB table: {}", table.getDescription().getTableName());
+  }
+
+  public void create(Path path, boolean isDir, String etag) {
+    Preconditions.checkArgument(path != null);
+    Preconditions.checkArgument(etag != null);
+    DynamoDBFileMetadata metadata = new DynamoDBFileMetadata();
+    metadata.setParent(path.getParent().toString());
+    metadata.setFileName(path.getName());
+    metadata.setIsDir(isDir);
+    metadata.setETag(etag);
+
+    mapper.save(metadata);
+  }
+
+  @Override
+  public boolean delete(Path f, boolean recursive) throws IOException {
+    return false;
+  }
+
+  @Override
+  public DynamoDBFileMetadata get(Path f) throws IOException {
+    final String hashKey = f.getParent().toString();
+    final String rangeKey = f.getName();
+    DynamoDBFileMetadata status = mapper.load(DynamoDBFileMetadata.class,
+        hashKey, rangeKey);
+    return status;
+  }
+
+  @Override
+  public FileStatus[] listStatus(Path f) throws FileNotFoundException, IOException {
+    return new FileStatus[0];
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/5e1370e3/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/MetadataStore.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/MetadataStore.java
b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/MetadataStore.java
new file mode 100644
index 0000000..29138c0
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/MetadataStore.java
@@ -0,0 +1,36 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.s3a;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.Path;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+
+@InterfaceAudience.LimitedPrivate("HDFS")
+@InterfaceStability.Unstable
+public interface MetadataStore {
+  boolean delete(Path f, boolean recursive) throws IOException;
+
+  DynamoDBMetadataStore.DynamoDBFileMetadata get(Path f) throws IOException;
+
+  FileStatus[] listStatus(Path f) throws FileNotFoundException, IOException;
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/5e1370e3/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
index 63bfb4f..d9f01b0 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
@@ -500,7 +500,7 @@ public class S3AFileSystem extends FileSystem {
     return path.toUri().getPath().substring(1);
   }
 
-  private Path keyToPath(String key) {
+  static Path keyToPath(String key) {
     return new Path("/" + key);
   }
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/5e1370e3/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestDynamoDBMetadataStore.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestDynamoDBMetadataStore.java
b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestDynamoDBMetadataStore.java
new file mode 100644
index 0000000..9307337
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestDynamoDBMetadataStore.java
@@ -0,0 +1,86 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a;
+
+import com.amazonaws.services.dynamodbv2.AmazonDynamoDBClient;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.s3a.DynamoDBMetadataStore.DynamoDBFileMetadata;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.IOException;
+
+import static org.apache.hadoop.fs.s3a.DynamoDBMetadataStore.S3_CONSISTENT_METADATA_TABLE_KEY;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+
+public class TestDynamoDBMetadataStore {
+  private static final String TABLE_NAME = "S3AMetadataStoreTest";
+
+  private Configuration conf = new Configuration();
+  private final AmazonDynamoDBClient client = new AmazonDynamoDBClient();
+  DynamoDBMetadataStore store;
+
+  @Before
+  public void setUp() {
+    client.setEndpoint("http://localhost:8000");
+    conf.set(S3_CONSISTENT_METADATA_TABLE_KEY,
+        TABLE_NAME);
+  }
+
+  @After
+  public void tearDown() throws IOException {
+    if (store != null) {
+      store.close();
+    }
+    DynamoDBMetadataStore.deleteTable(client, TABLE_NAME);
+  }
+
+  @Test
+  public void testCreateTable() {
+    store = new DynamoDBMetadataStore(conf);
+    assertTrue(DynamoDBMetadataStore.hasTable(client, TABLE_NAME));
+  }
+
+  @Test
+  public void testCreateFiles() throws IOException {
+    store = new DynamoDBMetadataStore(conf);
+    Path f = new Path("/foo");
+    store.create(f, false, "E12345");
+
+    DynamoDBFileMetadata status = store.get(f);
+    assertEquals("/", status.getParent());
+    assertEquals("foo", status.getFileName());
+    assertEquals("E12345", status.getETag());
+     assertFalse(status.getIsDir());
+
+    Path d = new Path("/dir");
+    store.create(d, true, "E2222");
+    DynamoDBFileMetadata metadata = store.get(d);
+    assertEquals("/", metadata.getParent());
+    assertEquals("dir", metadata.getFileName());
+    assertEquals(d, metadata.getPath());
+    assertTrue(metadata.getIsDir());
+    assertEquals("E2222", metadata.getETag());
+  }
+}
\ No newline at end of file


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message