hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ji...@apache.org
Subject [50/52] [abbrv] HADOOP-10400. Incorporate new S3A FileSystem implementation. Contributed by Jordan Mendelson and Dave Wang.
Date Mon, 15 Sep 2014 17:49:59 GMT
http://git-wip-us.apache.org/repos/asf/hadoop/blob/24d920b8/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/TestS3AContractSeek.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/TestS3AContractSeek.java
b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/TestS3AContractSeek.java
new file mode 100644
index 0000000..d677ec4
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/TestS3AContractSeek.java
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ *  or more contributor license agreements.  See the NOTICE file
+ *  distributed with this work for additional information
+ *  regarding copyright ownership.  The ASF licenses this file
+ *  to you under the Apache License, Version 2.0 (the
+ *  "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+package org.apache.hadoop.fs.contract.s3a;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.contract.AbstractContractSeekTest;
+import org.apache.hadoop.fs.contract.AbstractFSContract;
+
+public class TestS3AContractSeek extends AbstractContractSeekTest {
+
+  @Override
+  protected AbstractFSContract createContract(Configuration conf) {
+    return new S3AContract(conf);
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/24d920b8/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3AFileSystemContractBaseTest.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3AFileSystemContractBaseTest.java
b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3AFileSystemContractBaseTest.java
new file mode 100644
index 0000000..8455233
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3AFileSystemContractBaseTest.java
@@ -0,0 +1,327 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a;
+
+import static org.junit.Assume.*;
+
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FileSystemContractBaseTest;
+import org.apache.hadoop.fs.Path;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.net.URI;
+import java.util.UUID;
+
+/**
+ *  Tests a live S3 system. If you keys and bucket aren't specified, all tests 
+ *  are marked as passed 
+ *  
+ *  This uses BlockJUnit4ClassRunner because FileSystemContractBaseTest from 
+ *  TestCase which uses the old Junit3 runner that doesn't ignore assumptions 
+ *  properly making it impossible to skip the tests if we don't have a valid
+ *  bucket.
+ **/
+public class S3AFileSystemContractBaseTest extends FileSystemContractBaseTest {
+  private static final int TEST_BUFFER_SIZE = 128;
+  private static final int MODULUS = 128;
+
+  protected static final Logger LOG = LoggerFactory.getLogger(S3AFileSystemContractBaseTest.class);
+
+  @Override
+  public void setUp() throws Exception {
+    Configuration conf = new Configuration();
+
+    URI testURI = URI.create(conf.get("test.fs.s3a.name"));
+    
+    boolean liveTest = testURI != null && !testURI.equals("s3a:///");
+    
+    // This doesn't work with our JUnit 3 style test cases, so instead we'll 
+    // make this whole class not run by default
+    assumeTrue(liveTest);
+    
+    fs = new S3AFileSystem();
+    fs.initialize(testURI, conf);
+    super.setUp();
+  }
+
+  @Override
+  protected void tearDown() throws Exception {
+	if (fs != null) {
+	  fs.delete(path("/tests3a"), true);
+	}
+	super.tearDown();
+  }
+
+  @Test(timeout = 10000)
+  public void testMkdirs() throws IOException {
+    // No trailing slash
+    assertTrue(fs.mkdirs(path("/tests3a/a")));
+    assertTrue(fs.exists(path("/tests3a/a")));
+
+    // With trailing slash
+    assertTrue(fs.mkdirs(path("/tests3a/b/")));
+    assertTrue(fs.exists(path("/tests3a/b/")));
+
+    // Two levels deep
+    assertTrue(fs.mkdirs(path("/tests3a/c/a/")));
+    assertTrue(fs.exists(path("/tests3a/c/a/")));
+
+    // Mismatched slashes
+    assertTrue(fs.exists(path("/tests3a/c/a")));
+  }
+
+
+  @Test(timeout=20000)
+  public void testDelete() throws IOException {
+    // Test deleting an empty directory
+    assertTrue(fs.mkdirs(path("/tests3a/d")));
+    assertTrue(fs.delete(path("/tests3a/d"), true));
+    assertFalse(fs.exists(path("/tests3a/d")));
+
+    // Test deleting a deep empty directory
+    assertTrue(fs.mkdirs(path("/tests3a/e/f/g/h")));
+    assertTrue(fs.delete(path("/tests3a/e/f/g"), true));
+    assertFalse(fs.exists(path("/tests3a/e/f/g/h")));
+    assertFalse(fs.exists(path("/tests3a/e/f/g")));
+    assertTrue(fs.exists(path("/tests3a/e/f")));
+
+    // Test delete of just a file
+    writeFile(path("/tests3a/f/f/file"), 1000);
+    assertTrue(fs.exists(path("/tests3a/f/f/file")));
+    assertTrue(fs.delete(path("/tests3a/f/f/file"), false));
+    assertFalse(fs.exists(path("/tests3a/f/f/file")));
+
+
+    // Test delete of a path with files in various directories
+    writeFile(path("/tests3a/g/h/i/file"), 1000);
+    assertTrue(fs.exists(path("/tests3a/g/h/i/file")));
+    writeFile(path("/tests3a/g/h/j/file"), 1000);
+    assertTrue(fs.exists(path("/tests3a/g/h/j/file")));
+    try {
+      assertFalse(fs.delete(path("/tests3a/g/h"), false));
+      fail("Expected delete to fail with recursion turned off");
+    } catch (IOException e) {}
+    assertTrue(fs.exists(path("/tests3a/g/h/j/file")));
+    assertTrue(fs.delete(path("/tests3a/g/h"), true));
+    assertFalse(fs.exists(path("/tests3a/g/h/j")));
+  }
+
+
+  @Test(timeout = 3600000)
+  public void testOpenCreate() throws IOException {
+    try {
+      createAndReadFileTest(1024);
+    } catch (IOException e) {
+      fail(e.getMessage());
+    }
+
+    try {
+      createAndReadFileTest(5 * 1024 * 1024);
+    } catch (IOException e) {
+      fail(e.getMessage());
+    }
+
+    try {
+      createAndReadFileTest(20 * 1024 * 1024);
+    } catch (IOException e) {
+      fail(e.getMessage());
+    }
+
+    /*
+    Enable to test the multipart upload
+    try {
+      createAndReadFileTest((long)6 * 1024 * 1024 * 1024);
+    } catch (IOException e) {
+      fail(e.getMessage());
+    }
+    */
+  }
+
+  @Test(timeout = 1200000)
+  public void testRenameFile() throws IOException {
+    Path srcPath = path("/tests3a/a/srcfile");
+
+    final OutputStream outputStream = fs.create(srcPath, false);
+    generateTestData(outputStream, 11 * 1024 * 1024);
+    outputStream.close();
+
+    assertTrue(fs.exists(srcPath));
+
+    Path dstPath = path("/tests3a/b/dstfile");
+
+    assertFalse(fs.rename(srcPath, dstPath));
+    assertTrue(fs.mkdirs(dstPath.getParent()));
+    assertTrue(fs.rename(srcPath, dstPath));
+    assertTrue(fs.exists(dstPath));
+    assertFalse(fs.exists(srcPath));
+    assertTrue(fs.exists(srcPath.getParent()));
+  }
+
+
+  @Test(timeout = 10000)
+  public void testRenameDirectory() throws IOException {
+    Path srcPath = path("/tests3a/a");
+
+    assertTrue(fs.mkdirs(srcPath));
+    writeFile(new Path(srcPath, "b/testfile"), 1024);
+
+    Path nonEmptyPath = path("/tests3a/nonempty");
+    writeFile(new Path(nonEmptyPath, "b/testfile"), 1024);
+
+    assertFalse(fs.rename(srcPath, nonEmptyPath));
+
+    Path dstPath = path("/tests3a/b");
+    assertTrue(fs.rename(srcPath, dstPath));
+    assertFalse(fs.exists(srcPath));
+    assertTrue(fs.exists(new Path(dstPath, "b/testfile")));
+  }
+
+
+  @Test(timeout=10000)
+  public void testSeek() throws IOException {
+    Path path = path("/tests3a/testfile.seek");
+    writeFile(path, TEST_BUFFER_SIZE * 10);
+
+
+    FSDataInputStream inputStream = fs.open(path, TEST_BUFFER_SIZE);
+    inputStream.seek(inputStream.getPos() + MODULUS);
+
+    testReceivedData(inputStream, TEST_BUFFER_SIZE * 10 - MODULUS);
+  }
+
+  /**
+   * Creates and reads a file with the given size in S3. The test file is 
+   * generated according to a specific pattern.
+   * During the read phase the incoming data stream is also checked against this pattern.
+   *
+   * @param fileSize
+   * the size of the file to be generated in bytes
+   * @throws IOException
+   * thrown if an I/O error occurs while writing or reading the test file
+   */
+  private void createAndReadFileTest(final long fileSize) throws IOException {
+    final String objectName = UUID.randomUUID().toString();
+    final Path objectPath = new Path("/tests3a/", objectName);
+
+    // Write test file to S3
+    final OutputStream outputStream = fs.create(objectPath, false);
+    generateTestData(outputStream, fileSize);
+    outputStream.close();
+
+    // Now read the same file back from S3
+    final InputStream inputStream = fs.open(objectPath);
+    testReceivedData(inputStream, fileSize);
+    inputStream.close();
+
+    // Delete test file
+    fs.delete(objectPath, false);
+  }
+
+
+  /**
+   * Receives test data from the given input stream and checks the size of the 
+   * data as well as the pattern inside the received data.
+   *
+   * @param inputStream
+   * the input stream to read the test data from
+   * @param expectedSize
+   * the expected size of the data to be read from the input stream in bytes
+   * @throws IOException
+   * thrown if an error occurs while reading the data
+   */
+  private void testReceivedData(final InputStream inputStream, 
+    final long expectedSize) throws IOException {
+    final byte[] testBuffer = new byte[TEST_BUFFER_SIZE];
+
+    long totalBytesRead = 0;
+    int nextExpectedNumber = 0;
+    while (true) {
+      final int bytesRead = inputStream.read(testBuffer);
+      if (bytesRead < 0) {
+        break;
+      }
+
+      totalBytesRead += bytesRead;
+
+      for (int i = 0; i < bytesRead; ++i) {
+        if (testBuffer[i] != nextExpectedNumber) {
+          throw new IOException("Read number " + testBuffer[i] + " but expected "
+                                + nextExpectedNumber);
+        }
+
+        ++nextExpectedNumber;
+
+        if (nextExpectedNumber == MODULUS) {
+          nextExpectedNumber = 0;
+        }
+      }
+    }
+
+    if (totalBytesRead != expectedSize) {
+      throw new IOException("Expected to read " + expectedSize + 
+                            " bytes but only received " + totalBytesRead);
+    }
+  }
+
+
+  /**
+   * Generates test data of the given size according to some specific pattern
+   * and writes it to the provided output stream.
+   *
+   * @param outputStream
+   * the output stream to write the data to
+   * @param size
+   * the size of the test data to be generated in bytes
+   * @throws IOException
+   * thrown if an error occurs while writing the data
+   */
+  private void generateTestData(final OutputStream outputStream, 
+    final long size) throws IOException {
+
+    final byte[] testBuffer = new byte[TEST_BUFFER_SIZE];
+    for (int i = 0; i < testBuffer.length; ++i) {
+      testBuffer[i] = (byte) (i % MODULUS);
+    }
+
+    long bytesWritten = 0;
+    while (bytesWritten < size) {
+
+      final long diff = size - bytesWritten;
+      if (diff < testBuffer.length) {
+        outputStream.write(testBuffer, 0, (int)diff);
+        bytesWritten += diff;
+      } else {
+        outputStream.write(testBuffer);
+        bytesWritten += testBuffer.length;
+      }
+    }
+  }
+
+  private void writeFile(Path name, int fileSize) throws IOException {
+    final OutputStream outputStream = fs.create(name, false);
+    generateTestData(outputStream, fileSize);
+    outputStream.close();
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/24d920b8/hadoop-tools/hadoop-aws/src/test/resources/contract/s3a.xml
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/resources/contract/s3a.xml b/hadoop-tools/hadoop-aws/src/test/resources/contract/s3a.xml
new file mode 100644
index 0000000..4142471
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/resources/contract/s3a.xml
@@ -0,0 +1,105 @@
+<!--
+  ~ Licensed to the Apache Software Foundation (ASF) under one
+  ~  or more contributor license agreements.  See the NOTICE file
+  ~  distributed with this work for additional information
+  ~  regarding copyright ownership.  The ASF licenses this file
+  ~  to you under the Apache License, Version 2.0 (the
+  ~  "License"); you may not use this file except in compliance
+  ~  with the License.  You may obtain a copy of the License at
+  ~
+  ~       http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~  Unless required by applicable law or agreed to in writing, software
+  ~  distributed under the License is distributed on an "AS IS" BASIS,
+  ~  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  ~  See the License for the specific language governing permissions and
+  ~  limitations under the License.
+  -->
+
+<configuration>
+  <!--
+  S3A is a blobstore, with very different behavior than a
+  classic filesystem.
+  -->
+
+  <property>
+    <name>fs.contract.test.root-tests-enabled</name>
+    <value>true</value>
+  </property>
+
+  <property>
+    <name>fs.contract.test.random-seek-count</name>
+    <value>10</value>
+  </property>
+
+  <property>
+    <name>fs.contract.is-blobstore</name>
+    <value>true</value>
+  </property>
+
+  <property>
+    <name>fs.contract.is-case-sensitive</name>
+    <value>true</value>
+  </property>
+
+  <property>
+    <name>fs.contract.rename-returns-false-if-source-missing</name>
+    <value>true</value>
+  </property>
+
+  <property>
+    <name>fs.contract.supports-append</name>
+    <value>false</value>
+  </property>
+
+  <property>
+    <name>fs.contract.supports-atomic-directory-delete</name>
+    <value>false</value>
+  </property>
+
+  <property>
+    <name>fs.contract.supports-atomic-rename</name>
+    <value>false</value>
+  </property>
+
+  <property>
+    <name>fs.contract.supports-block-locality</name>
+    <value>false</value>
+  </property>
+
+  <property>
+    <name>fs.contract.supports-concat</name>
+    <value>false</value>
+  </property>
+
+  <property>
+    <name>fs.contract.supports-seek</name>
+    <value>true</value>
+  </property>
+
+  <property>
+    <name>fs.contract.supports-seek-on-closed-file</name>
+    <value>true</value>
+  </property>
+
+  <property>
+    <name>fs.contract.rejects-seek-past-eof</name>
+    <value>true</value>
+  </property>
+
+  <property>
+    <name>fs.contract.supports-strict-exceptions</name>
+    <value>true</value>
+  </property>
+
+  <property>
+    <name>fs.contract.supports-unix-permissions</name>
+    <value>false</value>
+  </property>
+
+  <property>
+    <name>fs.contract.rename-overwrites-dest</name>
+    <value>true</value>
+  </property>
+
+</configuration>

http://git-wip-us.apache.org/repos/asf/hadoop/blob/24d920b8/hadoop-tools/hadoop-aws/src/test/resources/contract/s3n.xml
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/resources/contract/s3n.xml b/hadoop-tools/hadoop-aws/src/test/resources/contract/s3n.xml
index ab46178..cb8aca7 100644
--- a/hadoop-tools/hadoop-aws/src/test/resources/contract/s3n.xml
+++ b/hadoop-tools/hadoop-aws/src/test/resources/contract/s3n.xml
@@ -78,6 +78,11 @@
   </property>
 
   <property>
+    <name>fs.contract.supports-seek-on-closed-file</name>
+    <value>true</value>
+  </property>
+
+  <property>
     <name>fs.contract.rejects-seek-past-eof</name>
     <value>true</value>
   </property>
@@ -92,4 +97,4 @@
     <value>false</value>
   </property>
 
-</configuration>
\ No newline at end of file
+</configuration>

http://git-wip-us.apache.org/repos/asf/hadoop/blob/24d920b8/hadoop-tools/hadoop-azure/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/pom.xml b/hadoop-tools/hadoop-azure/pom.xml
index a20fb86..fbdc132 100644
--- a/hadoop-tools/hadoop-azure/pom.xml
+++ b/hadoop-tools/hadoop-azure/pom.xml
@@ -132,14 +132,8 @@
     </dependency>
 
     <dependency>
-      <groupId>org.codehaus.jackson</groupId>
-      <artifactId>jackson-mapper-asl</artifactId>
-      <scope>compile</scope>
-    </dependency>
-    
-    <dependency>
-      <groupId>org.codehaus.jackson</groupId>
-      <artifactId>jackson-core-asl</artifactId>
+      <groupId>com.fasterxml.jackson.core</groupId>
+      <artifactId>jackson-core</artifactId>
       <scope>compile</scope>
     </dependency>
     


Mime
View raw message