hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ravip...@apache.org
Subject hadoop git commit: HADOOP-12696. Add tests for S3FileSystem Contract. Contributed by Matt Paduano
Date Tue, 19 Jan 2016 21:57:20 GMT
Repository: hadoop
Updated Branches:
  refs/heads/trunk 89d1fd5da -> 1acc509b4


HADOOP-12696. Add tests for S3FileSystem Contract. Contributed by Matt Paduano


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1acc509b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1acc509b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1acc509b

Branch: refs/heads/trunk
Commit: 1acc509b45d58c0eb7e83ea1ba13169410be0dbe
Parents: 89d1fd5
Author: Ravi Prakash <raviprak@altiscale.com>
Authored: Tue Jan 19 13:57:08 2016 -0800
Committer: Ravi Prakash <raviprak@altiscale.com>
Committed: Tue Jan 19 13:57:08 2016 -0800

----------------------------------------------------------------------
 hadoop-common-project/hadoop-common/CHANGES.txt |   2 +
 .../src/site/markdown/filesystem/testing.md     |   4 +-
 .../fs/contract/AbstractContractSeekTest.java   |   8 +-
 .../hadoop/fs/contract/ContractOptions.java     |   6 ++
 .../org/apache/hadoop/fs/s3/S3FileSystem.java   |  58 +++++++----
 .../org/apache/hadoop/fs/s3/S3InputStream.java  |  17 +--
 .../hadoop/fs/contract/s3/S3Contract.java       |  40 +++++++
 .../fs/contract/s3/TestS3ContractCreate.java    |  32 ++++++
 .../fs/contract/s3/TestS3ContractDelete.java    |  31 ++++++
 .../fs/contract/s3/TestS3ContractMkdir.java     |  32 ++++++
 .../fs/contract/s3/TestS3ContractOpen.java      |  32 ++++++
 .../fs/contract/s3/TestS3ContractRename.java    |  32 ++++++
 .../fs/contract/s3/TestS3ContractRootDir.java   |  34 ++++++
 .../fs/contract/s3/TestS3ContractSeek.java      |  32 ++++++
 .../src/test/resources/contract/s3.xml          | 104 +++++++++++++++++++
 15 files changed, 430 insertions(+), 34 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/1acc509b/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index bd6550c..a5dcf6a 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -1078,6 +1078,8 @@ Release 2.8.0 - UNRELEASED
     HADOOP-12604. Exception may be swallowed in KMSClientProvider.
     (Yongjun Zhang)
 
+    HADOOP-12696. Add tests for S3Filesystem Contract (Matt Paduano via raviprak)
+
   OPTIMIZATIONS
 
     HADOOP-11785. Reduce the number of listStatus operation in distcp

http://git-wip-us.apache.org/repos/asf/hadoop/blob/1acc509b/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/testing.md
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/testing.md b/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/testing.md
index 444fb60..99561cd 100644
--- a/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/testing.md
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/testing.md
@@ -190,7 +190,7 @@ tests against remote FileSystems that require login details require usernames/ID
 
 All these details MUST be required to be placed in the file `src/test/resources/contract-test-options.xml`,
and your SCM tools configured to never commit this file to subversion, git or
 equivalent. Furthermore, the build MUST be configured to never bundle this file in any `-test`
artifacts generated. The Hadoop build does this, excluding `src/test/**/*.xml` from the JAR
files.
-
+In addition, `src/test/resources/auth-keys.xml` will need to be created.  It can be a copy
of `contract-test-options.xml`.
 The `AbstractFSContract` class automatically loads this resource file if present; specific
keys for specific test cases can be added.
 
 As an example, here are what S3N test keys look like:
@@ -214,7 +214,7 @@ As an example, here are what S3N test keys look like:
 
 The `AbstractBondedFSContract` automatically skips a test suite if the FileSystem URL is
not defined in the property `fs.contract.test.fs.%s`, where `%s` matches the schema name of
the FileSystem.
 
-
+When running the tests `maven.test.skip` will need to be turned off since it is true by default
on these tests.  This can be done with a command like `mvn test -Ptests-on`.
 
 ### Important: passing the tests does not guarantee compatibility
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/1acc509b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractSeekTest.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractSeekTest.java
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractSeekTest.java
index 4a0560e..8f56510 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractSeekTest.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractSeekTest.java
@@ -116,15 +116,13 @@ public abstract class AbstractContractSeekTest extends AbstractFSContractTestBas
    */
   @Test
   public void testSeekReadClosedFile() throws Throwable {
-    boolean supportsSeekOnClosedFiles = isSupported(SUPPORTS_SEEK_ON_CLOSED_FILE);
-
     instream = getFileSystem().open(smallSeekFile);
     getLog().debug(
       "Stream is of type " + instream.getClass().getCanonicalName());
     instream.close();
     try {
       instream.seek(0);
-      if (!supportsSeekOnClosedFiles) {
+      if (!isSupported(SUPPORTS_SEEK_ON_CLOSED_FILE)) {
         fail("seek succeeded on a closed stream");
       }
     } catch (IOException e) {
@@ -132,7 +130,9 @@ public abstract class AbstractContractSeekTest extends AbstractFSContractTestBas
     }
     try {
       int data = instream.available();
-      fail("read() succeeded on a closed stream, got " + data);
+      if (!isSupported(SUPPORTS_AVAILABLE_ON_CLOSED_FILE)) {
+        fail("available() succeeded on a closed stream, got " + data);
+      }
     } catch (IOException e) {
       //expected a closed file
     }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/1acc509b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/ContractOptions.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/ContractOptions.java
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/ContractOptions.java
index 56caced..d8c2592 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/ContractOptions.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/ContractOptions.java
@@ -148,6 +148,12 @@ public interface ContractOptions {
   String SUPPORTS_SEEK_ON_CLOSED_FILE = "supports-seek-on-closed-file";
 
   /**
+   * Is available() on a closed InputStream supported?
+   * @{value}
+   */
+  String SUPPORTS_AVAILABLE_ON_CLOSED_FILE = "supports-available-on-closed-file";
+
+  /**
    * Flag to indicate that this FS expects to throw the strictest
    * exceptions it can, not generic IOEs, which, if returned,
    * must be rejected.

http://git-wip-us.apache.org/repos/asf/hadoop/blob/1acc509b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3/S3FileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3/S3FileSystem.java
b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3/S3FileSystem.java
index 8bdfe9a..e5147a3 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3/S3FileSystem.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3/S3FileSystem.java
@@ -35,6 +35,7 @@ import org.apache.hadoop.fs.FSDataOutputStream;
 import org.apache.hadoop.fs.FileAlreadyExistsException;
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.ParentNotDirectoryException;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.permission.FsPermission;
 import org.apache.hadoop.fs.s3native.NativeS3FileSystem;
@@ -62,7 +63,7 @@ public class S3FileSystem extends FileSystem {
   public S3FileSystem() {
     // set store in initialize()
   }
-  
+
   public S3FileSystem(FileSystemStore store) {
     this.store = store;
   }
@@ -90,14 +91,14 @@ public class S3FileSystem extends FileSystem {
     }
     store.initialize(uri, conf);
     setConf(conf);
-    this.uri = URI.create(uri.getScheme() + "://" + uri.getAuthority());    
+    this.uri = URI.create(uri.getScheme() + "://" + uri.getAuthority());
     this.workingDir =
       new Path("/user", System.getProperty("user.name")).makeQualified(this);
-  }  
+  }
 
   private static FileSystemStore createDefaultStore(Configuration conf) {
     FileSystemStore store = new Jets3tFileSystemStore();
-    
+
     RetryPolicy basePolicy = RetryPolicies.retryUpToMaximumCountWithFixedSleep(
                                                                                conf.getInt("fs.s3.maxRetries",
4),
                                                                                conf.getLong("fs.s3.sleepTimeSeconds",
10), TimeUnit.SECONDS);
@@ -105,13 +106,13 @@ public class S3FileSystem extends FileSystem {
       new HashMap<Class<? extends Exception>, RetryPolicy>();
     exceptionToPolicyMap.put(IOException.class, basePolicy);
     exceptionToPolicyMap.put(S3Exception.class, basePolicy);
-    
+
     RetryPolicy methodPolicy = RetryPolicies.retryByException(
                                                               RetryPolicies.TRY_ONCE_THEN_FAIL,
exceptionToPolicyMap);
     Map<String,RetryPolicy> methodNameToPolicyMap = new HashMap<String,RetryPolicy>();
     methodNameToPolicyMap.put("storeBlock", methodPolicy);
     methodNameToPolicyMap.put("retrieveBlock", methodPolicy);
-    
+
     return (FileSystemStore) RetryProxy.create(FileSystemStore.class,
                                                store, methodNameToPolicyMap);
   }
@@ -144,21 +145,29 @@ public class S3FileSystem extends FileSystem {
       paths.add(0, absolutePath);
       absolutePath = absolutePath.getParent();
     } while (absolutePath != null);
-    
+
     boolean result = true;
-    for (Path p : paths) {
-      result &= mkdir(p);
+    for (int i = 0; i < paths.size(); i++) {
+      Path p = paths.get(i);
+      try {
+        result &= mkdir(p);
+      } catch(FileAlreadyExistsException e) {
+        if (i + 1 < paths.size()) {
+          throw new ParentNotDirectoryException(e.getMessage());
+        }
+        throw e;
+      }
     }
     return result;
   }
-  
+
   private boolean mkdir(Path path) throws IOException {
     Path absolutePath = makeAbsolute(path);
     INode inode = store.retrieveINode(absolutePath);
     if (inode == null) {
       store.storeINode(absolutePath, INode.DIRECTORY_INODE);
     } else if (inode.isFile()) {
-      throw new IOException(String.format(
+      throw new FileAlreadyExistsException(String.format(
           "Can't make directory for path %s since it is a file.",
           absolutePath));
     }
@@ -176,11 +185,12 @@ public class S3FileSystem extends FileSystem {
 
   private INode checkFile(Path path) throws IOException {
     INode inode = store.retrieveINode(makeAbsolute(path));
+    String message = String.format("No such file: '%s'", path.toString());
     if (inode == null) {
-      throw new IOException("No such file.");
+      throw new FileNotFoundException(message + " does not exist");
     }
     if (inode.isDirectory()) {
-      throw new IOException("Path " + path + " is a directory.");
+      throw new FileNotFoundException(message + " is a directory");
     }
     return inode;
   }
@@ -222,10 +232,14 @@ public class S3FileSystem extends FileSystem {
 
     INode inode = store.retrieveINode(makeAbsolute(file));
     if (inode != null) {
-      if (overwrite) {
+      if (overwrite && !inode.isDirectory()) {
         delete(file, true);
       } else {
-        throw new FileAlreadyExistsException("File already exists: " + file);
+        String message = String.format("File already exists: '%s'", file);
+        if (inode.isDirectory()) {
+          message = message + " is a directory";
+        }
+        throw new FileAlreadyExistsException(message);
       }
     } else {
       Path parent = file.getParent();
@@ -233,7 +247,7 @@ public class S3FileSystem extends FileSystem {
         if (!mkdirs(parent)) {
           throw new IOException("Mkdirs failed to create " + parent.toString());
         }
-      }      
+      }
     }
     return new FSDataOutputStream
         (new S3OutputStream(getConf(), store, makeAbsolute(file),
@@ -259,7 +273,7 @@ public class S3FileSystem extends FileSystem {
       if (debugEnabled) {
         LOG.debug(debugPreamble + "returning false as src does not exist");
       }
-      return false; 
+      return false;
     }
 
     Path absoluteDst = makeAbsolute(dst);
@@ -404,7 +418,7 @@ public class S3FileSystem extends FileSystem {
        store.deleteBlock(block);
      }
    } else {
-     FileStatus[] contents = null; 
+     FileStatus[] contents = null;
      try {
        contents = listStatus(absolutePath);
      } catch(FileNotFoundException fnfe) {
@@ -412,7 +426,7 @@ public class S3FileSystem extends FileSystem {
      }
 
      if ((contents.length !=0) && (!recursive)) {
-       throw new IOException("Directory " + path.toString() 
+       throw new IOException("Directory " + path.toString()
            + " is not empty.");
      }
      for (FileStatus p:contents) {
@@ -424,9 +438,9 @@ public class S3FileSystem extends FileSystem {
    }
    return true;
   }
-  
+
   /**
-   * FileStatus for S3 file systems. 
+   * FileStatus for S3 file systems.
    */
   @Override
   public FileStatus getFileStatus(Path f)  throws IOException {
@@ -436,7 +450,7 @@ public class S3FileSystem extends FileSystem {
     }
     return new S3FileStatus(f.makeQualified(this), inode);
   }
-  
+
   @Override
   public long getDefaultBlockSize() {
     return getConf().getLong("fs.s3.block.size", 64 * 1024 * 1024);

http://git-wip-us.apache.org/repos/asf/hadoop/blob/1acc509b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3/S3InputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3/S3InputStream.java
b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3/S3InputStream.java
index 5af57e6..6f39f01 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3/S3InputStream.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3/S3InputStream.java
@@ -22,6 +22,7 @@ import java.io.DataInputStream;
 import java.io.File;
 import java.io.FileInputStream;
 import java.io.IOException;
+import java.io.EOFException;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
@@ -46,14 +47,14 @@ class S3InputStream extends FSInputStream {
   private long pos = 0;
 
   private File blockFile;
-  
+
   private DataInputStream blockStream;
 
   private long blockEnd = -1;
-  
+
   private FileSystem.Statistics stats;
-  
-  private static final Log LOG = 
+
+  private static final Log LOG =
     LogFactory.getLog(S3InputStream.class.getName());
 
 
@@ -65,7 +66,7 @@ class S3InputStream extends FSInputStream {
 
   public S3InputStream(Configuration conf, FileSystemStore store,
                        INode inode, FileSystem.Statistics stats) {
-    
+
     this.store = store;
     this.stats = stats;
     this.blocks = inode.getBlocks();
@@ -86,8 +87,12 @@ class S3InputStream extends FSInputStream {
 
   @Override
   public synchronized void seek(long targetPos) throws IOException {
+    String message = String.format("Cannot seek to %d", targetPos);
     if (targetPos > fileLength) {
-      throw new IOException("Cannot seek after EOF");
+      throw new EOFException(message + ": after EOF");
+    }
+    if (targetPos < 0) {
+      throw new EOFException(message + ": negative");
     }
     pos = targetPos;
     blockEnd = -1;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/1acc509b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3/S3Contract.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3/S3Contract.java
b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3/S3Contract.java
new file mode 100644
index 0000000..b388ce7
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3/S3Contract.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ *  or more contributor license agreements.  See the NOTICE file
+ *  distributed with this work for additional information
+ *  regarding copyright ownership.  The ASF licenses this file
+ *  to you under the Apache License, Version 2.0 (the
+ *  "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+package org.apache.hadoop.fs.contract.s3;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.contract.AbstractBondedFSContract;
+
+public class S3Contract extends AbstractBondedFSContract {
+
+  public static final String CONTRACT_XML = "contract/s3.xml";
+
+
+  public S3Contract(Configuration conf) {
+    super(conf);
+    //insert the base features
+    addConfResource(CONTRACT_XML);
+  }
+
+  @Override
+  public String getScheme() {
+    return "s3";
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/1acc509b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3/TestS3ContractCreate.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3/TestS3ContractCreate.java
b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3/TestS3ContractCreate.java
new file mode 100644
index 0000000..b8c758c
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3/TestS3ContractCreate.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ *  or more contributor license agreements.  See the NOTICE file
+ *  distributed with this work for additional information
+ *  regarding copyright ownership.  The ASF licenses this file
+ *  to you under the Apache License, Version 2.0 (the
+ *  "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+package org.apache.hadoop.fs.contract.s3;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.contract.AbstractContractCreateTest;
+import org.apache.hadoop.fs.contract.AbstractFSContract;
+import org.apache.hadoop.fs.contract.ContractTestUtils;
+
+public class TestS3ContractCreate extends AbstractContractCreateTest {
+
+  @Override
+  protected AbstractFSContract createContract(Configuration conf) {
+    return new S3Contract(conf);
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/1acc509b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3/TestS3ContractDelete.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3/TestS3ContractDelete.java
b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3/TestS3ContractDelete.java
new file mode 100644
index 0000000..2d3cec7
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3/TestS3ContractDelete.java
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ *  or more contributor license agreements.  See the NOTICE file
+ *  distributed with this work for additional information
+ *  regarding copyright ownership.  The ASF licenses this file
+ *  to you under the Apache License, Version 2.0 (the
+ *  "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+package org.apache.hadoop.fs.contract.s3;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.contract.AbstractContractDeleteTest;
+import org.apache.hadoop.fs.contract.AbstractFSContract;
+
+public class TestS3ContractDelete extends AbstractContractDeleteTest {
+
+  @Override
+  protected AbstractFSContract createContract(Configuration conf) {
+    return new S3Contract(conf);
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/1acc509b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3/TestS3ContractMkdir.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3/TestS3ContractMkdir.java
b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3/TestS3ContractMkdir.java
new file mode 100644
index 0000000..992ce53
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3/TestS3ContractMkdir.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ *  or more contributor license agreements.  See the NOTICE file
+ *  distributed with this work for additional information
+ *  regarding copyright ownership.  The ASF licenses this file
+ *  to you under the Apache License, Version 2.0 (the
+ *  "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+package org.apache.hadoop.fs.contract.s3;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.contract.AbstractContractMkdirTest;
+import org.apache.hadoop.fs.contract.AbstractFSContract;
+import org.apache.hadoop.fs.contract.ContractTestUtils;
+
+public class TestS3ContractMkdir extends AbstractContractMkdirTest {
+
+  @Override
+  protected AbstractFSContract createContract(Configuration conf) {
+    return new S3Contract(conf);
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/1acc509b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3/TestS3ContractOpen.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3/TestS3ContractOpen.java
b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3/TestS3ContractOpen.java
new file mode 100644
index 0000000..2a4ba31
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3/TestS3ContractOpen.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ *  or more contributor license agreements.  See the NOTICE file
+ *  distributed with this work for additional information
+ *  regarding copyright ownership.  The ASF licenses this file
+ *  to you under the Apache License, Version 2.0 (the
+ *  "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+package org.apache.hadoop.fs.contract.s3;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.contract.AbstractContractOpenTest;
+import org.apache.hadoop.fs.contract.AbstractFSContract;
+import org.apache.hadoop.fs.contract.ContractTestUtils;
+
+public class TestS3ContractOpen extends AbstractContractOpenTest {
+
+  @Override
+  protected AbstractFSContract createContract(Configuration conf) {
+    return new S3Contract(conf);
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/1acc509b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3/TestS3ContractRename.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3/TestS3ContractRename.java
b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3/TestS3ContractRename.java
new file mode 100644
index 0000000..68bdbda
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3/TestS3ContractRename.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ *  or more contributor license agreements.  See the NOTICE file
+ *  distributed with this work for additional information
+ *  regarding copyright ownership.  The ASF licenses this file
+ *  to you under the Apache License, Version 2.0 (the
+ *  "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+package org.apache.hadoop.fs.contract.s3;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.contract.AbstractContractRenameTest;
+import org.apache.hadoop.fs.contract.AbstractFSContract;
+
+public class TestS3ContractRename extends AbstractContractRenameTest {
+
+  @Override
+  protected AbstractFSContract createContract(Configuration conf) {
+    return new S3Contract(conf);
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/1acc509b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3/TestS3ContractRootDir.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3/TestS3ContractRootDir.java
b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3/TestS3ContractRootDir.java
new file mode 100644
index 0000000..b968081
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3/TestS3ContractRootDir.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ *  or more contributor license agreements.  See the NOTICE file
+ *  distributed with this work for additional information
+ *  regarding copyright ownership.  The ASF licenses this file
+ *  to you under the Apache License, Version 2.0 (the
+ *  "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+package org.apache.hadoop.fs.contract.s3;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.contract.AbstractContractRootDirectoryTest;
+import org.apache.hadoop.fs.contract.AbstractFSContract;
+
+/**
+ * root dir operations against an S3 bucket
+ */
+public class TestS3ContractRootDir extends AbstractContractRootDirectoryTest {
+
+  @Override
+  protected AbstractFSContract createContract(Configuration conf) {
+    return new S3Contract(conf);
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/1acc509b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3/TestS3ContractSeek.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3/TestS3ContractSeek.java
b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3/TestS3ContractSeek.java
new file mode 100644
index 0000000..bfcd163
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3/TestS3ContractSeek.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ *  or more contributor license agreements.  See the NOTICE file
+ *  distributed with this work for additional information
+ *  regarding copyright ownership.  The ASF licenses this file
+ *  to you under the Apache License, Version 2.0 (the
+ *  "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+package org.apache.hadoop.fs.contract.s3;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.contract.AbstractContractSeekTest;
+import org.apache.hadoop.fs.contract.AbstractFSContract;
+import org.apache.hadoop.fs.contract.ContractTestUtils;
+
+public class TestS3ContractSeek extends AbstractContractSeekTest {
+
+  @Override
+  protected AbstractFSContract createContract(Configuration conf) {
+    return new S3Contract(conf);
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/1acc509b/hadoop-tools/hadoop-aws/src/test/resources/contract/s3.xml
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/resources/contract/s3.xml b/hadoop-tools/hadoop-aws/src/test/resources/contract/s3.xml
new file mode 100644
index 0000000..4b742c1
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/resources/contract/s3.xml
@@ -0,0 +1,104 @@
+<!--
+  ~ Licensed to the Apache Software Foundation (ASF) under one
+  ~  or more contributor license agreements.  See the NOTICE file
+  ~  distributed with this work for additional information
+  ~  regarding copyright ownership.  The ASF licenses this file
+  ~  to you under the Apache License, Version 2.0 (the
+  ~  "License"); you may not use this file except in compliance
+  ~  with the License.  You may obtain a copy of the License at
+  ~
+  ~       http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~  Unless required by applicable law or agreed to in writing, software
+  ~  distributed under the License is distributed on an "AS IS" BASIS,
+  ~  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  ~  See the License for the specific language governing permissions and
+  ~  limitations under the License.
+  -->
+
+<configuration>
+  <!--
+  S3 is backed by a blobstore.
+  -->
+
+  <property>
+    <name>fs.contract.test.root-tests-enabled</name>
+    <value>true</value>
+  </property>
+
+  <property>
+    <name>fs.contract.test.random-seek-count</name>
+    <value>10</value>
+  </property>
+
+  <property>
+    <name>fs.contract.is-blobstore</name>
+    <value>true</value>
+  </property>
+
+  <property>
+    <name>fs.contract.is-case-sensitive</name>
+    <value>true</value>
+  </property>
+
+  <property>
+    <name>fs.contract.rename-returns-false-if-source-missing</name>
+    <value>true</value>
+  </property>
+
+  <property>
+    <name>fs.contract.supports-append</name>
+    <value>false</value>
+  </property>
+
+  <property>
+    <name>fs.contract.supports-atomic-directory-delete</name>
+    <value>false</value>
+  </property>
+
+  <property>
+    <name>fs.contract.supports-atomic-rename</name>
+    <value>false</value>
+  </property>
+
+  <property>
+    <name>fs.contract.supports-block-locality</name>
+    <value>false</value>
+  </property>
+
+  <property>
+    <name>fs.contract.supports-concat</name>
+    <value>false</value>
+  </property>
+
+  <property>
+    <name>fs.contract.supports-seek</name>
+    <value>true</value>
+  </property>
+
+  <property>
+    <name>fs.contract.supports-seek-on-closed-file</name>
+    <value>true</value>
+  </property>
+
+  <property>
+    <name>fs.contract.supports-available-on-closed-file</name>
+    <value>true</value>
+  </property>
+
+  <property>
+    <name>fs.contract.rejects-seek-past-eof</name>
+    <value>true</value>
+  </property>
+
+  <property>
+    <name>fs.contract.supports-strict-exceptions</name>
+    <value>true</value>
+  </property>
+
+  <property>
+    <name>fs.contract.supports-unix-permissions</name>
+    <value>false</value>
+  </property>
+
+</configuration>


Mime
View raw message