hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From cnaur...@apache.org
Subject [3/3] hadoop git commit: HADOOP-13502. Split fs.contract.is-blobstore flag into more descriptive flags for use by contract tests. Contributed by Chris Nauroth.
Date Wed, 26 Oct 2016 16:28:27 GMT
HADOOP-13502. Split fs.contract.is-blobstore flag into more descriptive flags for use by contract
tests. Contributed by Chris Nauroth.

(cherry picked from commit 1f8490a5bacd98d0d371447ada3b31f93ca40a4e)
(cherry picked from commit 082d69ee663d5c1a59f32c40b9fbde9996886de9)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4df17809
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4df17809
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4df17809

Branch: refs/heads/branch-2.8
Commit: 4df1780991db8262dce64d7bfeae38b99751a42a
Parents: c940c68
Author: Chris Nauroth <cnauroth@apache.org>
Authored: Wed Oct 26 08:57:29 2016 -0700
Committer: Chris Nauroth <cnauroth@apache.org>
Committed: Wed Oct 26 08:57:46 2016 -0700

----------------------------------------------------------------------
 .gitignore                                        |  1 +
 .../fs/contract/AbstractContractCreateTest.java   | 18 +++++++++---------
 .../hadoop/fs/contract/ContractOptions.java       | 18 ++++++++++++++++++
 .../src/test/resources/contract/ftp.xml           |  7 +------
 .../hadoop-aws/src/test/resources/contract/s3.xml | 10 ++++++++++
 .../src/test/resources/contract/s3a.xml           |  5 +++++
 .../src/test/resources/contract/s3n.xml           | 10 ++++++++++
 .../src/test/resources/contract/swift.xml         | 10 ++++++++++
 8 files changed, 64 insertions(+), 15 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/4df17809/.gitignore
----------------------------------------------------------------------
diff --git a/.gitignore b/.gitignore
index 5925ec4..eb1fc96 100644
--- a/.gitignore
+++ b/.gitignore
@@ -29,4 +29,5 @@ yarnregistry.pdf
 hadoop-tools/hadoop-aws/src/test/resources/auth-keys.xml
 hadoop-tools/hadoop-aws/src/test/resources/contract-test-options.xml
 hadoop-tools/hadoop-azure/src/test/resources/azure-auth-keys.xml
+hadoop-tools/hadoop-openstack/src/test/resources/auth-keys.xml
 patchprocess/

http://git-wip-us.apache.org/repos/asf/hadoop/blob/4df17809/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractCreateTest.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractCreateTest.java
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractCreateTest.java
index 9344225..84dc775 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractCreateTest.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractCreateTest.java
@@ -121,7 +121,7 @@ public abstract class AbstractContractCreateTest extends
     try {
       assertIsDirectory(path);
     } catch (AssertionError failure) {
-      if (isSupported(IS_BLOBSTORE)) {
+      if (isSupported(CREATE_OVERWRITES_DIRECTORY)) {
         // file/directory hack surfaces here
         throw new AssumptionViolatedException(failure.toString(), failure);
       }
@@ -137,10 +137,10 @@ public abstract class AbstractContractCreateTest extends
       FileStatus status = getFileSystem().getFileStatus(path);
 
       boolean isDir = status.isDirectory();
-      if (!isDir && isSupported(IS_BLOBSTORE)) {
-        // object store: downgrade to a skip so that the failure is visible
-        // in test results
-        skip("Object store allows a file to overwrite a directory");
+      if (!isDir && isSupported(CREATE_OVERWRITES_DIRECTORY)) {
+        // For some file systems, downgrade to a skip so that the failure is
+        // visible in test results.
+        skip("This Filesystem allows a file to overwrite a directory");
       }
       fail("write of file over dir succeeded");
     } catch (FileAlreadyExistsException expected) {
@@ -170,10 +170,10 @@ public abstract class AbstractContractCreateTest extends
                                    1024)) {
       if (!getFileSystem().exists(path)) {
 
-        if (isSupported(IS_BLOBSTORE)) {
-          // object store: downgrade to a skip so that the failure is visible
-          // in test results
-          skip("Filesystem is an object store and newly created files are not immediately
visible");
+        if (isSupported(CREATE_VISIBILITY_DELAYED)) {
+          // For some file systems, downgrade to a skip so that the failure is
+          // visible in test results.
+          skip("This Filesystem delays visibility of newly created files");
         }
         assertPathExists("expected path to be visible before anything written",
                          path);

http://git-wip-us.apache.org/repos/asf/hadoop/blob/4df17809/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/ContractOptions.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/ContractOptions.java
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/ContractOptions.java
index c8af062..b778f0d 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/ContractOptions.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/ContractOptions.java
@@ -37,16 +37,33 @@ public interface ContractOptions {
   String FS_CONTRACT_KEY = "fs.contract.";
 
   /**
+   * Flag to indicate that a newly created file may overwrite a pre-existing
+   * directory.
+   * {@value}
+   */
+  String CREATE_OVERWRITES_DIRECTORY = "create-overwrites-directory";
+
+  /**
+   * Flag to indicate that a newly created file is not made visible in the
+   * namespace immediately.  Instead, the file becomes visible at a later point
+   * in the file creation lifecycle, such as when the client closes it.
+   * {@value}
+   */
+  String CREATE_VISIBILITY_DELAYED = "create-visibility-delayed";
+
+  /**
    * Is a filesystem case sensitive.
    * Some of the filesystems that say "no" here may mean
    * that it varies from platform to platform -the localfs being the key
    * example.
+   * {@value}
    */
   String IS_CASE_SENSITIVE = "is-case-sensitive";
 
   /**
    * Blobstore flag. Implies it's not a real directory tree and
    * consistency is below that which Hadoop expects
+   * {@value}
    */
   String IS_BLOBSTORE = "is-blobstore";
 
@@ -196,6 +213,7 @@ public interface ContractOptions {
   /**
    * Limit for #of random seeks to perform.
    * Keep low for remote filesystems for faster tests
+   * {@value}
    */
   String TEST_RANDOM_SEEK_COUNT = "test.random-seek-count";
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/4df17809/hadoop-common-project/hadoop-common/src/test/resources/contract/ftp.xml
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/resources/contract/ftp.xml b/hadoop-common-project/hadoop-common/src/test/resources/contract/ftp.xml
index 9c3e380..f95446e 100644
--- a/hadoop-common-project/hadoop-common/src/test/resources/contract/ftp.xml
+++ b/hadoop-common-project/hadoop-common/src/test/resources/contract/ftp.xml
@@ -27,11 +27,6 @@
   </property>
 
   <property>
-    <name>fs.contract.is-blobstore</name>
-    <value>false</value>
-  </property>
-
-  <property>
     <name>fs.contract.is-case-sensitive</name>
     <value>true</value>
   </property>
@@ -81,4 +76,4 @@
     <value>false</value>
   </property>
 
-</configuration>
\ No newline at end of file
+</configuration>

http://git-wip-us.apache.org/repos/asf/hadoop/blob/4df17809/hadoop-tools/hadoop-aws/src/test/resources/contract/s3.xml
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/resources/contract/s3.xml b/hadoop-tools/hadoop-aws/src/test/resources/contract/s3.xml
index 4b742c1..49acd3b 100644
--- a/hadoop-tools/hadoop-aws/src/test/resources/contract/s3.xml
+++ b/hadoop-tools/hadoop-aws/src/test/resources/contract/s3.xml
@@ -37,6 +37,16 @@
   </property>
 
   <property>
+    <name>fs.contract.create-overwrites-directory</name>
+    <value>true</value>
+  </property>
+
+  <property>
+    <name>fs.contract.create-visibility-delayed</name>
+    <value>true</value>
+  </property>
+
+  <property>
     <name>fs.contract.is-case-sensitive</name>
     <value>true</value>
   </property>

http://git-wip-us.apache.org/repos/asf/hadoop/blob/4df17809/hadoop-tools/hadoop-aws/src/test/resources/contract/s3a.xml
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/resources/contract/s3a.xml b/hadoop-tools/hadoop-aws/src/test/resources/contract/s3a.xml
index be1e7ca..a534f0a 100644
--- a/hadoop-tools/hadoop-aws/src/test/resources/contract/s3a.xml
+++ b/hadoop-tools/hadoop-aws/src/test/resources/contract/s3a.xml
@@ -38,6 +38,11 @@
   </property>
 
   <property>
+    <name>fs.contract.create-visibility-delayed</name>
+    <value>true</value>
+  </property>
+
+  <property>
     <name>fs.contract.is-case-sensitive</name>
     <value>true</value>
   </property>

http://git-wip-us.apache.org/repos/asf/hadoop/blob/4df17809/hadoop-tools/hadoop-aws/src/test/resources/contract/s3n.xml
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/resources/contract/s3n.xml b/hadoop-tools/hadoop-aws/src/test/resources/contract/s3n.xml
index cb8aca7..0c6b8c6 100644
--- a/hadoop-tools/hadoop-aws/src/test/resources/contract/s3n.xml
+++ b/hadoop-tools/hadoop-aws/src/test/resources/contract/s3n.xml
@@ -38,6 +38,16 @@
   </property>
 
   <property>
+    <name>fs.contract.create-overwrites-directory</name>
+    <value>true</value>
+  </property>
+
+  <property>
+    <name>fs.contract.create-visibility-delayed</name>
+    <value>true</value>
+  </property>
+
+  <property>
     <name>fs.contract.is-case-sensitive</name>
     <value>true</value>
   </property>

http://git-wip-us.apache.org/repos/asf/hadoop/blob/4df17809/hadoop-tools/hadoop-openstack/src/test/resources/contract/swift.xml
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-openstack/src/test/resources/contract/swift.xml b/hadoop-tools/hadoop-openstack/src/test/resources/contract/swift.xml
index 12a67e0..fbf3a17 100644
--- a/hadoop-tools/hadoop-openstack/src/test/resources/contract/swift.xml
+++ b/hadoop-tools/hadoop-openstack/src/test/resources/contract/swift.xml
@@ -38,6 +38,16 @@
   </property>
 
   <property>
+    <name>fs.contract.create-overwrites-directory</name>
+    <value>true</value>
+  </property>
+
+  <property>
+    <name>fs.contract.create-visibility-delayed</name>
+    <value>true</value>
+  </property>
+
+  <property>
     <name>fs.contract.is-case-sensitive</name>
     <value>true</value>
   </property>


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message