hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ste...@apache.org
Subject [20/20] hadoop git commit: HADOOP-14553. Add (parallelized) integration tests to hadoop-azure Contributed by Steve Loughran
Date Fri, 15 Sep 2017 16:39:45 GMT
HADOOP-14553. Add (parallelized) integration tests to hadoop-azure
Contributed by Steve Loughran

(cherry picked from commit 2d2d97fa7d4224369b3c13bc4a45e8cc9e29afb1)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/9f6b08f8
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/9f6b08f8
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/9f6b08f8

Branch: refs/heads/branch-3.0
Commit: 9f6b08f8404ea8576bd20bfeeb92bc2c206ae0d9
Parents: b5e9982
Author: Steve Loughran <stevel@apache.org>
Authored: Fri Sep 15 17:04:43 2017 +0100
Committer: Steve Loughran <stevel@apache.org>
Committed: Fri Sep 15 17:04:43 2017 +0100

----------------------------------------------------------------------
 .../hadoop/fs/FileSystemContractBaseTest.java   |  11 +-
 .../fs/contract/AbstractContractOpenTest.java   |   4 +-
 .../fs/contract/AbstractContractSeekTest.java   |   2 +-
 hadoop-tools/hadoop-azure/pom.xml               | 251 ++++++
 .../fs/azure/AzureNativeFileSystemStore.java    |   2 +-
 .../hadoop-azure/src/site/markdown/index.md     |  94 +-
 .../src/site/markdown/testing_azure.md          | 576 ++++++++++++
 .../hadoop/fs/azure/AbstractWasbTestBase.java   | 136 ++-
 .../fs/azure/AbstractWasbTestWithTimeout.java   |  73 ++
 .../fs/azure/AzureBlobStorageTestAccount.java   |  42 +-
 .../azure/ITestAzureConcurrentOutOfBandIo.java  | 179 ++++
 ...zureConcurrentOutOfBandIoWithSecureMode.java |  33 +
 .../ITestAzureFileSystemErrorConditions.java    | 243 +++++
 .../fs/azure/ITestBlobDataValidation.java       | 244 ++++++
 .../fs/azure/ITestBlobTypeSpeedDifference.java  | 163 ++++
 .../fs/azure/ITestBlockBlobInputStream.java     | 874 ++++++++++++++++++
 .../hadoop/fs/azure/ITestContainerChecks.java   | 194 ++++
 ...estFileSystemOperationExceptionHandling.java | 283 ++++++
 ...TestFileSystemOperationExceptionMessage.java |  79 ++
 ...perationsExceptionHandlingMultiThreaded.java | 366 ++++++++
 .../ITestFileSystemOperationsWithThreads.java   | 821 +++++++++++++++++
 ...stNativeAzureFSAuthWithBlobSpecificKeys.java |  40 +
 .../ITestNativeAzureFSAuthorizationCaching.java |  53 ++
 .../azure/ITestNativeAzureFSPageBlobLive.java   |  43 +
 .../azure/ITestNativeAzureFileSystemAppend.java | 350 ++++++++
 ...ativeAzureFileSystemAtomicRenameDirList.java |  55 ++
 ...veAzureFileSystemAuthorizationWithOwner.java | 122 +++
 ...ITestNativeAzureFileSystemClientLogging.java | 136 +++
 ...estNativeAzureFileSystemConcurrencyLive.java | 185 ++++
 ...stNativeAzureFileSystemContractEmulator.java |  65 ++
 .../ITestNativeAzureFileSystemContractLive.java | 108 +++
 ...tiveAzureFileSystemContractPageBlobLive.java | 114 +++
 .../azure/ITestNativeAzureFileSystemLive.java   | 236 +++++
 .../ITestOutOfBandAzureBlobOperationsLive.java  | 185 ++++
 .../ITestReadAndSeekPageBlobAfterWrite.java     | 341 ++++++++
 .../fs/azure/ITestWasbRemoteCallHelper.java     | 568 ++++++++++++
 .../fs/azure/ITestWasbUriAndConfiguration.java  | 610 +++++++++++++
 .../hadoop/fs/azure/MockWasbAuthorizerImpl.java | 103 ++-
 .../fs/azure/NativeAzureFileSystemBaseTest.java | 115 ++-
 .../hadoop/fs/azure/RunningLiveWasbTests.txt    |  22 -
 .../azure/TestAzureConcurrentOutOfBandIo.java   | 195 -----
 ...zureConcurrentOutOfBandIoWithSecureMode.java |  50 --
 .../TestAzureFileSystemErrorConditions.java     | 244 ------
 .../hadoop/fs/azure/TestBlobDataValidation.java | 237 -----
 .../hadoop/fs/azure/TestBlobMetadata.java       |   7 +-
 .../fs/azure/TestBlobOperationDescriptor.java   |   3 -
 .../fs/azure/TestBlobTypeSpeedDifference.java   | 160 ----
 .../fs/azure/TestBlockBlobInputStream.java      | 875 -------------------
 .../fs/azure/TestClientThrottlingAnalyzer.java  |   5 +-
 .../hadoop/fs/azure/TestContainerChecks.java    | 185 ----
 ...estFileSystemOperationExceptionHandling.java | 269 ------
 ...TestFileSystemOperationExceptionMessage.java |  79 --
 ...perationsExceptionHandlingMultiThreaded.java | 330 -------
 .../TestFileSystemOperationsWithThreads.java    | 821 -----------------
 ...stNativeAzureFSAuthWithBlobSpecificKeys.java |  44 -
 .../TestNativeAzureFSAuthorizationCaching.java  |  60 --
 .../fs/azure/TestNativeAzureFSPageBlobLive.java |  43 -
 .../azure/TestNativeAzureFileSystemAppend.java  | 362 --------
 ...ativeAzureFileSystemAtomicRenameDirList.java |  50 --
 .../TestNativeAzureFileSystemAuthorization.java |  53 +-
 ...veAzureFileSystemAuthorizationWithOwner.java | 122 ---
 ...TestNativeAzureFileSystemBlockLocations.java |   8 +-
 .../TestNativeAzureFileSystemClientLogging.java | 140 ---
 .../TestNativeAzureFileSystemConcurrency.java   |  29 +-
 ...estNativeAzureFileSystemConcurrencyLive.java | 184 ----
 ...stNativeAzureFileSystemContractEmulator.java |  48 -
 .../TestNativeAzureFileSystemContractLive.java  |  80 --
 ...TestNativeAzureFileSystemContractMocked.java |   3 +
 ...tiveAzureFileSystemContractPageBlobLive.java |  93 --
 .../TestNativeAzureFileSystemFileNameCheck.java |  28 +-
 .../fs/azure/TestNativeAzureFileSystemLive.java | 242 -----
 .../azure/TestNativeAzureFileSystemMocked.java  |   4 +
 .../TestNativeAzureFileSystemUploadLogic.java   |  78 +-
 .../azure/TestOutOfBandAzureBlobOperations.java |   8 +-
 .../TestOutOfBandAzureBlobOperationsLive.java   | 203 -----
 .../TestReadAndSeekPageBlobAfterWrite.java      | 355 --------
 .../azure/TestShellDecryptionKeyProvider.java   |  15 +-
 .../apache/hadoop/fs/azure/TestWasbFsck.java    |   9 +-
 .../fs/azure/TestWasbRemoteCallHelper.java      | 569 ------------
 .../fs/azure/TestWasbUriAndConfiguration.java   | 617 -------------
 .../ITestAzureNativeContractAppend.java         |  41 +
 .../ITestAzureNativeContractCreate.java         |  34 +
 .../ITestAzureNativeContractDelete.java         |  33 +
 .../ITestAzureNativeContractDistCp.java         |  47 +
 .../ITestAzureNativeContractGetFileStatus.java  |  35 +
 .../contract/ITestAzureNativeContractMkdir.java |  33 +
 .../contract/ITestAzureNativeContractOpen.java  |  34 +
 .../ITestAzureNativeContractRename.java         |  34 +
 .../contract/ITestAzureNativeContractSeek.java  |  34 +
 .../contract/NativeAzureFileSystemContract.java |  19 +-
 .../contract/TestAzureNativeContractAppend.java |  37 -
 .../contract/TestAzureNativeContractCreate.java |  30 -
 .../contract/TestAzureNativeContractDelete.java |  30 -
 .../contract/TestAzureNativeContractDistCp.java |  33 -
 .../TestAzureNativeContractGetFileStatus.java   |  30 -
 .../contract/TestAzureNativeContractMkdir.java  |  30 -
 .../contract/TestAzureNativeContractOpen.java   |  30 -
 .../contract/TestAzureNativeContractRename.java |  30 -
 .../contract/TestAzureNativeContractSeek.java   |  30 -
 .../integration/AbstractAzureScaleTest.java     |  66 ++
 .../azure/integration/AzureTestConstants.java   | 180 ++++
 .../fs/azure/integration/AzureTestUtils.java    | 479 ++++++++++
 .../integration/CleanupTestContainers.java      |  87 ++
 .../azure/integration/ITestAzureHugeFiles.java  | 456 ++++++++++
 .../hadoop/fs/azure/integration/Sizes.java      |  43 +
 .../ITestAzureFileSystemInstrumentation.java    | 586 +++++++++++++
 .../TestAzureFileSystemInstrumentation.java     | 579 ------------
 107 files changed, 10227 insertions(+), 7901 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/9f6b08f8/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileSystemContractBaseTest.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileSystemContractBaseTest.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileSystemContractBaseTest.java
index b49dd53..a4ccee3 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileSystemContractBaseTest.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileSystemContractBaseTest.java
@@ -61,7 +61,16 @@ public abstract class FileSystemContractBaseTest {
   protected byte[] data = dataset(getBlockSize() * 2, 0, 255);
 
   @Rule
-  public Timeout globalTimeout = new Timeout(30000);
+  public Timeout globalTimeout = new Timeout(getGlobalTimeout());
+
+  /**
+   * Get the timeout in milliseconds for each test case.
+   * @return a time in milliseconds.
+   */
+  protected int getGlobalTimeout() {
+    return 30 * 1000;
+  }
+
   @Rule
   public ExpectedException thrown = ExpectedException.none();
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/9f6b08f8/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractOpenTest.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractOpenTest.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractOpenTest.java
index f9b16f4..ccf188f 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractOpenTest.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractOpenTest.java
@@ -122,7 +122,7 @@ public abstract class AbstractContractOpenTest extends AbstractFSContractTestBas
     Path path = path("testopenfiletwice.txt");
     byte[] block = dataset(TEST_FILE_LEN, 0, 255);
     //this file now has a simple rule: offset => value
-    createFile(getFileSystem(), path, false, block);
+    createFile(getFileSystem(), path, true, block);
     //open first
     FSDataInputStream instream1 = getFileSystem().open(path);
     FSDataInputStream instream2 = null;
@@ -150,7 +150,7 @@ public abstract class AbstractContractOpenTest extends AbstractFSContractTestBas
     int base = 0x40; // 64
     byte[] block = dataset(len, base, base + len);
     //this file now has a simple rule: offset => (value | 0x40)
-    createFile(getFileSystem(), path, false, block);
+    createFile(getFileSystem(), path, true, block);
     //open first
     instream = getFileSystem().open(path);
     assertEquals(base, instream.read());

http://git-wip-us.apache.org/repos/asf/hadoop/blob/9f6b08f8/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractSeekTest.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractSeekTest.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractSeekTest.java
index 3e71682..7af3cb0 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractSeekTest.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractSeekTest.java
@@ -341,7 +341,7 @@ public abstract class AbstractContractSeekTest extends AbstractFSContractTestBas
     int filesize = 10 * 1024;
     byte[] buf = dataset(filesize, 0, 255);
     Path randomSeekFile = path("testrandomseeks.bin");
-    createFile(getFileSystem(), randomSeekFile, false, buf);
+    createFile(getFileSystem(), randomSeekFile, true, buf);
     Random r = new Random();
 
     // Record the sequence of seeks and reads which trigger a failure.

http://git-wip-us.apache.org/repos/asf/hadoop/blob/9f6b08f8/hadoop-tools/hadoop-azure/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/pom.xml b/hadoop-tools/hadoop-azure/pom.xml
index 03b531b..1cde471 100644
--- a/hadoop-tools/hadoop-azure/pom.xml
+++ b/hadoop-tools/hadoop-azure/pom.xml
@@ -34,6 +34,15 @@
   <properties>
     <file.encoding>UTF-8</file.encoding>
     <downloadSources>true</downloadSources>
+    <hadoop.tmp.dir>${project.build.directory}/test</hadoop.tmp.dir>
+    <!-- are scale tests enabled ? -->
+    <fs.azure.scale.test.enabled>unset</fs.azure.scale.test.enabled>
+    <!-- Size in MB of huge files. -->
+    <fs.azure.scale.test.huge.filesize>unset</fs.azure.scale.test.huge.filesize>
+    <!-- Size in MB of the partion size in huge file uploads. -->
+    <fs.azure.scale.test.huge.partitionsize>unset</fs.azure.scale.test.huge.partitionsize>
+    <!-- Timeout in seconds for scale tests.-->
+    <fs.azure.scale.test.timeout>7200</fs.azure.scale.test.timeout>
   </properties>
 
   <build>
@@ -224,4 +233,246 @@
     </dependency>
 
   </dependencies>
+
+  <profiles>
+    <profile>
+      <id>parallel-tests</id>
+      <activation>
+        <property>
+          <name>parallel-tests</name>
+        </property>
+      </activation>
+      <build>
+        <plugins>
+          <plugin>
+            <artifactId>maven-antrun-plugin</artifactId>
+            <executions>
+              <execution>
+                <id>create-parallel-tests-dirs</id>
+                <phase>test-compile</phase>
+                <configuration>
+                  <target>
+                    <script language="javascript"><![CDATA[
+                      var baseDirs = [
+                        project.getProperty("test.build.data"),
+                        project.getProperty("test.build.dir"),
+                        project.getProperty("hadoop.tmp.dir")
+                      ];
+                      for (var i in baseDirs) {
+                        for (var j = 1; j <= ${testsThreadCount}; ++j) {
+                          var mkdir = project.createTask("mkdir");
+                          mkdir.setDir(new java.io.File(baseDirs[i], j));
+                          mkdir.perform();
+                        }
+                      }
+                    ]]></script>
+                  </target>
+                </configuration>
+                <goals>
+                  <goal>run</goal>
+                </goals>
+              </execution>
+            </executions>
+          </plugin>
+          <plugin>
+            <groupId>org.apache.maven.plugins</groupId>
+            <artifactId>maven-surefire-plugin</artifactId>
+            <executions>
+              <execution>
+                <id>default-test</id>
+                <goals>
+                  <goal>test</goal>
+                </goals>
+                <configuration>
+                  <forkCount>1</forkCount>
+                  <forkCount>${testsThreadCount}</forkCount>
+                  <reuseForks>false</reuseForks>
+                  <argLine>${maven-surefire-plugin.argLine} -DminiClusterDedicatedDirs=true</argLine>
+                  <forkedProcessTimeoutInSeconds>${fs.azure.scale.test.timeout}</forkedProcessTimeoutInSeconds>
+                  <systemPropertyVariables>
+                    <test.build.data>${test.build.data}/${surefire.forkNumber}</test.build.data>
+                    <test.build.dir>${test.build.dir}/${surefire.forkNumber}</test.build.dir>
+                    <hadoop.tmp.dir>${hadoop.tmp.dir}/${surefire.forkNumber}</hadoop.tmp.dir>
+                    <test.unique.fork.id>fork-${surefire.forkNumber}</test.unique.fork.id>
+                    <fs.azure.scale.test.enabled>${fs.azure.scale.test.enabled}</fs.azure.scale.test.enabled>
+                    <fs.azure.scale.test.huge.filesize>${fs.azure.scale.test.huge.filesize}</fs.azure.scale.test.huge.filesize>
+                    <fs.azure.scale.test.huge.huge.partitionsize>${fs.azure.scale.test.huge.partitionsize}</fs.azure.scale.test.huge.huge.partitionsize>
+                    <fs.azure.scale.test.timeout>${fs.azure.scale.test.timeout}</fs.azure.scale.test.timeout>
+                  </systemPropertyVariables>
+                  <includes>
+                    <include>**/Test*.java</include>
+                  </includes>
+                  <excludes>
+                    <exclude>**/TestRollingWindowAverage*.java</exclude>
+                  </excludes>
+                </configuration>
+              </execution>
+              <execution>
+                <id>serialized-test</id>
+                <goals>
+                  <goal>test</goal>
+                </goals>
+                <configuration>
+                  <forkCount>1</forkCount>
+                  <reuseForks>false</reuseForks>
+                  <argLine>${maven-surefire-plugin.argLine} -DminiClusterDedicatedDirs=true</argLine>
+                  <forkedProcessTimeoutInSeconds>${fs.azure.scale.test.timeout}</forkedProcessTimeoutInSeconds>
+                  <systemPropertyVariables>
+                    <test.build.data>${test.build.data}/${surefire.forkNumber}</test.build.data>
+                    <test.build.dir>${test.build.dir}/${surefire.forkNumber}</test.build.dir>
+                    <hadoop.tmp.dir>${hadoop.tmp.dir}/${surefire.forkNumber}</hadoop.tmp.dir>
+                    <test.unique.fork.id>fork-${surefire.forkNumber}</test.unique.fork.id>
+                    <fs.azure.scale.test.enabled>${fs.azure.scale.test.enabled}</fs.azure.scale.test.enabled>
+                    <fs.azure.scale.test.huge.filesize>${fs.azure.scale.test.huge.filesize}</fs.azure.scale.test.huge.filesize>
+                    <fs.azure.scale.test.huge.huge.partitionsize>${fs.azure.scale.test.huge.partitionsize}</fs.azure.scale.test.huge.huge.partitionsize>
+                    <fs.azure.scale.test.timeout>${fs.azure.scale.test.timeout}</fs.azure.scale.test.timeout>
+                  </systemPropertyVariables>
+                  <includes>
+                    <include>**/TestRollingWindowAverage*.java</include>
+                  </includes>
+                </configuration>
+              </execution>
+            </executions>
+          </plugin>
+          <plugin>
+            <groupId>org.apache.maven.plugins</groupId>
+            <artifactId>maven-failsafe-plugin</artifactId>
+            <executions>
+              <execution>
+                <id>default-integration-test</id>
+                <goals>
+                  <goal>integration-test</goal>
+                  <goal>verify</goal>
+                </goals>
+                <configuration>
+                  <forkCount>${testsThreadCount}</forkCount>
+                  <reuseForks>false</reuseForks>
+                  <argLine>${maven-surefire-plugin.argLine} -DminiClusterDedicatedDirs=true</argLine>
+                  <forkedProcessTimeoutInSeconds>${fs.azure.scale.test.timeout}</forkedProcessTimeoutInSeconds>
+                  <systemPropertyVariables>
+                    <!-- Tell tests that they are being executed in parallel -->
+                    <test.parallel.execution>true</test.parallel.execution>
+                    <test.build.data>${test.build.data}/${surefire.forkNumber}</test.build.data>
+                    <test.build.dir>${test.build.dir}/${surefire.forkNumber}</test.build.dir>
+                    <hadoop.tmp.dir>${hadoop.tmp.dir}/${surefire.forkNumber}</hadoop.tmp.dir>
+
+                    <!-- Due to a Maven quirk, setting this to just -->
+                    <!-- surefire.forkNumber won't do the parameter -->
+                    <!-- substitution.  Putting a prefix in front of it like -->
+                    <!-- "fork-" makes it work. -->
+                    <test.unique.fork.id>fork-${surefire.forkNumber}</test.unique.fork.id>
+                    <!-- Propagate scale parameters -->
+                    <fs.azure.scale.test.enabled>${fs.azure.scale.test.enabled}</fs.azure.scale.test.enabled>
+                    <fs.azure.scale.test.huge.filesize>${fs.azure.scale.test.huge.filesize}</fs.azure.scale.test.huge.filesize>
+                    <fs.azure.scale.test.huge.huge.partitionsize>${fs.azure.scale.test.huge.partitionsize}</fs.azure.scale.test.huge.huge.partitionsize>
+                    <fs.azure.scale.test.timeout>${fs.azure.scale.test.timeout}</fs.azure.scale.test.timeout>
+                  </systemPropertyVariables>
+                  <!-- Some tests cannot run in parallel.  Tests that cover -->
+                  <!-- access to the root directory must run in isolation -->
+                  <!-- from anything else that could modify the bucket. -->
+                  <!-- azure tests that cover multi-part upload must run in -->
+                  <!-- isolation, because the file system is configured to -->
+                  <!-- purge existing multi-part upload data on -->
+                  <!-- initialization.  MiniYARNCluster has not yet been -->
+                  <!-- changed to handle parallel test execution gracefully. -->
+                  <!-- Exclude all of these tests from parallel execution, -->
+                  <!-- and instead run them sequentially in a separate -->
+                  <!-- Surefire execution step later. -->
+                  <includes>
+                    <include>**/ITest*.java</include>
+                  </includes>
+                  <excludes>
+                    <exclude>**/ITestFileSystemOperationsExceptionHandlingMultiThreaded.java</exclude>
+                    <exclude>**/ITestFileSystemOperationsWithThreads.java</exclude>
+                    <exclude>**/ITestOutOfBandAzureBlobOperationsLive.java</exclude>
+                    <exclude>**/ITestNativeAzureFileSystemAuthorizationWithOwner.java</exclude>
+                    <exclude>**/ITestNativeAzureFileSystemConcurrencyLive.java</exclude>
+                    <exclude>**/ITestNativeAzureFileSystemLive.java</exclude>
+                    <exclude>**/ITestNativeAzureFSPageBlobLive.java</exclude>
+                    <exclude>**/ITestWasbRemoteCallHelper.java</exclude>
+                    <exclude>**/ITestBlockBlobInputStream.java</exclude>
+                  </excludes>
+                </configuration>
+              </execution>
+              <!-- Do a sequential run for tests that cannot handle -->
+              <!-- parallel execution. -->
+              <execution>
+                <id>sequential-integration-tests</id>
+                <goals>
+                  <goal>integration-test</goal>
+                  <goal>verify</goal>
+                </goals>
+                <configuration>
+                  <forkedProcessTimeoutInSeconds>${fs.azure.scale.test.timeout}</forkedProcessTimeoutInSeconds>
+                  <systemPropertyVariables>
+                    <test.parallel.execution>false</test.parallel.execution>
+                    <fs.azure.scale.test.enabled>${fs.azure.scale.test.enabled}</fs.azure.scale.test.enabled>
+                    <fs.azure.scale.test.huge.filesize>${fs.azure.scale.test.huge.filesize}</fs.azure.scale.test.huge.filesize>
+                    <fs.azure.scale.test.huge.huge.partitionsize>${fs.azure.scale.test.huge.partitionsize}</fs.azure.scale.test.huge.huge.partitionsize>
+                    <fs.azure.scale.test.timeout>${fs.azure.scale.test.timeout}</fs.azure.scale.test.timeout>
+                  </systemPropertyVariables>
+                  <includes>
+                    <include>**/ITestFileSystemOperationsExceptionHandlingMultiThreaded.java</include>
+                    <include>**/ITestFileSystemOperationsWithThreads.java</include>
+                    <include>**/ITestOutOfBandAzureBlobOperationsLive.java</include>
+                    <include>**/ITestNativeAzureFileSystemAuthorizationWithOwner.java</include>
+                    <include>**/ITestNativeAzureFileSystemConcurrencyLive.java</include>
+                    <include>**/ITestNativeAzureFileSystemLive.java</include>
+                    <include>**/ITestWasbRemoteCallHelper.java</include>
+                    <include>**/ITestBlockBlobInputStream.java</include>
+                  </includes>
+                </configuration>
+              </execution>
+            </executions>
+          </plugin>
+        </plugins>
+      </build>
+    </profile>
+    <profile>
+      <id>sequential-tests</id>
+      <activation>
+        <property>
+          <name>!parallel-tests</name>
+        </property>
+      </activation>
+      <build>
+        <plugins>
+          <plugin>
+            <groupId>org.apache.maven.plugins</groupId>
+            <artifactId>maven-failsafe-plugin</artifactId>
+            <executions>
+              <execution>
+                <goals>
+                  <goal>integration-test</goal>
+                  <goal>verify</goal>
+                </goals>
+                <configuration>
+                  <systemPropertyVariables>
+                    <!-- Propagate scale parameters -->
+                    <fs.azure.scale.test.enabled>${fs.azure.scale.test.enabled}</fs.azure.scale.test.enabled>
+                    <fs.azure.scale.test.huge.filesize>${fs.azure.scale.test.huge.filesize}</fs.azure.scale.test.huge.filesize>
+                    <fs.azure.scale.test.timeout>${fs.azure.scale.test.timeout}</fs.azure.scale.test.timeout>
+                  </systemPropertyVariables>
+                  <forkedProcessTimeoutInSeconds>${fs.azure.scale.test.timeout}</forkedProcessTimeoutInSeconds>
+                </configuration>
+              </execution>
+            </executions>
+          </plugin>
+        </plugins>
+      </build>
+    </profile>
+
+    <!-- Turn on scale tests-->
+    <profile>
+      <id>scale</id>
+      <activation>
+        <property>
+          <name>scale</name>
+        </property>
+      </activation>
+      <properties>
+        <fs.azure.scale.test.enabled>true</fs.azure.scale.test.enabled>
+      </properties>
+    </profile>
+  </profiles>
 </project>

http://git-wip-us.apache.org/repos/asf/hadoop/blob/9f6b08f8/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java
index 639862f..f1031b4 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java
@@ -346,7 +346,7 @@ public class AzureNativeFileSystemStore implements NativeFileSystemStore {
   private String delegationToken;
 
   /** The error message template when container is not accessible. */
-  static final String NO_ACCESS_TO_CONTAINER_MSG = "No credentials found for "
+  public static final String NO_ACCESS_TO_CONTAINER_MSG = "No credentials found for "
       + "account %s in the configuration, and its container %s is not "
       + "accessible using anonymous credentials. Please check if the container "
       + "exists first. If it is not publicly available, you have to provide "

http://git-wip-us.apache.org/repos/asf/hadoop/blob/9f6b08f8/hadoop-tools/hadoop-azure/src/site/markdown/index.md
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/site/markdown/index.md b/hadoop-tools/hadoop-azure/src/site/markdown/index.md
index 466bf0b..876d7cc 100644
--- a/hadoop-tools/hadoop-azure/src/site/markdown/index.md
+++ b/hadoop-tools/hadoop-azure/src/site/markdown/index.md
@@ -519,96 +519,8 @@ The maximum number of entries that that cache can hold can be customized using t
       <value>true</value>
     </property>
 ```
-## Testing the hadoop-azure Module
 
-The hadoop-azure module includes a full suite of unit tests.  Most of the tests
-will run without additional configuration by running `mvn test`.  This includes
-tests against mocked storage, which is an in-memory emulation of Azure Storage.
-
-A selection of tests can run against the
-[Azure Storage Emulator](http://msdn.microsoft.com/en-us/library/azure/hh403989.aspx)
-which is a high-fidelity emulation of live Azure Storage.  The emulator is
-sufficient for high-confidence testing.  The emulator is a Windows executable
-that runs on a local machine.
-
-To use the emulator, install Azure SDK 2.3 and start the storage emulator.  Then,
-edit `src/test/resources/azure-test.xml` and add the following property:
-
-```xml
-<property>
-  <name>fs.azure.test.emulator</name>
-  <value>true</value>
-</property>
-```
-
-There is a known issue when running tests with the emulator.  You may see the
-following failure message:
-
-    com.microsoft.windowsazure.storage.StorageException: The value for one of the HTTP headers is not in the correct format.
-
-To resolve this, restart the Azure Emulator.  Ensure it v3.2 or later.
-
-It's also possible to run tests against a live Azure Storage account by saving a
-file to `src/test/resources/azure-auth-keys.xml` and setting
-the name of the storage account and its access key.
-
-For example:
-
-```xml
-<?xml version="1.0"?>
-<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
-<configuration>
-  <property>
-    <name>fs.azure.test.account.name</name>
-    <value>{ACCOUNTNAME}.blob.core.windows.net</value>
-  </property>
-  <property>
-    <name>fs.azure.account.key.{ACCOUNTNAME}.blob.core.windows.net</name>
-    <value>{ACCOUNT ACCESS KEY}</value>
-  </property>
-</configuration>
-```
-
-To run contract tests, set the WASB file system URI in `src/test/resources/azure-auth-keys.xml`
-and the account access key. For example:
-
-```xml
-<?xml version="1.0"?>
-<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
-<configuration>
-  <property>
-    <name>fs.contract.test.fs.wasb</name>
-    <value>wasb://{CONTAINERNAME}@{ACCOUNTNAME}.blob.core.windows.net</value>
-    <description>The name of the azure file system for testing.</description>
-  </property>
-  <property>
-    <name>fs.azure.account.key.{ACCOUNTNAME}.blob.core.windows.net</name>
-    <value>{ACCOUNT ACCESS KEY}</value>
-  </property>
-</configuration>
-```
-
-Overall, to run all the tests using `mvn test`,  a sample `azure-auth-keys.xml` is like following:
-
-```xml
-<?xml version="1.0"?>
-<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
-<configuration>
-  <property>
-    <name>fs.azure.test.account.name</name>
-    <value>{ACCOUNTNAME}.blob.core.windows.net</value>
-  </property>
-  <property>
-    <name>fs.azure.account.key.{ACCOUNTNAME}.blob.core.windows.net</name>
-    <value>{ACCOUNT ACCESS KEY}</value>
-  </property>
-  <property>
-    <name>fs.contract.test.fs.wasb</name>
-    <value>wasb://{CONTAINERNAME}@{ACCOUNTNAME}.blob.core.windows.net</value>
-  </property>
-</configuration>
-```
-
-DO NOT ADD `azure-auth-keys.xml` TO REVISION CONTROL.  The keys to your Azure
-Storage account are a secret and must not be shared.
+## Further Reading
 
+* [Testing the Azure WASB client](testing_azure.html).
+* MSDN article, [Understanding Block Blobs, Append Blobs, and Page Blobs](https://docs.microsoft.com/en-us/rest/api/storageservices/understanding-block-blobs--append-blobs--and-page-blobs)

http://git-wip-us.apache.org/repos/asf/hadoop/blob/9f6b08f8/hadoop-tools/hadoop-azure/src/site/markdown/testing_azure.md
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/site/markdown/testing_azure.md b/hadoop-tools/hadoop-azure/src/site/markdown/testing_azure.md
new file mode 100644
index 0000000..b58e68b
--- /dev/null
+++ b/hadoop-tools/hadoop-azure/src/site/markdown/testing_azure.md
@@ -0,0 +1,576 @@
+<!---
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License. See accompanying LICENSE file.
+-->
+
+# Testing the Azure WASB client
+
+<!-- MACRO{toc|fromDepth=0|toDepth=5} -->
+
+This module includes both unit tests, which can run in isolation without
+connecting to the Azure Storage service, and integration tests, which require a working
+connection to interact with a container.  Unit test suites follow the naming
+convention `Test*.java`.  Integration tests follow the naming convention
+`ITest*.java`.
+
+## Policy for submitting patches which affect the `hadoop-azure` module.
+
+The Apache Jenkins infrastucture does not run any cloud integration tests,
+due to the need to keep credentials secure.
+
+### The submitter of any patch is required to run all the integration tests and declare which Azure region they used.
+
+This is important: **patches which do not include this declaration will be ignored**
+
+This policy has proven to be the only mechanism to guarantee full regression
+testing of code changes. Why the declaration of region? Two reasons
+
+1. It helps us identify regressions which only surface against specific endpoints.
+1. It forces the submitters to be more honest about their testing. It's easy
+to lie, "yes, I tested this". To say "yes, I tested this against Azure US-west"
+is a more specific lie and harder to make. And, if you get caught out: you
+lose all credibility with the project.
+
+You don't need to test from a VM within the Azure infrastructure, all you need
+are credentials.
+
+It's neither hard nor expensive to run the tests; if you can't,
+there's no guarantee your patch works. The reviewers have enough to do, and
+don't have the time to do these tests, especially as every failure will simply
+make for a slow iterative development.
+
+Please: run the tests. And if you don't, we are sorry for declining your
+patch, but we have to.
+
+
+### What if there's an intermittent failure of a test?
+
+Some of the tests do fail intermittently, especially in parallel runs.
+If this happens, try to run the test on its own to see if the test succeeds.
+
+If it still fails, include this fact in your declaration. We know some tests
+are intermittently unreliable.
+
+### What if the tests are timing out or failing over my network connection?
+
+The tests are designed to be configurable for different
+timeouts. If you are seeing problems and this configuration isn't working,
+that's a sign of the configuration mechanism isn't complete. If it's happening
+in the production code, that could be a sign of a problem which may surface
+over long-haul connections. Please help us identify and fix these problems
+&mdash; especially as you are the one best placed to verify the fixes work.
+
+## Setting up the tests
+
+## Testing the `hadoop-azure` Module
+
+The `hadoop-azure` module includes a full suite of unit tests.  Many of the tests
+will run without additional configuration by running `mvn test`.  This includes
+tests against mocked storage, which is an in-memory emulation of Azure Storage.
+
+The integration tests are designed to test directly against an Azure storage
+service, and require an account and credentials in order to run.
+
+This is done by creating the file to `src/test/resources/azure-auth-keys.xml`
+and setting the name of the storage account and its access key.
+
+For example:
+
+```xml
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<configuration>
+  <property>
+    <name>fs.azure.test.account.name</name>
+    <value>{ACCOUNTNAME}.blob.core.windows.net</value>
+  </property>
+  <property>
+    <name>fs.azure.account.key.{ACCOUNTNAME}.blob.core.windows.net</name>
+    <value>{ACCOUNT ACCESS KEY}</value>
+  </property>
+</configuration>
+```
+
+To run contract tests, set the WASB file system URI in `src/test/resources/azure-auth-keys.xml`
+and the account access key. For example:
+
+```xml
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<configuration>
+  <property>
+    <name>fs.contract.test.fs.wasb</name>
+    <value>wasb://{CONTAINERNAME}@{ACCOUNTNAME}.blob.core.windows.net</value>
+    <description>The name of the azure file system for testing.</description>
+  </property>
+  <property>
+    <name>fs.azure.account.key.{ACCOUNTNAME}.blob.core.windows.net</name>
+    <value>{ACCOUNT ACCESS KEY}</value>
+  </property>
+</configuration>
+```
+
+Overall, to run all the tests using `mvn test`,  a sample `azure-auth-keys.xml` is like following:
+
+```xml
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<configuration>
+  <property>
+    <name>fs.azure.test.account.name</name>
+    <value>{ACCOUNTNAME}.blob.core.windows.net</value>
+  </property>
+  <property>
+    <name>fs.azure.account.key.{ACCOUNTNAME}.blob.core.windows.net</name>
+    <value>{ACCOUNT ACCESS KEY}</value>
+  </property>
+  <property>
+    <name>fs.contract.test.fs.wasb</name>
+    <value>wasb://{CONTAINERNAME}@{ACCOUNTNAME}.blob.core.windows.net</value>
+  </property>
+</configuration>
+```
+
+DO NOT ADD `azure-auth-keys.xml` TO REVISION CONTROL.  The keys to your Azure
+Storage account are a secret and must not be shared.
+
+
+## Running the Tests
+
+After completing the configuration, execute the test run through Maven.
+
+```bash
+mvn -T 1C clean verify
+```
+
+It's also possible to execute multiple test suites in parallel by passing the
+`parallel-tests` property on the command line.  The tests spend most of their
+time blocked on network I/O, so running in parallel tends to
+complete full test runs faster.
+
+```bash
+mvn -T 1C -Dparallel-tests clean verify
+```
+
+Some tests must run with exclusive access to the storage container, so even with the
+`parallel-tests` property, several test suites will run in serial in a separate
+Maven execution step after the parallel tests.
+
+By default, `parallel-tests` runs 4 test suites concurrently.  This can be tuned
+by passing the `testsThreadCount` property.
+
+```bash
+mvn -T 1C -Dparallel-tests -DtestsThreadCount=8 clean verify
+```
+
+<!---
+To run just unit tests, which do not require Azure connectivity or credentials,
+use any of the above invocations, but switch the goal to `test` instead of
+`verify`.
+-->
+
+```bash
+mvn -T 1C clean test
+
+mvn -T 1C -Dparallel-tests clean test
+
+mvn -T 1C -Dparallel-tests -DtestsThreadCount=8 clean test
+```
+
+To run only a specific named subset of tests, pass the `test` property for unit
+tests or the `it.test` property for integration tests.
+
+```bash
+mvn -T 1C clean test -Dtest=TestRollingWindowAverage
+
+mvn -T 1C clean verify -Dscale -Dit.test=ITestFileSystemOperationExceptionMessage -Dtest=none
+
+mvn -T 1C clean verify -Dtest=none -Dit.test=ITest*
+
+```
+
+Note
+
+1. When running a specific subset of tests, the patterns passed in `test`
+and `it.test` override the configuration of which tests need to run in isolation
+in a separate serial phase (mentioned above).  This can cause unpredictable
+results, so the recommendation is to avoid passing `parallel-tests` in
+combination with `test` or `it.test`.  If you know that you are specifying only
+tests that can run safely in parallel, then it will work.  For wide patterns,
+like `ITest*` shown above, it may cause unpredictable test failures.
+
+2. The command line shell may try to expand the "*" and sometimes the "#" symbols
+in test patterns. In such situations, escape the character it with a "\\" prefix.
+Example:
+
+          mvn -T 1C clean verify -Dtest=none -Dit.test=ITest\*
+
+
+## Viewing the results
+
+Integration test results and logs are stored in `target/failsafe-reports/`.
+An HTML report can be generated during site generation, or with the `surefire-report`
+plugin:
+
+```bash
+
+# for the unit tests
+mvn -T 1C surefire-report:report-only
+
+# for the integration tests
+mvn -T 1C surefire-report:failsafe-report-only
+
+# all reports for this module
+mvn -T 1C site:site
+```
+
+## Scale Tests
+
+There are a set of tests designed to measure the scalability and performance
+at scale of the filesystem client, *Scale Tests*. Tests include: creating
+and traversing directory trees, uploading large files, renaming them,
+deleting them, seeking through the files, performing random IO, and others.
+This makes them a foundational part of the benchmarking.
+
+By their very nature they are slow. And, as their execution time is often
+limited by bandwidth between the computer running the tests and the Azure endpoint,
+parallel execution does not speed these tests up.
+
+### Enabling the Scale Tests
+
+The tests are enabled if the `scale` property is set in the maven build
+this can be done regardless of whether or not the parallel test profile
+is used
+
+```bash
+mvn -T 1C verify -Dscale
+
+mvn -T 1C verify -Dparallel-tests -Dscale -DtestsThreadCount=8
+```
+
+The most bandwidth intensive tests (those which upload data) always run
+sequentially; those which are slow due to HTTPS setup costs or server-side
+actions are included in the set of parallelized tests.
+
+
+### Scale test tuning options
+
+
+Some of the tests can be tuned from the maven build or from the
+configuration file used to run the tests.
+
+```bash
+mvn -T 1C verify -Dparallel-tests -Dscale -DtestsThreadCount=8 -Dfs.azure.scale.test.huge.filesize=128M
+```
+
+The algorithm is
+
+1. The value is queried from the configuration file, using a default value if
+it is not set.
+1. The value is queried from the JVM System Properties, where it is passed
+down by maven.
+1. If the system property is null, an empty string, or it has the value `unset`,
+then the configuration value is used. The `unset` option is used to
+[work round a quirk in maven property propagation](http://stackoverflow.com/questions/7773134/null-versus-empty-arguments-in-maven).
+
+Only a few properties can be set this way; more will be added.
+
+| Property | Meaninging |
+|-----------|-------------|
+| `fs.azure.scale.test.huge.filesize`| Size for huge file uploads |
+| `fs.azure.scale.test.huge.huge.partitionsize`| Size for partitions in huge file uploads |
+
+The file and partition sizes are numeric values with a k/m/g/t/p suffix depending
+on the desired size. For example: 128M, 128m, 2G, 2G, 4T or even 1P.
+
+#### Scale test configuration options
+
+Some scale tests perform multiple operations (such as creating many directories).
+
+The exact number of operations to perform is configurable in the option
+`scale.test.operation.count`
+
+```xml
+<property>
+  <name>scale.test.operation.count</name>
+  <value>10</value>
+</property>
+```
+
+Larger values generate more load, and are recommended when testing locally,
+or in batch runs.
+
+Smaller values results in faster test runs, especially when the object
+store is a long way away.
+
+Operations which work on directories have a separate option: this controls
+the width and depth of tests creating recursive directories. Larger
+values create exponentially more directories, with consequent performance
+impact.
+
+```xml
+<property>
+  <name>scale.test.directory.count</name>
+  <value>2</value>
+</property>
+```
+
+DistCp tests targeting Azure support a configurable file size.  The default is
+10 MB, but the configuration value is expressed in KB so that it can be tuned
+smaller to achieve faster test runs.
+
+```xml
+<property>
+  <name>scale.test.distcp.file.size.kb</name>
+  <value>10240</value>
+</property>
+```
+
+Azure-specific scale test properties are
+
+##### `fs.azure.scale.test.huge.filesize`: size in MB for "Huge file tests".
+
+The Huge File tests validate Azure storages's ability to handle large files —the property
+`fs.azure.scale.test.huge.filesize` declares the file size to use.
+
+```xml
+<property>
+  <name>fs.azure.scale.test.huge.filesize</name>
+  <value>200M</value>
+</property>
+```
+
+Tests at this scale are slow: they are best executed from hosts running in
+the cloud infrastructure where the storage endpoint is based.
+
+## Using the emulator
+
+A selection of tests can run against the
+[Azure Storage Emulator](http://msdn.microsoft.com/en-us/library/azure/hh403989.aspx)
+which is a high-fidelity emulation of live Azure Storage.  The emulator is
+sufficient for high-confidence testing.  The emulator is a Windows executable
+that runs on a local machine.
+
+To use the emulator, install Azure SDK 2.3 and start the storage emulator.  Then,
+edit `src/test/resources/azure-test.xml` and add the following property:
+
+```xml
+<property>
+  <name>fs.azure.test.emulator</name>
+  <value>true</value>
+</property>
+```
+
+There is a known issue when running tests with the emulator.  You may see the
+following failure message:
+
+    com.microsoft.windowsazure.storage.StorageException: The value for one of the HTTP headers is not in the correct format.
+
+To resolve this, restart the Azure Emulator.  Ensure it is v3.2 or later.
+
+
+## Debugging Test failures
+
+Logging at debug level is the standard way to provide more diagnostics output;
+after setting this rerun the tests
+
+```properties
+log4j.logger.org.apache.hadoop.fs.azure=DEBUG
+```
+
+## Adding new tests
+
+New tests are always welcome. Bear in mind that we need to keep costs
+and test time down, which is done by
+
+* Not duplicating tests.
+* Being efficient in your use of Hadoop API calls.
+* Isolating large/slow tests into the "scale" test group.
+* Designing all tests to execute in parallel (where possible).
+* Adding new probes and predicates into existing tests, albeit carefully.
+
+*No duplication*: if an operation is tested elsewhere, don't repeat it. This
+applies as much for metadata operations as it does for bulk IO. If a new
+test case is added which completely obsoletes an existing test, it is OK
+to cut the previous one —after showing that coverage is not worsened.
+
+*Efficient*: prefer the `getFileStatus()` and examining the results, rather than
+call to `exists()`, `isFile()`, etc.
+
+*Fail with useful information:* provide as much diagnostics as possible
+on a failure. Using `org.apache.hadoop.fs.contract.ContractTestUtils` to make
+assertions about the state of a filesystem helps here.
+
+*Isolating Scale tests*. Any test doing large amounts of IO MUST extend the
+class `AbstractAzureScaleTest`, so only running if `scale` is defined on a build,
+supporting test timeouts configurable by the user. Scale tests should also
+support configurability as to the actual size of objects/number of operations,
+so that behavior at different scale can be verified.
+
+*Designed for parallel execution*. A key need here is for each test suite to work
+on isolated parts of the filesystem. Subclasses of `AbstractWasbTestBase`
+SHOULD use the `path()`, `methodpath()` and `blobpath()` methods,
+to build isolated paths. Tests MUST NOT assume that they have exclusive access
+to a bucket.
+
+*Extending existing tests where appropriate*. This recommendation goes
+against normal testing best practise of "test one thing per method".
+Because it is so slow to create directory trees or upload large files, we do
+not have that luxury. All the tests against real endpoints are integration
+tests where sharing test setup and teardown saves time and money.
+
+A standard way to do this is to extend existing tests with some extra predicates,
+rather than write new tests. When doing this, make sure that the new predicates
+fail with meaningful diagnostics, so any new problems can be easily debugged
+from test logs.
+
+
+### Requirements of new Tests
+
+
+This is what we expect from new tests; they're an extension of the normal
+Hadoop requirements, based on the need to work with remote servers whose
+use requires the presence of secret credentials, where tests may be slow,
+and where finding out why something failed from nothing but the test output
+is critical.
+
+#### Subclasses Existing Shared Base Blasses
+
+There are a set of base classes which should be extended for Azure tests and
+integration tests.
+
+##### `org.apache.hadoop.fs.azure.AbstractWasbTestWithTimeout`
+
+This extends the junit `Assert` class with thread names and timeouts,
+the default timeout being set in `AzureTestConstants.AZURE_TEST_TIMEOUT` to
+ten minutes. The thread names are set to aid analyzing the stack trace of
+a test: a `jstack` call can be used to
+
+##### `org.apache.hadoop.fs.azure.AbstractWasbTestBase`
+
+The base class for tests which use `AzureBlobStorageTestAccount` to create
+mock or live Azure clients; in test teardown it tries to clean up store state.
+
+1. This class requires subclasses to implement `createTestAccount()` to create
+a mock or real test account.
+
+1. The configuration used to create a test account *should* be that from
+`createConfiguration()`; this can be extended in subclasses to tune the settings.
+
+
+##### `org.apache.hadoop.fs.azure.integration.AbstractAzureScaleTest`
+
+This extends `AbstractWasbTestBase` for scale tests; those test which
+only run when `-Dscale` is used to select the "scale" profile.
+These tests have a timeout of 30 minutes, so as to support slow test runs.
+
+Having shared base classes help reduces future maintenance. Please
+use them.
+
+#### Secure
+
+Don't ever log credentials. The credential tests go out of their way to
+not provide meaningful logs or assertion messages precisely to avoid this.
+
+#### Efficient of Time and Money
+
+This means efficient in test setup/teardown, and, ideally, making use of
+existing public datasets to save setup time and tester cost.
+
+
+The reference example is `ITestAzureHugeFiles`:. This marks the test suite as
+`@FixMethodOrder(MethodSorters.NAME_ASCENDING)` then orders the test cases such
+that each test case expects the previous test to have completed (here: uploaded a file,
+renamed a file, ...). This provides for independent tests in the reports, yet still
+permits an ordered sequence of operations. Do note the use of `Assume.assume()`
+to detect when the preconditions for a single test case are not met, hence,
+the tests become skipped, rather than fail with a trace which is really a false alarm.
+
+
+### Works Over Long-haul Links
+
+As well as making file size and operation counts scaleable, this includes
+making test timeouts adequate. The Scale tests make this configurable; it's
+hard coded to ten minutes in `AbstractAzureIntegrationTest()`; subclasses can
+change this by overriding `getTestTimeoutMillis()`.
+
+Equally importantly: support proxies, as some testers need them.
+
+
+### Provides Diagnostics and timing information
+
+1. Create logs, log things.
+1. you can use `AbstractWasbTestBase.describe(format-string, args)` here; it
+adds some newlines so as to be easier to spot.
+1. Use `ContractTestUtils.NanoTimer` to measure the duration of operations,
+and log the output.
+
+#### Fails Meaningfully
+
+The `ContractTestUtils` class contains a whole set of assertions for making
+statements about the expected state of a filesystem, e.g.
+`assertPathExists(FS, path)`, `assertPathDoesNotExists(FS, path)`, and others.
+These do their best to provide meaningful diagnostics on failures (e.g. directory
+listings, file status, ...), so help make failures easier to understand.
+
+At the very least, *do not use `assertTrue()` or `assertFalse()` without
+including error messages*.
+
+
+### Cleans Up Afterwards
+
+Keeps costs down.
+
+1. Do not only cleanup if a test case completes successfully; test suite
+teardown must do it.
+1. That teardown code must check for the filesystem and other fields being
+null before the cleanup. Why? If test setup fails, the teardown methods still
+get called.
+
+### Works Reliably
+
+We really appreciate this &mdash; you will too.
+
+
+## Tips
+
+### How to keep your credentials really safe
+
+Although the `auth-keys.xml` file is marged as ignored in git and subversion,
+it is still in your source tree, and there's always that risk that it may
+creep out.
+
+You can avoid this by keeping your keys outside the source tree and
+using an absolute XInclude reference to it.
+
+```xml
+<configuration>
+
+  <include xmlns="http://www.w3.org/2001/XInclude"
+    href="file:///users/qe/.auth-keys.xml" />
+
+</configuration>
+```
+
+### Cleaning up Containers
+
+The Azure tests create containers with the prefix `"wasbtests-"` and delete
+them after the test runs. If a test run is interrupted, these containers
+may not get deleted. There is a special test case which can be manually invoked
+to list and delete these, `CleanupTestContainers`
+
+```bash
+mvn test -Dtest=CleanupTestContainers
+```
+
+This will delete the containers; the output log of the test run will
+provide the details and summary of the operation.

http://git-wip-us.apache.org/repos/asf/hadoop/blob/9f6b08f8/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/AbstractWasbTestBase.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/AbstractWasbTestBase.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/AbstractWasbTestBase.java
index d04a19c..0d3a06c 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/AbstractWasbTestBase.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/AbstractWasbTestBase.java
@@ -18,15 +18,21 @@
 
 package org.apache.hadoop.fs.azure;
 
-import static org.junit.Assume.assumeNotNull;
+import java.io.IOException;
 
-import com.google.common.annotations.VisibleForTesting;
 import org.apache.hadoop.conf.Configuration;
 import org.junit.After;
 import org.junit.Before;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.azure.integration.AzureTestConstants;
+import org.apache.hadoop.io.IOUtils;
+
+import static org.junit.Assume.assumeNotNull;
+import static org.apache.hadoop.fs.azure.integration.AzureTestUtils.*;
+
 /**
  * Abstract test class that provides basic setup and teardown of testing Azure
  * Storage account.  Each subclass defines a different set of test cases to run
@@ -34,41 +40,137 @@ import org.slf4j.LoggerFactory;
  * to run those tests.  The returned account might integrate with Azure Storage
  * directly or it might be a mock implementation.
  */
-public abstract class AbstractWasbTestBase {
+public abstract class AbstractWasbTestBase extends AbstractWasbTestWithTimeout
+    implements AzureTestConstants {
 
   protected static final Logger LOG =
       LoggerFactory.getLogger(AbstractWasbTestBase.class);
 
-  @VisibleForTesting
   protected NativeAzureFileSystem fs;
-  private AzureBlobStorageTestAccount testAccount;
+  protected AzureBlobStorageTestAccount testAccount;
 
   @Before
   public void setUp() throws Exception {
-    testAccount = createTestAccount();
-    if (testAccount != null) {
-      fs = testAccount.getFileSystem();
-    }
-    assumeNotNull(testAccount);
+    AzureBlobStorageTestAccount account = createTestAccount();
+    assumeNotNull(account);
+    bindToTestAccount(account);
   }
 
   @After
   public void tearDown() throws Exception {
-    if (testAccount != null) {
-      testAccount.cleanup();
-      testAccount = null;
-      fs = null;
-    }
+    describe("closing test account and filesystem");
+    testAccount = cleanupTestAccount(testAccount);
+    IOUtils.closeStream(fs);
+    fs = null;
   }
 
-  public Configuration getConfiguration() {
-    return new Configuration();
+  /**
+   * Create the configuration to use when creating a test account.
+   * Subclasses can override this to tune the test account configuration.
+   * @return a configuration.
+   */
+  public Configuration createConfiguration() {
+    return AzureBlobStorageTestAccount.createTestConfiguration();
   }
 
+  /**
+   * Create the test account.
+   * Subclasses must implement this.
+   * @return the test account.
+   * @throws Exception
+   */
   protected abstract AzureBlobStorageTestAccount createTestAccount()
       throws Exception;
 
+  /**
+   * Get the test account.
+   * @return the current test account.
+   */
   protected AzureBlobStorageTestAccount getTestAccount() {
     return testAccount;
   }
+
+  /**
+   * Get the filesystem
+   * @return the current filesystem.
+   */
+  protected NativeAzureFileSystem getFileSystem() {
+    return fs;
+  }
+
+  /**
+   * Get the configuration used to create the filesystem
+   * @return the configuration of the test FS
+   */
+  protected Configuration getConfiguration() {
+    return getFileSystem().getConf();
+  }
+
+  /**
+   * Bind to a new test account; closing any existing one.
+   * This updates the test account returned in {@link #getTestAccount()}
+   * and the filesystem in {@link #getFileSystem()}.
+   * @param account new test account
+   */
+  protected void bindToTestAccount(AzureBlobStorageTestAccount account) {
+    // clean any existing test account
+    cleanupTestAccount(testAccount);
+    IOUtils.closeStream(fs);
+    testAccount = account;
+    if (testAccount != null) {
+      fs = testAccount.getFileSystem();
+    }
+  }
+
+  /**
+   * Return a path to a blob which will be unique for this fork.
+   * @param filepath filepath
+   * @return a path under the default blob directory
+   * @throws IOException
+   */
+  protected Path blobPath(String filepath) throws IOException {
+    return blobPathForTests(getFileSystem(), filepath);
+  }
+
+  /**
+   * Create a path under the test path provided by
+   * the FS contract.
+   * @param filepath path string in
+   * @return a path qualified by the test filesystem
+   * @throws IOException IO problems
+   */
+  protected Path path(String filepath) throws IOException {
+    return pathForTests(getFileSystem(), filepath);
+  }
+
+  /**
+   * Return a path bonded to this method name, unique to this fork during
+   * parallel execution.
+   * @return a method name unique to (fork, method).
+   * @throws IOException IO problems
+   */
+  protected Path methodPath() throws IOException {
+    return path(methodName.getMethodName());
+  }
+
+  /**
+   * Return a blob path bonded to this method name, unique to this fork during
+   * parallel execution.
+   * @return a method name unique to (fork, method).
+   * @throws IOException IO problems
+   */
+  protected Path methodBlobPath() throws IOException {
+    return blobPath(methodName.getMethodName());
+  }
+
+  /**
+   * Describe a test in the logs.
+   * @param text text to print
+   * @param args arguments to format in the printing
+   */
+  protected void describe(String text, Object... args) {
+    LOG.info("\n\n{}: {}\n",
+        methodName.getMethodName(),
+        String.format(text, args));
+  }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/9f6b08f8/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/AbstractWasbTestWithTimeout.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/AbstractWasbTestWithTimeout.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/AbstractWasbTestWithTimeout.java
new file mode 100644
index 0000000..b7076a4
--- /dev/null
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/AbstractWasbTestWithTimeout.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.azure;
+
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Rule;
+import org.junit.rules.TestName;
+import org.junit.rules.Timeout;
+
+import org.apache.hadoop.fs.azure.integration.AzureTestConstants;
+
+/**
+ * Base class for any Wasb test with timeouts & named threads.
+ * This class does not attempt to bind to Azure.
+ */
+public class AbstractWasbTestWithTimeout extends Assert {
+
+  /**
+   * The name of the current method.
+   */
+  @Rule
+  public TestName methodName = new TestName();
+  /**
+   * Set the timeout for every test.
+   * This is driven by the value returned by {@link #getTestTimeoutMillis()}.
+   */
+  @Rule
+  public Timeout testTimeout = new Timeout(getTestTimeoutMillis());
+
+  /**
+   * Name the junit thread for the class. This will overridden
+   * before the individual test methods are run.
+   */
+  @BeforeClass
+  public static void nameTestThread() {
+    Thread.currentThread().setName("JUnit");
+  }
+
+  /**
+   * Name the thread to the current test method.
+   */
+  @Before
+  public void nameThread() {
+    Thread.currentThread().setName("JUnit-" + methodName.getMethodName());
+  }
+
+  /**
+   * Override point: the test timeout in milliseconds.
+   * @return a timeout in milliseconds
+   */
+  protected int getTestTimeoutMillis() {
+    return AzureTestConstants.AZURE_TEST_TIMEOUT;
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/9f6b08f8/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/AzureBlobStorageTestAccount.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/AzureBlobStorageTestAccount.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/AzureBlobStorageTestAccount.java
index 7fa59ce..5b36c87 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/AzureBlobStorageTestAccount.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/AzureBlobStorageTestAccount.java
@@ -21,12 +21,15 @@ package org.apache.hadoop.fs.azure;
 import com.microsoft.azure.storage.*;
 import com.microsoft.azure.storage.blob.*;
 import com.microsoft.azure.storage.core.Base64;
-import org.apache.commons.configuration2.SubsetConfiguration;
+import org.junit.Assert;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import org.apache.commons.configuration2.SubsetConfiguration;
+
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.azure.integration.AzureTestConstants;
 import org.apache.hadoop.fs.azure.metrics.AzureFileSystemInstrumentation;
 import org.apache.hadoop.fs.azure.metrics.AzureFileSystemMetricsSystem;
 import org.apache.hadoop.metrics2.AbstractMetric;
@@ -35,6 +38,8 @@ import org.apache.hadoop.metrics2.MetricsSink;
 import org.apache.hadoop.metrics2.MetricsTag;
 import org.apache.hadoop.metrics2.impl.TestMetricsConfig;
 
+import java.io.File;
+import java.io.IOException;
 import java.net.URI;
 import java.net.URISyntaxException;
 import java.util.*;
@@ -46,10 +51,10 @@ import static org.apache.hadoop.fs.azure.AzureNativeFileSystemStore.KEY_USE_SECU
 
 /**
  * Helper class to create WASB file systems backed by either a mock in-memory
- * implementation or a real Azure Storage account. See RunningLiveWasbTests.txt
- * for instructions on how to connect to a real Azure Storage account.
+ * implementation or a real Azure Storage account.
  */
-public final class AzureBlobStorageTestAccount {
+public final class AzureBlobStorageTestAccount implements AutoCloseable,
+    AzureTestConstants {
   private static final Logger LOG = LoggerFactory.getLogger(
       AzureBlobStorageTestAccount.class);
 
@@ -166,6 +171,7 @@ public final class AzureBlobStorageTestAccount {
     return new Path("/" + DEFAULT_PAGE_BLOB_DIRECTORY);
   }
 
+  @Deprecated
   public static Path pageBlobPath(String fileName) {
     return new Path(pageBlobPath(), fileName);
   }
@@ -201,6 +207,9 @@ public final class AzureBlobStorageTestAccount {
    * @return
    */
   private boolean wasGeneratedByMe(MetricsRecord currentRecord) {
+    Assert.assertNotNull("null filesystem", fs);
+    Assert.assertNotNull("null filesystemn instance ID",
+        fs.getInstrumentation().getFileSystemInstanceId());
     String myFsId = fs.getInstrumentation().getFileSystemInstanceId().toString();
     for (MetricsTag currentTag : currentRecord.tags()) {
       if (currentTag.name().equalsIgnoreCase("wasbFileSystemId")) {
@@ -247,13 +256,16 @@ public final class AzureBlobStorageTestAccount {
     getBlobReference(blobKey).releaseLease(accessCondition);
   }
 
-  private static void saveMetricsConfigFile() {
+  private static void saveMetricsConfigFile() throws IOException {
     if (!metricsConfigSaved) {
+      String testFilename = TestMetricsConfig.getTestFilename(
+          "hadoop-metrics2-azure-file-system");
+      File dest = new File(testFilename).getCanonicalFile();
+      dest.getParentFile().mkdirs();
       new org.apache.hadoop.metrics2.impl.ConfigBuilder()
       .add("azure-file-system.sink.azuretestcollector.class",
           StandardCollector.class.getName())
-      .save(TestMetricsConfig.getTestFilename(
-          "hadoop-metrics2-azure-file-system.properties"));
+      .save(testFilename);
       metricsConfigSaved = true;
     }
   }
@@ -314,9 +326,8 @@ public final class AzureBlobStorageTestAccount {
     Configuration conf = createTestConfiguration();
     if (!conf.getBoolean(USE_EMULATOR_PROPERTY_NAME, false)) {
       // Not configured to test against the storage emulator.
-      LOG.warn("Skipping emulator Azure test because configuration doesn't "
-          + "indicate that it's running. Please see RunningLiveWasbTests.txt "
-          + "for guidance.");
+      LOG.warn("Skipping emulator Azure test because configuration "
+          + "doesn't indicate that it's running.");
       return null;
     }
     CloudStorageAccount account =
@@ -482,8 +493,7 @@ public final class AzureBlobStorageTestAccount {
         credentials = StorageCredentialsAnonymous.ANONYMOUS;
       } else {
         LOG.warn("Skipping live Azure test because of missing key for"
-            + " account '" + accountName + "'. "
-            + "Please see RunningLiveWasbTests.txt for guidance.");
+            + " account '" + accountName + "'.");
         return null;
       }
     } else {
@@ -517,8 +527,7 @@ public final class AzureBlobStorageTestAccount {
       throws URISyntaxException, KeyProviderException {
     String testAccountName = conf.get(TEST_ACCOUNT_NAME_PROPERTY_NAME);
     if (testAccountName == null) {
-      LOG.warn("Skipping live Azure test because of missing test account. "
-          + "Please see RunningLiveWasbTests.txt for guidance.");
+      LOG.warn("Skipping live Azure test because of missing test account");
       return null;
     }
     return createStorageAccount(testAccountName, conf, false);
@@ -863,6 +872,11 @@ public final class AzureBlobStorageTestAccount {
     }
   }
 
+  @Override
+  public void close() throws Exception {
+    cleanup();
+  }
+
   public NativeAzureFileSystem getFileSystem() {
     return fs;
   }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/9f6b08f8/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestAzureConcurrentOutOfBandIo.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestAzureConcurrentOutOfBandIo.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestAzureConcurrentOutOfBandIo.java
new file mode 100644
index 0000000..7e733dc
--- /dev/null
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestAzureConcurrentOutOfBandIo.java
@@ -0,0 +1,179 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.azure;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.util.Arrays;
+
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.hadoop.fs.azure.integration.AzureTestUtils;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.fs.permission.PermissionStatus;
+
+/**
+ * Handle OOB IO into a shared container.
+ */
+public class ITestAzureConcurrentOutOfBandIo extends AbstractWasbTestBase {
+
+  private static final Logger LOG =
+      LoggerFactory.getLogger(ITestAzureConcurrentOutOfBandIo.class);
+
+  // Class constants.
+  static final int DOWNLOAD_BLOCK_SIZE = 8 * 1024 * 1024;
+  static final int UPLOAD_BLOCK_SIZE = 4 * 1024 * 1024;
+  static final int BLOB_SIZE = 32 * 1024 * 1024;
+
+  // Number of blocks to be written before flush.
+  static final int NUMBER_OF_BLOCKS = 2;
+
+  @Override
+  protected AzureBlobStorageTestAccount createTestAccount() throws Exception {
+    return AzureBlobStorageTestAccount.createOutOfBandStore(
+        UPLOAD_BLOCK_SIZE, DOWNLOAD_BLOCK_SIZE);
+  }
+
+  class DataBlockWriter implements Runnable {
+
+    Thread runner;
+    AzureBlobStorageTestAccount writerStorageAccount;
+    String key;
+    boolean done = false;
+
+    /**
+     * Constructor captures the test account.
+     * 
+     * @param testAccount
+     */
+    public DataBlockWriter(AzureBlobStorageTestAccount testAccount, String key) {
+      writerStorageAccount = testAccount;
+      this.key = key;
+    }
+
+    /**
+     * Start writing blocks to Azure storage.
+     */
+    public void startWriting() {
+      runner = new Thread(this); // Create the block writer thread.
+      runner.start(); // Start the block writer thread.
+    }
+
+    /**
+     * Stop writing blocks to Azure storage.
+     */
+    public void stopWriting() {
+      done = true;
+    }
+
+    /**
+     * Implementation of the runnable interface. The run method is a tight loop
+     * which repeatedly updates the blob with a 4 MB block.
+     */
+    public void run() {
+      byte[] dataBlockWrite = new byte[UPLOAD_BLOCK_SIZE];
+
+      OutputStream outputStream = null;
+
+      try {
+        for (int i = 0; !done; i++) {
+          // Write two 4 MB blocks to the blob.
+          //
+          outputStream = writerStorageAccount.getStore().storefile(
+              key,
+              new PermissionStatus("", "", FsPermission.getDefault()),
+              key);
+
+          Arrays.fill(dataBlockWrite, (byte) (i % 256));
+          for (int j = 0; j < NUMBER_OF_BLOCKS; j++) {
+            outputStream.write(dataBlockWrite);
+          }
+
+          outputStream.flush();
+          outputStream.close();
+        }
+      } catch (AzureException e) {
+        LOG.error("DatablockWriter thread encountered a storage exception."
+            + e.getMessage(), e);
+      } catch (IOException e) {
+        LOG.error("DatablockWriter thread encountered an I/O exception."
+            + e.getMessage(), e);
+      }
+    }
+  }
+
+  @Test
+  public void testReadOOBWrites() throws Exception {
+
+    byte[] dataBlockWrite = new byte[UPLOAD_BLOCK_SIZE];
+    byte[] dataBlockRead = new byte[UPLOAD_BLOCK_SIZE];
+
+    // Write to blob to make sure it exists.
+    //
+   // Write five 4 MB blocks to the blob. To ensure there is data in the blob before
+   // reading.  This eliminates the race between the reader and writer threads.
+    String key = "WASB_String" + AzureTestUtils.getForkID() + ".txt";
+    OutputStream outputStream = testAccount.getStore().storefile(
+       key,
+       new PermissionStatus("", "", FsPermission.getDefault()),
+           key);
+   Arrays.fill(dataBlockWrite, (byte) 255);
+   for (int i = 0; i < NUMBER_OF_BLOCKS; i++) {
+     outputStream.write(dataBlockWrite);
+   }
+
+   outputStream.flush();
+   outputStream.close();
+
+   // Start writing blocks to Azure store using the DataBlockWriter thread.
+    DataBlockWriter writeBlockTask = new DataBlockWriter(testAccount, key);
+    writeBlockTask.startWriting();
+   int count = 0;
+
+   for (int i = 0; i < 5; i++) {
+     try(InputStream inputStream = testAccount.getStore().retrieve(key)) {
+        count = 0;
+        int c = 0;
+
+        while (c >= 0) {
+          c = inputStream.read(dataBlockRead, 0, UPLOAD_BLOCK_SIZE);
+          if (c < 0) {
+            break;
+          }
+
+          // Counting the number of bytes.
+          count += c;
+        }
+     } catch (IOException e) {
+       System.out.println(e.getCause().toString());
+       e.printStackTrace();
+       fail();
+     }
+   }
+
+    // Stop writing blocks.
+    writeBlockTask.stopWriting();
+
+    // Validate that a block was read.
+    assertEquals(NUMBER_OF_BLOCKS * UPLOAD_BLOCK_SIZE, count);
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/9f6b08f8/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestAzureConcurrentOutOfBandIoWithSecureMode.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestAzureConcurrentOutOfBandIoWithSecureMode.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestAzureConcurrentOutOfBandIoWithSecureMode.java
new file mode 100644
index 0000000..2b0ea56
--- /dev/null
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestAzureConcurrentOutOfBandIoWithSecureMode.java
@@ -0,0 +1,33 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.azure;
+
+/**
+ * Extends ITestAzureConcurrentOutOfBandIo in order to run testReadOOBWrites with secure mode
+ * (fs.azure.secure.mode) both enabled and disabled.
+ */
+public class ITestAzureConcurrentOutOfBandIoWithSecureMode
+    extends ITestAzureConcurrentOutOfBandIo {
+
+  @Override
+  protected AzureBlobStorageTestAccount createTestAccount() throws Exception {
+    return AzureBlobStorageTestAccount.createOutOfBandStore(
+        UPLOAD_BLOCK_SIZE, DOWNLOAD_BLOCK_SIZE, true);
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/9f6b08f8/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestAzureFileSystemErrorConditions.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestAzureFileSystemErrorConditions.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestAzureFileSystemErrorConditions.java
new file mode 100644
index 0000000..49e6730
--- /dev/null
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestAzureFileSystemErrorConditions.java
@@ -0,0 +1,243 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.azure;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.net.HttpURLConnection;
+import java.net.URI;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.concurrent.Callable;
+
+import com.microsoft.azure.storage.OperationContext;
+import com.microsoft.azure.storage.SendingRequestEvent;
+import com.microsoft.azure.storage.StorageEvent;
+import org.junit.Test;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.azure.AzureNativeFileSystemStore.TestHookOperationContext;
+import org.apache.hadoop.test.GenericTestUtils;
+
+import static org.apache.hadoop.fs.azure.AzureNativeFileSystemStore.NO_ACCESS_TO_CONTAINER_MSG;
+import static org.apache.hadoop.test.LambdaTestUtils.intercept;
+import static org.junit.Assume.assumeNotNull;
+
+/**
+ * Error handling.
+ */
+public class ITestAzureFileSystemErrorConditions extends
+    AbstractWasbTestWithTimeout {
+  private static final int ALL_THREE_FILE_SIZE = 1024;
+
+  @Test
+  public void testNoInitialize() throws Exception {
+    intercept(AssertionError.class,
+        new Callable<FileMetadata>() {
+          @Override
+          public FileMetadata call() throws Exception {
+            return new AzureNativeFileSystemStore()
+                .retrieveMetadata("foo");
+          }
+        });
+  }
+
+  /**
+   * Try accessing an unauthorized or non-existent (treated the same) container
+   * from WASB.
+   */
+  @Test
+  public void testAccessUnauthorizedPublicContainer() throws Exception {
+    final String container = "nonExistentContainer";
+    final String account = "hopefullyNonExistentAccount";
+    Path noAccessPath = new Path(
+        "wasb://" + container + "@" + account + "/someFile");
+    NativeAzureFileSystem.suppressRetryPolicy();
+    try {
+      FileSystem.get(noAccessPath.toUri(), new Configuration())
+        .open(noAccessPath);
+      assertTrue("Should've thrown.", false);
+    } catch (AzureException ex) {
+      GenericTestUtils.assertExceptionContains(
+          String.format(NO_ACCESS_TO_CONTAINER_MSG, account, container), ex);
+    } finally {
+      NativeAzureFileSystem.resumeRetryPolicy();
+    }
+  }
+
+  @Test
+  public void testAccessContainerWithWrongVersion() throws Exception {
+    AzureNativeFileSystemStore store = new AzureNativeFileSystemStore();
+    MockStorageInterface mockStorage = new MockStorageInterface();
+    store.setAzureStorageInteractionLayer(mockStorage);
+    try (FileSystem fs = new NativeAzureFileSystem(store)) {
+      Configuration conf = new Configuration();
+      AzureBlobStorageTestAccount.setMockAccountKey(conf);
+      HashMap<String, String> metadata = new HashMap<String, String>();
+      metadata.put(AzureNativeFileSystemStore.VERSION_METADATA_KEY,
+          "2090-04-05"); // It's from the future!
+      mockStorage.addPreExistingContainer(
+          AzureBlobStorageTestAccount.getMockContainerUri(), metadata);
+
+      AzureException ex = intercept(AzureException.class,
+          new Callable<FileStatus[]>() {
+            @Override
+            public FileStatus[] call() throws Exception {
+              fs.initialize(new URI(AzureBlobStorageTestAccount.MOCK_WASB_URI),
+                  conf);
+              return fs.listStatus(new Path("/"));
+            }
+          });
+      GenericTestUtils.assertExceptionContains(
+          "unsupported version: 2090-04-05.", ex);
+    }
+  }
+
+  private interface ConnectionRecognizer {
+    boolean isTargetConnection(HttpURLConnection connection);
+  }
+
+  private class TransientErrorInjector extends StorageEvent<SendingRequestEvent> {
+    private final ConnectionRecognizer connectionRecognizer;
+    private boolean injectedErrorOnce = false;
+
+    public TransientErrorInjector(ConnectionRecognizer connectionRecognizer) {
+      this.connectionRecognizer = connectionRecognizer;
+    }
+
+    @Override
+    public void eventOccurred(SendingRequestEvent eventArg) {
+      HttpURLConnection connection
+          = (HttpURLConnection) eventArg.getConnectionObject();
+      if (!connectionRecognizer.isTargetConnection(connection)) {
+        return;
+      }
+      if (!injectedErrorOnce) {
+        connection.setReadTimeout(1);
+        connection.disconnect();
+        injectedErrorOnce = true;
+      }
+    }
+  }
+
+  private void injectTransientError(NativeAzureFileSystem fs,
+      final ConnectionRecognizer connectionRecognizer) {
+    fs.getStore().addTestHookToOperationContext(new TestHookOperationContext() {
+      @Override
+      public OperationContext modifyOperationContext(OperationContext original) {
+        original.getSendingRequestEventHandler().addListener(
+            new TransientErrorInjector(connectionRecognizer));
+        return original;
+      }
+    });
+  }
+
+  @Test
+  public void testTransientErrorOnDelete() throws Exception {
+    // Need to do this test against a live storage account
+    AzureBlobStorageTestAccount testAccount =
+        AzureBlobStorageTestAccount.create();
+    assumeNotNull(testAccount);
+    try {
+      NativeAzureFileSystem fs = testAccount.getFileSystem();
+      injectTransientError(fs, new ConnectionRecognizer() {
+        @Override
+        public boolean isTargetConnection(HttpURLConnection connection) {
+          return connection.getRequestMethod().equals("DELETE");
+        }
+      });
+      Path testFile = new Path("/a/b");
+      assertTrue(fs.createNewFile(testFile));
+      assertTrue(fs.rename(testFile, new Path("/x")));
+    } finally {
+      testAccount.cleanup();
+    }
+  }
+
+  private void writeAllThreeFile(NativeAzureFileSystem fs, Path testFile)
+      throws IOException {
+    byte[] buffer = new byte[ALL_THREE_FILE_SIZE];
+    Arrays.fill(buffer, (byte) 3);
+    try(OutputStream stream = fs.create(testFile)) {
+      stream.write(buffer);
+    }
+  }
+
+  private void readAllThreeFile(NativeAzureFileSystem fs, Path testFile)
+      throws IOException {
+    byte[] buffer = new byte[ALL_THREE_FILE_SIZE];
+    InputStream inStream = fs.open(testFile);
+    assertEquals(buffer.length,
+        inStream.read(buffer, 0, buffer.length));
+    inStream.close();
+    for (int i = 0; i < buffer.length; i++) {
+      assertEquals(3, buffer[i]);
+    }
+  }
+
+  @Test
+  public void testTransientErrorOnCommitBlockList() throws Exception {
+    // Need to do this test against a live storage account
+    AzureBlobStorageTestAccount testAccount =
+        AzureBlobStorageTestAccount.create();
+    assumeNotNull(testAccount);
+    try {
+      NativeAzureFileSystem fs = testAccount.getFileSystem();
+      injectTransientError(fs, new ConnectionRecognizer() {
+        @Override
+        public boolean isTargetConnection(HttpURLConnection connection) {
+          return connection.getRequestMethod().equals("PUT")
+              && connection.getURL().getQuery() != null
+              && connection.getURL().getQuery().contains("blocklist");
+        }
+      });
+      Path testFile = new Path("/a/b");
+      writeAllThreeFile(fs, testFile);
+      readAllThreeFile(fs, testFile);
+    } finally {
+      testAccount.cleanup();
+    }
+  }
+
+  @Test
+  public void testTransientErrorOnRead() throws Exception {
+    // Need to do this test against a live storage account
+    AzureBlobStorageTestAccount testAccount =
+        AzureBlobStorageTestAccount.create();
+    assumeNotNull(testAccount);
+    try {
+      NativeAzureFileSystem fs = testAccount.getFileSystem();
+      Path testFile = new Path("/a/b");
+      writeAllThreeFile(fs, testFile);
+      injectTransientError(fs, new ConnectionRecognizer() {
+        @Override
+        public boolean isTargetConnection(HttpURLConnection connection) {
+          return connection.getRequestMethod().equals("GET");
+        }
+      });
+      readAllThreeFile(fs, testFile);
+    } finally {
+      testAccount.cleanup();
+    }
+  }
+}


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message