hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ste...@apache.org
Subject [02/20] hadoop git commit: HADOOP-14553. Add (parallelized) integration tests to hadoop-azure Contributed by Steve Loughran
Date Fri, 15 Sep 2017 16:39:27 GMT
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2d2d97fa/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/integration/AzureTestConstants.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/integration/AzureTestConstants.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/integration/AzureTestConstants.java
new file mode 100644
index 0000000..0b72f06
--- /dev/null
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/integration/AzureTestConstants.java
@@ -0,0 +1,180 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.azure.integration;
+
+import org.apache.hadoop.fs.Path;
+
+/**
+ * Constants for the Azure tests.
+ */
+public interface AzureTestConstants {
+
+  /**
+   * Prefix for any cross-filesystem scale test options.
+   */
+  String SCALE_TEST = "scale.test.";
+
+  /**
+   * Prefix for wasb-specific scale tests.
+   */
+  String AZURE_SCALE_TEST = "fs.azure.scale.test.";
+
+  /**
+   * Prefix for FS wasb tests.
+   */
+  String TEST_FS_WASB = "test.fs.azure.";
+
+  /**
+   * Name of the test filesystem.
+   */
+  String TEST_FS_WASB_NAME = TEST_FS_WASB + "name";
+
+  /**
+   * Tell tests that they are being executed in parallel: {@value}.
+   */
+  String KEY_PARALLEL_TEST_EXECUTION = "test.parallel.execution";
+
+  /**
+   * A property set to true in maven if scale tests are enabled: {@value}.
+   */
+  String KEY_SCALE_TESTS_ENABLED = AZURE_SCALE_TEST + "enabled";
+
+  /**
+   * The number of operations to perform: {@value}.
+   */
+  String KEY_OPERATION_COUNT = SCALE_TEST + "operation.count";
+
+  /**
+   * The number of directory operations to perform: {@value}.
+   */
+  String KEY_DIRECTORY_COUNT = SCALE_TEST + "directory.count";
+
+  /**
+   * The readahead buffer: {@value}.
+   */
+  String KEY_READ_BUFFER_SIZE = AZURE_SCALE_TEST + "read.buffer.size";
+
+  int DEFAULT_READ_BUFFER_SIZE = 16384;
+
+  /**
+   * Key for a multi MB test file: {@value}.
+   */
+  String KEY_CSVTEST_FILE = AZURE_SCALE_TEST + "csvfile";
+
+  /**
+   * Default path for the multi MB test file: {@value}.
+   */
+  String DEFAULT_CSVTEST_FILE = "wasb://datasets@azuremlsampleexperiments.blob.core.windows.net/network_intrusion_detection.csv";
+
+  /**
+   * Name of the property to define the timeout for scale tests: {@value}.
+   * Measured in seconds.
+   */
+  String KEY_TEST_TIMEOUT = AZURE_SCALE_TEST + "timeout";
+
+  /**
+   * Name of the property to define the file size for the huge file
+   * tests: {@value}.
+   * Measured in KB; a suffix like "M", or "G" will change the unit.
+   */
+  String KEY_HUGE_FILESIZE = AZURE_SCALE_TEST + "huge.filesize";
+
+  /**
+   * Name of the property to define the partition size for the huge file
+   * tests: {@value}.
+   * Measured in KB; a suffix like "M", or "G" will change the unit.
+   */
+  String KEY_HUGE_PARTITION_SIZE = AZURE_SCALE_TEST + "huge.partitionsize";
+
+  /**
+   * The default huge size is small —full 5GB+ scale tests are something
+   * to run in long test runs on EC2 VMs. {@value}.
+   */
+  String DEFAULT_HUGE_FILESIZE = "10M";
+
+  /**
+   * The default number of operations to perform: {@value}.
+   */
+  long DEFAULT_OPERATION_COUNT = 2005;
+
+  /**
+   * Default number of directories to create when performing
+   * directory performance/scale tests.
+   */
+  int DEFAULT_DIRECTORY_COUNT = 2;
+
+  /**
+   * Default policy on scale tests: {@value}.
+   */
+  boolean DEFAULT_SCALE_TESTS_ENABLED = false;
+
+  /**
+   * Fork ID passed down from maven if the test is running in parallel.
+   */
+  String TEST_UNIQUE_FORK_ID = "test.unique.fork.id";
+
+  /**
+   * Timeout in Milliseconds for standard tests: {@value}.
+   */
+  int AZURE_TEST_TIMEOUT = 10 * 60 * 1000;
+
+  /**
+   * Timeout in Seconds for Scale Tests: {@value}.
+   */
+  int SCALE_TEST_TIMEOUT_SECONDS = 30 * 60;
+
+  int SCALE_TEST_TIMEOUT_MILLIS = SCALE_TEST_TIMEOUT_SECONDS * 1000;
+
+
+
+  String ACCOUNT_KEY_PROPERTY_NAME
+      = "fs.azure.account.key.";
+  String SAS_PROPERTY_NAME = "fs.azure.sas.";
+  String TEST_CONFIGURATION_FILE_NAME = "azure-test.xml";
+  String TEST_ACCOUNT_NAME_PROPERTY_NAME
+      = "fs.azure.test.account.name";
+  String MOCK_ACCOUNT_NAME
+      = "mockAccount.blob.core.windows.net";
+  String MOCK_CONTAINER_NAME = "mockContainer";
+  String WASB_AUTHORITY_DELIMITER = "@";
+  String WASB_SCHEME = "wasb";
+  String PATH_DELIMITER = "/";
+  String AZURE_ROOT_CONTAINER = "$root";
+  String MOCK_WASB_URI = "wasb://" + MOCK_CONTAINER_NAME
+      + WASB_AUTHORITY_DELIMITER + MOCK_ACCOUNT_NAME + "/";
+  String USE_EMULATOR_PROPERTY_NAME
+      = "fs.azure.test.emulator";
+
+  String KEY_DISABLE_THROTTLING
+      = "fs.azure.disable.bandwidth.throttling";
+  String KEY_READ_TOLERATE_CONCURRENT_APPEND
+      = "fs.azure.io.read.tolerate.concurrent.append";
+  /**
+   * Path for page blobs: {@value}.
+   */
+  String DEFAULT_PAGE_BLOB_DIRECTORY = "pageBlobs";
+
+  String DEFAULT_ATOMIC_RENAME_DIRECTORIES
+      = "/atomicRenameDir1,/atomicRenameDir2";
+
+  /**
+   * Base directory for page blobs.
+   */
+  Path PAGE_BLOB_DIR = new Path("/" + DEFAULT_PAGE_BLOB_DIRECTORY);
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/2d2d97fa/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/integration/AzureTestUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/integration/AzureTestUtils.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/integration/AzureTestUtils.java
new file mode 100644
index 0000000..2fbbcd1
--- /dev/null
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/integration/AzureTestUtils.java
@@ -0,0 +1,479 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.azure.integration;
+
+import java.io.IOException;
+import java.net.URI;
+import java.util.List;
+
+import org.junit.Assert;
+import org.junit.Assume;
+import org.junit.internal.AssumptionViolatedException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileContext;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.azure.AzureBlobStorageTestAccount;
+import org.apache.hadoop.fs.azure.NativeAzureFileSystem;
+
+import static org.apache.hadoop.fs.azure.integration.AzureTestConstants.*;
+import static org.apache.hadoop.test.MetricsAsserts.getLongCounter;
+import static org.apache.hadoop.test.MetricsAsserts.getLongGauge;
+import static org.apache.hadoop.test.MetricsAsserts.getMetrics;
+
+/**
+ * Utilities for the Azure tests. Based on {@code S3ATestUtils}, so
+ * (initially) has unused method.
+ */
+public final class AzureTestUtils extends Assert {
+  private static final Logger LOG = LoggerFactory.getLogger(
+      AzureTestUtils.class);
+
+  /**
+   * Value to set a system property to (in maven) to declare that
+   * a property has been unset.
+   */
+  public static final String UNSET_PROPERTY = "unset";
+
+  /**
+   * Create the test filesystem.
+   *
+   * If the test.fs.wasb.name property is not set, this will
+   * raise a JUnit assumption exception
+   *
+   * @param conf configuration
+   * @return the FS
+   * @throws IOException IO Problems
+   * @throws AssumptionViolatedException if the FS is not named
+   */
+  public static NativeAzureFileSystem createTestFileSystem(Configuration conf)
+      throws IOException {
+
+    String fsname = conf.getTrimmed(TEST_FS_WASB_NAME, "");
+
+    boolean liveTest = !StringUtils.isEmpty(fsname);
+    URI testURI = null;
+    if (liveTest) {
+      testURI = URI.create(fsname);
+      liveTest = testURI.getScheme().equals(WASB_SCHEME);
+    }
+    if (!liveTest) {
+      // Skip the test
+      throw new AssumptionViolatedException(
+          "No test filesystem in " + TEST_FS_WASB_NAME);
+    }
+    NativeAzureFileSystem fs1 = new NativeAzureFileSystem();
+    fs1.initialize(testURI, conf);
+    return fs1;
+  }
+
+  /**
+   * Create a file context for tests.
+   *
+   * If the test.fs.wasb.name property is not set, this will
+   * trigger a JUnit failure.
+   *
+   * Multipart purging is enabled.
+   * @param conf configuration
+   * @return the FS
+   * @throws IOException IO Problems
+   * @throws AssumptionViolatedException if the FS is not named
+   */
+  public static FileContext createTestFileContext(Configuration conf)
+      throws IOException {
+    String fsname = conf.getTrimmed(TEST_FS_WASB_NAME, "");
+
+    boolean liveTest = !StringUtils.isEmpty(fsname);
+    URI testURI = null;
+    if (liveTest) {
+      testURI = URI.create(fsname);
+      liveTest = testURI.getScheme().equals(WASB_SCHEME);
+    }
+    if (!liveTest) {
+      // This doesn't work with our JUnit 3 style test cases, so instead we'll
+      // make this whole class not run by default
+      throw new AssumptionViolatedException("No test filesystem in "
+          + TEST_FS_WASB_NAME);
+    }
+    FileContext fc = FileContext.getFileContext(testURI, conf);
+    return fc;
+  }
+
+  /**
+   * Get a long test property.
+   * <ol>
+   *   <li>Look up configuration value (which can pick up core-default.xml),
+   *       using {@code defVal} as the default value (if conf != null).
+   *   </li>
+   *   <li>Fetch the system property.</li>
+   *   <li>If the system property is not empty or "(unset)":
+   *   it overrides the conf value.
+   *   </li>
+   * </ol>
+   * This puts the build properties in charge of everything. It's not a
+   * perfect design; having maven set properties based on a file, as ant let
+   * you do, is better for customization.
+   *
+   * As to why there's a special (unset) value, see
+   * {@link http://stackoverflow.com/questions/7773134/null-versus-empty-arguments-in-maven}
+   * @param conf config: may be null
+   * @param key key to look up
+   * @param defVal default value
+   * @return the evaluated test property.
+   */
+  public static long getTestPropertyLong(Configuration conf,
+      String key, long defVal) {
+    return Long.valueOf(
+        getTestProperty(conf, key, Long.toString(defVal)));
+  }
+  /**
+   * Get a test property value in bytes, using k, m, g, t, p, e suffixes.
+   * {@link org.apache.hadoop.util.StringUtils.TraditionalBinaryPrefix#string2long(String)}
+   * <ol>
+   *   <li>Look up configuration value (which can pick up core-default.xml),
+   *       using {@code defVal} as the default value (if conf != null).
+   *   </li>
+   *   <li>Fetch the system property.</li>
+   *   <li>If the system property is not empty or "(unset)":
+   *   it overrides the conf value.
+   *   </li>
+   * </ol>
+   * This puts the build properties in charge of everything. It's not a
+   * perfect design; having maven set properties based on a file, as ant let
+   * you do, is better for customization.
+   *
+   * As to why there's a special (unset) value, see
+   * {@link http://stackoverflow.com/questions/7773134/null-versus-empty-arguments-in-maven}
+   * @param conf config: may be null
+   * @param key key to look up
+   * @param defVal default value
+   * @return the evaluated test property.
+   */
+  public static long getTestPropertyBytes(Configuration conf,
+      String key, String defVal) {
+    return org.apache.hadoop.util.StringUtils.TraditionalBinaryPrefix
+        .string2long(getTestProperty(conf, key, defVal));
+  }
+
+  /**
+   * Get an integer test property; algorithm described in
+   * {@link #getTestPropertyLong(Configuration, String, long)}.
+   * @param key key to look up
+   * @param defVal default value
+   * @return the evaluated test property.
+   */
+  public static int getTestPropertyInt(Configuration conf,
+      String key, int defVal) {
+    return (int) getTestPropertyLong(conf, key, defVal);
+  }
+
+  /**
+   * Get a boolean test property; algorithm described in
+   * {@link #getTestPropertyLong(Configuration, String, long)}.
+   * @param key key to look up
+   * @param defVal default value
+   * @return the evaluated test property.
+   */
+  public static boolean getTestPropertyBool(Configuration conf,
+      String key,
+      boolean defVal) {
+    return Boolean.valueOf(
+        getTestProperty(conf, key, Boolean.toString(defVal)));
+  }
+
+  /**
+   * Get a string test property.
+   * <ol>
+   *   <li>Look up configuration value (which can pick up core-default.xml),
+   *       using {@code defVal} as the default value (if conf != null).
+   *   </li>
+   *   <li>Fetch the system property.</li>
+   *   <li>If the system property is not empty or "(unset)":
+   *   it overrides the conf value.
+   *   </li>
+   * </ol>
+   * This puts the build properties in charge of everything. It's not a
+   * perfect design; having maven set properties based on a file, as ant let
+   * you do, is better for customization.
+   *
+   * As to why there's a special (unset) value, see
+   * @see <a href="http://stackoverflow.com/questions/7773134/null-versus-empty-arguments-in-maven">
+   *   Stack Overflow</a>
+   * @param conf config: may be null
+   * @param key key to look up
+   * @param defVal default value
+   * @return the evaluated test property.
+   */
+
+  public static String getTestProperty(Configuration conf,
+      String key,
+      String defVal) {
+    String confVal = conf != null
+        ? conf.getTrimmed(key, defVal)
+        : defVal;
+    String propval = System.getProperty(key);
+    return StringUtils.isNotEmpty(propval) && !UNSET_PROPERTY.equals(propval)
+        ? propval : confVal;
+  }
+
+  /**
+   * Verify the class of an exception. If it is not as expected, rethrow it.
+   * Comparison is on the exact class, not subclass-of inference as
+   * offered by {@code instanceof}.
+   * @param clazz the expected exception class
+   * @param ex the exception caught
+   * @return the exception, if it is of the expected class
+   * @throws Exception the exception passed in.
+   */
+  public static Exception verifyExceptionClass(Class clazz,
+      Exception ex)
+      throws Exception {
+    if (!(ex.getClass().equals(clazz))) {
+      throw ex;
+    }
+    return ex;
+  }
+
+  /**
+   * Turn off FS Caching: use if a filesystem with different options from
+   * the default is required.
+   * @param conf configuration to patch
+   */
+  public static void disableFilesystemCaching(Configuration conf) {
+    conf.setBoolean("fs.wasb.impl.disable.cache", true);
+  }
+
+  /**
+   * Create a test path, using the value of
+   * {@link AzureTestUtils#TEST_UNIQUE_FORK_ID} if it is set.
+   * @param defVal default value
+   * @return a path
+   */
+  public static Path createTestPath(Path defVal) {
+    String testUniqueForkId = System.getProperty(
+        AzureTestConstants.TEST_UNIQUE_FORK_ID);
+    return testUniqueForkId == null
+        ? defVal
+        : new Path("/" + testUniqueForkId, "test");
+  }
+
+  /**
+   * Create a test page blob path using the value of
+   * {@link AzureTestConstants#TEST_UNIQUE_FORK_ID} if it is set.
+   * @param filename filename at the end of the path
+   * @return an absolute path
+   */
+  public static Path blobPathForTests(FileSystem fs, String filename) {
+    String testUniqueForkId = System.getProperty(
+        AzureTestConstants.TEST_UNIQUE_FORK_ID);
+    return fs.makeQualified(new Path(PAGE_BLOB_DIR,
+        testUniqueForkId == null
+            ? filename
+            : (testUniqueForkId + "/" + filename)));
+  }
+
+  /**
+   * Create a test path using the value of
+   * {@link AzureTestConstants#TEST_UNIQUE_FORK_ID} if it is set.
+   * @param filename filename at the end of the path
+   * @return an absolute path
+   */
+  public static Path pathForTests(FileSystem fs, String filename) {
+    String testUniqueForkId = System.getProperty(
+        AzureTestConstants.TEST_UNIQUE_FORK_ID);
+    return fs.makeQualified(new Path(
+        testUniqueForkId == null
+            ? ("/test/" + filename)
+            : (testUniqueForkId + "/" + filename)));
+  }
+
+  /**
+   * Get a unique fork ID.
+   * Returns a default value for non-parallel tests.
+   * @return a string unique for all test VMs running in this maven build.
+   */
+  public static String getForkID() {
+    return System.getProperty(
+        AzureTestConstants.TEST_UNIQUE_FORK_ID, "fork-1");
+  }
+
+  /**
+   * Flag to indicate that this test is being executed in parallel.
+   * This is used by some of the scale tests to validate test time expectations.
+   * @return true if the build indicates this test is being run in parallel.
+   */
+  public static boolean isParallelExecution() {
+    return Boolean.getBoolean(KEY_PARALLEL_TEST_EXECUTION);
+  }
+
+  /**
+   * Asserts that {@code obj} is an instance of {@code expectedClass} using a
+   * descriptive assertion message.
+   * @param expectedClass class
+   * @param obj object to check
+   */
+  public static void assertInstanceOf(Class<?> expectedClass, Object obj) {
+    Assert.assertTrue(String.format("Expected instance of class %s, but is %s.",
+        expectedClass, obj.getClass()),
+        expectedClass.isAssignableFrom(obj.getClass()));
+  }
+
+  /**
+   * Builds a comma-separated list of class names.
+   * @param classes list of classes
+   * @return comma-separated list of class names
+   */
+  public static <T extends Class<?>> String buildClassListString(
+      List<T> classes) {
+    StringBuilder sb = new StringBuilder();
+    for (int i = 0; i < classes.size(); ++i) {
+      if (i > 0) {
+        sb.append(',');
+      }
+      sb.append(classes.get(i).getName());
+    }
+    return sb.toString();
+  }
+
+  /**
+   * This class should not be instantiated.
+   */
+  private AzureTestUtils() {
+  }
+
+  /**
+   * Assert that a configuration option matches the expected value.
+   * @param conf configuration
+   * @param key option key
+   * @param expected expected value
+   */
+  public static void assertOptionEquals(Configuration conf,
+      String key,
+      String expected) {
+    assertEquals("Value of " + key, expected, conf.get(key));
+  }
+
+  /**
+   * Assume that a condition is met. If not: log at WARN and
+   * then throw an {@link AssumptionViolatedException}.
+   * @param message message in an assumption
+   * @param condition condition to probe
+   */
+  public static void assume(String message, boolean condition) {
+    if (!condition) {
+      LOG.warn(message);
+    }
+    Assume.assumeTrue(message, condition);
+  }
+
+  /**
+   * Gets the current value of the given gauge.
+   * @param fs filesystem
+   * @param gaugeName gauge name
+   * @return the gauge value
+   */
+  public static long getLongGaugeValue(NativeAzureFileSystem fs,
+      String gaugeName) {
+    return getLongGauge(gaugeName, getMetrics(fs.getInstrumentation()));
+  }
+
+  /**
+   * Gets the current value of the given counter.
+   * @param fs filesystem
+   * @param counterName counter name
+   * @return the counter value
+   */
+  public static long getLongCounterValue(NativeAzureFileSystem fs,
+      String counterName) {
+    return getLongCounter(counterName, getMetrics(fs.getInstrumentation()));
+  }
+
+
+  /**
+   * Delete a path, catching any exception and downgrading to a log message.
+   * @param fs filesystem
+   * @param path path to delete
+   * @param recursive recursive delete?
+   * @throws IOException IO failure.
+   */
+  public static void deleteQuietly(FileSystem fs,
+      Path path,
+      boolean recursive) throws IOException {
+    if (fs != null && path != null) {
+      try {
+        fs.delete(path, recursive);
+      } catch (IOException e) {
+        LOG.warn("When deleting {}", path, e);
+      }
+    }
+  }
+
+
+  /**
+   * Clean up the test account if non-null; return null to put in the
+   * field.
+   * @param testAccount test account to clean up
+   * @return null
+   * @throws Execption cleanup problems
+   */
+  public static AzureBlobStorageTestAccount cleanup(
+      AzureBlobStorageTestAccount testAccount) throws Exception {
+    if (testAccount != null) {
+      testAccount.cleanup();
+      testAccount = null;
+    }
+    return null;
+  }
+
+
+  /**
+   * Clean up the test account; any thrown exceptions are caught and
+   * logged.
+   * @param testAccount test account
+   * @return null, so that any fields can be reset.
+   */
+  public static AzureBlobStorageTestAccount cleanupTestAccount(
+      AzureBlobStorageTestAccount testAccount) {
+    if (testAccount != null) {
+      try {
+        testAccount.cleanup();
+      } catch (Exception e) {
+        LOG.error("While cleaning up test account: ", e);
+      }
+    }
+    return null;
+  }
+
+  /**
+   * Assume that the scale tests are enabled by the relevant system property.
+   */
+  public static void assumeScaleTestsEnabled(Configuration conf) {
+    boolean enabled = getTestPropertyBool(
+        conf,
+        KEY_SCALE_TESTS_ENABLED,
+        DEFAULT_SCALE_TESTS_ENABLED);
+    assume("Scale test disabled: to enable set property "
+            + KEY_SCALE_TESTS_ENABLED,
+        enabled);
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/2d2d97fa/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/integration/CleanupTestContainers.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/integration/CleanupTestContainers.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/integration/CleanupTestContainers.java
new file mode 100644
index 0000000..059a8c4
--- /dev/null
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/integration/CleanupTestContainers.java
@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.azure.integration;
+
+import java.util.EnumSet;
+
+import com.microsoft.azure.storage.CloudStorageAccount;
+import com.microsoft.azure.storage.blob.CloudBlobClient;
+import com.microsoft.azure.storage.blob.CloudBlobContainer;
+import org.junit.Test;
+
+import org.apache.hadoop.fs.azure.AbstractWasbTestBase;
+import org.apache.hadoop.fs.azure.AzureBlobStorageTestAccount;
+
+/**
+ * This looks like a test, but it is really a command to invoke to
+ * clean up containers created in other test runs.
+ *
+ */
+public class CleanupTestContainers extends AbstractWasbTestBase {
+
+  private static final String CONTAINER_PREFIX = "wasbtests-";
+
+  @Override
+  protected AzureBlobStorageTestAccount createTestAccount() throws Exception {
+    return AzureBlobStorageTestAccount.create(
+        "CleanupTestContainers",
+        EnumSet.noneOf(AzureBlobStorageTestAccount.CreateOptions.class),
+        createConfiguration(),
+        true);
+  }
+
+  @Test
+  public void testEnumContainers() throws Throwable {
+    describe("Enumerating all the WASB test containers");
+
+    int count = 0;
+    CloudStorageAccount storageAccount = getTestAccount().getRealAccount();
+    CloudBlobClient blobClient = storageAccount.createCloudBlobClient();
+    Iterable<CloudBlobContainer> containers
+        = blobClient.listContainers(CONTAINER_PREFIX);
+    for (CloudBlobContainer container : containers) {
+      count++;
+      LOG.info("Container {} URI {}",
+          container.getName(),
+          container.getUri());
+    }
+    LOG.info("Found {} test containers", count);
+  }
+
+  @Test
+  public void testDeleteContainers() throws Throwable {
+    describe("Delete all the WASB test containers");
+    int count = 0;
+    CloudStorageAccount storageAccount = getTestAccount().getRealAccount();
+    CloudBlobClient blobClient = storageAccount.createCloudBlobClient();
+    Iterable<CloudBlobContainer> containers
+        = blobClient.listContainers(CONTAINER_PREFIX);
+    for (CloudBlobContainer container : containers) {
+      LOG.info("Container {} URI {}",
+          container.getName(),
+          container.getUri());
+      if (container.deleteIfExists()) {
+        count++;
+      }
+    }
+    LOG.info("Deleted {} test containers", count);
+  }
+
+
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/2d2d97fa/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/integration/ITestAzureHugeFiles.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/integration/ITestAzureHugeFiles.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/integration/ITestAzureHugeFiles.java
new file mode 100644
index 0000000..850aca1
--- /dev/null
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/integration/ITestAzureHugeFiles.java
@@ -0,0 +1,456 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.azure.integration;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.util.EnumSet;
+import java.util.Iterator;
+
+import org.junit.Assert;
+import org.junit.Assume;
+import org.junit.FixMethodOrder;
+import org.junit.Test;
+import org.junit.runners.MethodSorters;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.StorageStatistics;
+import org.apache.hadoop.fs.azure.AzureBlobStorageTestAccount;
+import org.apache.hadoop.fs.azure.NativeAzureFileSystem;
+import org.apache.hadoop.fs.contract.ContractTestUtils;
+import org.apache.hadoop.io.IOUtils;
+
+import static org.apache.hadoop.fs.azure.integration.AzureTestUtils.*;
+import static org.apache.hadoop.fs.contract.ContractTestUtils.*;
+
+
+/**
+ * Scale test which creates a huge file.
+ *
+ * <b>Important:</b> the order in which these tests execute is fixed to
+ * alphabetical order. Test cases are numbered {@code test_123_} to impose
+ * an ordering based on the numbers.
+ *
+ * Having this ordering allows the tests to assume that the huge file
+ * exists. Even so: they should all have a {@link #assumeHugeFileExists()}
+ * check at the start, in case an individual test is executed.
+ *
+ * <b>Ignore checkstyle complaints about naming: we need a scheme with visible
+ * ordering.</b>
+ */
+
+@FixMethodOrder(MethodSorters.NAME_ASCENDING)
+public class ITestAzureHugeFiles extends AbstractAzureScaleTest {
+
+  private static final Logger LOG = LoggerFactory.getLogger(
+      ITestAzureHugeFiles.class);
+
+  private Path scaleTestDir;
+  private Path hugefile;
+  private Path hugefileRenamed;
+  private AzureBlobStorageTestAccount testAccountForCleanup;
+
+  private static final int UPLOAD_BLOCKSIZE = 64 * S_1K;
+  private static final byte[] SOURCE_DATA;
+
+  static {
+    SOURCE_DATA = dataset(UPLOAD_BLOCKSIZE, 0, S_256);
+  }
+
+  private Path testPath;
+
+  @Override
+  public void setUp() throws Exception {
+    super.setUp();
+    testPath = path("ITestAzureHugeFiles");
+    scaleTestDir = new Path(testPath, "scale");
+    hugefile = new Path(scaleTestDir, "hugefile");
+    hugefileRenamed = new Path(scaleTestDir, "hugefileRenamed");
+  }
+
+  /**
+   * Only clean up the test account (and delete the container) if the account
+   * is set in the field {@code testAccountForCleanup}.
+   * @throws Exception
+   */
+  @Override
+  public void tearDown() throws Exception {
+    testAccount = null;
+    super.tearDown();
+    if (testAccountForCleanup != null) {
+      cleanupTestAccount(testAccount);
+    }
+  }
+
+  @Override
+  protected AzureBlobStorageTestAccount createTestAccount() throws Exception {
+    return AzureBlobStorageTestAccount.create(
+        "testazurehugefiles",
+        EnumSet.of(AzureBlobStorageTestAccount.CreateOptions.CreateContainer),
+        createConfiguration(),
+        true);
+  }
+
+  /**
+   * Stop the test-case teardown from deleting the test path.
+   * @throws IOException never
+   */
+  protected void deleteTestDirInTeardown() throws IOException {
+    // this is a no-op, so the test file is preserved.
+    // the last test in the suite does the teardown
+  }
+
+  protected void deleteHugeFile() throws IOException {
+    describe("Deleting %s", hugefile);
+    ContractTestUtils.NanoTimer timer = new ContractTestUtils.NanoTimer();
+    getFileSystem().delete(hugefile, false);
+    timer.end("time to delete %s", hugefile);
+  }
+
+  /**
+   * Log how long an IOP took, by dividing the total time by the
+   * count of operations, printing in a human-readable form.
+   * @param operation operation being measured
+   * @param timer timing data
+   * @param count IOP count.
+   */
+  protected void logTimePerIOP(String operation,
+      ContractTestUtils.NanoTimer timer,
+      long count) {
+    LOG.info("Time per {}: {} nS",
+        operation, toHuman(timer.duration() / count));
+  }
+
+  /**
+   * Assume that the huge file exists, skip if not/empty.
+   * @return the file status
+   * @throws IOException IO failure
+   */
+  FileStatus assumeHugeFileExists() throws IOException {
+    assertPathExists(getFileSystem(), "huge file not created", hugefile);
+    try {
+      FileStatus status = getFileSystem().getFileStatus(hugefile);
+      Assume.assumeTrue("Not a file: " + status, status.isFile());
+      Assume.assumeTrue("File " + hugefile + " is empty", status.getLen() > 0);
+      return status;
+    } catch (FileNotFoundException e) {
+      skip("huge file not created: " + hugefile);
+    }
+    return null;
+  }
+
+  /**
+   * If/when {@link NativeAzureFileSystem#getStorageStatistics()} returns
+   * statistics, this will be interesting.
+   */
+  private void logFSState() {
+    StorageStatistics statistics = getFileSystem().getStorageStatistics();
+    Iterator<StorageStatistics.LongStatistic> longStatistics
+        = statistics.getLongStatistics();
+    while (longStatistics.hasNext()) {
+      StorageStatistics.LongStatistic next = longStatistics.next();
+      LOG.info("{} = {}", next.getName(), next.getValue());
+    }
+  }
+
+  @Test
+  public void test_010_CreateHugeFile() throws IOException {
+    long filesize = getTestPropertyBytes(getConfiguration(),
+        KEY_HUGE_FILESIZE,
+        DEFAULT_HUGE_FILESIZE);
+    long filesizeMB = filesize / S_1M;
+
+    // clean up from any previous attempts
+    deleteHugeFile();
+
+    describe("Creating file %s of size %d MB", hugefile, filesizeMB);
+
+    // now do a check of available upload time, with a pessimistic bandwidth
+    // (that of remote upload tests). If the test times out then not only is
+    // the test outcome lost, as the follow-on tests continue, they will
+    // overlap with the ongoing upload test, for much confusion.
+/*
+    int timeout = getTestTimeoutSeconds();
+    // assume 1 MB/s upload bandwidth
+    int bandwidth = _1MB;
+    long uploadTime = filesize / bandwidth;
+    assertTrue(String.format("Timeout set in %s seconds is too low;" +
+            " estimating upload time of %d seconds at 1 MB/s." +
+            " Rerun tests with -D%s=%d",
+        timeout, uploadTime, KEY_TEST_TIMEOUT, uploadTime * 2),
+        uploadTime < timeout);
+*/
+    assertEquals("File size set in " + KEY_HUGE_FILESIZE + " = " + filesize
+            + " is not a multiple of " + UPLOAD_BLOCKSIZE,
+        0, filesize % UPLOAD_BLOCKSIZE);
+
+    byte[] data = SOURCE_DATA;
+
+    long blocks = filesize / UPLOAD_BLOCKSIZE;
+    long blocksPerMB = S_1M / UPLOAD_BLOCKSIZE;
+
+    // perform the upload.
+    // there's lots of logging here, so that a tail -f on the output log
+    // can give a view of what is happening.
+    NativeAzureFileSystem fs = getFileSystem();
+
+    ContractTestUtils.NanoTimer timer = new ContractTestUtils.NanoTimer();
+    long blocksPer10MB = blocksPerMB * 10;
+    fs.mkdirs(hugefile.getParent());
+    try (FSDataOutputStream out = fs.create(hugefile,
+        true,
+        UPLOAD_BLOCKSIZE,
+        null)) {
+      for (long block = 1; block <= blocks; block++) {
+        out.write(data);
+        long written = block * UPLOAD_BLOCKSIZE;
+        // every 10 MB and on file upload @ 100%, print some stats
+        if (block % blocksPer10MB == 0 || written == filesize) {
+          long percentage = written * 100 / filesize;
+          double elapsedTime = timer.elapsedTime() / NANOSEC;
+          double writtenMB = 1.0 * written / S_1M;
+          LOG.info(String.format("[%02d%%] Buffered %.2f MB out of %d MB;"
+                  + " elapsedTime=%.2fs; write to buffer bandwidth=%.2f MB/s",
+              percentage,
+              writtenMB,
+              filesizeMB,
+              elapsedTime,
+              writtenMB / elapsedTime));
+        }
+      }
+      // now close the file
+      LOG.info("Closing stream {}", out);
+      ContractTestUtils.NanoTimer closeTimer
+          = new ContractTestUtils.NanoTimer();
+      out.close();
+      closeTimer.end("time to close() output stream");
+    }
+
+    timer.end("time to write %d MB in blocks of %d",
+        filesizeMB, UPLOAD_BLOCKSIZE);
+    logFSState();
+    bandwidth(timer, filesize);
+    ContractTestUtils.assertPathExists(fs, "Huge file", hugefile);
+    FileStatus status = fs.getFileStatus(hugefile);
+    ContractTestUtils.assertIsFile(hugefile, status);
+    assertEquals("File size in " + status, filesize, status.getLen());
+  }
+
+  @Test
+  public void test_040_PositionedReadHugeFile() throws Throwable {
+    assumeHugeFileExists();
+    describe("Positioned reads of file %s", hugefile);
+    NativeAzureFileSystem fs = getFileSystem();
+    FileStatus status = fs.getFileStatus(hugefile);
+    long filesize = status.getLen();
+    int ops = 0;
+    final int bufferSize = 8192;
+    byte[] buffer = new byte[bufferSize];
+    long eof = filesize - 1;
+
+    ContractTestUtils.NanoTimer timer = new ContractTestUtils.NanoTimer();
+    ContractTestUtils.NanoTimer readAtByte0, readAtByte0Again, readAtEOF;
+    try (FSDataInputStream in = openDataFile()) {
+      readAtByte0 = new ContractTestUtils.NanoTimer();
+      in.readFully(0, buffer);
+      readAtByte0.end("time to read data at start of file");
+      ops++;
+
+      readAtEOF = new ContractTestUtils.NanoTimer();
+      in.readFully(eof - bufferSize, buffer);
+      readAtEOF.end("time to read data at end of file");
+      ops++;
+
+      readAtByte0Again = new ContractTestUtils.NanoTimer();
+      in.readFully(0, buffer);
+      readAtByte0Again.end("time to read data at start of file again");
+      ops++;
+      LOG.info("Final stream state: {}", in);
+    }
+    long mb = Math.max(filesize / S_1M, 1);
+
+    logFSState();
+    timer.end("time to performed positioned reads of %d MB ", mb);
+    LOG.info("Time per positioned read = {} nS",
+        toHuman(timer.nanosPerOperation(ops)));
+  }
+
+  protected FSDataInputStream openDataFile() throws IOException {
+    NanoTimer openTimer = new NanoTimer();
+    FSDataInputStream inputStream = getFileSystem().open(hugefile,
+        UPLOAD_BLOCKSIZE);
+    openTimer.end("open data file");
+    return inputStream;
+  }
+
+
+  /**
+   * Work out the bandwidth in bytes/second.
+   * @param timer timer measuring the duration
+   * @param bytes bytes
+   * @return the number of bytes/second of the recorded operation
+   */
+  public static double bandwidthInBytes(NanoTimer timer, long bytes) {
+    return bytes * NANOSEC / timer.duration();
+  }
+
+  @Test
+  public void test_050_readHugeFile() throws Throwable {
+    assumeHugeFileExists();
+    describe("Reading %s", hugefile);
+    NativeAzureFileSystem fs = getFileSystem();
+    FileStatus status = fs.getFileStatus(hugefile);
+    long filesize = status.getLen();
+    long blocks = filesize / UPLOAD_BLOCKSIZE;
+    byte[] data = new byte[UPLOAD_BLOCKSIZE];
+
+    ContractTestUtils.NanoTimer timer = new ContractTestUtils.NanoTimer();
+    try (FSDataInputStream in = openDataFile()) {
+      for (long block = 0; block < blocks; block++) {
+        in.readFully(data);
+      }
+      LOG.info("Final stream state: {}", in);
+    }
+
+    long mb = Math.max(filesize / S_1M, 1);
+    timer.end("time to read file of %d MB ", mb);
+    LOG.info("Time per MB to read = {} nS",
+        toHuman(timer.nanosPerOperation(mb)));
+    bandwidth(timer, filesize);
+    logFSState();
+  }
+
+  @Test
+  public void test_060_openAndReadWholeFileBlocks() throws Throwable {
+    FileStatus status = assumeHugeFileExists();
+    int blockSize = S_1M;
+    describe("Open the test file and read it in blocks of size %d",
+        blockSize);
+    long len =  status.getLen();
+    FSDataInputStream in = openDataFile();
+    NanoTimer timer2 = null;
+    long blockCount = 0;
+    long totalToRead = 0;
+    int resetCount = 0;
+    try {
+      byte[] block = new byte[blockSize];
+      timer2 = new NanoTimer();
+      long count = 0;
+      // implicitly rounding down here
+      blockCount = len / blockSize;
+      totalToRead = blockCount * blockSize;
+      long minimumBandwidth = S_128K;
+      int maxResetCount = 4;
+      resetCount = 0;
+      for (long i = 0; i < blockCount; i++) {
+        int offset = 0;
+        int remaining = blockSize;
+        long blockId = i + 1;
+        NanoTimer blockTimer = new NanoTimer();
+        int reads = 0;
+        while (remaining > 0) {
+          NanoTimer readTimer = new NanoTimer();
+          int bytesRead = in.read(block, offset, remaining);
+          reads++;
+          if (bytesRead == 1) {
+            break;
+          }
+          remaining -= bytesRead;
+          offset += bytesRead;
+          count += bytesRead;
+          readTimer.end();
+          if (bytesRead != 0) {
+            LOG.debug("Bytes in read #{}: {} , block bytes: {},"
+                    + " remaining in block: {}"
+                    + " duration={} nS; ns/byte: {}, bandwidth={} MB/s",
+                reads, bytesRead, blockSize - remaining, remaining,
+                readTimer.duration(),
+                readTimer.nanosPerOperation(bytesRead),
+                readTimer.bandwidthDescription(bytesRead));
+          } else {
+            LOG.warn("0 bytes returned by read() operation #{}", reads);
+          }
+        }
+        blockTimer.end("Reading block %d in %d reads", blockId, reads);
+        String bw = blockTimer.bandwidthDescription(blockSize);
+        LOG.info("Bandwidth of block {}: {} MB/s: ", blockId, bw);
+        if (bandwidthInBytes(blockTimer, blockSize) < minimumBandwidth) {
+          LOG.warn("Bandwidth {} too low on block {}: resetting connection",
+              bw, blockId);
+          Assert.assertTrue("Bandwidth of " + bw + " too low after "
+              + resetCount + " attempts", resetCount <= maxResetCount);
+          resetCount++;
+          // reset the connection
+        }
+      }
+    } finally {
+      IOUtils.closeStream(in);
+    }
+    timer2.end("Time to read %d bytes in %d blocks", totalToRead, blockCount);
+    LOG.info("Overall Bandwidth {} MB/s; reset connections {}",
+        timer2.bandwidth(totalToRead), resetCount);
+  }
+
+  @Test
+  public void test_100_renameHugeFile() throws Throwable {
+    assumeHugeFileExists();
+    describe("renaming %s to %s", hugefile, hugefileRenamed);
+    NativeAzureFileSystem fs = getFileSystem();
+    FileStatus status = fs.getFileStatus(hugefile);
+    long filesize = status.getLen();
+    fs.delete(hugefileRenamed, false);
+    ContractTestUtils.NanoTimer timer = new ContractTestUtils.NanoTimer();
+    fs.rename(hugefile, hugefileRenamed);
+    long mb = Math.max(filesize / S_1M, 1);
+    timer.end("time to rename file of %d MB", mb);
+    LOG.info("Time per MB to rename = {} nS",
+        toHuman(timer.nanosPerOperation(mb)));
+    bandwidth(timer, filesize);
+    logFSState();
+    FileStatus destFileStatus = fs.getFileStatus(hugefileRenamed);
+    assertEquals(filesize, destFileStatus.getLen());
+
+    // rename back
+    ContractTestUtils.NanoTimer timer2 = new ContractTestUtils.NanoTimer();
+    fs.rename(hugefileRenamed, hugefile);
+    timer2.end("Renaming back");
+    LOG.info("Time per MB to rename = {} nS",
+        toHuman(timer2.nanosPerOperation(mb)));
+    bandwidth(timer2, filesize);
+  }
+
+  @Test
+  public void test_999_deleteHugeFiles() throws IOException {
+    // mark the test account for cleanup after this test
+    testAccountForCleanup = testAccount;
+    deleteHugeFile();
+    ContractTestUtils.NanoTimer timer2 = new ContractTestUtils.NanoTimer();
+    NativeAzureFileSystem fs = getFileSystem();
+    fs.delete(hugefileRenamed, false);
+    timer2.end("time to delete %s", hugefileRenamed);
+    rm(fs, testPath, true, false);
+    assertPathDoesNotExist(fs, "deleted huge file", testPath);
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/2d2d97fa/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/integration/Sizes.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/integration/Sizes.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/integration/Sizes.java
new file mode 100644
index 0000000..92b10cf
--- /dev/null
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/integration/Sizes.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.azure.integration;
+
+/**
+ * Sizes of data.
+ * Checkstyle doesn't like the naming scheme or the fact its an interface.
+ */
+public interface Sizes {
+
+  int S_256 = 256;
+  int S_512 = 512;
+  int S_1K = 1024;
+  int S_4K = 4 * S_1K;
+  int S_8K = 8 * S_1K;
+  int S_16K = 16 * S_1K;
+  int S_32K = 32 * S_1K;
+  int S_64K = 64 * S_1K;
+  int S_128K = 128 * S_1K;
+  int S_256K = 256 * S_1K;
+  int S_1M = S_1K * S_1K;
+  int S_2M = 2 * S_1M;
+  int S_5M = 5 * S_1M;
+  int S_10M = 10* S_1M;
+  double NANOSEC = 1.0e9;
+
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/2d2d97fa/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/metrics/ITestAzureFileSystemInstrumentation.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/metrics/ITestAzureFileSystemInstrumentation.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/metrics/ITestAzureFileSystemInstrumentation.java
new file mode 100644
index 0000000..60e24ee
--- /dev/null
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/metrics/ITestAzureFileSystemInstrumentation.java
@@ -0,0 +1,586 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.azure.metrics;
+
+import static org.apache.hadoop.fs.azure.metrics.AzureFileSystemInstrumentation.WASB_CLIENT_ERRORS;
+import static org.apache.hadoop.fs.azure.metrics.AzureFileSystemInstrumentation.WASB_DIRECTORIES_CREATED;
+import static org.apache.hadoop.fs.azure.metrics.AzureFileSystemInstrumentation.WASB_DOWNLOAD_LATENCY;
+import static org.apache.hadoop.fs.azure.metrics.AzureFileSystemInstrumentation.WASB_DOWNLOAD_RATE;
+import static org.apache.hadoop.fs.azure.metrics.AzureFileSystemInstrumentation.WASB_FILES_CREATED;
+import static org.apache.hadoop.fs.azure.metrics.AzureFileSystemInstrumentation.WASB_FILES_DELETED;
+import static org.apache.hadoop.fs.azure.metrics.AzureFileSystemInstrumentation.WASB_SERVER_ERRORS;
+import static org.apache.hadoop.fs.azure.metrics.AzureFileSystemInstrumentation.WASB_UPLOAD_LATENCY;
+import static org.apache.hadoop.fs.azure.metrics.AzureFileSystemInstrumentation.WASB_UPLOAD_RATE;
+import static org.apache.hadoop.fs.azure.metrics.AzureFileSystemInstrumentation.WASB_WEB_RESPONSES;
+import static org.apache.hadoop.test.MetricsAsserts.assertCounter;
+import static org.apache.hadoop.test.MetricsAsserts.getMetrics;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.mockito.Matchers.argThat;
+import static org.mockito.Mockito.verify;
+
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.util.Arrays;
+import java.util.Date;
+
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.azure.AbstractWasbTestBase;
+import org.apache.hadoop.fs.azure.AzureBlobStorageTestAccount;
+import org.apache.hadoop.fs.azure.AzureException;
+import org.apache.hadoop.fs.azure.AzureNativeFileSystemStore;
+import org.apache.hadoop.fs.azure.NativeAzureFileSystem;
+import org.apache.hadoop.io.IOUtils;
+import org.apache.hadoop.metrics2.MetricsRecordBuilder;
+import org.apache.hadoop.metrics2.MetricsTag;
+import org.hamcrest.BaseMatcher;
+import org.hamcrest.Description;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Instrumentation test, changing state of time and verifying metrics are
+ * consistent.
+ */
+public class ITestAzureFileSystemInstrumentation extends AbstractWasbTestBase {
+
+  protected static final Logger LOG =
+      LoggerFactory.getLogger(ITestAzureFileSystemInstrumentation.class);
+
+  @Override
+  protected AzureBlobStorageTestAccount createTestAccount() throws Exception {
+    return AzureBlobStorageTestAccount.create();
+  }
+
+  @Test
+  public void testMetricTags() throws Exception {
+    String accountName =
+        getTestAccount().getRealAccount().getBlobEndpoint()
+        .getAuthority();
+    String containerName =
+        getTestAccount().getRealContainer().getName();
+    MetricsRecordBuilder myMetrics = getMyMetrics();
+    verify(myMetrics).add(argThat(
+        new TagMatcher("accountName", accountName)
+        ));
+    verify(myMetrics).add(argThat(
+        new TagMatcher("containerName", containerName)
+        ));
+    verify(myMetrics).add(argThat(
+        new TagMatcher("Context", "azureFileSystem")
+        ));
+    verify(myMetrics).add(argThat(
+        new TagExistsMatcher("wasbFileSystemId")
+        ));
+  }
+  
+
+  @Test
+  public void testMetricsOnMkdirList() throws Exception {
+    long base = getBaseWebResponses();
+
+    // Create a directory
+    assertTrue(fs.mkdirs(new Path("a")));
+    // At the time of writing
+    // getAncestor uses 2 calls for each folder level /user/<name>/a
+    // plus 1 call made by checkContainer
+    // mkdir checks the hierarchy with 2 calls per level
+    // mkdirs calls storeEmptyDir to create the empty folder, which makes 5 calls
+    // For a total of 7 + 6 + 5 = 18 web responses
+    base = assertWebResponsesInRange(base, 1, 18);
+    assertEquals(1,
+        AzureMetricsTestUtil.getLongCounterValue(getInstrumentation(), WASB_DIRECTORIES_CREATED));
+
+    // List the root contents
+    assertEquals(1, getFileSystem().listStatus(new Path("/")).length);
+    base = assertWebResponsesEquals(base, 1);
+
+    assertNoErrors();
+  }
+
+  private BandwidthGaugeUpdater getBandwidthGaugeUpdater() {
+    NativeAzureFileSystem azureFs = (NativeAzureFileSystem) getFileSystem();
+    AzureNativeFileSystemStore azureStore = azureFs.getStore();
+    return azureStore.getBandwidthGaugeUpdater();
+  }
+
+  private static byte[] nonZeroByteArray(int size) {
+    byte[] data = new byte[size];
+    Arrays.fill(data, (byte)5);
+    return data;
+  }
+
+  @Test
+  public void testMetricsOnFileCreateRead() throws Exception {
+    long base = getBaseWebResponses();
+
+    assertEquals(0, AzureMetricsTestUtil.getCurrentBytesWritten(getInstrumentation()));
+
+    Path filePath = new Path("/metricsTest_webResponses");
+    final int FILE_SIZE = 1000;
+
+    // Suppress auto-update of bandwidth metrics so we get
+    // to update them exactly when we want to.
+    getBandwidthGaugeUpdater().suppressAutoUpdate();
+
+    // Create a file
+    Date start = new Date();
+    OutputStream outputStream = getFileSystem().create(filePath);
+    outputStream.write(nonZeroByteArray(FILE_SIZE));
+    outputStream.close();
+    long uploadDurationMs = new Date().getTime() - start.getTime();
+
+    // The exact number of requests/responses that happen to create a file
+    // can vary  - at the time of writing this code it takes 10
+    // requests/responses for the 1000 byte file (33 for 100 MB),
+    // plus the initial container-check request but that
+    // can very easily change in the future. Just assert that we do roughly
+    // more than 2 but less than 15.
+    logOpResponseCount("Creating a 1K file", base);
+    base = assertWebResponsesInRange(base, 2, 15);
+    getBandwidthGaugeUpdater().triggerUpdate(true);
+    long bytesWritten = AzureMetricsTestUtil.getCurrentBytesWritten(getInstrumentation());
+    assertTrue("The bytes written in the last second " + bytesWritten +
+        " is pretty far from the expected range of around " + FILE_SIZE +
+        " bytes plus a little overhead.",
+        bytesWritten > (FILE_SIZE / 2) && bytesWritten < (FILE_SIZE * 2));
+    long totalBytesWritten = AzureMetricsTestUtil.getCurrentTotalBytesWritten(getInstrumentation());
+    assertTrue("The total bytes written  " + totalBytesWritten +
+        " is pretty far from the expected range of around " + FILE_SIZE +
+        " bytes plus a little overhead.",
+        totalBytesWritten >= FILE_SIZE && totalBytesWritten < (FILE_SIZE * 2));
+    long uploadRate = AzureMetricsTestUtil.getLongGaugeValue(getInstrumentation(), WASB_UPLOAD_RATE);
+    LOG.info("Upload rate: " + uploadRate + " bytes/second.");
+    long expectedRate = (FILE_SIZE * 1000L) / uploadDurationMs;
+    assertTrue("The upload rate " + uploadRate +
+        " is below the expected range of around " + expectedRate +
+        " bytes/second that the unit test observed. This should never be" +
+        " the case since the test underestimates the rate by looking at " +
+        " end-to-end time instead of just block upload time.",
+        uploadRate >= expectedRate);
+    long uploadLatency = AzureMetricsTestUtil.getLongGaugeValue(getInstrumentation(),
+        WASB_UPLOAD_LATENCY);
+    LOG.info("Upload latency: {}", uploadLatency);
+    long expectedLatency = uploadDurationMs; // We're uploading less than a block.
+    assertTrue("The upload latency " + uploadLatency +
+        " should be greater than zero now that I've just uploaded a file.",
+        uploadLatency > 0);
+    assertTrue("The upload latency " + uploadLatency +
+        " is more than the expected range of around " + expectedLatency +
+        " milliseconds that the unit test observed. This should never be" +
+        " the case since the test overestimates the latency by looking at " +
+        " end-to-end time instead of just block upload time.",
+        uploadLatency <= expectedLatency);
+
+    // Read the file
+    start = new Date();
+    InputStream inputStream = getFileSystem().open(filePath);
+    int count = 0;
+    while (inputStream.read() >= 0) {
+      count++;
+    }
+    inputStream.close();
+    long downloadDurationMs = new Date().getTime() - start.getTime();
+    assertEquals(FILE_SIZE, count);
+
+    // Again, exact number varies. At the time of writing this code
+    // it takes 4 request/responses, so just assert a rough range between
+    // 1 and 10.
+    logOpResponseCount("Reading a 1K file", base);
+    base = assertWebResponsesInRange(base, 1, 10);
+    getBandwidthGaugeUpdater().triggerUpdate(false);
+    long totalBytesRead = AzureMetricsTestUtil.getCurrentTotalBytesRead(getInstrumentation());
+    assertEquals(FILE_SIZE, totalBytesRead);
+    long bytesRead = AzureMetricsTestUtil.getCurrentBytesRead(getInstrumentation());
+    assertTrue("The bytes read in the last second " + bytesRead +
+        " is pretty far from the expected range of around " + FILE_SIZE +
+        " bytes plus a little overhead.",
+        bytesRead > (FILE_SIZE / 2) && bytesRead < (FILE_SIZE * 2));
+    long downloadRate = AzureMetricsTestUtil.getLongGaugeValue(getInstrumentation(), WASB_DOWNLOAD_RATE);
+    LOG.info("Download rate: " + downloadRate + " bytes/second.");
+    expectedRate = (FILE_SIZE * 1000L) / downloadDurationMs;
+    assertTrue("The download rate " + downloadRate +
+        " is below the expected range of around " + expectedRate +
+        " bytes/second that the unit test observed. This should never be" +
+        " the case since the test underestimates the rate by looking at " +
+        " end-to-end time instead of just block download time.",
+        downloadRate >= expectedRate);
+    long downloadLatency = AzureMetricsTestUtil.getLongGaugeValue(getInstrumentation(),
+        WASB_DOWNLOAD_LATENCY);
+    LOG.info("Download latency: " + downloadLatency);
+    expectedLatency = downloadDurationMs; // We're downloading less than a block.
+    assertTrue("The download latency " + downloadLatency +
+        " should be greater than zero now that I've just downloaded a file.",
+        downloadLatency > 0);
+    assertTrue("The download latency " + downloadLatency +
+        " is more than the expected range of around " + expectedLatency +
+        " milliseconds that the unit test observed. This should never be" +
+        " the case since the test overestimates the latency by looking at " +
+        " end-to-end time instead of just block download time.",
+        downloadLatency <= expectedLatency);
+
+    assertNoErrors();
+  }
+
+  @Test
+  public void testMetricsOnBigFileCreateRead() throws Exception {
+    long base = getBaseWebResponses();
+
+    assertEquals(0, AzureMetricsTestUtil.getCurrentBytesWritten(getInstrumentation()));
+
+    Path filePath = new Path("/metricsTest_webResponses");
+    final int FILE_SIZE = 100 * 1024 * 1024;
+
+    // Suppress auto-update of bandwidth metrics so we get
+    // to update them exactly when we want to.
+    getBandwidthGaugeUpdater().suppressAutoUpdate();
+
+    // Create a file
+    OutputStream outputStream = getFileSystem().create(filePath);
+    outputStream.write(new byte[FILE_SIZE]);
+    outputStream.close();
+
+    // The exact number of requests/responses that happen to create a file
+    // can vary  - at the time of writing this code it takes 34
+    // requests/responses for the 100 MB file,
+    // plus the initial container check request, but that
+    // can very easily change in the future. Just assert that we do roughly
+    // more than 20 but less than 50.
+    logOpResponseCount("Creating a 100 MB file", base);
+    base = assertWebResponsesInRange(base, 20, 50);
+    getBandwidthGaugeUpdater().triggerUpdate(true);
+    long totalBytesWritten = AzureMetricsTestUtil.getCurrentTotalBytesWritten(getInstrumentation());
+    assertTrue("The total bytes written  " + totalBytesWritten +
+        " is pretty far from the expected range of around " + FILE_SIZE +
+        " bytes plus a little overhead.",
+        totalBytesWritten >= FILE_SIZE && totalBytesWritten < (FILE_SIZE * 2));
+    long uploadRate = AzureMetricsTestUtil.getLongGaugeValue(getInstrumentation(), WASB_UPLOAD_RATE);
+    LOG.info("Upload rate: " + uploadRate + " bytes/second.");
+    long uploadLatency = AzureMetricsTestUtil.getLongGaugeValue(getInstrumentation(),
+        WASB_UPLOAD_LATENCY);
+    LOG.info("Upload latency: " + uploadLatency);
+    assertTrue("The upload latency " + uploadLatency +
+        " should be greater than zero now that I've just uploaded a file.",
+        uploadLatency > 0);
+
+    // Read the file
+    InputStream inputStream = getFileSystem().open(filePath);
+    int count = 0;
+    while (inputStream.read() >= 0) {
+      count++;
+    }
+    inputStream.close();
+    assertEquals(FILE_SIZE, count);
+
+    // Again, exact number varies. At the time of writing this code
+    // it takes 27 request/responses, so just assert a rough range between
+    // 20 and 40.
+    logOpResponseCount("Reading a 100 MB file", base);
+    base = assertWebResponsesInRange(base, 20, 40);
+    getBandwidthGaugeUpdater().triggerUpdate(false);
+    long totalBytesRead = AzureMetricsTestUtil.getCurrentTotalBytesRead(getInstrumentation());
+    assertEquals(FILE_SIZE, totalBytesRead);
+    long downloadRate = AzureMetricsTestUtil.getLongGaugeValue(getInstrumentation(), WASB_DOWNLOAD_RATE);
+    LOG.info("Download rate: " + downloadRate + " bytes/second.");
+    long downloadLatency = AzureMetricsTestUtil.getLongGaugeValue(getInstrumentation(),
+        WASB_DOWNLOAD_LATENCY);
+    LOG.info("Download latency: " + downloadLatency);
+    assertTrue("The download latency " + downloadLatency +
+        " should be greater than zero now that I've just downloaded a file.",
+        downloadLatency > 0);
+  }
+
+  @Test
+  public void testMetricsOnFileRename() throws Exception {
+    long base = getBaseWebResponses();
+
+    Path originalPath = new Path("/metricsTest_RenameStart");
+    Path destinationPath = new Path("/metricsTest_RenameFinal");
+
+    // Create an empty file
+    assertEquals(0, AzureMetricsTestUtil.getLongCounterValue(getInstrumentation(), WASB_FILES_CREATED));
+    assertTrue(getFileSystem().createNewFile(originalPath));
+    logOpResponseCount("Creating an empty file", base);
+    base = assertWebResponsesInRange(base, 2, 20);
+    assertEquals(1, AzureMetricsTestUtil.getLongCounterValue(getInstrumentation(), WASB_FILES_CREATED));
+
+    // Rename the file
+    assertTrue(
+        ((FileSystem) getFileSystem()).rename(originalPath, destinationPath));
+    // Varies: at the time of writing this code it takes 7 requests/responses.
+    logOpResponseCount("Renaming a file", base);
+    base = assertWebResponsesInRange(base, 2, 15);
+
+    assertNoErrors();
+  }
+
+  @Test
+  public void testMetricsOnFileExistsDelete() throws Exception {
+    long base = getBaseWebResponses();
+
+    Path filePath = new Path("/metricsTest_delete");
+
+    // Check existence
+    assertFalse(getFileSystem().exists(filePath));
+    // At the time of writing this code it takes 2 requests/responses to
+    // check existence, which seems excessive, plus initial request for
+    // container check.
+    logOpResponseCount("Checking file existence for non-existent file", base);
+    base = assertWebResponsesInRange(base, 1, 3);
+
+    // Create an empty file
+    assertTrue(getFileSystem().createNewFile(filePath));
+    base = getCurrentWebResponses();
+
+    // Check existence again
+    assertTrue(getFileSystem().exists(filePath));
+    logOpResponseCount("Checking file existence for existent file", base);
+    base = assertWebResponsesInRange(base, 1, 2);
+
+    // Delete the file
+    assertEquals(0, AzureMetricsTestUtil.getLongCounterValue(getInstrumentation(), WASB_FILES_DELETED));
+    assertTrue(getFileSystem().delete(filePath, false));
+    // At the time of writing this code it takes 4 requests/responses to
+    // delete, which seems excessive. Check for range 1-4 for now.
+    logOpResponseCount("Deleting a file", base);
+    base = assertWebResponsesInRange(base, 1, 4);
+    assertEquals(1, AzureMetricsTestUtil.getLongCounterValue(getInstrumentation(), WASB_FILES_DELETED));
+
+    assertNoErrors();
+  }
+
+  @Test
+  public void testMetricsOnDirRename() throws Exception {
+    long base = getBaseWebResponses();
+
+    Path originalDirName = new Path("/metricsTestDirectory_RenameStart");
+    Path innerFileName = new Path(originalDirName, "innerFile");
+    Path destDirName = new Path("/metricsTestDirectory_RenameFinal");
+
+    // Create an empty directory
+    assertTrue(getFileSystem().mkdirs(originalDirName));
+    base = getCurrentWebResponses();
+
+    // Create an inner file
+    assertTrue(getFileSystem().createNewFile(innerFileName));
+    base = getCurrentWebResponses();
+
+    // Rename the directory
+    assertTrue(getFileSystem().rename(originalDirName, destDirName));
+
+    // At the time of writing this code it takes 11 requests/responses
+    // to rename the directory with one file. Check for range 1-20 for now.
+    logOpResponseCount("Renaming a directory", base);
+    base = assertWebResponsesInRange(base, 1, 20);
+
+    assertNoErrors();
+  }
+
+  /**
+   * Recursive discovery of path depth
+   * @param path path to measure.
+   * @return depth, where "/" == 0.
+   */
+  int depth(Path path) {
+    if (path.isRoot()) {
+      return 0;
+    } else {
+      return 1 + depth(path.getParent());
+    }
+  }
+
+  @Test
+  public void testClientErrorMetrics() throws Exception {
+    String fileName = "metricsTestFile_ClientError";
+    Path filePath = new Path("/"+fileName);
+    final int FILE_SIZE = 100;
+    OutputStream outputStream = null;
+    String leaseID = null;
+    try {
+      // Create a file
+      outputStream = getFileSystem().create(filePath);
+      leaseID = getTestAccount().acquireShortLease(fileName);
+      try {
+        outputStream.write(new byte[FILE_SIZE]);
+        outputStream.close();
+        assertTrue("Should've thrown", false);
+      } catch (AzureException ex) {
+        assertTrue("Unexpected exception: " + ex,
+          ex.getMessage().contains("lease"));
+      }
+      assertEquals(1, AzureMetricsTestUtil.getLongCounterValue(getInstrumentation(), WASB_CLIENT_ERRORS));
+      assertEquals(0, AzureMetricsTestUtil.getLongCounterValue(getInstrumentation(), WASB_SERVER_ERRORS));
+    } finally {
+      if(leaseID != null){
+        getTestAccount().releaseLease(leaseID, fileName);
+      }
+      IOUtils.closeStream(outputStream);
+    }
+  }
+
+  private void logOpResponseCount(String opName, long base) {
+    LOG.info("{}  took {} web responses to complete.",
+        opName, getCurrentWebResponses() - base);
+  }
+
+  /**
+   * Gets (and asserts) the value of the wasb_web_responses counter just
+   * after the creation of the file system object.
+   */
+  private long getBaseWebResponses() {
+    // The number of requests should start at 0
+    return assertWebResponsesEquals(0, 0);
+  }
+
+  /**
+   * Gets the current value of the wasb_web_responses counter.
+   */
+  private long getCurrentWebResponses() {
+      return AzureMetricsTestUtil.getCurrentWebResponses(getInstrumentation());
+  }
+
+  /**
+   * Checks that the wasb_web_responses counter is at the given value.
+   * @param base The base value (before the operation of interest).
+   * @param expected The expected value for the operation of interest.
+   * @return The new base value now.
+   */
+  private long assertWebResponsesEquals(long base, long expected) {
+    assertCounter(WASB_WEB_RESPONSES, base + expected, getMyMetrics());
+    return base + expected;
+  }
+
+  private void assertNoErrors() {
+    assertEquals(0, AzureMetricsTestUtil.getLongCounterValue(getInstrumentation(), WASB_CLIENT_ERRORS));
+    assertEquals(0, AzureMetricsTestUtil.getLongCounterValue(getInstrumentation(), WASB_SERVER_ERRORS));
+  }
+
+  /**
+   * Checks that the wasb_web_responses counter is in the given range.
+   * @param base The base value (before the operation of interest).
+   * @param inclusiveLowerLimit The lower limit for what it should increase by.
+   * @param inclusiveUpperLimit The upper limit for what it should increase by.
+   * @return The new base value now.
+   */
+  private long assertWebResponsesInRange(long base,
+      long inclusiveLowerLimit,
+      long inclusiveUpperLimit) {
+    long currentResponses = getCurrentWebResponses();
+    long justOperation = currentResponses - base;
+    assertTrue(String.format(
+        "Web responses expected in range [%d, %d], but was %d.",
+        inclusiveLowerLimit, inclusiveUpperLimit, justOperation),
+        justOperation >= inclusiveLowerLimit &&
+        justOperation <= inclusiveUpperLimit);
+    return currentResponses;
+  }  
+
+  /**
+   * Gets the metrics for the file system object.
+   * @return The metrics record.
+   */
+  private MetricsRecordBuilder getMyMetrics() {
+    return getMetrics(getInstrumentation());
+  }
+
+  private AzureFileSystemInstrumentation getInstrumentation() {
+    return getFileSystem().getInstrumentation();
+  }
+
+  /**
+   * A matcher class for asserting that we got a tag with a given
+   * value.
+   */
+  private static class TagMatcher extends TagExistsMatcher {
+    private final String tagValue;
+
+    public TagMatcher(String tagName, String tagValue) {
+      super(tagName);
+      this.tagValue = tagValue;
+    }
+
+    @Override
+    public boolean matches(MetricsTag toMatch) {
+      return toMatch.value().equals(tagValue);
+    }
+
+    @Override
+    public void describeTo(Description desc) {
+      super.describeTo(desc);
+      desc.appendText(" with value " + tagValue);
+    }
+  }
+
+  /**
+   * A matcher class for asserting that we got a tag with any value.
+   */
+  private static class TagExistsMatcher extends BaseMatcher<MetricsTag> {
+    private final String tagName;
+
+    public TagExistsMatcher(String tagName) {
+      this.tagName = tagName;
+    }
+
+    @Override
+    public boolean matches(Object toMatch) {
+      MetricsTag asTag = (MetricsTag)toMatch;
+      return asTag.name().equals(tagName) && matches(asTag);
+    }
+
+    protected boolean matches(MetricsTag toMatch) {
+      return true;
+    }
+
+    @Override
+    public void describeTo(Description desc) {
+      desc.appendText("Has tag " + tagName);
+    }
+  }
+
+  /**
+   * A matcher class for asserting that a long value is in a
+   * given range.
+   */
+  private static class InRange extends BaseMatcher<Long> {
+    private final long inclusiveLowerLimit;
+    private final long inclusiveUpperLimit;
+    private long obtained;
+
+    public InRange(long inclusiveLowerLimit, long inclusiveUpperLimit) {
+      this.inclusiveLowerLimit = inclusiveLowerLimit;
+      this.inclusiveUpperLimit = inclusiveUpperLimit;
+    }
+
+    @Override
+    public boolean matches(Object number) {
+      obtained = (Long)number;
+      return obtained >= inclusiveLowerLimit &&
+          obtained <= inclusiveUpperLimit;
+    }
+
+    @Override
+    public void describeTo(Description description) {
+      description.appendText("Between " + inclusiveLowerLimit +
+          " and " + inclusiveUpperLimit + " inclusively");
+    }
+  }
+}


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message