hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From cnaur...@apache.org
Subject [03/24] hadoop git commit: HADOOP-9629. Support Windows Azure Storage - Blob as a file system in Hadoop. Contributed by Dexter Bradshaw, Mostafa Elhemali, Xi Fang, Johannes Klein, David Lao, Mike Liddell, Chuan Liu, Lengning Liu, Ivan Mitic, Michael Rys,
Date Wed, 17 Dec 2014 22:59:49 GMT
http://git-wip-us.apache.org/repos/asf/hadoop/blob/82268d87/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/StorageInterface.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/StorageInterface.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/StorageInterface.java
new file mode 100644
index 0000000..87cef86
--- /dev/null
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/StorageInterface.java
@@ -0,0 +1,566 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.azure;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.util.EnumSet;
+import java.util.HashMap;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+
+import com.microsoft.windowsazure.storage.CloudStorageAccount;
+import com.microsoft.windowsazure.storage.OperationContext;
+import com.microsoft.windowsazure.storage.RetryPolicyFactory;
+import com.microsoft.windowsazure.storage.StorageCredentials;
+import com.microsoft.windowsazure.storage.StorageException;
+import com.microsoft.windowsazure.storage.blob.BlobListingDetails;
+import com.microsoft.windowsazure.storage.blob.BlobProperties;
+import com.microsoft.windowsazure.storage.blob.BlobRequestOptions;
+import com.microsoft.windowsazure.storage.blob.CopyState;
+import com.microsoft.windowsazure.storage.blob.ListBlobItem;
+
+/**
+ * This is a very thin layer over the methods exposed by the Windows Azure
+ * Storage SDK that we need for WASB implementation. This base class has a real
+ * implementation that just simply redirects to the SDK, and a memory-backed one
+ * that's used for unit tests.
+ * 
+ * IMPORTANT: all the methods here must remain very simple redirects since code
+ * written here can't be properly unit tested.
+ */
+@InterfaceAudience.Private
+abstract class StorageInterface {
+
+  /**
+   * Sets the timeout to use when making requests to the storage service.
+   * <p>
+   * The server timeout interval begins at the time that the complete request
+   * has been received by the service, and the server begins processing the
+   * response. If the timeout interval elapses before the response is returned
+   * to the client, the operation times out. The timeout interval resets with
+   * each retry, if the request is retried.
+   * 
+   * The default timeout interval for a request made via the service client is
+   * 90 seconds. You can change this value on the service client by setting this
+   * property, so that all subsequent requests made via the service client will
+   * use the new timeout interval. You can also change this value for an
+   * individual request, by setting the
+   * {@link RequestOptions#timeoutIntervalInMs} property.
+   * 
+   * If you are downloading a large blob, you should increase the value of the
+   * timeout beyond the default value.
+   * 
+   * @param timeoutInMs
+   *          The timeout, in milliseconds, to use when making requests to the
+   *          storage service.
+   */
+  public abstract void setTimeoutInMs(int timeoutInMs);
+
+  /**
+   * Sets the RetryPolicyFactory object to use when making service requests.
+   * 
+   * @param retryPolicyFactory
+   *          the RetryPolicyFactory object to use when making service requests.
+   */
+  public abstract void setRetryPolicyFactory(
+      final RetryPolicyFactory retryPolicyFactory);
+
+  /**
+   * Creates a new Blob service client.
+   * 
+   */
+  public abstract void createBlobClient(CloudStorageAccount account);
+
+  /**
+   * Creates an instance of the <code>CloudBlobClient</code> class using the
+   * specified Blob service endpoint.
+   * 
+   * @param baseUri
+   *          A <code>java.net.URI</code> object that represents the Blob
+   *          service endpoint used to create the client.
+   */
+  public abstract void createBlobClient(URI baseUri);
+
+  /**
+   * Creates an instance of the <code>CloudBlobClient</code> class using the
+   * specified Blob service endpoint and account credentials.
+   * 
+   * @param baseUri
+   *          A <code>java.net.URI</code> object that represents the Blob
+   *          service endpoint used to create the client.
+   * @param credentials
+   *          A {@link StorageCredentials} object that represents the account
+   *          credentials.
+   */
+  public abstract void createBlobClient(URI baseUri,
+      StorageCredentials credentials);
+
+  /**
+   * Returns the credentials for the Blob service, as configured for the storage
+   * account.
+   * 
+   * @return A {@link StorageCredentials} object that represents the credentials
+   *         for this storage account.
+   */
+  public abstract StorageCredentials getCredentials();
+
+  /**
+   * Returns a reference to a {@link CloudBlobContainerWrapper} object that
+   * represents the cloud blob container for the specified address.
+   * 
+   * @param name
+   *          A <code>String</code> that represents the name of the container.
+   * @return A {@link CloudBlobContainerWrapper} object that represents a
+   *         reference to the cloud blob container.
+   * 
+   * @throws URISyntaxException
+   *           If the resource URI is invalid.
+   * @throws StorageException
+   *           If a storage service error occurred.
+   */
+  public abstract CloudBlobContainerWrapper getContainerReference(String name)
+      throws URISyntaxException, StorageException;
+
+  /**
+   * A thin wrapper over the {@link CloudBlobDirectory} class that simply
+   * redirects calls to the real object except in unit tests.
+   */
+  @InterfaceAudience.Private
+  public abstract static class CloudBlobDirectoryWrapper implements
+      ListBlobItem {
+    /**
+     * Returns the URI for this directory.
+     * 
+     * @return A <code>java.net.URI</code> object that represents the URI for
+     *         this directory.
+     */
+    public abstract URI getUri();
+
+    /**
+     * Returns an enumerable collection of blob items whose names begin with the
+     * specified prefix, using the specified flat or hierarchical option,
+     * listing details options, request options, and operation context.
+     * 
+     * @param prefix
+     *          A <code>String</code> that represents the prefix of the blob
+     *          name.
+     * @param useFlatBlobListing
+     *          <code>true</code> to indicate that the returned list will be
+     *          flat; <code>false</code> to indicate that the returned list will
+     *          be hierarchical.
+     * @param listingDetails
+     *          A <code>java.util.EnumSet</code> object that contains
+     *          {@link BlobListingDetails} values that indicate whether
+     *          snapshots, metadata, and/or uncommitted blocks are returned.
+     *          Committed blocks are always returned.
+     * @param options
+     *          A {@link BlobRequestOptions} object that specifies any
+     *          additional options for the request. Specifying <code>null</code>
+     *          will use the default request options from the associated service
+     *          client ( {@link CloudBlobClient}).
+     * @param opContext
+     *          An {@link OperationContext} object that represents the context
+     *          for the current operation. This object is used to track requests
+     *          to the storage service, and to provide additional runtime
+     *          information about the operation.
+     * 
+     * @return An enumerable collection of {@link ListBlobItem} objects that
+     *         represent the block items whose names begin with the specified
+     *         prefix in this directory.
+     * 
+     * @throws StorageException
+     *           If a storage service error occurred.
+     * @throws URISyntaxException
+     *           If the resource URI is invalid.
+     */
+    public abstract Iterable<ListBlobItem> listBlobs(String prefix,
+        boolean useFlatBlobListing, EnumSet<BlobListingDetails> listingDetails,
+        BlobRequestOptions options, OperationContext opContext)
+        throws URISyntaxException, StorageException;
+  }
+
+  /**
+   * A thin wrapper over the {@link CloudBlobContainer} class that simply
+   * redirects calls to the real object except in unit tests.
+   */
+  @InterfaceAudience.Private
+  public abstract static class CloudBlobContainerWrapper {
+    /**
+     * Returns the name of the container.
+     * 
+     * @return A <code>String</code> that represents the name of the container.
+     */
+    public abstract String getName();
+
+    /**
+     * Returns a value that indicates whether the container exists, using the
+     * specified operation context.
+     * 
+     * @param opContext
+     *          An {@link OperationContext} object that represents the context
+     *          for the current operation. This object is used to track requests
+     *          to the storage service, and to provide additional runtime
+     *          information about the operation.
+     * 
+     * @return <code>true</code> if the container exists, otherwise
+     *         <code>false</code>.
+     * 
+     * @throws StorageException
+     *           If a storage service error occurred.
+     */
+    public abstract boolean exists(OperationContext opContext)
+        throws StorageException;
+
+    /**
+     * Returns the metadata for the container.
+     * 
+     * @return A <code>java.util.HashMap</code> object that represents the
+     *         metadata for the container.
+     */
+    public abstract HashMap<String, String> getMetadata();
+
+    /**
+     * Sets the metadata for the container.
+     * 
+     * @param metadata
+     *          A <code>java.util.HashMap</code> object that represents the
+     *          metadata being assigned to the container.
+     */
+    public abstract void setMetadata(HashMap<String, String> metadata);
+
+    /**
+     * Downloads the container's attributes, which consist of metadata and
+     * properties, using the specified operation context.
+     * 
+     * @param opContext
+     *          An {@link OperationContext} object that represents the context
+     *          for the current operation. This object is used to track requests
+     *          to the storage service, and to provide additional runtime
+     *          information about the operation.
+     * 
+     * @throws StorageException
+     *           If a storage service error occurred.
+     */
+    public abstract void downloadAttributes(OperationContext opContext)
+        throws StorageException;
+
+    /**
+     * Uploads the container's metadata using the specified operation context.
+     * 
+     * @param opContext
+     *          An {@link OperationContext} object that represents the context
+     *          for the current operation. This object is used to track requests
+     *          to the storage service, and to provide additional runtime
+     *          information about the operation.
+     * 
+     * @throws StorageException
+     *           If a storage service error occurred.
+     */
+    public abstract void uploadMetadata(OperationContext opContext)
+        throws StorageException;
+
+    /**
+     * Creates the container using the specified operation context.
+     * 
+     * @param opContext
+     *          An {@link OperationContext} object that represents the context
+     *          for the current operation. This object is used to track requests
+     *          to the storage service, and to provide additional runtime
+     *          information about the operation.
+     * 
+     * @throws StorageException
+     *           If a storage service error occurred.
+     */
+    public abstract void create(OperationContext opContext)
+        throws StorageException;
+
+    /**
+     * Returns a wrapper for a CloudBlobDirectory.
+     * 
+     * @param relativePath
+     *          A <code>String</code> that represents the name of the directory,
+     *          relative to the container
+     * 
+     * @throws StorageException
+     *           If a storage service error occurred.
+     * 
+     * @throws URISyntaxException
+     *           If URI syntax exception occurred.
+     */
+    public abstract CloudBlobDirectoryWrapper getDirectoryReference(
+        String relativePath) throws URISyntaxException, StorageException;
+
+    /**
+     * Returns a wrapper for a CloudBlockBlob.
+     * 
+     * @param relativePath
+     *          A <code>String</code> that represents the name of the blob,
+     *          relative to the container
+     * 
+     * @throws StorageException
+     *           If a storage service error occurred.
+     * 
+     * @throws URISyntaxException
+     *           If URI syntax exception occurred.
+     */
+    public abstract CloudBlockBlobWrapper getBlockBlobReference(
+        String relativePath) throws URISyntaxException, StorageException;
+  }
+
+  /**
+   * A thin wrapper over the {@link CloudBlockBlob} class that simply redirects
+   * calls to the real object except in unit tests.
+   */
+  @InterfaceAudience.Private
+  public abstract static class CloudBlockBlobWrapper implements ListBlobItem {
+    /**
+     * Returns the URI for this blob.
+     * 
+     * @return A <code>java.net.URI</code> object that represents the URI for
+     *         the blob.
+     */
+    public abstract URI getUri();
+
+    /**
+     * Returns the metadata for the blob.
+     * 
+     * @return A <code>java.util.HashMap</code> object that represents the
+     *         metadata for the blob.
+     */
+    public abstract HashMap<String, String> getMetadata();
+
+    /**
+     * Sets the metadata for the blob.
+     * 
+     * @param metadata
+     *          A <code>java.util.HashMap</code> object that contains the
+     *          metadata being assigned to the blob.
+     */
+    public abstract void setMetadata(HashMap<String, String> metadata);
+
+    /**
+     * Copies an existing blob's contents, properties, and metadata to this
+     * instance of the <code>CloudBlob</code> class, using the specified
+     * operation context.
+     * 
+     * @param sourceBlob
+     *          A <code>CloudBlob</code> object that represents the source blob
+     *          to copy.
+     * @param opContext
+     *          An {@link OperationContext} object that represents the context
+     *          for the current operation. This object is used to track requests
+     *          to the storage service, and to provide additional runtime
+     *          information about the operation.
+     * 
+     * @throws StorageException
+     *           If a storage service error occurred.
+     * @throws URISyntaxException
+     * 
+     */
+    public abstract void startCopyFromBlob(CloudBlockBlobWrapper sourceBlob,
+        OperationContext opContext) throws StorageException, URISyntaxException;
+
+    /**
+     * Returns the blob's copy state.
+     * 
+     * @return A {@link CopyState} object that represents the copy state of the
+     *         blob.
+     */
+    public abstract CopyState getCopyState();
+
+    /**
+     * Deletes the blob using the specified operation context.
+     * <p>
+     * A blob that has snapshots cannot be deleted unless the snapshots are also
+     * deleted. If a blob has snapshots, use the
+     * {@link DeleteSnapshotsOption#DELETE_SNAPSHOTS_ONLY} or
+     * {@link DeleteSnapshotsOption#INCLUDE_SNAPSHOTS} value in the
+     * <code>deleteSnapshotsOption</code> parameter to specify how the snapshots
+     * should be handled when the blob is deleted.
+     * 
+     * @param opContext
+     *          An {@link OperationContext} object that represents the context
+     *          for the current operation. This object is used to track requests
+     *          to the storage service, and to provide additional runtime
+     *          information about the operation.
+     * 
+     * @throws StorageException
+     *           If a storage service error occurred.
+     */
+    public abstract void delete(OperationContext opContext)
+        throws StorageException;
+
+    /**
+     * Checks to see if the blob exists, using the specified operation context.
+     * 
+     * @param opContext
+     *          An {@link OperationContext} object that represents the context
+     *          for the current operation. This object is used to track requests
+     *          to the storage service, and to provide additional runtime
+     *          information about the operation.
+     * 
+     * @return <code>true</code> if the blob exists, other wise
+     *         <code>false</code>.
+     * 
+     * @throws StorageException
+     *           f a storage service error occurred.
+     */
+    public abstract boolean exists(OperationContext opContext)
+        throws StorageException;
+
+    /**
+     * Populates a blob's properties and metadata using the specified operation
+     * context.
+     * <p>
+     * This method populates the blob's system properties and user-defined
+     * metadata. Before reading a blob's properties or metadata, call this
+     * method or its overload to retrieve the latest values for the blob's
+     * properties and metadata from the Windows Azure storage service.
+     * 
+     * @param opContext
+     *          An {@link OperationContext} object that represents the context
+     *          for the current operation. This object is used to track requests
+     *          to the storage service, and to provide additional runtime
+     *          information about the operation.
+     * 
+     * @throws StorageException
+     *           If a storage service error occurred.
+     */
+    public abstract void downloadAttributes(OperationContext opContext)
+        throws StorageException;
+
+    /**
+     * Returns the blob's properties.
+     * 
+     * @return A {@link BlobProperties} object that represents the properties of
+     *         the blob.
+     */
+    public abstract BlobProperties getProperties();
+
+    /**
+     * Opens a blob input stream to download the blob using the specified
+     * operation context.
+     * <p>
+     * Use {@link CloudBlobClient#setStreamMinimumReadSizeInBytes} to configure
+     * the read size.
+     * 
+     * @param opContext
+     *          An {@link OperationContext} object that represents the context
+     *          for the current operation. This object is used to track requests
+     *          to the storage service, and to provide additional runtime
+     *          information about the operation.
+     * 
+     * @return An <code>InputStream</code> object that represents the stream to
+     *         use for reading from the blob.
+     * 
+     * @throws StorageException
+     *           If a storage service error occurred.
+     */
+    public abstract InputStream openInputStream(BlobRequestOptions options,
+        OperationContext opContext) throws StorageException;
+
+    /**
+     * Creates and opens an output stream to write data to the block blob using
+     * the specified operation context.
+     * 
+     * @param opContext
+     *          An {@link OperationContext} object that represents the context
+     *          for the current operation. This object is used to track requests
+     *          to the storage service, and to provide additional runtime
+     *          information about the operation.
+     * 
+     * @return A {@link BlobOutputStream} object used to write data to the blob.
+     * 
+     * @throws StorageException
+     *           If a storage service error occurred.
+     */
+    public abstract OutputStream openOutputStream(BlobRequestOptions options,
+        OperationContext opContext) throws StorageException;
+
+    /**
+     * Uploads the source stream data to the blob, using the specified operation
+     * context.
+     * 
+     * @param sourceStream
+     *          An <code>InputStream</code> object that represents the input
+     *          stream to write to the block blob.
+     * @param opContext
+     *          An {@link OperationContext} object that represents the context
+     *          for the current operation. This object is used to track requests
+     *          to the storage service, and to provide additional runtime
+     *          information about the operation.
+     * 
+     * @throws IOException
+     *           If an I/O error occurred.
+     * @throws StorageException
+     *           If a storage service error occurred.
+     */
+    public abstract void upload(InputStream sourceStream,
+        OperationContext opContext) throws StorageException, IOException;
+
+    /**
+     * Uploads the blob's metadata to the storage service using the specified
+     * lease ID, request options, and operation context.
+     * 
+     * @param opContext
+     *          An {@link OperationContext} object that represents the context
+     *          for the current operation. This object is used to track requests
+     *          to the storage service, and to provide additional runtime
+     *          information about the operation.
+     * 
+     * @throws StorageException
+     *           If a storage service error occurred.
+     */
+    public abstract void uploadMetadata(OperationContext opContext)
+        throws StorageException;
+
+    public abstract void uploadProperties(OperationContext opContext)
+        throws StorageException;
+
+    /**
+     * Sets the minimum read block size to use with this Blob.
+     * 
+     * @param minimumReadSizeBytes
+     *          The maximum block size, in bytes, for reading from a block blob
+     *          while using a {@link BlobInputStream} object, ranging from 512
+     *          bytes to 64 MB, inclusive.
+     */
+    public abstract void setStreamMinimumReadSizeInBytes(
+        int minimumReadSizeBytes);
+
+    /**
+     * Sets the write block size to use with this Blob.
+     * 
+     * @param writeBlockSizeBytes
+     *          The maximum block size, in bytes, for writing to a block blob
+     *          while using a {@link BlobOutputStream} object, ranging from 1 MB
+     *          to 4 MB, inclusive.
+     * 
+     * @throws IllegalArgumentException
+     *           If <code>writeBlockSizeInBytes</code> is less than 1 MB or
+     *           greater than 4 MB.
+     */
+    public abstract void setWriteBlockSizeInBytes(int writeBlockSizeBytes);
+
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/82268d87/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/StorageInterfaceImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/StorageInterfaceImpl.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/StorageInterfaceImpl.java
new file mode 100644
index 0000000..935bf71
--- /dev/null
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/StorageInterfaceImpl.java
@@ -0,0 +1,372 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.azure;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.util.EnumSet;
+import java.util.HashMap;
+import java.util.Iterator;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+
+import com.microsoft.windowsazure.storage.AccessCondition;
+import com.microsoft.windowsazure.storage.CloudStorageAccount;
+import com.microsoft.windowsazure.storage.OperationContext;
+import com.microsoft.windowsazure.storage.RetryPolicyFactory;
+import com.microsoft.windowsazure.storage.StorageCredentials;
+import com.microsoft.windowsazure.storage.StorageException;
+import com.microsoft.windowsazure.storage.StorageUri;
+import com.microsoft.windowsazure.storage.blob.BlobListingDetails;
+import com.microsoft.windowsazure.storage.blob.BlobProperties;
+import com.microsoft.windowsazure.storage.blob.BlobRequestOptions;
+import com.microsoft.windowsazure.storage.blob.CloudBlobClient;
+import com.microsoft.windowsazure.storage.blob.CloudBlobContainer;
+import com.microsoft.windowsazure.storage.blob.CloudBlobDirectory;
+import com.microsoft.windowsazure.storage.blob.CloudBlockBlob;
+import com.microsoft.windowsazure.storage.blob.CopyState;
+import com.microsoft.windowsazure.storage.blob.DeleteSnapshotsOption;
+import com.microsoft.windowsazure.storage.blob.ListBlobItem;
+
+/**
+ * A real implementation of the Azure interaction layer that just redirects
+ * calls to the Windows Azure storage SDK.
+ */
+@InterfaceAudience.Private
+class StorageInterfaceImpl extends StorageInterface {
+  private CloudBlobClient serviceClient;
+
+  @Override
+  public void setRetryPolicyFactory(final RetryPolicyFactory retryPolicyFactory) {
+    serviceClient.setRetryPolicyFactory(retryPolicyFactory);
+  }
+
+  @Override
+  public void setTimeoutInMs(int timeoutInMs) {
+    serviceClient.setTimeoutInMs(timeoutInMs);
+  }
+
+  @Override
+  public void createBlobClient(CloudStorageAccount account) {
+    serviceClient = account.createCloudBlobClient();
+  }
+
+  @Override
+  public void createBlobClient(URI baseUri) {
+    serviceClient = new CloudBlobClient(baseUri);
+  }
+
+  @Override
+  public void createBlobClient(URI baseUri, StorageCredentials credentials) {
+    serviceClient = new CloudBlobClient(baseUri, credentials);
+  }
+
+  @Override
+  public StorageCredentials getCredentials() {
+    return serviceClient.getCredentials();
+  }
+
+  @Override
+  public CloudBlobContainerWrapper getContainerReference(String uri)
+      throws URISyntaxException, StorageException {
+    return new CloudBlobContainerWrapperImpl(
+        serviceClient.getContainerReference(uri));
+  }
+
+  //
+  // WrappingIterator
+  //
+
+  /**
+   * This iterator wraps every ListBlobItem as they come from the listBlobs()
+   * calls to their proper wrapping objects.
+   */
+  private static class WrappingIterator implements Iterator<ListBlobItem> {
+    private final Iterator<ListBlobItem> present;
+
+    public WrappingIterator(Iterator<ListBlobItem> present) {
+      this.present = present;
+    }
+
+    public static Iterable<ListBlobItem> wrap(
+        final Iterable<ListBlobItem> present) {
+      return new Iterable<ListBlobItem>() {
+        @Override
+        public Iterator<ListBlobItem> iterator() {
+          return new WrappingIterator(present.iterator());
+        }
+      };
+    }
+
+    @Override
+    public boolean hasNext() {
+      return present.hasNext();
+    }
+
+    @Override
+    public ListBlobItem next() {
+      ListBlobItem unwrapped = present.next();
+      if (unwrapped instanceof CloudBlobDirectory) {
+        return new CloudBlobDirectoryWrapperImpl((CloudBlobDirectory) unwrapped);
+      } else if (unwrapped instanceof CloudBlockBlob) {
+        return new CloudBlockBlobWrapperImpl((CloudBlockBlob) unwrapped);
+      } else {
+        return unwrapped;
+      }
+    }
+
+    @Override
+    public void remove() {
+      present.remove();
+    }
+  }
+
+  //
+  // CloudBlobDirectoryWrapperImpl
+  //
+  @InterfaceAudience.Private
+  static class CloudBlobDirectoryWrapperImpl extends CloudBlobDirectoryWrapper {
+    private final CloudBlobDirectory directory;
+
+    public CloudBlobDirectoryWrapperImpl(CloudBlobDirectory directory) {
+      this.directory = directory;
+    }
+
+    @Override
+    public URI getUri() {
+      return directory.getUri();
+    }
+
+    @Override
+    public Iterable<ListBlobItem> listBlobs(String prefix,
+        boolean useFlatBlobListing, EnumSet<BlobListingDetails> listingDetails,
+        BlobRequestOptions options, OperationContext opContext)
+        throws URISyntaxException, StorageException {
+      return WrappingIterator.wrap(directory.listBlobs(prefix,
+          useFlatBlobListing, listingDetails, options, opContext));
+    }
+
+    @Override
+    public CloudBlobContainer getContainer() throws URISyntaxException,
+        StorageException {
+      return directory.getContainer();
+    }
+
+    @Override
+    public CloudBlobDirectory getParent() throws URISyntaxException,
+        StorageException {
+      return directory.getParent();
+    }
+
+    @Override
+    public StorageUri getStorageUri() {
+      return directory.getStorageUri();
+    }
+
+  }
+
+  //
+  // CloudBlobContainerWrapperImpl
+  //
+  @InterfaceAudience.Private
+  static class CloudBlobContainerWrapperImpl extends CloudBlobContainerWrapper {
+    private final CloudBlobContainer container;
+
+    public CloudBlobContainerWrapperImpl(CloudBlobContainer container) {
+      this.container = container;
+    }
+
+    @Override
+    public String getName() {
+      return container.getName();
+    }
+
+    @Override
+    public boolean exists(OperationContext opContext) throws StorageException {
+      return container.exists(AccessCondition.generateEmptyCondition(), null,
+          opContext);
+    }
+
+    @Override
+    public void create(OperationContext opContext) throws StorageException {
+      container.create(null, opContext);
+    }
+
+    @Override
+    public HashMap<String, String> getMetadata() {
+      return container.getMetadata();
+    }
+
+    @Override
+    public void setMetadata(HashMap<String, String> metadata) {
+      container.setMetadata(metadata);
+    }
+
+    @Override
+    public void downloadAttributes(OperationContext opContext)
+        throws StorageException {
+      container.downloadAttributes(AccessCondition.generateEmptyCondition(),
+          null, opContext);
+    }
+
+    @Override
+    public void uploadMetadata(OperationContext opContext)
+        throws StorageException {
+      container.uploadMetadata(AccessCondition.generateEmptyCondition(), null,
+          opContext);
+    }
+
+    @Override
+    public CloudBlobDirectoryWrapper getDirectoryReference(String relativePath)
+        throws URISyntaxException, StorageException {
+
+      CloudBlobDirectory dir = container.getDirectoryReference(relativePath);
+      return new CloudBlobDirectoryWrapperImpl(dir);
+    }
+
+    @Override
+    public CloudBlockBlobWrapper getBlockBlobReference(String relativePath)
+        throws URISyntaxException, StorageException {
+
+      return new CloudBlockBlobWrapperImpl(
+          container.getBlockBlobReference(relativePath));
+    }
+  }
+
+  //
+  // CloudBlockBlobWrapperImpl
+  //
+  @InterfaceAudience.Private
+  static class CloudBlockBlobWrapperImpl extends CloudBlockBlobWrapper {
+    private final CloudBlockBlob blob;
+
+    public URI getUri() {
+      return blob.getUri();
+    }
+
+    public CloudBlockBlobWrapperImpl(CloudBlockBlob blob) {
+      this.blob = blob;
+    }
+
+    @Override
+    public HashMap<String, String> getMetadata() {
+      return blob.getMetadata();
+    }
+
+    @Override
+    public void startCopyFromBlob(CloudBlockBlobWrapper sourceBlob,
+        OperationContext opContext) throws StorageException, URISyntaxException {
+
+      blob.startCopyFromBlob(((CloudBlockBlobWrapperImpl) sourceBlob).blob,
+          null, null, null, opContext);
+
+    }
+
+    @Override
+    public void delete(OperationContext opContext) throws StorageException {
+      blob.delete(DeleteSnapshotsOption.NONE, null, null, opContext);
+    }
+
+    @Override
+    public boolean exists(OperationContext opContext) throws StorageException {
+      return blob.exists(null, null, opContext);
+    }
+
+    @Override
+    public void downloadAttributes(OperationContext opContext)
+        throws StorageException {
+      blob.downloadAttributes(null, null, opContext);
+    }
+
+    @Override
+    public BlobProperties getProperties() {
+      return blob.getProperties();
+    }
+
+    @Override
+    public void setMetadata(HashMap<String, String> metadata) {
+      blob.setMetadata(metadata);
+    }
+
+    @Override
+    public InputStream openInputStream(BlobRequestOptions options,
+        OperationContext opContext) throws StorageException {
+      return blob.openInputStream(null, options, opContext);
+    }
+
+    @Override
+    public OutputStream openOutputStream(BlobRequestOptions options,
+        OperationContext opContext) throws StorageException {
+      return blob.openOutputStream(null, options, opContext);
+    }
+
+    @Override
+    public void upload(InputStream sourceStream, OperationContext opContext)
+        throws StorageException, IOException {
+      blob.upload(sourceStream, 0, null, null, opContext);
+    }
+
+    @Override
+    public CloudBlobContainer getContainer() throws URISyntaxException,
+        StorageException {
+      return blob.getContainer();
+    }
+
+    @Override
+    public CloudBlobDirectory getParent() throws URISyntaxException,
+        StorageException {
+      return blob.getParent();
+    }
+
+    @Override
+    public void uploadMetadata(OperationContext opContext)
+        throws StorageException {
+      blob.uploadMetadata(null, null, opContext);
+    }
+
+    @Override
+    public void uploadProperties(OperationContext opContext)
+        throws StorageException {
+      blob.uploadProperties(null, null, opContext);
+    }
+
+    @Override
+    public void setStreamMinimumReadSizeInBytes(int minimumReadSizeBytes) {
+      blob.setStreamMinimumReadSizeInBytes(minimumReadSizeBytes);
+    }
+
+    @Override
+    public void setWriteBlockSizeInBytes(int writeBlockSizeBytes) {
+      blob.setStreamWriteSizeInBytes(writeBlockSizeBytes);
+    }
+
+    @Override
+    public StorageUri getStorageUri() {
+      return blob.getStorageUri();
+    }
+
+    @Override
+    public CopyState getCopyState() {
+      return blob.getCopyState();
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/82268d87/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/Wasb.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/Wasb.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/Wasb.java
new file mode 100644
index 0000000..e098cef
--- /dev/null
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/Wasb.java
@@ -0,0 +1,48 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.azure;
+
+import java.io.IOException;
+import java.net.URI;
+import java.net.URISyntaxException;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.DelegateToFileSystem;
+
+
+/**
+ * WASB implementation of AbstractFileSystem.
+ * This impl delegates to the old FileSystem
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
+public class Wasb extends DelegateToFileSystem {
+
+  Wasb(final URI theUri, final Configuration conf) throws IOException,
+      URISyntaxException {
+    super(theUri, new NativeAzureFileSystem(), conf, "wasb", false);
+  }
+
+  @Override
+  public int getUriDefaultPort() {
+    return -1;
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/82268d87/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/WasbFsck.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/WasbFsck.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/WasbFsck.java
new file mode 100644
index 0000000..d311550
--- /dev/null
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/WasbFsck.java
@@ -0,0 +1,196 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.azure;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.List;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.util.Tool;
+import org.apache.hadoop.util.ToolRunner;
+
+import com.google.common.annotations.VisibleForTesting;
+
+/**
+ * An fsck tool implementation for WASB that does various admin/cleanup/recovery
+ * tasks on the WASB file system.
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
+public class WasbFsck extends Configured implements Tool {
+  private FileSystem mockFileSystemForTesting = null;
+  private static final String LOST_AND_FOUND_PATH = "/lost+found";
+  private boolean pathNameWarning = false;
+
+  public WasbFsck(Configuration conf) {
+    super(conf);
+  }
+
+  /**
+   * For testing purposes, set the file system to use here instead of relying on
+   * getting it from the FileSystem class based on the URI.
+   * 
+   * @param fileSystem
+   *          The file system to use.
+   */
+  @VisibleForTesting
+  public void setMockFileSystemForTesting(FileSystem fileSystem) {
+    this.mockFileSystemForTesting = fileSystem;
+  }
+
+  @Override
+  public int run(String[] args) throws Exception {
+    if (doPrintUsage(Arrays.asList(args))) {
+      printUsage();
+      return -1;
+    }
+    Path pathToCheck = null;
+    boolean doRecover = false;
+    boolean doDelete = false;
+    for (String arg : args) {
+      if (!arg.startsWith("-")) {
+        if (pathToCheck != null) {
+          System.err
+              .println("Can't specify multiple paths to check on the command-line");
+          return 1;
+        }
+        pathToCheck = new Path(arg);
+      } else if (arg.equals("-move")) {
+        doRecover = true;
+      } else if (arg.equals("-delete")) {
+        doDelete = true;
+      }
+    }
+    if (doRecover && doDelete) {
+      System.err
+          .println("Conflicting options: can't specify both -move and -delete.");
+      return 1;
+    }
+    if (pathToCheck == null) {
+      pathToCheck = new Path("/"); // Check everything.
+    }
+    FileSystem fs;
+    if (mockFileSystemForTesting == null) {
+      fs = FileSystem.get(pathToCheck.toUri(), getConf());
+    } else {
+      fs = mockFileSystemForTesting;
+    }
+
+    if (!recursiveCheckChildPathName(fs, fs.makeQualified(pathToCheck))) {
+      pathNameWarning = true;
+    }
+
+    if (!(fs instanceof NativeAzureFileSystem)) {
+      System.err
+          .println("Can only check WASB file system. Instead I'm asked to"
+              + " check: " + fs.getUri());
+      return 2;
+    }
+    NativeAzureFileSystem wasbFs = (NativeAzureFileSystem) fs;
+    if (doRecover) {
+      System.out.println("Recovering files with dangling data under: "
+          + pathToCheck);
+      wasbFs.recoverFilesWithDanglingTempData(pathToCheck, new Path(
+          LOST_AND_FOUND_PATH));
+    } else if (doDelete) {
+      System.out.println("Deleting temp files with dangling data under: "
+          + pathToCheck);
+      wasbFs.deleteFilesWithDanglingTempData(pathToCheck);
+    } else {
+      System.out.println("Please specify -move or -delete");
+    }
+    return 0;
+  }
+
+  public boolean getPathNameWarning() {
+    return pathNameWarning;
+  }
+
+  /**
+   * Recursively check if a given path and its child paths have colons in their
+   * names. It returns true if none of them has a colon or this path does not
+   * exist, and false otherwise.
+   */
+  private boolean recursiveCheckChildPathName(FileSystem fs, Path p)
+      throws IOException {
+    if (p == null) {
+      return true;
+    }
+    if (!fs.exists(p)) {
+      System.out.println("Path " + p + " does not exist!");
+      return true;
+    }
+
+    if (fs.isFile(p)) {
+      if (containsColon(p)) {
+        System.out.println("Warning: file " + p + " has a colon in its name.");
+        return false;
+      } else {
+        return true;
+      }
+    } else {
+      boolean flag;
+      if (containsColon(p)) {
+        System.out.println("Warning: directory " + p
+            + " has a colon in its name.");
+        flag = false;
+      } else {
+        flag = true;
+      }
+      FileStatus[] listed = fs.listStatus(p);
+      for (FileStatus l : listed) {
+        if (!recursiveCheckChildPathName(fs, l.getPath())) {
+          flag = false;
+        }
+      }
+      return flag;
+    }
+  }
+
+  private boolean containsColon(Path p) {
+    return p.toUri().getPath().toString().contains(":");
+  }
+
+  private static void printUsage() {
+    System.out.println("Usage: WasbFSck [<path>] [-move | -delete]");
+    System.out.println("\t<path>\tstart checking from this path");
+    System.out.println("\t-move\tmove any files whose upload was interrupted"
+        + " mid-stream to " + LOST_AND_FOUND_PATH);
+    System.out
+        .println("\t-delete\tdelete any files whose upload was interrupted"
+            + " mid-stream");
+    ToolRunner.printGenericCommandUsage(System.out);
+  }
+
+  private boolean doPrintUsage(List<String> args) {
+    return args.contains("-H");
+  }
+
+  public static void main(String[] args) throws Exception {
+    int res = ToolRunner.run(new WasbFsck(new Configuration()), args);
+    System.exit(res);
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/82268d87/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/package.html
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/package.html b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/package.html
new file mode 100644
index 0000000..de01683
--- /dev/null
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/package.html
@@ -0,0 +1,31 @@
+<html>
+
+<!--
+   Licensed to the Apache Software Foundation (ASF) under one or more
+   contributor license agreements.  See the NOTICE file distributed with
+   this work for additional information regarding copyright ownership.
+   The ASF licenses this file to You under the Apache License, Version 2.0
+   (the "License"); you may not use this file except in compliance with
+   the License.  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+-->
+
+<body>
+
+<p>
+A distributed implementation of {@link
+org.apache.hadoop.fs.FileSystem} for reading and writing files on
+<a href="http://store.azure.com">Azure Block Storage</a>.
+This implementation is blob-based and stores files on Azure in their native form for
+interoperability with other Azure tools.
+</p>
+
+</body>
+</html>

http://git-wip-us.apache.org/repos/asf/hadoop/blob/82268d87/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/AzureBlobStorageTestAccount.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/AzureBlobStorageTestAccount.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/AzureBlobStorageTestAccount.java
new file mode 100644
index 0000000..8133954
--- /dev/null
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/AzureBlobStorageTestAccount.java
@@ -0,0 +1,726 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ **/
+
+package org.apache.hadoop.fs.azure;
+
+import static org.apache.hadoop.fs.azure.AzureNativeFileSystemStore.DEFAULT_STORAGE_EMULATOR_ACCOUNT_NAME;
+
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.util.Calendar;
+import java.util.Date;
+import java.util.EnumSet;
+import java.util.GregorianCalendar;
+import java.util.TimeZone;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+
+import com.microsoft.windowsazure.storage.AccessCondition;
+import com.microsoft.windowsazure.storage.CloudStorageAccount;
+import com.microsoft.windowsazure.storage.StorageCredentials;
+import com.microsoft.windowsazure.storage.StorageCredentialsAccountAndKey;
+import com.microsoft.windowsazure.storage.StorageCredentialsAnonymous;
+import com.microsoft.windowsazure.storage.blob.BlobContainerPermissions;
+import com.microsoft.windowsazure.storage.blob.BlobContainerPublicAccessType;
+import com.microsoft.windowsazure.storage.blob.BlobOutputStream;
+import com.microsoft.windowsazure.storage.blob.CloudBlobClient;
+import com.microsoft.windowsazure.storage.blob.CloudBlobContainer;
+import com.microsoft.windowsazure.storage.blob.CloudBlockBlob;
+import com.microsoft.windowsazure.storage.blob.SharedAccessBlobPermissions;
+import com.microsoft.windowsazure.storage.blob.SharedAccessBlobPolicy;
+import com.microsoft.windowsazure.storage.core.Base64;
+
+/**
+ * Helper class to create WASB file systems backed by either a mock in-memory
+ * implementation or a real Azure Storage account. See RunningLiveWasbTests.txt
+ * for instructions on how to connect to a real Azure Storage account.
+ */
+public final class AzureBlobStorageTestAccount {
+
+  private static final String ACCOUNT_KEY_PROPERTY_NAME = "fs.azure.account.key.";
+  private static final String SAS_PROPERTY_NAME = "fs.azure.sas.";
+  private static final String TEST_CONFIGURATION_FILE_NAME = "azure-test.xml";
+  private static final String TEST_ACCOUNT_NAME_PROPERTY_NAME = "fs.azure.test.account.name";
+  public static final String MOCK_ACCOUNT_NAME = "mockAccount.blob.core.windows.net";
+  public static final String MOCK_CONTAINER_NAME = "mockContainer";
+  public static final String WASB_AUTHORITY_DELIMITER = "@";
+  public static final String WASB_SCHEME = "wasb";
+  public static final String PATH_DELIMITER = "/";
+  public static final String AZURE_ROOT_CONTAINER = "$root";
+  public static final String MOCK_WASB_URI = "wasb://" + MOCK_CONTAINER_NAME
+      + WASB_AUTHORITY_DELIMITER + MOCK_ACCOUNT_NAME + "/";
+  private static final String USE_EMULATOR_PROPERTY_NAME = "fs.azure.test.emulator";
+
+  private static final String KEY_DISABLE_THROTTLING = "fs.azure.disable.bandwidth.throttling";
+  private static final String KEY_READ_TOLERATE_CONCURRENT_APPEND = "fs.azure.io.read.tolerate.concurrent.append";
+
+  private CloudStorageAccount account;
+  private CloudBlobContainer container;
+  private CloudBlockBlob blob;
+  private NativeAzureFileSystem fs;
+  private AzureNativeFileSystemStore storage;
+  private MockStorageInterface mockStorage;
+
+  private AzureBlobStorageTestAccount(NativeAzureFileSystem fs,
+      CloudStorageAccount account, CloudBlobContainer container) {
+    this.account = account;
+    this.container = container;
+    this.fs = fs;
+  }
+
+  /**
+   * Create a test account with an initialized storage reference.
+   * 
+   * @param storage
+   *          -- store to be accessed by the account
+   * @param account
+   *          -- Windows Azure account object
+   * @param container
+   *          -- Windows Azure container object
+   */
+  private AzureBlobStorageTestAccount(AzureNativeFileSystemStore storage,
+      CloudStorageAccount account, CloudBlobContainer container) {
+    this.account = account;
+    this.container = container;
+    this.storage = storage;
+  }
+
+  /**
+   * Create a test account sessions with the default root container.
+   * 
+   * @param fs
+   *          - file system, namely WASB file system
+   * @param account
+   *          - Windows Azure account object
+   * @param blob
+   *          - block blob reference
+   */
+  private AzureBlobStorageTestAccount(NativeAzureFileSystem fs,
+      CloudStorageAccount account, CloudBlockBlob blob) {
+
+    this.account = account;
+    this.blob = blob;
+    this.fs = fs;
+  }
+
+  private AzureBlobStorageTestAccount(NativeAzureFileSystem fs,
+      MockStorageInterface mockStorage) {
+    this.fs = fs;
+    this.mockStorage = mockStorage;
+  }
+
+  public static String getMockContainerUri() {
+    return String.format("http://%s/%s",
+        AzureBlobStorageTestAccount.MOCK_ACCOUNT_NAME,
+        AzureBlobStorageTestAccount.MOCK_CONTAINER_NAME);
+  }
+
+  public static String toMockUri(String path) {
+    return String.format("http://%s/%s/%s",
+        AzureBlobStorageTestAccount.MOCK_ACCOUNT_NAME,
+        AzureBlobStorageTestAccount.MOCK_CONTAINER_NAME, path);
+  }
+
+  public static String toMockUri(Path path) {
+    // Remove the first SEPARATOR
+    return toMockUri(path.toUri().getRawPath().substring(1)); 
+  }
+
+  /**
+   * Gets the blob reference to the given blob key.
+   * 
+   * @param blobKey
+   *          The blob key (no initial slash).
+   * @return The blob reference.
+   */
+  public CloudBlockBlob getBlobReference(String blobKey) throws Exception {
+    return container.getBlockBlobReference(String.format(blobKey));
+  }
+
+  /**
+   * Acquires a short lease on the given blob in this test account.
+   * 
+   * @param blobKey
+   *          The key to the blob (no initial slash).
+   * @return The lease ID.
+   */
+  public String acquireShortLease(String blobKey) throws Exception {
+    return getBlobReference(blobKey).acquireLease(60, null);
+  }
+
+  /**
+   * Releases the lease on the container.
+   * 
+   * @param leaseID
+   *          The lease ID.
+   */
+  public void releaseLease(String leaseID, String blobKey) throws Exception {
+    AccessCondition accessCondition = new AccessCondition();
+    accessCondition.setLeaseID(leaseID);
+    getBlobReference(blobKey).releaseLease(accessCondition);
+  }
+
+  public static AzureBlobStorageTestAccount createMock() throws Exception {
+    return createMock(new Configuration());
+  }
+
+  public static AzureBlobStorageTestAccount createMock(Configuration conf)
+      throws Exception {
+    AzureNativeFileSystemStore store = new AzureNativeFileSystemStore();
+    MockStorageInterface mockStorage = new MockStorageInterface();
+    store.setAzureStorageInteractionLayer(mockStorage);
+    NativeAzureFileSystem fs = new NativeAzureFileSystem(store);
+    addWasbToConfiguration(conf);
+    setMockAccountKey(conf);
+    // register the fs provider.
+
+    fs.initialize(new URI(MOCK_WASB_URI), conf);
+    AzureBlobStorageTestAccount testAcct = new AzureBlobStorageTestAccount(fs,
+        mockStorage);
+    return testAcct;
+  }
+
+  /**
+   * Creates a test account that goes against the storage emulator.
+   * 
+   * @return The test account, or null if the emulator isn't setup.
+   */
+  public static AzureBlobStorageTestAccount createForEmulator()
+      throws Exception {
+    NativeAzureFileSystem fs = null;
+    CloudBlobContainer container = null;
+    Configuration conf = createTestConfiguration();
+    if (!conf.getBoolean(USE_EMULATOR_PROPERTY_NAME, false)) {
+      // Not configured to test against the storage emulator.
+      System.out.println("Skipping emulator Azure test because configuration "
+          + "doesn't indicate that it's running."
+          + " Please see README.txt for guidance.");
+      return null;
+    }
+    CloudStorageAccount account = CloudStorageAccount
+        .getDevelopmentStorageAccount();
+    fs = new NativeAzureFileSystem();
+    String containerName = String.format("wasbtests-%s-%tQ",
+        System.getProperty("user.name"), new Date());
+    container = account.createCloudBlobClient().getContainerReference(
+        containerName);
+    container.create();
+
+    // Set account URI and initialize Azure file system.
+    URI accountUri = createAccountUri(DEFAULT_STORAGE_EMULATOR_ACCOUNT_NAME,
+        containerName);
+    fs.initialize(accountUri, conf);
+
+    // Create test account initializing the appropriate member variables.
+    AzureBlobStorageTestAccount testAcct = new AzureBlobStorageTestAccount(fs,
+        account, container);
+
+    return testAcct;
+  }
+
+  public static AzureBlobStorageTestAccount createOutOfBandStore(
+      int uploadBlockSize, int downloadBlockSize) throws Exception {
+
+    CloudBlobContainer container = null;
+    Configuration conf = createTestConfiguration();
+    CloudStorageAccount account = createTestAccount(conf);
+    if (null == account) {
+      return null;
+    }
+
+    String containerName = String.format("wasbtests-%s-%tQ",
+        System.getProperty("user.name"), new Date());
+
+    // Create the container.
+    container = account.createCloudBlobClient().getContainerReference(
+        containerName);
+    container.create();
+
+    String accountName = conf.get(TEST_ACCOUNT_NAME_PROPERTY_NAME);
+
+    // Ensure that custom throttling is disabled and tolerate concurrent
+    // out-of-band appends.
+    conf.setBoolean(KEY_DISABLE_THROTTLING, true);
+    conf.setBoolean(KEY_READ_TOLERATE_CONCURRENT_APPEND, true);
+
+    // Set account URI and initialize Azure file system.
+    URI accountUri = createAccountUri(accountName, containerName);
+
+    // Create a new AzureNativeFileSystemStore object.
+    AzureNativeFileSystemStore testStorage = new AzureNativeFileSystemStore();
+
+    // Initialize the store with the throttling feedback interfaces.
+    testStorage.initialize(accountUri, conf);
+
+    // Create test account initializing the appropriate member variables.
+    AzureBlobStorageTestAccount testAcct = new AzureBlobStorageTestAccount(
+        testStorage, account, container);
+
+    return testAcct;
+  }
+
+  /**
+   * Sets the mock account key in the given configuration.
+   * 
+   * @param conf
+   *          The configuration.
+   */
+  public static void setMockAccountKey(Configuration conf) {
+    setMockAccountKey(conf, MOCK_ACCOUNT_NAME);
+  }
+
+  /**
+   * Sets the mock account key in the given configuration.
+   * 
+   * @param conf
+   *          The configuration.
+   */
+  public static void setMockAccountKey(Configuration conf, String accountName) {
+    conf.set(ACCOUNT_KEY_PROPERTY_NAME + accountName,
+        Base64.encode(new byte[] { 1, 2, 3 }));  
+  }
+
+  private static URI createAccountUri(String accountName)
+      throws URISyntaxException {
+    return new URI(WASB_SCHEME + ":" + PATH_DELIMITER + PATH_DELIMITER
+        + accountName);
+  }
+
+  private static URI createAccountUri(String accountName, String containerName)
+      throws URISyntaxException {
+    return new URI(WASB_SCHEME + ":" + PATH_DELIMITER + PATH_DELIMITER
+        + containerName + WASB_AUTHORITY_DELIMITER + accountName);
+  }
+
+  public static AzureBlobStorageTestAccount create() throws Exception {
+    return create("");
+  }
+
+  public static AzureBlobStorageTestAccount create(String containerNameSuffix)
+      throws Exception {
+    return create(containerNameSuffix,
+        EnumSet.of(CreateOptions.CreateContainer));
+  }
+
+  // Create a test account which uses throttling.
+  public static AzureBlobStorageTestAccount createThrottled() throws Exception {
+    return create("",
+        EnumSet.of(CreateOptions.useThrottling, CreateOptions.CreateContainer));
+  }
+
+  public static AzureBlobStorageTestAccount create(Configuration conf)
+      throws Exception {
+    return create("", EnumSet.of(CreateOptions.CreateContainer), conf);
+  }
+
+  static CloudStorageAccount createStorageAccount(String accountName,
+      Configuration conf, boolean allowAnonymous) throws URISyntaxException,
+      KeyProviderException {
+    String accountKey = AzureNativeFileSystemStore
+        .getAccountKeyFromConfiguration(accountName, conf);
+    StorageCredentials credentials;
+    if (accountKey == null && allowAnonymous) {
+      credentials = StorageCredentialsAnonymous.ANONYMOUS;
+    } else {
+      credentials = new StorageCredentialsAccountAndKey(
+          accountName.split("\\.")[0], accountKey);
+    }
+    if (credentials == null) {
+      return null;
+    } else {
+      return new CloudStorageAccount(credentials);
+    }
+  }
+
+  private static Configuration createTestConfiguration() {
+    return createTestConfiguration(null);
+  }
+
+  protected static Configuration createTestConfiguration(Configuration conf) {
+    if (conf == null) {
+      conf = new Configuration();
+    }
+
+    conf.addResource(TEST_CONFIGURATION_FILE_NAME);
+    return conf;
+  }
+
+  // for programmatic setting of the wasb configuration.
+  // note that tests can also get the
+  public static void addWasbToConfiguration(Configuration conf) {
+    conf.set("fs.wasb.impl", "org.apache.hadoop.fs.azure.NativeAzureFileSystem");
+    conf.set("fs.wasbs.impl",
+        "org.apache.hadoop.fs.azure.NativeAzureFileSystem");
+  }
+
+  static CloudStorageAccount createTestAccount() throws URISyntaxException,
+      KeyProviderException {
+    return createTestAccount(createTestConfiguration());
+  }
+
+  static CloudStorageAccount createTestAccount(Configuration conf)
+      throws URISyntaxException, KeyProviderException {
+    String testAccountName = conf.get(TEST_ACCOUNT_NAME_PROPERTY_NAME);
+    if (testAccountName == null) {
+      System.out
+          .println("Skipping live Azure test because of missing test account."
+              + " Please see README.txt for guidance.");
+      return null;
+    }
+    return createStorageAccount(testAccountName, conf, false);
+  }
+
+  public static enum CreateOptions {
+    UseSas, Readonly, CreateContainer, useThrottling
+  }
+
+  public static AzureBlobStorageTestAccount create(String containerNameSuffix,
+      EnumSet<CreateOptions> createOptions) throws Exception {
+    return create(containerNameSuffix, createOptions, null);
+  }
+
+  public static AzureBlobStorageTestAccount create(String containerNameSuffix,
+      EnumSet<CreateOptions> createOptions, Configuration initialConfiguration)
+      throws Exception {
+    NativeAzureFileSystem fs = null;
+    CloudBlobContainer container = null;
+    Configuration conf = createTestConfiguration(initialConfiguration);
+    CloudStorageAccount account = createTestAccount(conf);
+    if (account == null) {
+      return null;
+    }
+    fs = new NativeAzureFileSystem();
+    String containerName = String.format("wasbtests-%s-%tQ%s",
+        System.getProperty("user.name"), new Date(), containerNameSuffix);
+    container = account.createCloudBlobClient().getContainerReference(
+        containerName);
+    if (createOptions.contains(CreateOptions.CreateContainer)) {
+      container.create();
+    }
+    String accountName = conf.get(TEST_ACCOUNT_NAME_PROPERTY_NAME);
+    if (createOptions.contains(CreateOptions.UseSas)) {
+      String sas = generateSAS(container,
+          createOptions.contains(CreateOptions.Readonly));
+      if (!createOptions.contains(CreateOptions.CreateContainer)) {
+        // The caller doesn't want the container to be pre-created,
+        // so delete it now that we have generated the SAS.
+        container.delete();
+      }
+      // Remove the account key from the configuration to make sure we don't
+      // cheat and use that.
+      conf.set(ACCOUNT_KEY_PROPERTY_NAME + accountName, "");
+      // Set the SAS key.
+      conf.set(SAS_PROPERTY_NAME + containerName + "." + accountName, sas);
+    }
+
+    // Check if throttling is turned on and set throttling parameters
+    // appropriately.
+    if (createOptions.contains(CreateOptions.useThrottling)) {
+      conf.setBoolean(KEY_DISABLE_THROTTLING, false);
+    } else {
+      conf.setBoolean(KEY_DISABLE_THROTTLING, true);
+    }
+
+    // Set account URI and initialize Azure file system.
+    URI accountUri = createAccountUri(accountName, containerName);
+    fs.initialize(accountUri, conf);
+
+    // Create test account initializing the appropriate member variables.
+    AzureBlobStorageTestAccount testAcct = new AzureBlobStorageTestAccount(fs,
+        account, container);
+
+    return testAcct;
+  }
+
+  private static String generateContainerName() throws Exception {
+    String containerName = String.format("wasbtests-%s-%tQ",
+        System.getProperty("user.name"), new Date());
+    return containerName;
+  }
+
+  private static String generateSAS(CloudBlobContainer container,
+      boolean readonly) throws Exception {
+
+    // Create a container if it does not exist.
+    container.createIfNotExists();
+
+    // Create a new shared access policy.
+    SharedAccessBlobPolicy sasPolicy = new SharedAccessBlobPolicy();
+
+    // Create a UTC Gregorian calendar value.
+    GregorianCalendar calendar = new GregorianCalendar(
+        TimeZone.getTimeZone("UTC"));
+
+    // Specify the current time as the start time for the shared access
+    // signature.
+    //
+    calendar.setTime(new Date());
+    sasPolicy.setSharedAccessStartTime(calendar.getTime());
+
+    // Use the start time delta one hour as the end time for the shared
+    // access signature.
+    calendar.add(Calendar.HOUR, 10);
+    sasPolicy.setSharedAccessExpiryTime(calendar.getTime());
+
+    if (readonly) {
+      // Set READ permissions
+      sasPolicy.setPermissions(EnumSet.of(SharedAccessBlobPermissions.READ,
+          SharedAccessBlobPermissions.LIST));
+    } else {
+      // Set READ and WRITE permissions.
+      sasPolicy.setPermissions(EnumSet.of(SharedAccessBlobPermissions.READ,
+          SharedAccessBlobPermissions.WRITE, SharedAccessBlobPermissions.LIST));
+    }
+
+    // Create the container permissions.
+    BlobContainerPermissions containerPermissions = new BlobContainerPermissions();
+
+    // Turn public access to the container off.
+    containerPermissions.setPublicAccess(BlobContainerPublicAccessType.OFF);
+
+    container.uploadPermissions(containerPermissions);
+
+    // Create a shared access signature for the container.
+    String sas = container.generateSharedAccessSignature(sasPolicy, null);
+    // HACK: when the just generated SAS is used straight away, we get an
+    // authorization error intermittently. Sleeping for 1.5 seconds fixes that
+    // on my box.
+    Thread.sleep(1500);
+
+    // Return to caller with the shared access signature.
+    return sas;
+  }
+
+  public static void primePublicContainer(CloudBlobClient blobClient,
+      String accountName, String containerName, String blobName, int fileSize)
+      throws Exception {
+
+    // Create a container if it does not exist. The container name
+    // must be lower case.
+    CloudBlobContainer container = blobClient
+        .getContainerReference(containerName);
+
+    container.createIfNotExists();
+
+    // Create a new shared access policy.
+    SharedAccessBlobPolicy sasPolicy = new SharedAccessBlobPolicy();
+
+    // Set READ and WRITE permissions.
+    sasPolicy.setPermissions(EnumSet.of(SharedAccessBlobPermissions.READ,
+        SharedAccessBlobPermissions.WRITE, SharedAccessBlobPermissions.LIST,
+        SharedAccessBlobPermissions.DELETE));
+
+    // Create the container permissions.
+    BlobContainerPermissions containerPermissions = new BlobContainerPermissions();
+
+    // Turn public access to the container off.
+    containerPermissions
+        .setPublicAccess(BlobContainerPublicAccessType.CONTAINER);
+
+    // Set the policy using the values set above.
+    containerPermissions.getSharedAccessPolicies().put("testwasbpolicy",
+        sasPolicy);
+    container.uploadPermissions(containerPermissions);
+
+    // Create a blob output stream.
+    CloudBlockBlob blob = container.getBlockBlobReference(blobName);
+    BlobOutputStream outputStream = blob.openOutputStream();
+
+    outputStream.write(new byte[fileSize]);
+    outputStream.close();
+  }
+
+  public static AzureBlobStorageTestAccount createAnonymous(
+      final String blobName, final int fileSize) throws Exception {
+
+    NativeAzureFileSystem fs = null;
+    CloudBlobContainer container = null;
+    Configuration conf = createTestConfiguration(), noTestAccountConf = new Configuration();
+
+    // Set up a session with the cloud blob client to generate SAS and check the
+    // existence of a container and capture the container object.
+    CloudStorageAccount account = createTestAccount(conf);
+    if (account == null) {
+      return null;
+    }
+    CloudBlobClient blobClient = account.createCloudBlobClient();
+
+    // Capture the account URL and the account name.
+    String accountName = conf.get(TEST_ACCOUNT_NAME_PROPERTY_NAME);
+
+    // Generate a container name and create a shared access signature string for
+    // it.
+    //
+    String containerName = generateContainerName();
+
+    // Set up public container with the specified blob name.
+    primePublicContainer(blobClient, accountName, containerName, blobName,
+        fileSize);
+
+    // Capture the blob container object. It should exist after generating the
+    // shared access signature.
+    container = blobClient.getContainerReference(containerName);
+    if (null == container || !container.exists()) {
+      final String errMsg = String
+          .format("Container '%s' expected but not found while creating SAS account.");
+      throw new Exception(errMsg);
+    }
+
+    // Set the account URI.
+    URI accountUri = createAccountUri(accountName, containerName);
+
+    // Initialize the Native Azure file system with anonymous credentials.
+    fs = new NativeAzureFileSystem();
+    fs.initialize(accountUri, noTestAccountConf);
+
+    // Create test account initializing the appropriate member variables.
+    AzureBlobStorageTestAccount testAcct = new AzureBlobStorageTestAccount(fs,
+        account, container);
+
+    // Return to caller with test account.
+    return testAcct;
+  }
+
+  private static CloudBlockBlob primeRootContainer(CloudBlobClient blobClient,
+      String accountName, String blobName, int fileSize) throws Exception {
+
+    // Create a container if it does not exist. The container name
+    // must be lower case.
+    CloudBlobContainer container = blobClient.getContainerReference("https://"
+        + accountName + "/" + "$root");
+    container.createIfNotExists();
+
+    // Create a blob output stream.
+    CloudBlockBlob blob = container.getBlockBlobReference(blobName);
+    BlobOutputStream outputStream = blob.openOutputStream();
+
+    outputStream.write(new byte[fileSize]);
+    outputStream.close();
+
+    // Return a reference to the block blob object.
+    return blob;
+  }
+
+  public static AzureBlobStorageTestAccount createRoot(final String blobName,
+      final int fileSize) throws Exception {
+
+    NativeAzureFileSystem fs = null;
+    CloudBlobContainer container = null;
+    Configuration conf = createTestConfiguration();
+
+    // Set up a session with the cloud blob client to generate SAS and check the
+    // existence of a container and capture the container object.
+    CloudStorageAccount account = createTestAccount(conf);
+    if (account == null) {
+      return null;
+    }
+    CloudBlobClient blobClient = account.createCloudBlobClient();
+
+    // Capture the account URL and the account name.
+    String accountName = conf.get(TEST_ACCOUNT_NAME_PROPERTY_NAME);
+
+    // Set up public container with the specified blob name.
+    CloudBlockBlob blobRoot = primeRootContainer(blobClient, accountName,
+        blobName, fileSize);
+
+    // Capture the blob container object. It should exist after generating the
+    // shared access signature.
+    container = blobClient.getContainerReference(AZURE_ROOT_CONTAINER);
+    if (null == container || !container.exists()) {
+      final String errMsg = String
+          .format("Container '%s' expected but not found while creating SAS account.");
+      throw new Exception(errMsg);
+    }
+
+    // Set the account URI without a container name.
+    URI accountUri = createAccountUri(accountName);
+
+    // Initialize the Native Azure file system with anonymous credentials.
+    fs = new NativeAzureFileSystem();
+    fs.initialize(accountUri, conf);
+
+    // Create test account initializing the appropriate member variables.
+    // Set the container value to null for the default root container.
+    AzureBlobStorageTestAccount testAcct = new AzureBlobStorageTestAccount(fs,
+        account, blobRoot);
+
+    // Return to caller with test account.
+    return testAcct;
+  }
+
+  public void closeFileSystem() throws Exception {
+    if (fs != null) {
+      fs.close();
+    }
+  }
+
+  public void cleanup() throws Exception {
+    if (fs != null) {
+      fs.close();
+      fs = null;
+    }
+    if (container != null) {
+      container.deleteIfExists();
+      container = null;
+    }
+    if (blob != null) {
+      // The blob member variable is set for blobs under root containers.
+      // Delete blob objects created for root container tests when cleaning
+      // up the test account.
+      blob.delete();
+      blob = null;
+    }
+  }
+
+  public NativeAzureFileSystem getFileSystem() {
+    return fs;
+  }
+
+  public AzureNativeFileSystemStore getStore() {
+    return this.storage;
+  }
+
+  /**
+   * Gets the real blob container backing this account if it's not a mock.
+   * 
+   * @return A container, or null if it's a mock.
+   */
+  public CloudBlobContainer getRealContainer() {
+    return container;
+  }
+
+  /**
+   * Gets the real blob account backing this account if it's not a mock.
+   * 
+   * @return An account, or null if it's a mock.
+   */
+  public CloudStorageAccount getRealAccount() {
+    return account;
+  }
+
+  /**
+   * Gets the mock storage interface if this account is backed by a mock.
+   * 
+   * @return The mock storage, or null if it's backed by a real account.
+   */
+  public MockStorageInterface getMockStorage() {
+    return mockStorage;
+  }
+ 
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/82268d87/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/InMemoryBlockBlobStore.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/InMemoryBlockBlobStore.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/InMemoryBlockBlobStore.java
new file mode 100644
index 0000000..ab35961
--- /dev/null
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/InMemoryBlockBlobStore.java
@@ -0,0 +1,146 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.azure;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * A simple memory key-value store to help mock the Windows Azure Storage
+ * implementation for unit testing.
+ */
+public class InMemoryBlockBlobStore {
+  private final HashMap<String, Entry> blobs = new HashMap<String, Entry>();
+  private HashMap<String, String> containerMetadata;
+
+  public synchronized Iterable<String> getKeys() {
+    return new ArrayList<String>(blobs.keySet());
+  }
+
+  public static class ListBlobEntry {
+    private final String key;
+    private final HashMap<String, String> metadata;
+    private final int contentLength;
+
+    ListBlobEntry(String key, HashMap<String, String> metadata,
+        int contentLength) {
+      this.key = key;
+      this.metadata = metadata;
+      this.contentLength = contentLength;
+    }
+
+    public String getKey() {
+      return key;
+    }
+
+    public HashMap<String, String> getMetadata() {
+      return metadata;
+    }
+
+    public int getContentLength() {
+      return contentLength;
+    }
+  }
+
+  /**
+   * List all the blobs whose key starts with the given prefix.
+   * 
+   * @param prefix
+   *          The prefix to check.
+   * @param includeMetadata
+   *          If set, the metadata in the returned listing will be populated;
+   *          otherwise it'll be null.
+   * @return The listing.
+   */
+  public synchronized Iterable<ListBlobEntry> listBlobs(String prefix,
+      boolean includeMetadata) {
+    ArrayList<ListBlobEntry> list = new ArrayList<ListBlobEntry>();
+    for (Map.Entry<String, Entry> entry : blobs.entrySet()) {
+      if (entry.getKey().startsWith(prefix)) {
+        list.add(new ListBlobEntry(entry.getKey(),
+            includeMetadata ? new HashMap<String, String>(
+                entry.getValue().metadata) : null,
+            entry.getValue().content.length));
+      }
+    }
+    return list;
+  }
+
+  public synchronized byte[] getContent(String key) {
+    return blobs.get(key).content;
+  }
+
+  @SuppressWarnings("unchecked")
+  public synchronized void setContent(String key, byte[] value,
+      HashMap<String, String> metadata) {
+    blobs
+        .put(key, new Entry(value, (HashMap<String, String>) metadata.clone()));
+  }
+
+  public OutputStream upload(final String key,
+      final HashMap<String, String> metadata) {
+    setContent(key, new byte[0], metadata);
+    return new ByteArrayOutputStream() {
+      @Override
+      public void flush() throws IOException {
+        super.flush();
+        setContent(key, toByteArray(), metadata);
+      }
+    };
+  }
+
+  public synchronized void copy(String sourceKey, String destKey) {
+    blobs.put(destKey, blobs.get(sourceKey));
+  }
+
+  public synchronized void delete(String key) {
+    blobs.remove(key);
+  }
+
+  public synchronized boolean exists(String key) {
+    return blobs.containsKey(key);
+  }
+
+  @SuppressWarnings("unchecked")
+  public synchronized HashMap<String, String> getMetadata(String key) {
+    return (HashMap<String, String>) blobs.get(key).metadata.clone();
+  }
+
+  public synchronized HashMap<String, String> getContainerMetadata() {
+    return containerMetadata;
+  }
+
+  public synchronized void setContainerMetadata(HashMap<String, String> metadata) {
+    containerMetadata = metadata;
+  }
+
+  private static class Entry {
+    private byte[] content;
+    private HashMap<String, String> metadata;
+
+    public Entry(byte[] content, HashMap<String, String> metadata) {
+      this.content = content;
+      this.metadata = metadata;
+    }
+  }
+}


Mime
View raw message