hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From cnaur...@apache.org
Subject svn commit: r1601781 [4/6] - in /hadoop/common/trunk: hadoop-project/ hadoop-tools/ hadoop-tools/hadoop-azure/ hadoop-tools/hadoop-azure/dev-support/ hadoop-tools/hadoop-azure/src/ hadoop-tools/hadoop-azure/src/config/ hadoop-tools/hadoop-azure/src/mai...
Date Tue, 10 Jun 2014 22:26:47 GMT
Added: hadoop/common/trunk/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/StorageInterface.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/StorageInterface.java?rev=1601781&view=auto
==============================================================================
--- hadoop/common/trunk/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/StorageInterface.java (added)
+++ hadoop/common/trunk/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/StorageInterface.java Tue Jun 10 22:26:45 2014
@@ -0,0 +1,566 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.azure;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.util.EnumSet;
+import java.util.HashMap;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+
+import com.microsoft.windowsazure.storage.CloudStorageAccount;
+import com.microsoft.windowsazure.storage.OperationContext;
+import com.microsoft.windowsazure.storage.RetryPolicyFactory;
+import com.microsoft.windowsazure.storage.StorageCredentials;
+import com.microsoft.windowsazure.storage.StorageException;
+import com.microsoft.windowsazure.storage.blob.BlobListingDetails;
+import com.microsoft.windowsazure.storage.blob.BlobProperties;
+import com.microsoft.windowsazure.storage.blob.BlobRequestOptions;
+import com.microsoft.windowsazure.storage.blob.CopyState;
+import com.microsoft.windowsazure.storage.blob.ListBlobItem;
+
+/**
+ * This is a very thin layer over the methods exposed by the Windows Azure
+ * Storage SDK that we need for WASB implementation. This base class has a real
+ * implementation that just simply redirects to the SDK, and a memory-backed one
+ * that's used for unit tests.
+ * 
+ * IMPORTANT: all the methods here must remain very simple redirects since code
+ * written here can't be properly unit tested.
+ */
+@InterfaceAudience.Private
+abstract class StorageInterface {
+
+  /**
+   * Sets the timeout to use when making requests to the storage service.
+   * <p>
+   * The server timeout interval begins at the time that the complete request
+   * has been received by the service, and the server begins processing the
+   * response. If the timeout interval elapses before the response is returned
+   * to the client, the operation times out. The timeout interval resets with
+   * each retry, if the request is retried.
+   * 
+   * The default timeout interval for a request made via the service client is
+   * 90 seconds. You can change this value on the service client by setting this
+   * property, so that all subsequent requests made via the service client will
+   * use the new timeout interval. You can also change this value for an
+   * individual request, by setting the
+   * {@link RequestOptions#timeoutIntervalInMs} property.
+   * 
+   * If you are downloading a large blob, you should increase the value of the
+   * timeout beyond the default value.
+   * 
+   * @param timeoutInMs
+   *          The timeout, in milliseconds, to use when making requests to the
+   *          storage service.
+   */
+  public abstract void setTimeoutInMs(int timeoutInMs);
+
+  /**
+   * Sets the RetryPolicyFactory object to use when making service requests.
+   * 
+   * @param retryPolicyFactory
+   *          the RetryPolicyFactory object to use when making service requests.
+   */
+  public abstract void setRetryPolicyFactory(
+      final RetryPolicyFactory retryPolicyFactory);
+
+  /**
+   * Creates a new Blob service client.
+   * 
+   */
+  public abstract void createBlobClient(CloudStorageAccount account);
+
+  /**
+   * Creates an instance of the <code>CloudBlobClient</code> class using the
+   * specified Blob service endpoint.
+   * 
+   * @param baseUri
+   *          A <code>java.net.URI</code> object that represents the Blob
+   *          service endpoint used to create the client.
+   */
+  public abstract void createBlobClient(URI baseUri);
+
+  /**
+   * Creates an instance of the <code>CloudBlobClient</code> class using the
+   * specified Blob service endpoint and account credentials.
+   * 
+   * @param baseUri
+   *          A <code>java.net.URI</code> object that represents the Blob
+   *          service endpoint used to create the client.
+   * @param credentials
+   *          A {@link StorageCredentials} object that represents the account
+   *          credentials.
+   */
+  public abstract void createBlobClient(URI baseUri,
+      StorageCredentials credentials);
+
+  /**
+   * Returns the credentials for the Blob service, as configured for the storage
+   * account.
+   * 
+   * @return A {@link StorageCredentials} object that represents the credentials
+   *         for this storage account.
+   */
+  public abstract StorageCredentials getCredentials();
+
+  /**
+   * Returns a reference to a {@link CloudBlobContainerWrapper} object that
+   * represents the cloud blob container for the specified address.
+   * 
+   * @param name
+   *          A <code>String</code> that represents the name of the container.
+   * @return A {@link CloudBlobContainerWrapper} object that represents a
+   *         reference to the cloud blob container.
+   * 
+   * @throws URISyntaxException
+   *           If the resource URI is invalid.
+   * @throws StorageException
+   *           If a storage service error occurred.
+   */
+  public abstract CloudBlobContainerWrapper getContainerReference(String name)
+      throws URISyntaxException, StorageException;
+
+  /**
+   * A thin wrapper over the {@link CloudBlobDirectory} class that simply
+   * redirects calls to the real object except in unit tests.
+   */
+  @InterfaceAudience.Private
+  public abstract static class CloudBlobDirectoryWrapper implements
+      ListBlobItem {
+    /**
+     * Returns the URI for this directory.
+     * 
+     * @return A <code>java.net.URI</code> object that represents the URI for
+     *         this directory.
+     */
+    public abstract URI getUri();
+
+    /**
+     * Returns an enumerable collection of blob items whose names begin with the
+     * specified prefix, using the specified flat or hierarchical option,
+     * listing details options, request options, and operation context.
+     * 
+     * @param prefix
+     *          A <code>String</code> that represents the prefix of the blob
+     *          name.
+     * @param useFlatBlobListing
+     *          <code>true</code> to indicate that the returned list will be
+     *          flat; <code>false</code> to indicate that the returned list will
+     *          be hierarchical.
+     * @param listingDetails
+     *          A <code>java.util.EnumSet</code> object that contains
+     *          {@link BlobListingDetails} values that indicate whether
+     *          snapshots, metadata, and/or uncommitted blocks are returned.
+     *          Committed blocks are always returned.
+     * @param options
+     *          A {@link BlobRequestOptions} object that specifies any
+     *          additional options for the request. Specifying <code>null</code>
+     *          will use the default request options from the associated service
+     *          client ( {@link CloudBlobClient}).
+     * @param opContext
+     *          An {@link OperationContext} object that represents the context
+     *          for the current operation. This object is used to track requests
+     *          to the storage service, and to provide additional runtime
+     *          information about the operation.
+     * 
+     * @return An enumerable collection of {@link ListBlobItem} objects that
+     *         represent the block items whose names begin with the specified
+     *         prefix in this directory.
+     * 
+     * @throws StorageException
+     *           If a storage service error occurred.
+     * @throws URISyntaxException
+     *           If the resource URI is invalid.
+     */
+    public abstract Iterable<ListBlobItem> listBlobs(String prefix,
+        boolean useFlatBlobListing, EnumSet<BlobListingDetails> listingDetails,
+        BlobRequestOptions options, OperationContext opContext)
+        throws URISyntaxException, StorageException;
+  }
+
+  /**
+   * A thin wrapper over the {@link CloudBlobContainer} class that simply
+   * redirects calls to the real object except in unit tests.
+   */
+  @InterfaceAudience.Private
+  public abstract static class CloudBlobContainerWrapper {
+    /**
+     * Returns the name of the container.
+     * 
+     * @return A <code>String</code> that represents the name of the container.
+     */
+    public abstract String getName();
+
+    /**
+     * Returns a value that indicates whether the container exists, using the
+     * specified operation context.
+     * 
+     * @param opContext
+     *          An {@link OperationContext} object that represents the context
+     *          for the current operation. This object is used to track requests
+     *          to the storage service, and to provide additional runtime
+     *          information about the operation.
+     * 
+     * @return <code>true</code> if the container exists, otherwise
+     *         <code>false</code>.
+     * 
+     * @throws StorageException
+     *           If a storage service error occurred.
+     */
+    public abstract boolean exists(OperationContext opContext)
+        throws StorageException;
+
+    /**
+     * Returns the metadata for the container.
+     * 
+     * @return A <code>java.util.HashMap</code> object that represents the
+     *         metadata for the container.
+     */
+    public abstract HashMap<String, String> getMetadata();
+
+    /**
+     * Sets the metadata for the container.
+     * 
+     * @param metadata
+     *          A <code>java.util.HashMap</code> object that represents the
+     *          metadata being assigned to the container.
+     */
+    public abstract void setMetadata(HashMap<String, String> metadata);
+
+    /**
+     * Downloads the container's attributes, which consist of metadata and
+     * properties, using the specified operation context.
+     * 
+     * @param opContext
+     *          An {@link OperationContext} object that represents the context
+     *          for the current operation. This object is used to track requests
+     *          to the storage service, and to provide additional runtime
+     *          information about the operation.
+     * 
+     * @throws StorageException
+     *           If a storage service error occurred.
+     */
+    public abstract void downloadAttributes(OperationContext opContext)
+        throws StorageException;
+
+    /**
+     * Uploads the container's metadata using the specified operation context.
+     * 
+     * @param opContext
+     *          An {@link OperationContext} object that represents the context
+     *          for the current operation. This object is used to track requests
+     *          to the storage service, and to provide additional runtime
+     *          information about the operation.
+     * 
+     * @throws StorageException
+     *           If a storage service error occurred.
+     */
+    public abstract void uploadMetadata(OperationContext opContext)
+        throws StorageException;
+
+    /**
+     * Creates the container using the specified operation context.
+     * 
+     * @param opContext
+     *          An {@link OperationContext} object that represents the context
+     *          for the current operation. This object is used to track requests
+     *          to the storage service, and to provide additional runtime
+     *          information about the operation.
+     * 
+     * @throws StorageException
+     *           If a storage service error occurred.
+     */
+    public abstract void create(OperationContext opContext)
+        throws StorageException;
+
+    /**
+     * Returns a wrapper for a CloudBlobDirectory.
+     * 
+     * @param relativePath
+     *          A <code>String</code> that represents the name of the directory,
+     *          relative to the container
+     * 
+     * @throws StorageException
+     *           If a storage service error occurred.
+     * 
+     * @throws URISyntaxException
+     *           If URI syntax exception occurred.
+     */
+    public abstract CloudBlobDirectoryWrapper getDirectoryReference(
+        String relativePath) throws URISyntaxException, StorageException;
+
+    /**
+     * Returns a wrapper for a CloudBlockBlob.
+     * 
+     * @param relativePath
+     *          A <code>String</code> that represents the name of the blob,
+     *          relative to the container
+     * 
+     * @throws StorageException
+     *           If a storage service error occurred.
+     * 
+     * @throws URISyntaxException
+     *           If URI syntax exception occurred.
+     */
+    public abstract CloudBlockBlobWrapper getBlockBlobReference(
+        String relativePath) throws URISyntaxException, StorageException;
+  }
+
+  /**
+   * A thin wrapper over the {@link CloudBlockBlob} class that simply redirects
+   * calls to the real object except in unit tests.
+   */
+  @InterfaceAudience.Private
+  public abstract static class CloudBlockBlobWrapper implements ListBlobItem {
+    /**
+     * Returns the URI for this blob.
+     * 
+     * @return A <code>java.net.URI</code> object that represents the URI for
+     *         the blob.
+     */
+    public abstract URI getUri();
+
+    /**
+     * Returns the metadata for the blob.
+     * 
+     * @return A <code>java.util.HashMap</code> object that represents the
+     *         metadata for the blob.
+     */
+    public abstract HashMap<String, String> getMetadata();
+
+    /**
+     * Sets the metadata for the blob.
+     * 
+     * @param metadata
+     *          A <code>java.util.HashMap</code> object that contains the
+     *          metadata being assigned to the blob.
+     */
+    public abstract void setMetadata(HashMap<String, String> metadata);
+
+    /**
+     * Copies an existing blob's contents, properties, and metadata to this
+     * instance of the <code>CloudBlob</code> class, using the specified
+     * operation context.
+     * 
+     * @param sourceBlob
+     *          A <code>CloudBlob</code> object that represents the source blob
+     *          to copy.
+     * @param opContext
+     *          An {@link OperationContext} object that represents the context
+     *          for the current operation. This object is used to track requests
+     *          to the storage service, and to provide additional runtime
+     *          information about the operation.
+     * 
+     * @throws StorageException
+     *           If a storage service error occurred.
+     * @throws URISyntaxException
+     * 
+     */
+    public abstract void startCopyFromBlob(CloudBlockBlobWrapper sourceBlob,
+        OperationContext opContext) throws StorageException, URISyntaxException;
+
+    /**
+     * Returns the blob's copy state.
+     * 
+     * @return A {@link CopyState} object that represents the copy state of the
+     *         blob.
+     */
+    public abstract CopyState getCopyState();
+
+    /**
+     * Deletes the blob using the specified operation context.
+     * <p>
+     * A blob that has snapshots cannot be deleted unless the snapshots are also
+     * deleted. If a blob has snapshots, use the
+     * {@link DeleteSnapshotsOption#DELETE_SNAPSHOTS_ONLY} or
+     * {@link DeleteSnapshotsOption#INCLUDE_SNAPSHOTS} value in the
+     * <code>deleteSnapshotsOption</code> parameter to specify how the snapshots
+     * should be handled when the blob is deleted.
+     * 
+     * @param opContext
+     *          An {@link OperationContext} object that represents the context
+     *          for the current operation. This object is used to track requests
+     *          to the storage service, and to provide additional runtime
+     *          information about the operation.
+     * 
+     * @throws StorageException
+     *           If a storage service error occurred.
+     */
+    public abstract void delete(OperationContext opContext)
+        throws StorageException;
+
+    /**
+     * Checks to see if the blob exists, using the specified operation context.
+     * 
+     * @param opContext
+     *          An {@link OperationContext} object that represents the context
+     *          for the current operation. This object is used to track requests
+     *          to the storage service, and to provide additional runtime
+     *          information about the operation.
+     * 
+     * @return <code>true</code> if the blob exists, other wise
+     *         <code>false</code>.
+     * 
+     * @throws StorageException
+     *           f a storage service error occurred.
+     */
+    public abstract boolean exists(OperationContext opContext)
+        throws StorageException;
+
+    /**
+     * Populates a blob's properties and metadata using the specified operation
+     * context.
+     * <p>
+     * This method populates the blob's system properties and user-defined
+     * metadata. Before reading a blob's properties or metadata, call this
+     * method or its overload to retrieve the latest values for the blob's
+     * properties and metadata from the Windows Azure storage service.
+     * 
+     * @param opContext
+     *          An {@link OperationContext} object that represents the context
+     *          for the current operation. This object is used to track requests
+     *          to the storage service, and to provide additional runtime
+     *          information about the operation.
+     * 
+     * @throws StorageException
+     *           If a storage service error occurred.
+     */
+    public abstract void downloadAttributes(OperationContext opContext)
+        throws StorageException;
+
+    /**
+     * Returns the blob's properties.
+     * 
+     * @return A {@link BlobProperties} object that represents the properties of
+     *         the blob.
+     */
+    public abstract BlobProperties getProperties();
+
+    /**
+     * Opens a blob input stream to download the blob using the specified
+     * operation context.
+     * <p>
+     * Use {@link CloudBlobClient#setStreamMinimumReadSizeInBytes} to configure
+     * the read size.
+     * 
+     * @param opContext
+     *          An {@link OperationContext} object that represents the context
+     *          for the current operation. This object is used to track requests
+     *          to the storage service, and to provide additional runtime
+     *          information about the operation.
+     * 
+     * @return An <code>InputStream</code> object that represents the stream to
+     *         use for reading from the blob.
+     * 
+     * @throws StorageException
+     *           If a storage service error occurred.
+     */
+    public abstract InputStream openInputStream(BlobRequestOptions options,
+        OperationContext opContext) throws StorageException;
+
+    /**
+     * Creates and opens an output stream to write data to the block blob using
+     * the specified operation context.
+     * 
+     * @param opContext
+     *          An {@link OperationContext} object that represents the context
+     *          for the current operation. This object is used to track requests
+     *          to the storage service, and to provide additional runtime
+     *          information about the operation.
+     * 
+     * @return A {@link BlobOutputStream} object used to write data to the blob.
+     * 
+     * @throws StorageException
+     *           If a storage service error occurred.
+     */
+    public abstract OutputStream openOutputStream(BlobRequestOptions options,
+        OperationContext opContext) throws StorageException;
+
+    /**
+     * Uploads the source stream data to the blob, using the specified operation
+     * context.
+     * 
+     * @param sourceStream
+     *          An <code>InputStream</code> object that represents the input
+     *          stream to write to the block blob.
+     * @param opContext
+     *          An {@link OperationContext} object that represents the context
+     *          for the current operation. This object is used to track requests
+     *          to the storage service, and to provide additional runtime
+     *          information about the operation.
+     * 
+     * @throws IOException
+     *           If an I/O error occurred.
+     * @throws StorageException
+     *           If a storage service error occurred.
+     */
+    public abstract void upload(InputStream sourceStream,
+        OperationContext opContext) throws StorageException, IOException;
+
+    /**
+     * Uploads the blob's metadata to the storage service using the specified
+     * lease ID, request options, and operation context.
+     * 
+     * @param opContext
+     *          An {@link OperationContext} object that represents the context
+     *          for the current operation. This object is used to track requests
+     *          to the storage service, and to provide additional runtime
+     *          information about the operation.
+     * 
+     * @throws StorageException
+     *           If a storage service error occurred.
+     */
+    public abstract void uploadMetadata(OperationContext opContext)
+        throws StorageException;
+
+    public abstract void uploadProperties(OperationContext opContext)
+        throws StorageException;
+
+    /**
+     * Sets the minimum read block size to use with this Blob.
+     * 
+     * @param minimumReadSizeBytes
+     *          The maximum block size, in bytes, for reading from a block blob
+     *          while using a {@link BlobInputStream} object, ranging from 512
+     *          bytes to 64 MB, inclusive.
+     */
+    public abstract void setStreamMinimumReadSizeInBytes(
+        int minimumReadSizeBytes);
+
+    /**
+     * Sets the write block size to use with this Blob.
+     * 
+     * @param writeBlockSizeBytes
+     *          The maximum block size, in bytes, for writing to a block blob
+     *          while using a {@link BlobOutputStream} object, ranging from 1 MB
+     *          to 4 MB, inclusive.
+     * 
+     * @throws IllegalArgumentException
+     *           If <code>writeBlockSizeInBytes</code> is less than 1 MB or
+     *           greater than 4 MB.
+     */
+    public abstract void setWriteBlockSizeInBytes(int writeBlockSizeBytes);
+
+  }
+}

Added: hadoop/common/trunk/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/StorageInterfaceImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/StorageInterfaceImpl.java?rev=1601781&view=auto
==============================================================================
--- hadoop/common/trunk/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/StorageInterfaceImpl.java (added)
+++ hadoop/common/trunk/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/StorageInterfaceImpl.java Tue Jun 10 22:26:45 2014
@@ -0,0 +1,372 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.azure;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.util.EnumSet;
+import java.util.HashMap;
+import java.util.Iterator;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+
+import com.microsoft.windowsazure.storage.AccessCondition;
+import com.microsoft.windowsazure.storage.CloudStorageAccount;
+import com.microsoft.windowsazure.storage.OperationContext;
+import com.microsoft.windowsazure.storage.RetryPolicyFactory;
+import com.microsoft.windowsazure.storage.StorageCredentials;
+import com.microsoft.windowsazure.storage.StorageException;
+import com.microsoft.windowsazure.storage.StorageUri;
+import com.microsoft.windowsazure.storage.blob.BlobListingDetails;
+import com.microsoft.windowsazure.storage.blob.BlobProperties;
+import com.microsoft.windowsazure.storage.blob.BlobRequestOptions;
+import com.microsoft.windowsazure.storage.blob.CloudBlobClient;
+import com.microsoft.windowsazure.storage.blob.CloudBlobContainer;
+import com.microsoft.windowsazure.storage.blob.CloudBlobDirectory;
+import com.microsoft.windowsazure.storage.blob.CloudBlockBlob;
+import com.microsoft.windowsazure.storage.blob.CopyState;
+import com.microsoft.windowsazure.storage.blob.DeleteSnapshotsOption;
+import com.microsoft.windowsazure.storage.blob.ListBlobItem;
+
+/**
+ * A real implementation of the Azure interaction layer that just redirects
+ * calls to the Windows Azure storage SDK.
+ */
+@InterfaceAudience.Private
+class StorageInterfaceImpl extends StorageInterface {
+  private CloudBlobClient serviceClient;
+
+  @Override
+  public void setRetryPolicyFactory(final RetryPolicyFactory retryPolicyFactory) {
+    serviceClient.setRetryPolicyFactory(retryPolicyFactory);
+  }
+
+  @Override
+  public void setTimeoutInMs(int timeoutInMs) {
+    serviceClient.setTimeoutInMs(timeoutInMs);
+  }
+
+  @Override
+  public void createBlobClient(CloudStorageAccount account) {
+    serviceClient = account.createCloudBlobClient();
+  }
+
+  @Override
+  public void createBlobClient(URI baseUri) {
+    serviceClient = new CloudBlobClient(baseUri);
+  }
+
+  @Override
+  public void createBlobClient(URI baseUri, StorageCredentials credentials) {
+    serviceClient = new CloudBlobClient(baseUri, credentials);
+  }
+
+  @Override
+  public StorageCredentials getCredentials() {
+    return serviceClient.getCredentials();
+  }
+
+  @Override
+  public CloudBlobContainerWrapper getContainerReference(String uri)
+      throws URISyntaxException, StorageException {
+    return new CloudBlobContainerWrapperImpl(
+        serviceClient.getContainerReference(uri));
+  }
+
+  //
+  // WrappingIterator
+  //
+
+  /**
+   * This iterator wraps every ListBlobItem as they come from the listBlobs()
+   * calls to their proper wrapping objects.
+   */
+  private static class WrappingIterator implements Iterator<ListBlobItem> {
+    private final Iterator<ListBlobItem> present;
+
+    public WrappingIterator(Iterator<ListBlobItem> present) {
+      this.present = present;
+    }
+
+    public static Iterable<ListBlobItem> wrap(
+        final Iterable<ListBlobItem> present) {
+      return new Iterable<ListBlobItem>() {
+        @Override
+        public Iterator<ListBlobItem> iterator() {
+          return new WrappingIterator(present.iterator());
+        }
+      };
+    }
+
+    @Override
+    public boolean hasNext() {
+      return present.hasNext();
+    }
+
+    @Override
+    public ListBlobItem next() {
+      ListBlobItem unwrapped = present.next();
+      if (unwrapped instanceof CloudBlobDirectory) {
+        return new CloudBlobDirectoryWrapperImpl((CloudBlobDirectory) unwrapped);
+      } else if (unwrapped instanceof CloudBlockBlob) {
+        return new CloudBlockBlobWrapperImpl((CloudBlockBlob) unwrapped);
+      } else {
+        return unwrapped;
+      }
+    }
+
+    @Override
+    public void remove() {
+      present.remove();
+    }
+  }
+
+  //
+  // CloudBlobDirectoryWrapperImpl
+  //
+  @InterfaceAudience.Private
+  static class CloudBlobDirectoryWrapperImpl extends CloudBlobDirectoryWrapper {
+    private final CloudBlobDirectory directory;
+
+    public CloudBlobDirectoryWrapperImpl(CloudBlobDirectory directory) {
+      this.directory = directory;
+    }
+
+    @Override
+    public URI getUri() {
+      return directory.getUri();
+    }
+
+    @Override
+    public Iterable<ListBlobItem> listBlobs(String prefix,
+        boolean useFlatBlobListing, EnumSet<BlobListingDetails> listingDetails,
+        BlobRequestOptions options, OperationContext opContext)
+        throws URISyntaxException, StorageException {
+      return WrappingIterator.wrap(directory.listBlobs(prefix,
+          useFlatBlobListing, listingDetails, options, opContext));
+    }
+
+    @Override
+    public CloudBlobContainer getContainer() throws URISyntaxException,
+        StorageException {
+      return directory.getContainer();
+    }
+
+    @Override
+    public CloudBlobDirectory getParent() throws URISyntaxException,
+        StorageException {
+      return directory.getParent();
+    }
+
+    @Override
+    public StorageUri getStorageUri() {
+      return directory.getStorageUri();
+    }
+
+  }
+
+  //
+  // CloudBlobContainerWrapperImpl
+  //
+  @InterfaceAudience.Private
+  static class CloudBlobContainerWrapperImpl extends CloudBlobContainerWrapper {
+    private final CloudBlobContainer container;
+
+    public CloudBlobContainerWrapperImpl(CloudBlobContainer container) {
+      this.container = container;
+    }
+
+    @Override
+    public String getName() {
+      return container.getName();
+    }
+
+    @Override
+    public boolean exists(OperationContext opContext) throws StorageException {
+      return container.exists(AccessCondition.generateEmptyCondition(), null,
+          opContext);
+    }
+
+    @Override
+    public void create(OperationContext opContext) throws StorageException {
+      container.create(null, opContext);
+    }
+
+    @Override
+    public HashMap<String, String> getMetadata() {
+      return container.getMetadata();
+    }
+
+    @Override
+    public void setMetadata(HashMap<String, String> metadata) {
+      container.setMetadata(metadata);
+    }
+
+    @Override
+    public void downloadAttributes(OperationContext opContext)
+        throws StorageException {
+      container.downloadAttributes(AccessCondition.generateEmptyCondition(),
+          null, opContext);
+    }
+
+    @Override
+    public void uploadMetadata(OperationContext opContext)
+        throws StorageException {
+      container.uploadMetadata(AccessCondition.generateEmptyCondition(), null,
+          opContext);
+    }
+
+    @Override
+    public CloudBlobDirectoryWrapper getDirectoryReference(String relativePath)
+        throws URISyntaxException, StorageException {
+
+      CloudBlobDirectory dir = container.getDirectoryReference(relativePath);
+      return new CloudBlobDirectoryWrapperImpl(dir);
+    }
+
+    @Override
+    public CloudBlockBlobWrapper getBlockBlobReference(String relativePath)
+        throws URISyntaxException, StorageException {
+
+      return new CloudBlockBlobWrapperImpl(
+          container.getBlockBlobReference(relativePath));
+    }
+  }
+
+  //
+  // CloudBlockBlobWrapperImpl
+  //
+  @InterfaceAudience.Private
+  static class CloudBlockBlobWrapperImpl extends CloudBlockBlobWrapper {
+    private final CloudBlockBlob blob;
+
+    public URI getUri() {
+      return blob.getUri();
+    }
+
+    public CloudBlockBlobWrapperImpl(CloudBlockBlob blob) {
+      this.blob = blob;
+    }
+
+    @Override
+    public HashMap<String, String> getMetadata() {
+      return blob.getMetadata();
+    }
+
+    @Override
+    public void startCopyFromBlob(CloudBlockBlobWrapper sourceBlob,
+        OperationContext opContext) throws StorageException, URISyntaxException {
+
+      blob.startCopyFromBlob(((CloudBlockBlobWrapperImpl) sourceBlob).blob,
+          null, null, null, opContext);
+
+    }
+
+    @Override
+    public void delete(OperationContext opContext) throws StorageException {
+      blob.delete(DeleteSnapshotsOption.NONE, null, null, opContext);
+    }
+
+    @Override
+    public boolean exists(OperationContext opContext) throws StorageException {
+      return blob.exists(null, null, opContext);
+    }
+
+    @Override
+    public void downloadAttributes(OperationContext opContext)
+        throws StorageException {
+      blob.downloadAttributes(null, null, opContext);
+    }
+
+    @Override
+    public BlobProperties getProperties() {
+      return blob.getProperties();
+    }
+
+    @Override
+    public void setMetadata(HashMap<String, String> metadata) {
+      blob.setMetadata(metadata);
+    }
+
+    @Override
+    public InputStream openInputStream(BlobRequestOptions options,
+        OperationContext opContext) throws StorageException {
+      return blob.openInputStream(null, options, opContext);
+    }
+
+    @Override
+    public OutputStream openOutputStream(BlobRequestOptions options,
+        OperationContext opContext) throws StorageException {
+      return blob.openOutputStream(null, options, opContext);
+    }
+
+    @Override
+    public void upload(InputStream sourceStream, OperationContext opContext)
+        throws StorageException, IOException {
+      blob.upload(sourceStream, 0, null, null, opContext);
+    }
+
+    @Override
+    public CloudBlobContainer getContainer() throws URISyntaxException,
+        StorageException {
+      return blob.getContainer();
+    }
+
+    @Override
+    public CloudBlobDirectory getParent() throws URISyntaxException,
+        StorageException {
+      return blob.getParent();
+    }
+
+    @Override
+    public void uploadMetadata(OperationContext opContext)
+        throws StorageException {
+      blob.uploadMetadata(null, null, opContext);
+    }
+
+    @Override
+    public void uploadProperties(OperationContext opContext)
+        throws StorageException {
+      blob.uploadProperties(null, null, opContext);
+    }
+
+    @Override
+    public void setStreamMinimumReadSizeInBytes(int minimumReadSizeBytes) {
+      blob.setStreamMinimumReadSizeInBytes(minimumReadSizeBytes);
+    }
+
+    @Override
+    public void setWriteBlockSizeInBytes(int writeBlockSizeBytes) {
+      blob.setStreamWriteSizeInBytes(writeBlockSizeBytes);
+    }
+
+    @Override
+    public StorageUri getStorageUri() {
+      return blob.getStorageUri();
+    }
+
+    @Override
+    public CopyState getCopyState() {
+      return blob.getCopyState();
+    }
+  }
+}

Added: hadoop/common/trunk/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/Wasb.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/Wasb.java?rev=1601781&view=auto
==============================================================================
--- hadoop/common/trunk/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/Wasb.java (added)
+++ hadoop/common/trunk/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/Wasb.java Tue Jun 10 22:26:45 2014
@@ -0,0 +1,48 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.azure;
+
+import java.io.IOException;
+import java.net.URI;
+import java.net.URISyntaxException;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.DelegateToFileSystem;
+
+
+/**
+ * WASB implementation of AbstractFileSystem.
+ * This impl delegates to the old FileSystem
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
+public class Wasb extends DelegateToFileSystem {
+
+  Wasb(final URI theUri, final Configuration conf) throws IOException,
+      URISyntaxException {
+    super(theUri, new NativeAzureFileSystem(), conf, "wasb", false);
+  }
+
+  @Override
+  public int getUriDefaultPort() {
+    return -1;
+  }
+}

Added: hadoop/common/trunk/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/WasbFsck.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/WasbFsck.java?rev=1601781&view=auto
==============================================================================
--- hadoop/common/trunk/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/WasbFsck.java (added)
+++ hadoop/common/trunk/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/WasbFsck.java Tue Jun 10 22:26:45 2014
@@ -0,0 +1,196 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.azure;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.List;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.util.Tool;
+import org.apache.hadoop.util.ToolRunner;
+
+import com.google.common.annotations.VisibleForTesting;
+
+/**
+ * An fsck tool implementation for WASB that does various admin/cleanup/recovery
+ * tasks on the WASB file system.
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
+public class WasbFsck extends Configured implements Tool {
+  private FileSystem mockFileSystemForTesting = null;
+  private static final String LOST_AND_FOUND_PATH = "/lost+found";
+  private boolean pathNameWarning = false;
+
+  public WasbFsck(Configuration conf) {
+    super(conf);
+  }
+
+  /**
+   * For testing purposes, set the file system to use here instead of relying on
+   * getting it from the FileSystem class based on the URI.
+   * 
+   * @param fileSystem
+   *          The file system to use.
+   */
+  @VisibleForTesting
+  public void setMockFileSystemForTesting(FileSystem fileSystem) {
+    this.mockFileSystemForTesting = fileSystem;
+  }
+
+  @Override
+  public int run(String[] args) throws Exception {
+    if (doPrintUsage(Arrays.asList(args))) {
+      printUsage();
+      return -1;
+    }
+    Path pathToCheck = null;
+    boolean doRecover = false;
+    boolean doDelete = false;
+    for (String arg : args) {
+      if (!arg.startsWith("-")) {
+        if (pathToCheck != null) {
+          System.err
+              .println("Can't specify multiple paths to check on the command-line");
+          return 1;
+        }
+        pathToCheck = new Path(arg);
+      } else if (arg.equals("-move")) {
+        doRecover = true;
+      } else if (arg.equals("-delete")) {
+        doDelete = true;
+      }
+    }
+    if (doRecover && doDelete) {
+      System.err
+          .println("Conflicting options: can't specify both -move and -delete.");
+      return 1;
+    }
+    if (pathToCheck == null) {
+      pathToCheck = new Path("/"); // Check everything.
+    }
+    FileSystem fs;
+    if (mockFileSystemForTesting == null) {
+      fs = FileSystem.get(pathToCheck.toUri(), getConf());
+    } else {
+      fs = mockFileSystemForTesting;
+    }
+
+    if (!recursiveCheckChildPathName(fs, fs.makeQualified(pathToCheck))) {
+      pathNameWarning = true;
+    }
+
+    if (!(fs instanceof NativeAzureFileSystem)) {
+      System.err
+          .println("Can only check WASB file system. Instead I'm asked to"
+              + " check: " + fs.getUri());
+      return 2;
+    }
+    NativeAzureFileSystem wasbFs = (NativeAzureFileSystem) fs;
+    if (doRecover) {
+      System.out.println("Recovering files with dangling data under: "
+          + pathToCheck);
+      wasbFs.recoverFilesWithDanglingTempData(pathToCheck, new Path(
+          LOST_AND_FOUND_PATH));
+    } else if (doDelete) {
+      System.out.println("Deleting temp files with dangling data under: "
+          + pathToCheck);
+      wasbFs.deleteFilesWithDanglingTempData(pathToCheck);
+    } else {
+      System.out.println("Please specify -move or -delete");
+    }
+    return 0;
+  }
+
+  public boolean getPathNameWarning() {
+    return pathNameWarning;
+  }
+
+  /**
+   * Recursively check if a given path and its child paths have colons in their
+   * names. It returns true if none of them has a colon or this path does not
+   * exist, and false otherwise.
+   */
+  private boolean recursiveCheckChildPathName(FileSystem fs, Path p)
+      throws IOException {
+    if (p == null) {
+      return true;
+    }
+    if (!fs.exists(p)) {
+      System.out.println("Path " + p + " does not exist!");
+      return true;
+    }
+
+    if (fs.isFile(p)) {
+      if (containsColon(p)) {
+        System.out.println("Warning: file " + p + " has a colon in its name.");
+        return false;
+      } else {
+        return true;
+      }
+    } else {
+      boolean flag;
+      if (containsColon(p)) {
+        System.out.println("Warning: directory " + p
+            + " has a colon in its name.");
+        flag = false;
+      } else {
+        flag = true;
+      }
+      FileStatus[] listed = fs.listStatus(p);
+      for (FileStatus l : listed) {
+        if (!recursiveCheckChildPathName(fs, l.getPath())) {
+          flag = false;
+        }
+      }
+      return flag;
+    }
+  }
+
+  private boolean containsColon(Path p) {
+    return p.toUri().getPath().toString().contains(":");
+  }
+
+  private static void printUsage() {
+    System.out.println("Usage: WasbFSck [<path>] [-move | -delete]");
+    System.out.println("\t<path>\tstart checking from this path");
+    System.out.println("\t-move\tmove any files whose upload was interrupted"
+        + " mid-stream to " + LOST_AND_FOUND_PATH);
+    System.out
+        .println("\t-delete\tdelete any files whose upload was interrupted"
+            + " mid-stream");
+    ToolRunner.printGenericCommandUsage(System.out);
+  }
+
+  private boolean doPrintUsage(List<String> args) {
+    return args.contains("-H");
+  }
+
+  public static void main(String[] args) throws Exception {
+    int res = ToolRunner.run(new WasbFsck(new Configuration()), args);
+    System.exit(res);
+  }
+}

Added: hadoop/common/trunk/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/package.html
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/package.html?rev=1601781&view=auto
==============================================================================
--- hadoop/common/trunk/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/package.html (added)
+++ hadoop/common/trunk/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/package.html Tue Jun 10 22:26:45 2014
@@ -0,0 +1,31 @@
+<html>
+
+<!--
+   Licensed to the Apache Software Foundation (ASF) under one or more
+   contributor license agreements.  See the NOTICE file distributed with
+   this work for additional information regarding copyright ownership.
+   The ASF licenses this file to You under the Apache License, Version 2.0
+   (the "License"); you may not use this file except in compliance with
+   the License.  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+-->
+
+<body>
+
+<p>
+A distributed implementation of {@link
+org.apache.hadoop.fs.FileSystem} for reading and writing files on
+<a href="http://store.azure.com">Azure Block Storage</a>.
+This implementation is blob-based and stores files on Azure in their native form for
+interoperability with other Azure tools.
+</p>
+
+</body>
+</html>

Added: hadoop/common/trunk/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/AzureBlobStorageTestAccount.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/AzureBlobStorageTestAccount.java?rev=1601781&view=auto
==============================================================================
--- hadoop/common/trunk/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/AzureBlobStorageTestAccount.java (added)
+++ hadoop/common/trunk/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/AzureBlobStorageTestAccount.java Tue Jun 10 22:26:45 2014
@@ -0,0 +1,726 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ **/
+
+package org.apache.hadoop.fs.azure;
+
+import static org.apache.hadoop.fs.azure.AzureNativeFileSystemStore.DEFAULT_STORAGE_EMULATOR_ACCOUNT_NAME;
+
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.util.Calendar;
+import java.util.Date;
+import java.util.EnumSet;
+import java.util.GregorianCalendar;
+import java.util.TimeZone;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+
+import com.microsoft.windowsazure.storage.AccessCondition;
+import com.microsoft.windowsazure.storage.CloudStorageAccount;
+import com.microsoft.windowsazure.storage.StorageCredentials;
+import com.microsoft.windowsazure.storage.StorageCredentialsAccountAndKey;
+import com.microsoft.windowsazure.storage.StorageCredentialsAnonymous;
+import com.microsoft.windowsazure.storage.blob.BlobContainerPermissions;
+import com.microsoft.windowsazure.storage.blob.BlobContainerPublicAccessType;
+import com.microsoft.windowsazure.storage.blob.BlobOutputStream;
+import com.microsoft.windowsazure.storage.blob.CloudBlobClient;
+import com.microsoft.windowsazure.storage.blob.CloudBlobContainer;
+import com.microsoft.windowsazure.storage.blob.CloudBlockBlob;
+import com.microsoft.windowsazure.storage.blob.SharedAccessBlobPermissions;
+import com.microsoft.windowsazure.storage.blob.SharedAccessBlobPolicy;
+import com.microsoft.windowsazure.storage.core.Base64;
+
+/**
+ * Helper class to create WASB file systems backed by either a mock in-memory
+ * implementation or a real Azure Storage account. See RunningLiveWasbTests.txt
+ * for instructions on how to connect to a real Azure Storage account.
+ */
+public final class AzureBlobStorageTestAccount {
+
+  private static final String ACCOUNT_KEY_PROPERTY_NAME = "fs.azure.account.key.";
+  private static final String SAS_PROPERTY_NAME = "fs.azure.sas.";
+  private static final String TEST_CONFIGURATION_FILE_NAME = "azure-test.xml";
+  private static final String TEST_ACCOUNT_NAME_PROPERTY_NAME = "fs.azure.test.account.name";
+  public static final String MOCK_ACCOUNT_NAME = "mockAccount.blob.core.windows.net";
+  public static final String MOCK_CONTAINER_NAME = "mockContainer";
+  public static final String WASB_AUTHORITY_DELIMITER = "@";
+  public static final String WASB_SCHEME = "wasb";
+  public static final String PATH_DELIMITER = "/";
+  public static final String AZURE_ROOT_CONTAINER = "$root";
+  public static final String MOCK_WASB_URI = "wasb://" + MOCK_CONTAINER_NAME
+      + WASB_AUTHORITY_DELIMITER + MOCK_ACCOUNT_NAME + "/";
+  private static final String USE_EMULATOR_PROPERTY_NAME = "fs.azure.test.emulator";
+
+  private static final String KEY_DISABLE_THROTTLING = "fs.azure.disable.bandwidth.throttling";
+  private static final String KEY_READ_TOLERATE_CONCURRENT_APPEND = "fs.azure.io.read.tolerate.concurrent.append";
+
+  private CloudStorageAccount account;
+  private CloudBlobContainer container;
+  private CloudBlockBlob blob;
+  private NativeAzureFileSystem fs;
+  private AzureNativeFileSystemStore storage;
+  private MockStorageInterface mockStorage;
+
+  private AzureBlobStorageTestAccount(NativeAzureFileSystem fs,
+      CloudStorageAccount account, CloudBlobContainer container) {
+    this.account = account;
+    this.container = container;
+    this.fs = fs;
+  }
+
+  /**
+   * Create a test account with an initialized storage reference.
+   * 
+   * @param storage
+   *          -- store to be accessed by the account
+   * @param account
+   *          -- Windows Azure account object
+   * @param container
+   *          -- Windows Azure container object
+   */
+  private AzureBlobStorageTestAccount(AzureNativeFileSystemStore storage,
+      CloudStorageAccount account, CloudBlobContainer container) {
+    this.account = account;
+    this.container = container;
+    this.storage = storage;
+  }
+
+  /**
+   * Create a test account sessions with the default root container.
+   * 
+   * @param fs
+   *          - file system, namely WASB file system
+   * @param account
+   *          - Windows Azure account object
+   * @param blob
+   *          - block blob reference
+   */
+  private AzureBlobStorageTestAccount(NativeAzureFileSystem fs,
+      CloudStorageAccount account, CloudBlockBlob blob) {
+
+    this.account = account;
+    this.blob = blob;
+    this.fs = fs;
+  }
+
+  private AzureBlobStorageTestAccount(NativeAzureFileSystem fs,
+      MockStorageInterface mockStorage) {
+    this.fs = fs;
+    this.mockStorage = mockStorage;
+  }
+
+  public static String getMockContainerUri() {
+    return String.format("http://%s/%s",
+        AzureBlobStorageTestAccount.MOCK_ACCOUNT_NAME,
+        AzureBlobStorageTestAccount.MOCK_CONTAINER_NAME);
+  }
+
+  public static String toMockUri(String path) {
+    return String.format("http://%s/%s/%s",
+        AzureBlobStorageTestAccount.MOCK_ACCOUNT_NAME,
+        AzureBlobStorageTestAccount.MOCK_CONTAINER_NAME, path);
+  }
+
+  public static String toMockUri(Path path) {
+    // Remove the first SEPARATOR
+    return toMockUri(path.toUri().getRawPath().substring(1)); 
+  }
+
+  /**
+   * Gets the blob reference to the given blob key.
+   * 
+   * @param blobKey
+   *          The blob key (no initial slash).
+   * @return The blob reference.
+   */
+  public CloudBlockBlob getBlobReference(String blobKey) throws Exception {
+    return container.getBlockBlobReference(String.format(blobKey));
+  }
+
+  /**
+   * Acquires a short lease on the given blob in this test account.
+   * 
+   * @param blobKey
+   *          The key to the blob (no initial slash).
+   * @return The lease ID.
+   */
+  public String acquireShortLease(String blobKey) throws Exception {
+    return getBlobReference(blobKey).acquireLease(60, null);
+  }
+
+  /**
+   * Releases the lease on the container.
+   * 
+   * @param leaseID
+   *          The lease ID.
+   */
+  public void releaseLease(String leaseID, String blobKey) throws Exception {
+    AccessCondition accessCondition = new AccessCondition();
+    accessCondition.setLeaseID(leaseID);
+    getBlobReference(blobKey).releaseLease(accessCondition);
+  }
+
+  public static AzureBlobStorageTestAccount createMock() throws Exception {
+    return createMock(new Configuration());
+  }
+
+  public static AzureBlobStorageTestAccount createMock(Configuration conf)
+      throws Exception {
+    AzureNativeFileSystemStore store = new AzureNativeFileSystemStore();
+    MockStorageInterface mockStorage = new MockStorageInterface();
+    store.setAzureStorageInteractionLayer(mockStorage);
+    NativeAzureFileSystem fs = new NativeAzureFileSystem(store);
+    addWasbToConfiguration(conf);
+    setMockAccountKey(conf);
+    // register the fs provider.
+
+    fs.initialize(new URI(MOCK_WASB_URI), conf);
+    AzureBlobStorageTestAccount testAcct = new AzureBlobStorageTestAccount(fs,
+        mockStorage);
+    return testAcct;
+  }
+
+  /**
+   * Creates a test account that goes against the storage emulator.
+   * 
+   * @return The test account, or null if the emulator isn't setup.
+   */
+  public static AzureBlobStorageTestAccount createForEmulator()
+      throws Exception {
+    NativeAzureFileSystem fs = null;
+    CloudBlobContainer container = null;
+    Configuration conf = createTestConfiguration();
+    if (!conf.getBoolean(USE_EMULATOR_PROPERTY_NAME, false)) {
+      // Not configured to test against the storage emulator.
+      System.out.println("Skipping emulator Azure test because configuration "
+          + "doesn't indicate that it's running."
+          + " Please see README.txt for guidance.");
+      return null;
+    }
+    CloudStorageAccount account = CloudStorageAccount
+        .getDevelopmentStorageAccount();
+    fs = new NativeAzureFileSystem();
+    String containerName = String.format("wasbtests-%s-%tQ",
+        System.getProperty("user.name"), new Date());
+    container = account.createCloudBlobClient().getContainerReference(
+        containerName);
+    container.create();
+
+    // Set account URI and initialize Azure file system.
+    URI accountUri = createAccountUri(DEFAULT_STORAGE_EMULATOR_ACCOUNT_NAME,
+        containerName);
+    fs.initialize(accountUri, conf);
+
+    // Create test account initializing the appropriate member variables.
+    AzureBlobStorageTestAccount testAcct = new AzureBlobStorageTestAccount(fs,
+        account, container);
+
+    return testAcct;
+  }
+
+  public static AzureBlobStorageTestAccount createOutOfBandStore(
+      int uploadBlockSize, int downloadBlockSize) throws Exception {
+
+    CloudBlobContainer container = null;
+    Configuration conf = createTestConfiguration();
+    CloudStorageAccount account = createTestAccount(conf);
+    if (null == account) {
+      return null;
+    }
+
+    String containerName = String.format("wasbtests-%s-%tQ",
+        System.getProperty("user.name"), new Date());
+
+    // Create the container.
+    container = account.createCloudBlobClient().getContainerReference(
+        containerName);
+    container.create();
+
+    String accountName = conf.get(TEST_ACCOUNT_NAME_PROPERTY_NAME);
+
+    // Ensure that custom throttling is disabled and tolerate concurrent
+    // out-of-band appends.
+    conf.setBoolean(KEY_DISABLE_THROTTLING, true);
+    conf.setBoolean(KEY_READ_TOLERATE_CONCURRENT_APPEND, true);
+
+    // Set account URI and initialize Azure file system.
+    URI accountUri = createAccountUri(accountName, containerName);
+
+    // Create a new AzureNativeFileSystemStore object.
+    AzureNativeFileSystemStore testStorage = new AzureNativeFileSystemStore();
+
+    // Initialize the store with the throttling feedback interfaces.
+    testStorage.initialize(accountUri, conf);
+
+    // Create test account initializing the appropriate member variables.
+    AzureBlobStorageTestAccount testAcct = new AzureBlobStorageTestAccount(
+        testStorage, account, container);
+
+    return testAcct;
+  }
+
+  /**
+   * Sets the mock account key in the given configuration.
+   * 
+   * @param conf
+   *          The configuration.
+   */
+  public static void setMockAccountKey(Configuration conf) {
+    setMockAccountKey(conf, MOCK_ACCOUNT_NAME);
+  }
+
+  /**
+   * Sets the mock account key in the given configuration.
+   * 
+   * @param conf
+   *          The configuration.
+   */
+  public static void setMockAccountKey(Configuration conf, String accountName) {
+    conf.set(ACCOUNT_KEY_PROPERTY_NAME + accountName,
+        Base64.encode(new byte[] { 1, 2, 3 }));  
+  }
+
+  private static URI createAccountUri(String accountName)
+      throws URISyntaxException {
+    return new URI(WASB_SCHEME + ":" + PATH_DELIMITER + PATH_DELIMITER
+        + accountName);
+  }
+
+  private static URI createAccountUri(String accountName, String containerName)
+      throws URISyntaxException {
+    return new URI(WASB_SCHEME + ":" + PATH_DELIMITER + PATH_DELIMITER
+        + containerName + WASB_AUTHORITY_DELIMITER + accountName);
+  }
+
+  public static AzureBlobStorageTestAccount create() throws Exception {
+    return create("");
+  }
+
+  public static AzureBlobStorageTestAccount create(String containerNameSuffix)
+      throws Exception {
+    return create(containerNameSuffix,
+        EnumSet.of(CreateOptions.CreateContainer));
+  }
+
+  // Create a test account which uses throttling.
+  public static AzureBlobStorageTestAccount createThrottled() throws Exception {
+    return create("",
+        EnumSet.of(CreateOptions.useThrottling, CreateOptions.CreateContainer));
+  }
+
+  public static AzureBlobStorageTestAccount create(Configuration conf)
+      throws Exception {
+    return create("", EnumSet.of(CreateOptions.CreateContainer), conf);
+  }
+
+  static CloudStorageAccount createStorageAccount(String accountName,
+      Configuration conf, boolean allowAnonymous) throws URISyntaxException,
+      KeyProviderException {
+    String accountKey = AzureNativeFileSystemStore
+        .getAccountKeyFromConfiguration(accountName, conf);
+    StorageCredentials credentials;
+    if (accountKey == null && allowAnonymous) {
+      credentials = StorageCredentialsAnonymous.ANONYMOUS;
+    } else {
+      credentials = new StorageCredentialsAccountAndKey(
+          accountName.split("\\.")[0], accountKey);
+    }
+    if (credentials == null) {
+      return null;
+    } else {
+      return new CloudStorageAccount(credentials);
+    }
+  }
+
+  private static Configuration createTestConfiguration() {
+    return createTestConfiguration(null);
+  }
+
+  protected static Configuration createTestConfiguration(Configuration conf) {
+    if (conf == null) {
+      conf = new Configuration();
+    }
+
+    conf.addResource(TEST_CONFIGURATION_FILE_NAME);
+    return conf;
+  }
+
+  // for programmatic setting of the wasb configuration.
+  // note that tests can also get the
+  public static void addWasbToConfiguration(Configuration conf) {
+    conf.set("fs.wasb.impl", "org.apache.hadoop.fs.azure.NativeAzureFileSystem");
+    conf.set("fs.wasbs.impl",
+        "org.apache.hadoop.fs.azure.NativeAzureFileSystem");
+  }
+
+  static CloudStorageAccount createTestAccount() throws URISyntaxException,
+      KeyProviderException {
+    return createTestAccount(createTestConfiguration());
+  }
+
+  static CloudStorageAccount createTestAccount(Configuration conf)
+      throws URISyntaxException, KeyProviderException {
+    String testAccountName = conf.get(TEST_ACCOUNT_NAME_PROPERTY_NAME);
+    if (testAccountName == null) {
+      System.out
+          .println("Skipping live Azure test because of missing test account."
+              + " Please see README.txt for guidance.");
+      return null;
+    }
+    return createStorageAccount(testAccountName, conf, false);
+  }
+
+  public static enum CreateOptions {
+    UseSas, Readonly, CreateContainer, useThrottling
+  }
+
+  public static AzureBlobStorageTestAccount create(String containerNameSuffix,
+      EnumSet<CreateOptions> createOptions) throws Exception {
+    return create(containerNameSuffix, createOptions, null);
+  }
+
+  public static AzureBlobStorageTestAccount create(String containerNameSuffix,
+      EnumSet<CreateOptions> createOptions, Configuration initialConfiguration)
+      throws Exception {
+    NativeAzureFileSystem fs = null;
+    CloudBlobContainer container = null;
+    Configuration conf = createTestConfiguration(initialConfiguration);
+    CloudStorageAccount account = createTestAccount(conf);
+    if (account == null) {
+      return null;
+    }
+    fs = new NativeAzureFileSystem();
+    String containerName = String.format("wasbtests-%s-%tQ%s",
+        System.getProperty("user.name"), new Date(), containerNameSuffix);
+    container = account.createCloudBlobClient().getContainerReference(
+        containerName);
+    if (createOptions.contains(CreateOptions.CreateContainer)) {
+      container.create();
+    }
+    String accountName = conf.get(TEST_ACCOUNT_NAME_PROPERTY_NAME);
+    if (createOptions.contains(CreateOptions.UseSas)) {
+      String sas = generateSAS(container,
+          createOptions.contains(CreateOptions.Readonly));
+      if (!createOptions.contains(CreateOptions.CreateContainer)) {
+        // The caller doesn't want the container to be pre-created,
+        // so delete it now that we have generated the SAS.
+        container.delete();
+      }
+      // Remove the account key from the configuration to make sure we don't
+      // cheat and use that.
+      conf.set(ACCOUNT_KEY_PROPERTY_NAME + accountName, "");
+      // Set the SAS key.
+      conf.set(SAS_PROPERTY_NAME + containerName + "." + accountName, sas);
+    }
+
+    // Check if throttling is turned on and set throttling parameters
+    // appropriately.
+    if (createOptions.contains(CreateOptions.useThrottling)) {
+      conf.setBoolean(KEY_DISABLE_THROTTLING, false);
+    } else {
+      conf.setBoolean(KEY_DISABLE_THROTTLING, true);
+    }
+
+    // Set account URI and initialize Azure file system.
+    URI accountUri = createAccountUri(accountName, containerName);
+    fs.initialize(accountUri, conf);
+
+    // Create test account initializing the appropriate member variables.
+    AzureBlobStorageTestAccount testAcct = new AzureBlobStorageTestAccount(fs,
+        account, container);
+
+    return testAcct;
+  }
+
+  private static String generateContainerName() throws Exception {
+    String containerName = String.format("wasbtests-%s-%tQ",
+        System.getProperty("user.name"), new Date());
+    return containerName;
+  }
+
+  private static String generateSAS(CloudBlobContainer container,
+      boolean readonly) throws Exception {
+
+    // Create a container if it does not exist.
+    container.createIfNotExists();
+
+    // Create a new shared access policy.
+    SharedAccessBlobPolicy sasPolicy = new SharedAccessBlobPolicy();
+
+    // Create a UTC Gregorian calendar value.
+    GregorianCalendar calendar = new GregorianCalendar(
+        TimeZone.getTimeZone("UTC"));
+
+    // Specify the current time as the start time for the shared access
+    // signature.
+    //
+    calendar.setTime(new Date());
+    sasPolicy.setSharedAccessStartTime(calendar.getTime());
+
+    // Use the start time delta one hour as the end time for the shared
+    // access signature.
+    calendar.add(Calendar.HOUR, 10);
+    sasPolicy.setSharedAccessExpiryTime(calendar.getTime());
+
+    if (readonly) {
+      // Set READ permissions
+      sasPolicy.setPermissions(EnumSet.of(SharedAccessBlobPermissions.READ,
+          SharedAccessBlobPermissions.LIST));
+    } else {
+      // Set READ and WRITE permissions.
+      sasPolicy.setPermissions(EnumSet.of(SharedAccessBlobPermissions.READ,
+          SharedAccessBlobPermissions.WRITE, SharedAccessBlobPermissions.LIST));
+    }
+
+    // Create the container permissions.
+    BlobContainerPermissions containerPermissions = new BlobContainerPermissions();
+
+    // Turn public access to the container off.
+    containerPermissions.setPublicAccess(BlobContainerPublicAccessType.OFF);
+
+    container.uploadPermissions(containerPermissions);
+
+    // Create a shared access signature for the container.
+    String sas = container.generateSharedAccessSignature(sasPolicy, null);
+    // HACK: when the just generated SAS is used straight away, we get an
+    // authorization error intermittently. Sleeping for 1.5 seconds fixes that
+    // on my box.
+    Thread.sleep(1500);
+
+    // Return to caller with the shared access signature.
+    return sas;
+  }
+
+  public static void primePublicContainer(CloudBlobClient blobClient,
+      String accountName, String containerName, String blobName, int fileSize)
+      throws Exception {
+
+    // Create a container if it does not exist. The container name
+    // must be lower case.
+    CloudBlobContainer container = blobClient
+        .getContainerReference(containerName);
+
+    container.createIfNotExists();
+
+    // Create a new shared access policy.
+    SharedAccessBlobPolicy sasPolicy = new SharedAccessBlobPolicy();
+
+    // Set READ and WRITE permissions.
+    sasPolicy.setPermissions(EnumSet.of(SharedAccessBlobPermissions.READ,
+        SharedAccessBlobPermissions.WRITE, SharedAccessBlobPermissions.LIST,
+        SharedAccessBlobPermissions.DELETE));
+
+    // Create the container permissions.
+    BlobContainerPermissions containerPermissions = new BlobContainerPermissions();
+
+    // Turn public access to the container off.
+    containerPermissions
+        .setPublicAccess(BlobContainerPublicAccessType.CONTAINER);
+
+    // Set the policy using the values set above.
+    containerPermissions.getSharedAccessPolicies().put("testwasbpolicy",
+        sasPolicy);
+    container.uploadPermissions(containerPermissions);
+
+    // Create a blob output stream.
+    CloudBlockBlob blob = container.getBlockBlobReference(blobName);
+    BlobOutputStream outputStream = blob.openOutputStream();
+
+    outputStream.write(new byte[fileSize]);
+    outputStream.close();
+  }
+
+  public static AzureBlobStorageTestAccount createAnonymous(
+      final String blobName, final int fileSize) throws Exception {
+
+    NativeAzureFileSystem fs = null;
+    CloudBlobContainer container = null;
+    Configuration conf = createTestConfiguration(), noTestAccountConf = new Configuration();
+
+    // Set up a session with the cloud blob client to generate SAS and check the
+    // existence of a container and capture the container object.
+    CloudStorageAccount account = createTestAccount(conf);
+    if (account == null) {
+      return null;
+    }
+    CloudBlobClient blobClient = account.createCloudBlobClient();
+
+    // Capture the account URL and the account name.
+    String accountName = conf.get(TEST_ACCOUNT_NAME_PROPERTY_NAME);
+
+    // Generate a container name and create a shared access signature string for
+    // it.
+    //
+    String containerName = generateContainerName();
+
+    // Set up public container with the specified blob name.
+    primePublicContainer(blobClient, accountName, containerName, blobName,
+        fileSize);
+
+    // Capture the blob container object. It should exist after generating the
+    // shared access signature.
+    container = blobClient.getContainerReference(containerName);
+    if (null == container || !container.exists()) {
+      final String errMsg = String
+          .format("Container '%s' expected but not found while creating SAS account.");
+      throw new Exception(errMsg);
+    }
+
+    // Set the account URI.
+    URI accountUri = createAccountUri(accountName, containerName);
+
+    // Initialize the Native Azure file system with anonymous credentials.
+    fs = new NativeAzureFileSystem();
+    fs.initialize(accountUri, noTestAccountConf);
+
+    // Create test account initializing the appropriate member variables.
+    AzureBlobStorageTestAccount testAcct = new AzureBlobStorageTestAccount(fs,
+        account, container);
+
+    // Return to caller with test account.
+    return testAcct;
+  }
+
+  private static CloudBlockBlob primeRootContainer(CloudBlobClient blobClient,
+      String accountName, String blobName, int fileSize) throws Exception {
+
+    // Create a container if it does not exist. The container name
+    // must be lower case.
+    CloudBlobContainer container = blobClient.getContainerReference("https://"
+        + accountName + "/" + "$root");
+    container.createIfNotExists();
+
+    // Create a blob output stream.
+    CloudBlockBlob blob = container.getBlockBlobReference(blobName);
+    BlobOutputStream outputStream = blob.openOutputStream();
+
+    outputStream.write(new byte[fileSize]);
+    outputStream.close();
+
+    // Return a reference to the block blob object.
+    return blob;
+  }
+
+  public static AzureBlobStorageTestAccount createRoot(final String blobName,
+      final int fileSize) throws Exception {
+
+    NativeAzureFileSystem fs = null;
+    CloudBlobContainer container = null;
+    Configuration conf = createTestConfiguration();
+
+    // Set up a session with the cloud blob client to generate SAS and check the
+    // existence of a container and capture the container object.
+    CloudStorageAccount account = createTestAccount(conf);
+    if (account == null) {
+      return null;
+    }
+    CloudBlobClient blobClient = account.createCloudBlobClient();
+
+    // Capture the account URL and the account name.
+    String accountName = conf.get(TEST_ACCOUNT_NAME_PROPERTY_NAME);
+
+    // Set up public container with the specified blob name.
+    CloudBlockBlob blobRoot = primeRootContainer(blobClient, accountName,
+        blobName, fileSize);
+
+    // Capture the blob container object. It should exist after generating the
+    // shared access signature.
+    container = blobClient.getContainerReference(AZURE_ROOT_CONTAINER);
+    if (null == container || !container.exists()) {
+      final String errMsg = String
+          .format("Container '%s' expected but not found while creating SAS account.");
+      throw new Exception(errMsg);
+    }
+
+    // Set the account URI without a container name.
+    URI accountUri = createAccountUri(accountName);
+
+    // Initialize the Native Azure file system with anonymous credentials.
+    fs = new NativeAzureFileSystem();
+    fs.initialize(accountUri, conf);
+
+    // Create test account initializing the appropriate member variables.
+    // Set the container value to null for the default root container.
+    AzureBlobStorageTestAccount testAcct = new AzureBlobStorageTestAccount(fs,
+        account, blobRoot);
+
+    // Return to caller with test account.
+    return testAcct;
+  }
+
+  public void closeFileSystem() throws Exception {
+    if (fs != null) {
+      fs.close();
+    }
+  }
+
+  public void cleanup() throws Exception {
+    if (fs != null) {
+      fs.close();
+      fs = null;
+    }
+    if (container != null) {
+      container.deleteIfExists();
+      container = null;
+    }
+    if (blob != null) {
+      // The blob member variable is set for blobs under root containers.
+      // Delete blob objects created for root container tests when cleaning
+      // up the test account.
+      blob.delete();
+      blob = null;
+    }
+  }
+
+  public NativeAzureFileSystem getFileSystem() {
+    return fs;
+  }
+
+  public AzureNativeFileSystemStore getStore() {
+    return this.storage;
+  }
+
+  /**
+   * Gets the real blob container backing this account if it's not a mock.
+   * 
+   * @return A container, or null if it's a mock.
+   */
+  public CloudBlobContainer getRealContainer() {
+    return container;
+  }
+
+  /**
+   * Gets the real blob account backing this account if it's not a mock.
+   * 
+   * @return An account, or null if it's a mock.
+   */
+  public CloudStorageAccount getRealAccount() {
+    return account;
+  }
+
+  /**
+   * Gets the mock storage interface if this account is backed by a mock.
+   * 
+   * @return The mock storage, or null if it's backed by a real account.
+   */
+  public MockStorageInterface getMockStorage() {
+    return mockStorage;
+  }
+ 
+}

Added: hadoop/common/trunk/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/InMemoryBlockBlobStore.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/InMemoryBlockBlobStore.java?rev=1601781&view=auto
==============================================================================
--- hadoop/common/trunk/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/InMemoryBlockBlobStore.java (added)
+++ hadoop/common/trunk/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/InMemoryBlockBlobStore.java Tue Jun 10 22:26:45 2014
@@ -0,0 +1,146 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.azure;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * A simple memory key-value store to help mock the Windows Azure Storage
+ * implementation for unit testing.
+ */
+public class InMemoryBlockBlobStore {
+  private final HashMap<String, Entry> blobs = new HashMap<String, Entry>();
+  private HashMap<String, String> containerMetadata;
+
+  public synchronized Iterable<String> getKeys() {
+    return new ArrayList<String>(blobs.keySet());
+  }
+
+  public static class ListBlobEntry {
+    private final String key;
+    private final HashMap<String, String> metadata;
+    private final int contentLength;
+
+    ListBlobEntry(String key, HashMap<String, String> metadata,
+        int contentLength) {
+      this.key = key;
+      this.metadata = metadata;
+      this.contentLength = contentLength;
+    }
+
+    public String getKey() {
+      return key;
+    }
+
+    public HashMap<String, String> getMetadata() {
+      return metadata;
+    }
+
+    public int getContentLength() {
+      return contentLength;
+    }
+  }
+
+  /**
+   * List all the blobs whose key starts with the given prefix.
+   * 
+   * @param prefix
+   *          The prefix to check.
+   * @param includeMetadata
+   *          If set, the metadata in the returned listing will be populated;
+   *          otherwise it'll be null.
+   * @return The listing.
+   */
+  public synchronized Iterable<ListBlobEntry> listBlobs(String prefix,
+      boolean includeMetadata) {
+    ArrayList<ListBlobEntry> list = new ArrayList<ListBlobEntry>();
+    for (Map.Entry<String, Entry> entry : blobs.entrySet()) {
+      if (entry.getKey().startsWith(prefix)) {
+        list.add(new ListBlobEntry(entry.getKey(),
+            includeMetadata ? new HashMap<String, String>(
+                entry.getValue().metadata) : null,
+            entry.getValue().content.length));
+      }
+    }
+    return list;
+  }
+
+  public synchronized byte[] getContent(String key) {
+    return blobs.get(key).content;
+  }
+
+  @SuppressWarnings("unchecked")
+  public synchronized void setContent(String key, byte[] value,
+      HashMap<String, String> metadata) {
+    blobs
+        .put(key, new Entry(value, (HashMap<String, String>) metadata.clone()));
+  }
+
+  public OutputStream upload(final String key,
+      final HashMap<String, String> metadata) {
+    setContent(key, new byte[0], metadata);
+    return new ByteArrayOutputStream() {
+      @Override
+      public void flush() throws IOException {
+        super.flush();
+        setContent(key, toByteArray(), metadata);
+      }
+    };
+  }
+
+  public synchronized void copy(String sourceKey, String destKey) {
+    blobs.put(destKey, blobs.get(sourceKey));
+  }
+
+  public synchronized void delete(String key) {
+    blobs.remove(key);
+  }
+
+  public synchronized boolean exists(String key) {
+    return blobs.containsKey(key);
+  }
+
+  @SuppressWarnings("unchecked")
+  public synchronized HashMap<String, String> getMetadata(String key) {
+    return (HashMap<String, String>) blobs.get(key).metadata.clone();
+  }
+
+  public synchronized HashMap<String, String> getContainerMetadata() {
+    return containerMetadata;
+  }
+
+  public synchronized void setContainerMetadata(HashMap<String, String> metadata) {
+    containerMetadata = metadata;
+  }
+
+  private static class Entry {
+    private byte[] content;
+    private HashMap<String, String> metadata;
+
+    public Entry(byte[] content, HashMap<String, String> metadata) {
+      this.content = content;
+      this.metadata = metadata;
+    }
+  }
+}



Mime
View raw message