Mailing-List: contact commits-help@brooklyn.incubator.apache.org; run by ezmlm
Precedence: bulk
Reply-To: dev@brooklyn.incubator.apache.org
Content-Type: text/plain; charset="us-ascii"
MIME-Version: 1.0
Content-Transfer-Encoding: 7bit
From: heneveld@apache.org
To: commits@brooklyn.incubator.apache.org
Date: Thu, 26 Jun 2014 11:23:45 -0000
Message-Id: <bd5c2b9f7dae46b2aa5d7f3c4e3a724b@git.apache.org>
Subject: [1/3] git commit: fixes the "RetryOnRenew" lease-renewal needed bug
 in jclouds, locally, with some guice magic.

Repository: incubator-brooklyn
Updated Branches:
  refs/heads/master ab19dce3d -> a926e8a72


fixes the "RetryOnRenew" lease-renewal needed bug in jclouds, locally, with some guice magic.

https://issues.apache.org/jira/browse/JCLOUDS-615 / https://issues.apache.org/jira/browse/BROOKLYN-6

when the former above is fixed properly upstream we can undo this.


Project: http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/commit/a5a9de81
Tree: http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/tree/a5a9de81
Diff: http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/diff/a5a9de81

Branch: refs/heads/master
Commit: a5a9de81873d2f7cbf3cf4a9f4ee2b673dd15899
Parents: 985951c
Author: Alex Heneveld <alex.heneveld@cloudsoftcorp.com>
Authored: Wed Jun 25 15:03:41 2014 +0100
Committer: Alex Heneveld <alex.heneveld@cloudsoftcorp.com>
Committed: Wed Jun 25 15:09:38 2014 +0100

----------------------------------------------------------------------
 .../brooklyn/util/http/HttpToolResponse.java    |   5 +
 .../JcloudsBlobStoreBasedObjectStore.java       |  19 +-
 .../jclouds/BlobStoreCapturingError.java        | 119 +++++++++++
 .../brooklyn/location/jclouds/JcloudsUtil.java  |  58 +++++-
 .../jclouds/config/AlwaysRetryOnRenew.java      | 121 ++++++++++++
 .../persister/jclouds/BlobStoreExpiryTest.java  | 195 +++++++++++++++++++
 .../rebind/persister/jclouds/BlobStoreTest.java |  19 +-
 7 files changed, 508 insertions(+), 28 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/blob/a5a9de81/core/src/main/java/brooklyn/util/http/HttpToolResponse.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/brooklyn/util/http/HttpToolResponse.java b/core/src/main/java/brooklyn/util/http/HttpToolResponse.java
index 96c58e8..45599bc 100644
--- a/core/src/main/java/brooklyn/util/http/HttpToolResponse.java
+++ b/core/src/main/java/brooklyn/util/http/HttpToolResponse.java
@@ -14,6 +14,7 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import brooklyn.event.feed.http.HttpPollValue;
+import brooklyn.util.guava.Maybe;
 import brooklyn.util.stream.Streams;
 import brooklyn.util.time.Time;
 
@@ -143,6 +144,10 @@ public class HttpToolResponse implements HttpPollValue {
     public String getContentAsString() {
         return new String(getContent());
     }
+    
+    public Maybe<HttpResponse> getResponse() {
+        return Maybe.fromNullable(response);
+    }
 
     @Override
     public String toString() {

http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/blob/a5a9de81/locations/jclouds/src/main/java/brooklyn/entity/rebind/persister/jclouds/JcloudsBlobStoreBasedObjectStore.java
----------------------------------------------------------------------
diff --git a/locations/jclouds/src/main/java/brooklyn/entity/rebind/persister/jclouds/JcloudsBlobStoreBasedObjectStore.java b/locations/jclouds/src/main/java/brooklyn/entity/rebind/persister/jclouds/JcloudsBlobStoreBasedObjectStore.java
index a91bca0..c9aa309 100644
--- a/locations/jclouds/src/main/java/brooklyn/entity/rebind/persister/jclouds/JcloudsBlobStoreBasedObjectStore.java
+++ b/locations/jclouds/src/main/java/brooklyn/entity/rebind/persister/jclouds/JcloudsBlobStoreBasedObjectStore.java
@@ -6,28 +6,27 @@ import java.util.List;
 
 import javax.annotation.Nullable;
 
-import org.jclouds.ContextBuilder;
 import org.jclouds.blobstore.BlobStoreContext;
 import org.jclouds.blobstore.domain.StorageMetadata;
 import org.jclouds.blobstore.options.ListContainerOptions;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import com.google.common.base.Function;
-import com.google.common.base.Objects;
-import com.google.common.base.Preconditions;
-import com.google.common.collect.FluentIterable;
-
 import brooklyn.config.BrooklynServerConfig;
 import brooklyn.entity.rebind.persister.PersistMode;
 import brooklyn.entity.rebind.persister.PersistenceObjectStore;
 import brooklyn.location.basic.LocationConfigKeys;
 import brooklyn.location.cloud.CloudLocationConfig;
 import brooklyn.location.jclouds.JcloudsLocation;
+import brooklyn.location.jclouds.JcloudsUtil;
 import brooklyn.management.ManagementContext;
 import brooklyn.management.ha.HighAvailabilityMode;
 import brooklyn.util.exceptions.FatalConfigurationRuntimeException;
-import brooklyn.util.text.Strings;
+
+import com.google.common.base.Function;
+import com.google.common.base.Objects;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.FluentIterable;
 
 /**
  * @author Andrea Turli
@@ -72,11 +71,7 @@ public class JcloudsBlobStoreBasedObjectStore implements PersistenceObjectStore
             String provider = checkNotNull(location.getConfig(LocationConfigKeys.CLOUD_PROVIDER), "provider must not be null");
             String endpoint = location.getConfig(CloudLocationConfig.CLOUD_ENDPOINT);
 
-            ContextBuilder contextBuilder = ContextBuilder.newBuilder(provider).credentials(identity, credential);
-            if (!Strings.isBlank(endpoint)) {
-                contextBuilder.endpoint(endpoint);
-            }
-            context = contextBuilder.buildView(BlobStoreContext.class);
+            context = JcloudsUtil.newBlobstoreContext(provider, endpoint, identity, credential, true);
      
             // TODO do we need to get location from region? can't see the jclouds API.
             // doesn't matter in some places because it's already in the endpoint

http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/blob/a5a9de81/locations/jclouds/src/main/java/brooklyn/location/jclouds/BlobStoreCapturingError.java
----------------------------------------------------------------------
diff --git a/locations/jclouds/src/main/java/brooklyn/location/jclouds/BlobStoreCapturingError.java b/locations/jclouds/src/main/java/brooklyn/location/jclouds/BlobStoreCapturingError.java
new file mode 100644
index 0000000..af1543d
--- /dev/null
+++ b/locations/jclouds/src/main/java/brooklyn/location/jclouds/BlobStoreCapturingError.java
@@ -0,0 +1,119 @@
+package brooklyn.location.jclouds;
+
+import java.util.Set;
+
+import org.jclouds.blobstore.BlobStore;
+import org.jclouds.blobstore.BlobStoreContext;
+import org.jclouds.blobstore.domain.Blob;
+import org.jclouds.blobstore.domain.BlobBuilder;
+import org.jclouds.blobstore.domain.BlobMetadata;
+import org.jclouds.blobstore.domain.PageSet;
+import org.jclouds.blobstore.domain.StorageMetadata;
+import org.jclouds.blobstore.options.CreateContainerOptions;
+import org.jclouds.blobstore.options.GetOptions;
+import org.jclouds.blobstore.options.ListContainerOptions;
+import org.jclouds.blobstore.options.PutOptions;
+import org.jclouds.domain.Location;
+
+public class BlobStoreCapturingError implements BlobStore {
+
+    BlobStore delegate;
+
+    public BlobStoreContext getContext() {
+        return delegate.getContext();
+    }
+
+    public BlobBuilder blobBuilder(String name) {
+        return delegate.blobBuilder(name);
+    }
+
+    public Set<? extends Location> listAssignableLocations() {
+        return delegate.listAssignableLocations();
+    }
+
+    public PageSet<? extends StorageMetadata> list() {
+        return delegate.list();
+    }
+
+    public boolean containerExists(String container) {
+        return delegate.containerExists(container);
+    }
+
+    public boolean createContainerInLocation(Location location, String container) {
+        return delegate.createContainerInLocation(location, container);
+    }
+
+    public boolean createContainerInLocation(Location location, String container, CreateContainerOptions options) {
+        return delegate.createContainerInLocation(location, container, options);
+    }
+
+    public PageSet<? extends StorageMetadata> list(String container) {
+        return delegate.list(container);
+    }
+
+    public PageSet<? extends StorageMetadata> list(String container, ListContainerOptions options) {
+        return delegate.list(container, options);
+    }
+
+    public void clearContainer(String container) {
+        delegate.clearContainer(container);
+    }
+
+    public void clearContainer(String container, ListContainerOptions options) {
+        delegate.clearContainer(container, options);
+    }
+
+    public void deleteContainer(String container) {
+        delegate.deleteContainer(container);
+    }
+
+    public boolean directoryExists(String container, String directory) {
+        return delegate.directoryExists(container, directory);
+    }
+
+    public void createDirectory(String container, String directory) {
+        delegate.createDirectory(container, directory);
+    }
+
+    public void deleteDirectory(String containerName, String name) {
+        delegate.deleteDirectory(containerName, name);
+    }
+
+    public boolean blobExists(String container, String name) {
+        return delegate.blobExists(container, name);
+    }
+
+    public String putBlob(String container, Blob blob) {
+        return delegate.putBlob(container, blob);
+    }
+
+    public String putBlob(String container, Blob blob, PutOptions options) {
+        return delegate.putBlob(container, blob, options);
+    }
+
+    public BlobMetadata blobMetadata(String container, String name) {
+        return delegate.blobMetadata(container, name);
+    }
+
+    public Blob getBlob(String container, String name) {
+        return delegate.getBlob(container, name);
+    }
+
+    public Blob getBlob(String container, String name, GetOptions options) {
+        return delegate.getBlob(container, name, options);
+    }
+
+    public void removeBlob(String container, String name) {
+        delegate.removeBlob(container, name);
+    }
+
+    public long countBlobs(String container) {
+        return delegate.countBlobs(container);
+    }
+
+    public long countBlobs(String container, ListContainerOptions options) {
+        return delegate.countBlobs(container, options);
+    }
+    
+    
+}

http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/blob/a5a9de81/locations/jclouds/src/main/java/brooklyn/location/jclouds/JcloudsUtil.java
----------------------------------------------------------------------
diff --git a/locations/jclouds/src/main/java/brooklyn/location/jclouds/JcloudsUtil.java b/locations/jclouds/src/main/java/brooklyn/location/jclouds/JcloudsUtil.java
index fb02493..31bbd1b 100644
--- a/locations/jclouds/src/main/java/brooklyn/location/jclouds/JcloudsUtil.java
+++ b/locations/jclouds/src/main/java/brooklyn/location/jclouds/JcloudsUtil.java
@@ -6,7 +6,10 @@ import static org.jclouds.aws.ec2.reference.AWSEC2Constants.PROPERTY_EC2_AMI_QUE
 import static org.jclouds.aws.ec2.reference.AWSEC2Constants.PROPERTY_EC2_CC_AMI_QUERY;
 import static org.jclouds.compute.options.RunScriptOptions.Builder.overrideLoginCredentials;
 import static org.jclouds.compute.util.ComputeServiceUtils.execHttpResponse;
-import static org.jclouds.scriptbuilder.domain.Statements.*;
+import static org.jclouds.scriptbuilder.domain.Statements.appendFile;
+import static org.jclouds.scriptbuilder.domain.Statements.exec;
+import static org.jclouds.scriptbuilder.domain.Statements.interpret;
+import static org.jclouds.scriptbuilder.domain.Statements.newStatementList;
 
 import java.io.File;
 import java.io.IOException;
@@ -18,9 +21,12 @@ import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.TimeoutException;
 
+import javax.annotation.Nullable;
+
 import org.jclouds.Constants;
 import org.jclouds.ContextBuilder;
 import org.jclouds.aws.ec2.AWSEC2Api;
+import org.jclouds.blobstore.BlobStoreContext;
 import org.jclouds.compute.ComputeService;
 import org.jclouds.compute.ComputeServiceContext;
 import org.jclouds.compute.RunScriptOnNodesException;
@@ -47,7 +53,9 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import brooklyn.entity.basic.Entities;
+import brooklyn.location.jclouds.config.AlwaysRetryOnRenew;
 import brooklyn.location.jclouds.config.BrooklynStandardJcloudsGuiceModule;
+import brooklyn.util.collections.MutableList;
 import brooklyn.util.collections.MutableMap;
 import brooklyn.util.config.ConfigBag;
 import brooklyn.util.net.Protocol;
@@ -55,6 +63,7 @@ import brooklyn.util.ssh.IptablesCommands;
 import brooklyn.util.ssh.IptablesCommands.Chain;
 import brooklyn.util.ssh.IptablesCommands.Policy;
 
+import com.google.common.annotations.Beta;
 import com.google.common.base.Charsets;
 import com.google.common.base.Function;
 import com.google.common.base.Predicate;
@@ -239,11 +248,7 @@ public class JcloudsUtil implements JcloudsLocationConfig {
             LOG.debug("jclouds ComputeService cache miss for compute service, creating, for "+Entities.sanitize(properties));
         }
 
-        Iterable<Module> modules = ImmutableSet.<Module> of(
-                new SshjSshClientModule(), 
-                new SLF4JLoggingModule(),
-                new BouncyCastleCryptoModule(),
-                new BrooklynStandardJcloudsGuiceModule());
+        Iterable<Module> modules = getCommonModules();
 
         // Synchronizing to avoid deadlock from sun.reflect.annotation.AnnotationType.
         // See https://github.com/brooklyncentral/brooklyn/issues/974
@@ -271,7 +276,45 @@ public class JcloudsUtil implements JcloudsLocationConfig {
         }
         return computeService;
      }
+
+    /** returns the jclouds modules we typically install */ 
+    public static ImmutableSet<Module> getCommonModules() {
+        return ImmutableSet.<Module> of(
+                new SshjSshClientModule(), 
+                new SLF4JLoggingModule(),
+                new BouncyCastleCryptoModule(),
+                new BrooklynStandardJcloudsGuiceModule());
+    }
      
+    /** 
+     *  Temporary constructor to address https://issues.apache.org/jira/browse/JCLOUDS-615.
+     *  <p>
+     *  See https://issues.apache.org/jira/browse/BROOKLYN-6 .
+     *  When https://issues.apache.org/jira/browse/JCLOUDS-615 is fixed in the jclouds we use,
+     *  we can remove the useSoftlayerFix argument. 
+     *  <p>
+     *  (Marked Beta as that argument will likely be removed.)
+     *  
+     *  @since 1.7.0 */
+    @Beta
+    public static BlobStoreContext newBlobstoreContext(String provider, @Nullable String endpoint, String identity, String credential, boolean useSoftlayerFix) {
+        AlwaysRetryOnRenew.InterceptRetryOnRenewModule fix = 
+            useSoftlayerFix ? new AlwaysRetryOnRenew.InterceptRetryOnRenewModule() : null;
+            
+        ContextBuilder contextBuilder = ContextBuilder.newBuilder(provider).credentials(identity, credential);
+        contextBuilder.modules(MutableList.copyOf(JcloudsUtil.getCommonModules())
+            .appendIfNotNull(fix));
+        if (!brooklyn.util.text.Strings.isBlank(endpoint)) {
+            contextBuilder.endpoint(endpoint);
+        }
+        BlobStoreContext context = contextBuilder.buildView(BlobStoreContext.class);
+
+        if (useSoftlayerFix)
+            fix.inject(context.utils().injector());
+        
+        return context;
+    }
+
      protected static String getDeprecatedProperty(ConfigBag conf, String key) {
         if (conf.containsKey(key)) {
             LOG.warn("Jclouds using deprecated brooklyn-jclouds property "+key+": "+Entities.sanitize(conf.getAllConfig()));
@@ -332,7 +375,7 @@ public class JcloudsUtil implements JcloudsLocationConfig {
         };
         
         LOG.info("Waiting for password, for "+node.getProviderId()+":"+node.getId());
-        Predicate passwordReadyRetryable = Predicates2.retry(passwordReady, timeUnit.toMillis(timeout), 10*1000, TimeUnit.MILLISECONDS);
+        Predicate<String> passwordReadyRetryable = Predicates2.retry(passwordReady, timeUnit.toMillis(timeout), 10*1000, TimeUnit.MILLISECONDS);
         boolean ready = passwordReadyRetryable.apply(node.getProviderId());
         if (!ready) throw new TimeoutException("Password not available for "+node+" in region "+region+" after "+timeout+" "+timeUnit.name());
 
@@ -398,4 +441,5 @@ public class JcloudsUtil implements JcloudsLocationConfig {
             }
         }
     }
+
 }

http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/blob/a5a9de81/locations/jclouds/src/main/java/brooklyn/location/jclouds/config/AlwaysRetryOnRenew.java
----------------------------------------------------------------------
diff --git a/locations/jclouds/src/main/java/brooklyn/location/jclouds/config/AlwaysRetryOnRenew.java b/locations/jclouds/src/main/java/brooklyn/location/jclouds/config/AlwaysRetryOnRenew.java
new file mode 100644
index 0000000..2fa666b
--- /dev/null
+++ b/locations/jclouds/src/main/java/brooklyn/location/jclouds/config/AlwaysRetryOnRenew.java
@@ -0,0 +1,121 @@
+package brooklyn.location.jclouds.config;
+
+import static org.jclouds.http.HttpUtils.closeClientButKeepContentStream;
+import static org.jclouds.http.HttpUtils.releasePayload;
+
+import java.lang.reflect.Method;
+
+import javax.annotation.Resource;
+
+import org.aopalliance.intercept.MethodInterceptor;
+import org.aopalliance.intercept.MethodInvocation;
+import org.jclouds.domain.Credentials;
+import org.jclouds.http.HttpCommand;
+import org.jclouds.http.HttpResponse;
+import org.jclouds.http.HttpRetryHandler;
+import org.jclouds.logging.Logger;
+import org.jclouds.openstack.domain.AuthenticationResponse;
+import org.jclouds.openstack.handlers.RetryOnRenew;
+import org.jclouds.openstack.reference.AuthHeaders;
+
+import com.google.common.annotations.Beta;
+import com.google.common.cache.LoadingCache;
+import com.google.common.collect.Multimap;
+import com.google.inject.AbstractModule;
+import com.google.inject.Inject;
+import com.google.inject.Injector;
+import com.google.inject.Singleton;
+import com.google.inject.matcher.AbstractMatcher;
+import com.google.inject.matcher.Matcher;
+import com.google.inject.matcher.Matchers;
+
+/** Fix for RetryOnRenew so that it always retries on 401 when using a token.
+ *  The "lease renew" text is not necessarily returned from swift servers.
+ *  <p>
+ *  See https://issues.apache.org/jira/browse/BROOKLYN-6 .
+ *  When https://issues.apache.org/jira/browse/JCLOUDS-615 is fixed in the jclouds we use,
+ *  we can remove this. 
+ *  <p>
+ *  (Marked Beta as this will likely be removed.)
+ *  
+ *  @since 1.7.0 */
+@Beta
+@Singleton
+public class AlwaysRetryOnRenew implements HttpRetryHandler {
+   @Resource
+   protected Logger logger = Logger.NULL;
+
+   private final LoadingCache<Credentials, AuthenticationResponse> authenticationResponseCache;
+
+   @Inject
+   protected AlwaysRetryOnRenew(LoadingCache<Credentials, AuthenticationResponse> authenticationResponseCache) {
+      this.authenticationResponseCache = authenticationResponseCache;
+   }
+
+   @Override
+   public boolean shouldRetryRequest(HttpCommand command, HttpResponse response) {
+      boolean retry = false; // default
+      try {
+         switch (response.getStatusCode()) {
+            case 401:
+               // Do not retry on 401 from authentication request
+               Multimap<String, String> headers = command.getCurrentRequest().getHeaders();
+               if (headers != null && headers.containsKey(AuthHeaders.AUTH_USER)
+                        && headers.containsKey(AuthHeaders.AUTH_KEY) && !headers.containsKey(AuthHeaders.AUTH_TOKEN)) {
+                  retry = false;
+               } else {
+                  closeClientButKeepContentStream(response);
+                  authenticationResponseCache.invalidateAll();
+                  retry = true;
+                  
+                  // always retry. not all swift servers say 'lease renew', e.g. softlayer
+                  
+//                  byte[] content = closeClientButKeepContentStream(response);
+//                  if (content != null && new String(content).contains("lease renew")) {
+//                     logger.debug("invalidating authentication token");
+//                     authenticationResponseCache.invalidateAll();
+//                     retry = true;
+//                  } else {
+//                     retry = false;
+//                  }
+               }
+               break;
+         }
+         return retry;
+
+      } finally {
+         releasePayload(response);
+      }
+   }
+
+   /** 
+    * Intercepts calls to the *other* RetryOnRenew instance, and uses the one above.
+    * It's messy, but the only way I could find in the maze of guice. */
+   public static class InterceptRetryOnRenewModule extends AbstractModule {
+       AlwaysRetryOnRenew intereceptingRetryOnRenew;
+
+       public void inject(Injector injector) {
+           intereceptingRetryOnRenew = injector.getInstance(AlwaysRetryOnRenew.class);
+       }
+       
+       @Override
+       protected void configure() {
+           Matcher<? super Class<?>> classMatcher = Matchers.subclassesOf(RetryOnRenew.class);
+           Matcher<? super Method> methodMatcher = new AbstractMatcher<Method>() {
+               @Override
+               public boolean matches(Method t) {
+                   return "shouldRetryRequest".matches(t.getName());
+               }
+           };
+           MethodInterceptor interceptors = new MethodInterceptor() {
+               @Override
+               public Object invoke(MethodInvocation invocation) throws Throwable {
+                   if (intereceptingRetryOnRenew==null)
+                       throw new IllegalStateException("inject() must be called to set up before use");
+                   return intereceptingRetryOnRenew.shouldRetryRequest((HttpCommand)invocation.getArguments()[0], (HttpResponse)invocation.getArguments()[1]);
+               }
+           };
+           bindInterceptor(classMatcher, methodMatcher, interceptors);
+       }
+   }
+}

http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/blob/a5a9de81/locations/jclouds/src/test/java/brooklyn/entity/rebind/persister/jclouds/BlobStoreExpiryTest.java
----------------------------------------------------------------------
diff --git a/locations/jclouds/src/test/java/brooklyn/entity/rebind/persister/jclouds/BlobStoreExpiryTest.java b/locations/jclouds/src/test/java/brooklyn/entity/rebind/persister/jclouds/BlobStoreExpiryTest.java
new file mode 100644
index 0000000..b687104
--- /dev/null
+++ b/locations/jclouds/src/test/java/brooklyn/entity/rebind/persister/jclouds/BlobStoreExpiryTest.java
@@ -0,0 +1,195 @@
+package brooklyn.entity.rebind.persister.jclouds;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+import static org.jclouds.openstack.reference.AuthHeaders.URL_SUFFIX;
+
+import java.io.IOException;
+import java.net.URI;
+import java.util.List;
+import java.util.Map.Entry;
+
+import org.apache.http.client.HttpClient;
+import org.jclouds.blobstore.BlobStoreContext;
+import org.jclouds.blobstore.domain.PageSet;
+import org.jclouds.blobstore.domain.StorageMetadata;
+import org.jclouds.domain.Credentials;
+import org.jclouds.openstack.domain.AuthenticationResponse;
+import org.jclouds.openstack.handlers.RetryOnRenew;
+import org.jclouds.openstack.reference.AuthHeaders;
+import org.junit.Assert;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.testng.annotations.AfterMethod;
+import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.Test;
+
+import brooklyn.config.BrooklynProperties;
+import brooklyn.entity.basic.Entities;
+import brooklyn.location.basic.LocationConfigKeys;
+import brooklyn.location.cloud.CloudLocationConfig;
+import brooklyn.location.jclouds.JcloudsLocation;
+import brooklyn.location.jclouds.JcloudsUtil;
+import brooklyn.management.ManagementContext;
+import brooklyn.test.entity.LocalManagementContextForTests;
+import brooklyn.util.collections.MutableMap;
+import brooklyn.util.http.HttpTool;
+import brooklyn.util.http.HttpToolResponse;
+import brooklyn.util.text.Identifiers;
+import brooklyn.util.time.Duration;
+import brooklyn.util.time.Time;
+
+import com.google.common.base.Preconditions;
+import com.google.common.cache.LoadingCache;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableMap.Builder;
+import com.google.inject.Inject;
+import com.google.inject.Module;
+
+@Test(groups="Integration")
+public class BlobStoreExpiryTest {
+
+    private static final Logger log = LoggerFactory.getLogger(BlobStoreExpiryTest.class);
+    
+    /**
+     * Integration tests as written require a location defined as follows:
+     * 
+     * <code>
+     * brooklyn.location.named.brooklyn-jclouds-objstore-test-1==jclouds:swift:https://ams01.objectstorage.softlayer.net/auth/v1.0
+     * brooklyn.location.named.brooklyn-jclouds-objstore-test-1.identity=IBMOS1234-5:yourname
+     * brooklyn.location.named.brooklyn-jclouds-objstore-test-1.credential=0123abcd.......
+     * </code>
+     */
+    
+    public static final String PERSIST_TO_OBJECT_STORE_FOR_TEST_SPEC = BlobStoreTest.PERSIST_TO_OBJECT_STORE_FOR_TEST_SPEC;
+    public static final String CONTAINER_PREFIX = "brooklyn-persistence-test";
+    private String locationSpec = PERSIST_TO_OBJECT_STORE_FOR_TEST_SPEC;
+    
+    private JcloudsLocation location;
+    private BlobStoreContext context;
+
+    private ManagementContext mgmt;
+    private String testContainerName;
+
+    Module myAuth;
+    private String identity;
+    private String credential;
+    private String provider;
+    private String endpoint;
+
+    public synchronized BlobStoreContext getBlobStoreContext(boolean applyFix) {
+        if (context==null) {
+            if (location==null) {
+                Preconditions.checkNotNull(locationSpec, "locationSpec required for remote object store when location is null");
+                Preconditions.checkNotNull(mgmt, "mgmt required for remote object store when location is null");
+                location = (JcloudsLocation) mgmt.getLocationRegistry().resolve(locationSpec);
+            }
+            
+            identity = checkNotNull(location.getConfig(LocationConfigKeys.ACCESS_IDENTITY), "identity must not be null");
+            credential = checkNotNull(location.getConfig(LocationConfigKeys.ACCESS_CREDENTIAL), "credential must not be null");
+            provider = checkNotNull(location.getConfig(LocationConfigKeys.CLOUD_PROVIDER), "provider must not be null");
+            endpoint = location.getConfig(CloudLocationConfig.CLOUD_ENDPOINT);
+
+            context = JcloudsUtil.newBlobstoreContext(provider, endpoint, identity, credential, applyFix);
+        }
+        return context;
+    }
+    
+    @BeforeMethod(alwaysRun=true)
+    public void setup() {
+        testContainerName = CONTAINER_PREFIX+"-"+Identifiers.makeRandomId(8);
+        mgmt = new LocalManagementContextForTests(BrooklynProperties.Factory.newDefault());
+    }
+    
+    @AfterMethod(alwaysRun=true)
+    public void teardown() {
+        Entities.destroyAll(mgmt);
+        if (context!=null) context.close();
+        context = null;
+    }
+    
+    public void testRenewAuthFailsInSoftlayer() throws IOException {
+        doTestRenewAuth(false);
+    }
+
+    public void testRenewAuthSucceedsWithOurOverride() throws IOException {
+        doTestRenewAuth(true);
+    }
+    
+    protected void doTestRenewAuth(boolean applyFix) throws IOException {
+        getBlobStoreContext(applyFix);
+        
+        injectShortLivedTokenForSoftlayerAmsterdam();
+        
+        context.getBlobStore().createContainerInLocation(null, testContainerName);
+        
+        assertContainerFound();
+        
+        log.info("created container, now sleeping for expiration");
+        
+        Time.sleep(Duration.TEN_SECONDS);
+        
+        if (!applyFix) {
+            // with the fix not applied, we have to invalidate the cache manually
+            try {
+                assertContainerFound();
+                Assert.fail("should fail as long as "+RetryOnRenew.class+" is not working");
+            } catch (Exception e) {
+                log.info("failed, as expected: "+e);
+            }
+            getAuthCache().invalidateAll();
+            log.info("invalidated, should now succeed");
+        }
+
+        assertContainerFound();
+
+        context.getBlobStore().deleteContainer(testContainerName);
+    }
+
+    private void assertContainerFound() {
+        PageSet<? extends StorageMetadata> ps = context.getBlobStore().list();
+        BlobStoreTest.assertHasItemNamed(ps, testContainerName);
+    }
+
+    private void injectShortLivedTokenForSoftlayerAmsterdam() {
+        HttpToolResponse tokenHttpResponse1 = requestTokenWithExplicitLifetime("https://ams01.objectstorage.softlayer.net/auth/v1.0/v1.0", "ams01.objectstorage.softlayer.net", 
+            identity, credential, Duration.FIVE_SECONDS);
+        
+        Builder<String, URI> servicesMapBuilder = ImmutableMap.builder();
+        for (Entry<String, List<String>> entry : tokenHttpResponse1.getHeaderLists().entrySet()) {
+           if (entry.getKey().toLowerCase().endsWith(URL_SUFFIX.toLowerCase()))
+              servicesMapBuilder.put(entry.getKey(), URI.create(entry.getValue().iterator().next()));
+        }
+        AuthenticationResponse authResponse = new AuthenticationResponse(tokenHttpResponse1.getHeaderLists().get(AuthHeaders.AUTH_TOKEN).get(0), servicesMapBuilder.build());
+        
+        getAuthCache().put(new Credentials(identity, credential), authResponse);
+    }
+
+    private LoadingCache<Credentials, AuthenticationResponse> getAuthCache() {
+        return context.utils().injector().getInstance(CachePeeker.class).authenticationResponseCache;
+    }
+    
+    public static class CachePeeker {
+        private final LoadingCache<Credentials, AuthenticationResponse> authenticationResponseCache;
+
+        @Inject
+        protected CachePeeker(LoadingCache<Credentials, AuthenticationResponse> authenticationResponseCache) {
+           this.authenticationResponseCache = authenticationResponseCache;
+        }
+    }
+    
+    public static HttpToolResponse requestTokenWithExplicitLifetime(String url, String host, String user, String key, Duration expiration) {
+        HttpClient client = HttpTool.httpClientBuilder().build();
+        HttpToolResponse response = HttpTool.httpGet(client, URI.create(url), MutableMap.<String,String>of()
+            .add(AuthHeaders.AUTH_USER, user)
+            .add(AuthHeaders.AUTH_KEY, key)
+            .add("Host", host)
+            .add("X-Auth-New-Token", ""+true)
+            .add("X-Auth-Token-Lifetime", ""+expiration.toSeconds())
+            );
+//        curl -v https://ams01.objectstorage.softlayer.net/auth/v1.0/v1.0 -H "X-Auth-User: IBMOS321366-2:cloudsoft" -H "X-Auth-Key: 06cef1beff5432cc9453934e06beb85de5f0a53a2340d7e0cd4a4705655e8132" -H "Host: ams01.objectstorage.softlayer.net" -H "X-Auth-New-Token: true" -H "X-Auth-Token-Lifetime: 15"
+//            -H "Host: ams01.objectstorage.softlayer.net" -H "X-Auth-New-Token: true" -H "X-Auth-Token-Lifetime: 15"
+        log.info("Requested token with explicit lifetime: "+expiration+" at "+url+"\n"+response+"\n"+response.getHeaderLists());
+        return response;
+    }
+    
+}

http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/blob/a5a9de81/locations/jclouds/src/test/java/brooklyn/entity/rebind/persister/jclouds/BlobStoreTest.java
----------------------------------------------------------------------
diff --git a/locations/jclouds/src/test/java/brooklyn/entity/rebind/persister/jclouds/BlobStoreTest.java b/locations/jclouds/src/test/java/brooklyn/entity/rebind/persister/jclouds/BlobStoreTest.java
index ef95fb8..d7bfae3 100644
--- a/locations/jclouds/src/test/java/brooklyn/entity/rebind/persister/jclouds/BlobStoreTest.java
+++ b/locations/jclouds/src/test/java/brooklyn/entity/rebind/persister/jclouds/BlobStoreTest.java
@@ -4,7 +4,6 @@ import static com.google.common.base.Preconditions.checkNotNull;
 
 import java.io.IOException;
 
-import org.jclouds.ContextBuilder;
 import org.jclouds.blobstore.BlobStoreContext;
 import org.jclouds.blobstore.domain.Blob;
 import org.jclouds.blobstore.domain.PageSet;
@@ -20,6 +19,7 @@ import brooklyn.entity.basic.Entities;
 import brooklyn.location.basic.LocationConfigKeys;
 import brooklyn.location.cloud.CloudLocationConfig;
 import brooklyn.location.jclouds.JcloudsLocation;
+import brooklyn.location.jclouds.JcloudsUtil;
 import brooklyn.management.ManagementContext;
 import brooklyn.test.entity.LocalManagementContextForTests;
 import brooklyn.util.stream.Streams;
@@ -64,10 +64,7 @@ public class BlobStoreTest {
             String provider = checkNotNull(location.getConfig(LocationConfigKeys.CLOUD_PROVIDER), "provider must not be null");
             String endpoint = location.getConfig(CloudLocationConfig.CLOUD_ENDPOINT);
 
-            context = ContextBuilder.newBuilder(provider)
-                .credentials(identity, credential)
-                .endpoint(endpoint)
-                .buildView(BlobStoreContext.class);
+            context = JcloudsUtil.newBlobstoreContext(provider, endpoint, identity, credential, true);
         }
         return context;
     }
@@ -84,7 +81,6 @@ public class BlobStoreTest {
         Entities.destroyAll(mgmt);
     }
     
-    
     public void testCreateListDestroyContainer() throws IOException {
         context.getBlobStore().createContainerInLocation(null, testContainerName);
         context.getBlobStore().list(testContainerName);
@@ -123,10 +119,15 @@ public class BlobStoreTest {
         context.getBlobStore().deleteContainer(testContainerName);
     }
 
-    private void assertHasItemNamed(PageSet<? extends StorageMetadata> ps, String name) {
+    static void assertHasItemNamed(PageSet<? extends StorageMetadata> ps, String name) {
+        if (!hasItemNamed(ps, name))
+            Assert.fail("No item named '"+name+"' in "+ps);
+    }
+
+    static boolean hasItemNamed(PageSet<? extends StorageMetadata> ps, String name) {
         for (StorageMetadata sm: ps)
-            if (name==null || name.equals(sm.getName())) return;
-        Assert.fail("No item named '"+name+"' in "+ps);
+            if (name==null || name.equals(sm.getName())) return true;
+        return false;
     }
 
 }