brooklyn-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From aleds...@apache.org
Subject [1/4] brooklyn-server git commit: BROOKLYN-359: support openstack-swift with jclouds2
Date Wed, 18 Jan 2017 10:41:08 GMT
Repository: brooklyn-server
Updated Branches:
  refs/heads/master fabfcdb9f -> db7418c6e


BROOKLYN-359: support openstack-swift with jclouds2

* Adds BlobStoreContextFactoryImpl (instead of using
  JcloudsUtil.newBlobstoreContext, which is now deprecated).
* Extracts “jclouds.*” from the location config, and passes it through
  to jclouds when creating the blobstore context.
* Re-enables/fixes BlobStoreExpiryTest, to work with jclouds2


Project: http://git-wip-us.apache.org/repos/asf/brooklyn-server/repo
Commit: http://git-wip-us.apache.org/repos/asf/brooklyn-server/commit/013b38bf
Tree: http://git-wip-us.apache.org/repos/asf/brooklyn-server/tree/013b38bf
Diff: http://git-wip-us.apache.org/repos/asf/brooklyn-server/diff/013b38bf

Branch: refs/heads/master
Commit: 013b38bf0f146114d3a67423c0dcc3661b50f88c
Parents: 810a711
Author: Aled Sage <aled.sage@gmail.com>
Authored: Tue Jan 17 22:04:14 2017 +0000
Committer: Aled Sage <aled.sage@gmail.com>
Committed: Wed Jan 18 09:48:53 2017 +0000

----------------------------------------------------------------------
 .../command/support/CloudExplorerSupport.java   |  13 +-
 .../JcloudsBlobStoreBasedObjectStore.java       |  13 +-
 .../jclouds/BlobStoreContextFactory.java        |  37 ++++
 .../jclouds/BlobStoreContextFactoryImpl.java    | 113 ++++++++++++
 .../brooklyn/location/jclouds/JcloudsUtil.java  |  39 ++--
 .../persist/jclouds/BlobStoreExpiryTest.java    | 184 +++++++++++++------
 .../mgmt/persist/jclouds/BlobStoreTest.java     |  34 +---
 .../jclouds/JcloudsExpect100ContinueTest.java   |  12 +-
 .../brooklyn/cli/CloudExplorerLiveTest.java     |   3 +-
 9 files changed, 307 insertions(+), 141 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/brooklyn-server/blob/013b38bf/launcher-common/src/main/java/org/apache/brooklyn/launcher/command/support/CloudExplorerSupport.java
----------------------------------------------------------------------
diff --git a/launcher-common/src/main/java/org/apache/brooklyn/launcher/command/support/CloudExplorerSupport.java
b/launcher-common/src/main/java/org/apache/brooklyn/launcher/command/support/CloudExplorerSupport.java
index 6708d81..544ad92 100644
--- a/launcher-common/src/main/java/org/apache/brooklyn/launcher/command/support/CloudExplorerSupport.java
+++ b/launcher-common/src/main/java/org/apache/brooklyn/launcher/command/support/CloudExplorerSupport.java
@@ -18,8 +18,6 @@
  */
 package org.apache.brooklyn.launcher.command.support;
 
-import static com.google.common.base.Preconditions.checkNotNull;
-
 import java.io.InputStream;
 import java.io.PrintStream;
 import java.util.Collection;
@@ -31,11 +29,9 @@ import java.util.concurrent.Callable;
 import org.apache.brooklyn.api.location.Location;
 import org.apache.brooklyn.api.location.LocationDefinition;
 import org.apache.brooklyn.api.mgmt.ManagementContext;
-import org.apache.brooklyn.core.location.LocationConfigKeys;
-import org.apache.brooklyn.core.location.cloud.CloudLocationConfig;
+import org.apache.brooklyn.location.jclouds.BlobStoreContextFactoryImpl;
 import org.apache.brooklyn.location.jclouds.JcloudsLocation;
 import org.apache.brooklyn.location.jclouds.JcloudsLocationCustomizer;
-import org.apache.brooklyn.location.jclouds.JcloudsUtil;
 import org.apache.brooklyn.util.core.config.ConfigBag;
 import org.apache.brooklyn.util.exceptions.FatalConfigurationRuntimeException;
 import org.apache.brooklyn.util.stream.Streams;
@@ -333,12 +329,7 @@ public abstract class CloudExplorerSupport implements Callable<Void>
{
 
         @Override
         protected void doCall(JcloudsLocation loc, String indent) throws Exception {
-            String identity = checkNotNull(loc.getConfig(LocationConfigKeys.ACCESS_IDENTITY),
"identity must not be null");
-            String credential = checkNotNull(loc.getConfig(LocationConfigKeys.ACCESS_CREDENTIAL),
"credential must not be null");
-            String provider = checkNotNull(loc.getConfig(LocationConfigKeys.CLOUD_PROVIDER),
"provider must not be null");
-            String endpoint = loc.getConfig(CloudLocationConfig.CLOUD_ENDPOINT);
-
-            BlobStoreContext context = JcloudsUtil.newBlobstoreContext(provider, endpoint,
identity, credential);
+            BlobStoreContext context = BlobStoreContextFactoryImpl.INSTANCE.newBlobStoreContext(loc.config().getBag());
             try {
                 org.jclouds.blobstore.BlobStore blobStore = context.getBlobStore();
                 doCall(blobStore, indent);

http://git-wip-us.apache.org/repos/asf/brooklyn-server/blob/013b38bf/locations/jclouds/src/main/java/org/apache/brooklyn/core/mgmt/persist/jclouds/JcloudsBlobStoreBasedObjectStore.java
----------------------------------------------------------------------
diff --git a/locations/jclouds/src/main/java/org/apache/brooklyn/core/mgmt/persist/jclouds/JcloudsBlobStoreBasedObjectStore.java
b/locations/jclouds/src/main/java/org/apache/brooklyn/core/mgmt/persist/jclouds/JcloudsBlobStoreBasedObjectStore.java
index 7705e50..25558a2 100644
--- a/locations/jclouds/src/main/java/org/apache/brooklyn/core/mgmt/persist/jclouds/JcloudsBlobStoreBasedObjectStore.java
+++ b/locations/jclouds/src/main/java/org/apache/brooklyn/core/mgmt/persist/jclouds/JcloudsBlobStoreBasedObjectStore.java
@@ -18,16 +18,12 @@
  */
 package org.apache.brooklyn.core.mgmt.persist.jclouds;
 
-import static com.google.common.base.Preconditions.checkNotNull;
-
 import java.util.List;
 
 import javax.annotation.Nullable;
 
 import org.apache.brooklyn.api.mgmt.ManagementContext;
 import org.apache.brooklyn.api.mgmt.ha.HighAvailabilityMode;
-import org.apache.brooklyn.core.location.LocationConfigKeys;
-import org.apache.brooklyn.core.location.cloud.CloudLocationConfig;
 import org.apache.brooklyn.core.mgmt.persist.PersistMode;
 import org.apache.brooklyn.core.mgmt.persist.PersistenceObjectStore;
 import org.apache.brooklyn.core.server.BrooklynServerConfig;
@@ -36,8 +32,8 @@ import org.jclouds.blobstore.domain.StorageMetadata;
 import org.jclouds.blobstore.options.ListContainerOptions;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
+import org.apache.brooklyn.location.jclouds.BlobStoreContextFactoryImpl;
 import org.apache.brooklyn.location.jclouds.JcloudsLocation;
-import org.apache.brooklyn.location.jclouds.JcloudsUtil;
 import org.apache.brooklyn.util.exceptions.FatalConfigurationRuntimeException;
 import org.apache.brooklyn.util.text.Strings;
 
@@ -96,12 +92,7 @@ public class JcloudsBlobStoreBasedObjectStore implements PersistenceObjectStore
                 location = (JcloudsLocation) mgmt.getLocationRegistry().getLocationManaged(locationSpec);
             }
             
-            String identity = checkNotNull(location.getConfig(LocationConfigKeys.ACCESS_IDENTITY),
"identity must not be null");
-            String credential = checkNotNull(location.getConfig(LocationConfigKeys.ACCESS_CREDENTIAL),
"credential must not be null");
-            String provider = checkNotNull(location.getConfig(LocationConfigKeys.CLOUD_PROVIDER),
"provider must not be null");
-            String endpoint = location.getConfig(CloudLocationConfig.CLOUD_ENDPOINT);
-
-            context = JcloudsUtil.newBlobstoreContext(provider, endpoint, identity, credential);
+            context = BlobStoreContextFactoryImpl.INSTANCE.newBlobStoreContext(location.config().getBag());
      
             // TODO do we need to get location from region? can't see the jclouds API.
             // doesn't matter in some places because it's already in the endpoint

http://git-wip-us.apache.org/repos/asf/brooklyn-server/blob/013b38bf/locations/jclouds/src/main/java/org/apache/brooklyn/location/jclouds/BlobStoreContextFactory.java
----------------------------------------------------------------------
diff --git a/locations/jclouds/src/main/java/org/apache/brooklyn/location/jclouds/BlobStoreContextFactory.java
b/locations/jclouds/src/main/java/org/apache/brooklyn/location/jclouds/BlobStoreContextFactory.java
new file mode 100644
index 0000000..0a2dd26
--- /dev/null
+++ b/locations/jclouds/src/main/java/org/apache/brooklyn/location/jclouds/BlobStoreContextFactory.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.brooklyn.location.jclouds;
+
+import org.apache.brooklyn.util.core.config.ConfigBag;
+import org.jclouds.blobstore.BlobStoreContext;
+
+/**
+ * For creating a new {@link BlobStoreContext}, to access object stores such as S3 or Swift.

+ * 
+ * @see {@link BlobStoreContextFactoryImpl}
+ */
+public interface BlobStoreContextFactory {
+
+    // TODO Longer term, we could make this more testable by having the BlobStoreContextFactory
configurable.
+    // See the pattern used in {@link JcloudsLocationConfig#COMPUTE_SERVICE_REGISTRY}.
+    //
+    // However, for now we have just kept the separation of interface and implementation.
+    
+    public BlobStoreContext newBlobStoreContext(ConfigBag conf);
+}

http://git-wip-us.apache.org/repos/asf/brooklyn-server/blob/013b38bf/locations/jclouds/src/main/java/org/apache/brooklyn/location/jclouds/BlobStoreContextFactoryImpl.java
----------------------------------------------------------------------
diff --git a/locations/jclouds/src/main/java/org/apache/brooklyn/location/jclouds/BlobStoreContextFactoryImpl.java
b/locations/jclouds/src/main/java/org/apache/brooklyn/location/jclouds/BlobStoreContextFactoryImpl.java
new file mode 100644
index 0000000..94f6110
--- /dev/null
+++ b/locations/jclouds/src/main/java/org/apache/brooklyn/location/jclouds/BlobStoreContextFactoryImpl.java
@@ -0,0 +1,113 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.brooklyn.location.jclouds;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+
+import java.util.Map;
+import java.util.Properties;
+
+import org.apache.brooklyn.core.config.Sanitizer;
+import org.apache.brooklyn.core.location.LocationConfigKeys;
+import org.apache.brooklyn.core.location.cloud.CloudLocationConfig;
+import org.apache.brooklyn.core.mgmt.persist.DeserializingJcloudsRenamesProvider;
+import org.apache.brooklyn.util.collections.MutableList;
+import org.apache.brooklyn.util.core.config.ConfigBag;
+import org.jclouds.Constants;
+import org.jclouds.ContextBuilder;
+import org.jclouds.blobstore.BlobStoreContext;
+import org.jclouds.encryption.bouncycastle.config.BouncyCastleCryptoModule;
+import org.jclouds.logging.slf4j.config.SLF4JLoggingModule;
+import org.jclouds.sshj.config.SshjSshClientModule;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.base.Predicates;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Maps;
+import com.google.inject.Module;
+
+public class BlobStoreContextFactoryImpl implements BlobStoreContextFactory {
+
+    // TODO Comparing this with {@link JcloudsLocation#getComputeService()}, that calls
+    // uses {@code ResolvingConfigBag.newInstanceExtending(getManagementContext(), config)}.
+    // The reason is to allow DSL to be handled better - in particular when the context in
+    // which the DSL must be resolved depends upon the entity on which it is defined (e.g.
using
+    // {@code provisioning.properties}).
+    //
+    // For blobstore usage (which is used for configuring persistence), we'll still want
to 
+    // support the use of DSL (especially for externalised configuration), but we won't have
to
+    // evaluate it in the context of a particular entity. We can therefore probably get away
without
+    // using {@link ResolvingConfigBag}. However, it's worth improving the test coverage
for this!
+
+    // TODO We could try to remove some duplication compared to {@link ComputeServiceRegistryImpl},
+    // e.g. for {@link #getCommonModules()}. However, there's not that much duplication (and
the 
+    // common-modules could be different for blob-store versus compute-service).
+
+    private static final Logger LOG = LoggerFactory.getLogger(BlobStoreContextFactoryImpl.class);
+
+    public static final BlobStoreContextFactory INSTANCE = new BlobStoreContextFactoryImpl();
+    
+    protected BlobStoreContextFactoryImpl() {
+    }
+
+    @Override
+    public BlobStoreContext newBlobStoreContext(ConfigBag conf) {
+        String rawProvider = checkNotNull(conf.get(LocationConfigKeys.CLOUD_PROVIDER), "provider
must not be null");
+        String provider = DeserializingJcloudsRenamesProvider.INSTANCE.applyJcloudsRenames(rawProvider);
+        String identity = checkNotNull(conf.get(LocationConfigKeys.ACCESS_IDENTITY), "identity
must not be null");
+        String credential = checkNotNull(conf.get(LocationConfigKeys.ACCESS_CREDENTIAL),
"credential must not be null");
+        String endpoint = conf.get(CloudLocationConfig.CLOUD_ENDPOINT);
+
+        Properties overrides = new Properties();
+        // * Java 7,8 bug workaround - sockets closed by GC break the internal bookkeeping
+        //   of HttpUrlConnection, leading to invalid handling of the "HTTP/1.1 100 Continue"
+        //   response. Coupled with a bug when using SSL sockets reads will block
+        //   indefinitely even though a read timeout is explicitly set.
+        // * Java 6 ignores the header anyways as it is included in its restricted headers
black list.
+        // * Also there's a bug in SL object store which still expects Content-Length bytes
+        //   even when it responds with a 408 timeout response, leading to incorrectly
+        //   interpreting the next request (triggered by above problem).
+        overrides.setProperty(Constants.PROPERTY_STRIP_EXPECT_HEADER, "true");
+
+        // Add extra jclouds-specific configuration
+        Map<String, Object> extra = Maps.filterKeys(conf.getAllConfig(), Predicates.containsPattern("^jclouds\\."));
+        if (extra.size() > 0) {
+            LOG.debug("Configuring custom jclouds property overrides for {}: {}", provider,
Sanitizer.sanitize(extra));
+        }
+        overrides.putAll(Maps.filterValues(extra, Predicates.notNull()));
+
+        ContextBuilder contextBuilder = ContextBuilder.newBuilder(provider).credentials(identity,
credential);
+        contextBuilder.modules(MutableList.copyOf(getCommonModules()));
+        if (!org.apache.brooklyn.util.text.Strings.isBlank(endpoint)) {
+            contextBuilder.endpoint(endpoint);
+        }
+        contextBuilder.overrides(overrides);
+        BlobStoreContext context = contextBuilder.buildView(BlobStoreContext.class);
+        return context;
+    }
+
+    /** returns the jclouds modules we typically install */ 
+    protected ImmutableSet<Module> getCommonModules() {
+        return ImmutableSet.<Module> of(
+                new SshjSshClientModule(),
+                new SLF4JLoggingModule(),
+                new BouncyCastleCryptoModule());
+    }
+}

http://git-wip-us.apache.org/repos/asf/brooklyn-server/blob/013b38bf/locations/jclouds/src/main/java/org/apache/brooklyn/location/jclouds/JcloudsUtil.java
----------------------------------------------------------------------
diff --git a/locations/jclouds/src/main/java/org/apache/brooklyn/location/jclouds/JcloudsUtil.java
b/locations/jclouds/src/main/java/org/apache/brooklyn/location/jclouds/JcloudsUtil.java
index 88f4127..2336c0f 100644
--- a/locations/jclouds/src/main/java/org/apache/brooklyn/location/jclouds/JcloudsUtil.java
+++ b/locations/jclouds/src/main/java/org/apache/brooklyn/location/jclouds/JcloudsUtil.java
@@ -18,6 +18,7 @@
  */
 package org.apache.brooklyn.location.jclouds;
 
+import static com.google.common.base.Preconditions.checkNotNull;
 import static org.jclouds.compute.options.RunScriptOptions.Builder.overrideLoginCredentials;
 import static org.jclouds.compute.util.ComputeServiceUtils.execHttpResponse;
 import static org.jclouds.scriptbuilder.domain.Statements.appendFile;
@@ -38,6 +39,8 @@ import java.util.concurrent.TimeoutException;
 import javax.annotation.Nullable;
 
 import org.apache.brooklyn.core.config.Sanitizer;
+import org.apache.brooklyn.core.location.LocationConfigKeys;
+import org.apache.brooklyn.core.location.cloud.CloudLocationConfig;
 import org.apache.brooklyn.core.mgmt.persist.DeserializingJcloudsRenamesProvider;
 import org.apache.brooklyn.util.collections.MutableList;
 import org.apache.brooklyn.util.core.config.ConfigBag;
@@ -82,6 +85,7 @@ import com.google.common.annotations.Beta;
 import com.google.common.base.Charsets;
 import com.google.common.base.Function;
 import com.google.common.base.Predicate;
+import com.google.common.base.Predicates;
 import com.google.common.base.Splitter;
 import com.google.common.base.Strings;
 import com.google.common.collect.FluentIterable;
@@ -269,30 +273,19 @@ public class JcloudsUtil implements JcloudsLocationConfig {
      *  <p>
      *  (Marked Beta as that argument will likely be removed.)
      *
-     *  @since 0.7.0 */
+     *  @since 0.7.0
+     *  @deprecated since 0.11.0; instead use BlobStoreContextFactoryImpl.INSTANCE
+     */
     @Beta
-    public static BlobStoreContext newBlobstoreContext(String rawProvider, @Nullable String
endpoint, String identity, String credential) {
-        String provider = DeserializingJcloudsRenamesProvider.INSTANCE.applyJcloudsRenames(rawProvider);
-        
-        Properties overrides = new Properties();
-        // * Java 7,8 bug workaround - sockets closed by GC break the internal bookkeeping
-        //   of HttpUrlConnection, leading to invalid handling of the "HTTP/1.1 100 Continue"
-        //   response. Coupled with a bug when using SSL sockets reads will block
-        //   indefinitely even though a read timeout is explicitly set.
-        // * Java 6 ignores the header anyways as it is included in its restricted headers
black list.
-        // * Also there's a bug in SL object store which still expects Content-Length bytes
-        //   even when it responds with a 408 timeout response, leading to incorrectly
-        //   interpreting the next request (triggered by above problem).
-        overrides.setProperty(Constants.PROPERTY_STRIP_EXPECT_HEADER, "true");
-
-        ContextBuilder contextBuilder = ContextBuilder.newBuilder(provider).credentials(identity,
credential);
-        contextBuilder.modules(MutableList.copyOf(JcloudsUtil.getCommonModules()));
-        if (!org.apache.brooklyn.util.text.Strings.isBlank(endpoint)) {
-            contextBuilder.endpoint(endpoint);
-        }
-        contextBuilder.overrides(overrides);
-        BlobStoreContext context = contextBuilder.buildView(BlobStoreContext.class);
-        return context;
+    @Deprecated
+    public static BlobStoreContext newBlobstoreContext(String provider, @Nullable String
endpoint, String identity, String credential) {
+        ConfigBag conf = ConfigBag.newInstance();
+        conf.put(LocationConfigKeys.CLOUD_PROVIDER, provider);
+        conf.put(LocationConfigKeys.ACCESS_IDENTITY, identity);
+        conf.put(LocationConfigKeys.ACCESS_CREDENTIAL, credential);
+        conf.put(CloudLocationConfig.CLOUD_ENDPOINT, endpoint);
+
+        return BlobStoreContextFactoryImpl.INSTANCE.newBlobStoreContext(conf);
     }
 
     /**

http://git-wip-us.apache.org/repos/asf/brooklyn-server/blob/013b38bf/locations/jclouds/src/test/java/org/apache/brooklyn/core/mgmt/persist/jclouds/BlobStoreExpiryTest.java
----------------------------------------------------------------------
diff --git a/locations/jclouds/src/test/java/org/apache/brooklyn/core/mgmt/persist/jclouds/BlobStoreExpiryTest.java
b/locations/jclouds/src/test/java/org/apache/brooklyn/core/mgmt/persist/jclouds/BlobStoreExpiryTest.java
index 460d705..1ff6930 100644
--- a/locations/jclouds/src/test/java/org/apache/brooklyn/core/mgmt/persist/jclouds/BlobStoreExpiryTest.java
+++ b/locations/jclouds/src/test/java/org/apache/brooklyn/core/mgmt/persist/jclouds/BlobStoreExpiryTest.java
@@ -20,26 +20,91 @@ package org.apache.brooklyn.core.mgmt.persist.jclouds;
 
 import static com.google.common.base.Preconditions.checkNotNull;
 
+import java.net.URISyntaxException;
+import java.net.URL;
+import java.util.Date;
+import java.util.Set;
+
 import org.apache.brooklyn.api.mgmt.ManagementContext;
 import org.apache.brooklyn.core.entity.Entities;
 import org.apache.brooklyn.core.internal.BrooklynProperties;
 import org.apache.brooklyn.core.location.LocationConfigKeys;
 import org.apache.brooklyn.core.location.cloud.CloudLocationConfig;
 import org.apache.brooklyn.core.test.entity.LocalManagementContextForTests;
+import org.apache.brooklyn.location.jclouds.BlobStoreContextFactoryImpl;
 import org.apache.brooklyn.location.jclouds.JcloudsLocation;
-import org.apache.brooklyn.location.jclouds.JcloudsUtil;
+import org.apache.brooklyn.util.collections.MutableMap;
+import org.apache.brooklyn.util.http.HttpTool;
+import org.apache.brooklyn.util.http.HttpToolResponse;
 import org.apache.brooklyn.util.text.Identifiers;
+import org.apache.brooklyn.util.time.Duration;
+import org.apache.brooklyn.util.time.Time;
+import org.apache.http.client.HttpClient;
 import org.jclouds.blobstore.BlobStoreContext;
+import org.jclouds.blobstore.domain.PageSet;
+import org.jclouds.blobstore.domain.StorageMetadata;
+import org.jclouds.domain.Credentials;
+import org.jclouds.openstack.keystone.v2_0.domain.Access;
+import org.jclouds.openstack.keystone.v2_0.domain.Service;
+import org.jclouds.openstack.keystone.v2_0.domain.Token;
+import org.jclouds.openstack.keystone.v2_0.domain.User;
+import org.jclouds.openstack.v2_0.reference.AuthHeaders;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.testng.annotations.AfterMethod;
 import org.testng.annotations.BeforeMethod;
 import org.testng.annotations.Test;
 
-import com.google.common.base.Preconditions;
-
-// Disabled as `swift` is not supported in jclouds 2.0.0+ 
-@Test(groups={"Live", "Live-sanity"} , enabled = false)
+import com.google.common.cache.LoadingCache;
+import com.google.common.collect.ImmutableSet;
+import com.google.inject.Inject;
+
+/**
+ * Tests that the token is automatically renewed. It does this by requesting a short-lived
token, 
+ * and injecting that into the guts of jclouds!
+ * 
+ * Expect to see in the log something like the following:
+ * <ol>
+ *   <li>Requesting a token:
+ *       <pre>
+ *       2017-01-17 21:30:18,832 INFO  o.a.b.c.m.p.j.BlobStoreExpiryTest [main]: Requested
token with explicit lifetime: 5s at https://ams01.objectstorage.softlayer.net/auth/v1.0
+ *       HttpToolResponse{responseCode=0}
+ *       {Content-Length=[1472], X-Auth-Token-Expires=[4], X-Auth-Token=[AUTH_temptemptemptemptemptemptemptempte
...
+ *       </pre>
+ *   <li>First request with that token succeeds:
+ *       <br>
+ *       <pre>
+ *       2017-01-17 21:30:18,863 DEBUG jclouds.headers [main]: >> PUT https://ams01.objectstorage.softlayer.net/v1/AUTH_xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx/brooklyn-persistence-test-W6eNdNXp
HTTP/1.1
+ *       2017-01-17 21:30:18,863 DEBUG jclouds.headers [main]: >> Accept: application/json
+ *       2017-01-17 21:30:18,863 DEBUG jclouds.headers [main]: >> X-Auth-Token: AUTH_temptemptemptemptemptemptemptempte
+ *       2017-01-17 21:30:18,925 DEBUG jclouds.headers [main]: << HTTP/1.1 201 Created
+ *       </pre>
+ *   <li>After 10 seconds, next request fails with 401 Unauthorized:
+ *       <pre>
+ *       2017-01-17 21:30:29,023 DEBUG jclouds.headers [main]: >> GET https://ams01.objectstorage.softlayer.net/v1/AUTH_xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx?format=json
HTTP/1.1
+ *       2017-01-17 21:30:29,023 DEBUG jclouds.headers [main]: >> Accept: application/json
+ *       2017-01-17 21:30:29,023 DEBUG jclouds.headers [main]: >> X-Auth-Token: AUTH_temptemptemptemptemptemptemptempte
+ *       2017-01-17 21:30:29,140 DEBUG jclouds.headers [main]: << HTTP/1.1 401 Unauthorized
+ *       </pre>
+ *   <li>Automatically renews:
+ *       <pre>
+ *       2017-01-17 21:30:29,141 DEBUG o.j.o.k.v.h.RetryOnRenew [main]: invalidating authentication
token - first time for [method=org.jclouds.openstack.swift.v1.features.ContainerApi.public
abstract com.google.common.collect.FluentIterable org.jclouds.openstack.swift.v1.features.ContainerApi.list()[],
request=GET https://ams01.objectstorage.softlayer.net/v1/AUTH_xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx?format=json
HTTP/1.1]
+ *       2017-01-17 21:30:29,146 DEBUG o.j.r.i.InvokeHttpMethod [main]: >> invoking
TempAuthApi.auth
+ *       2017-01-17 21:30:29,146 DEBUG o.j.h.i.JavaUrlHttpCommandExecutorService [main]:
Sending request -203921621: GET https://ams01.objectstorage.softlayer.net/auth/v1.0 HTTP/1.1
+ *       2017-01-17 21:30:29,146 DEBUG jclouds.headers [main]: >> GET https://ams01.objectstorage.softlayer.net/auth/v1.0
HTTP/1.1
+ *       2017-01-17 21:30:29,288 DEBUG jclouds.headers [main]: << HTTP/1.1 200 OK
+ *       2017-01-17 21:30:29,288 DEBUG jclouds.headers [main]: << X-Storage-Token:
AUTH_newnewnewnewnewnewnewnewnewnewnewn
+ *       </pre>
+ *   <li>Request is repeated automatically with the new token, and works:
+ *       <pre>
+ *       2017-01-17 21:30:29,290 DEBUG jclouds.headers [main]: >> GET https://ams01.objectstorage.softlayer.net/v1/AUTH_xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx?format=json
HTTP/1.1
+ *       2017-01-17 21:30:29,290 DEBUG jclouds.headers [main]: >> Accept: application/json
+ *       2017-01-17 21:30:29,290 DEBUG jclouds.headers [main]: >> X-Auth-Token: AUTH_newnewnewnewnewnewnewnewnewnewnewn
+ *       2017-01-17 21:30:29,347 DEBUG jclouds.headers [main]: << HTTP/1.1 200 OK
+ *       </pre>
+ * </ol>
+ */
+@Test(groups={"Live", "Live-sanity"})
 public class BlobStoreExpiryTest {
 
     private static final Logger log = LoggerFactory.getLogger(BlobStoreExpiryTest.class);
@@ -48,49 +113,36 @@ public class BlobStoreExpiryTest {
      * Live tests as written require a location defined as follows:
      * 
      * <code>
-     * brooklyn.location.named.brooklyn-jclouds-objstore-test-1==jclouds:swift:https://ams01.objectstorage.softlayer.net/auth/v1.0
+     * brooklyn.location.named.brooklyn-jclouds-objstore-test-1==jclouds:openstack-swift:https://ams01.objectstorage.softlayer.net/auth/v1.0
      * brooklyn.location.named.brooklyn-jclouds-objstore-test-1.identity=IBMOS1234-5:yourname
      * brooklyn.location.named.brooklyn-jclouds-objstore-test-1.credential=0123abcd.......
+     * brooklyn.location.named.brooklyn-jclouds-objstore-test-1.jclouds.keystone.credential-type=tempAuthCredentials
      * </code>
      */
     
     public static final String PERSIST_TO_OBJECT_STORE_FOR_TEST_SPEC = BlobStoreTest.PERSIST_TO_OBJECT_STORE_FOR_TEST_SPEC;
-    public static final String CONTAINER_PREFIX = "brooklyn-persistence-test";
     private String locationSpec = PERSIST_TO_OBJECT_STORE_FOR_TEST_SPEC;
+    public static final String CONTAINER_PREFIX = "brooklyn-persistence-test";
     
-    private JcloudsLocation location;
-    private BlobStoreContext context;
-
     private ManagementContext mgmt;
-    private String testContainerName;
+    private JcloudsLocation location;
 
     private String identity;
     private String credential;
-    private String provider;
     private String endpoint;
 
-    public synchronized BlobStoreContext getSwiftBlobStoreContext() {
-        if (context==null) {
-            if (location==null) {
-                Preconditions.checkNotNull(locationSpec, "locationSpec required for remote
object store when location is null");
-                Preconditions.checkNotNull(mgmt, "mgmt required for remote object store when
location is null");
-                location = (JcloudsLocation) mgmt.getLocationRegistry().getLocationManaged(locationSpec);
-            }
-            
-            identity = checkNotNull(location.getConfig(LocationConfigKeys.ACCESS_IDENTITY),
"identity must not be null");
-            credential = checkNotNull(location.getConfig(LocationConfigKeys.ACCESS_CREDENTIAL),
"credential must not be null");
-            provider = checkNotNull(location.getConfig(LocationConfigKeys.CLOUD_PROVIDER),
"provider must not be null");
-            endpoint = location.getConfig(CloudLocationConfig.CLOUD_ENDPOINT);
-
-            context = JcloudsUtil.newBlobstoreContext(provider, endpoint, identity, credential);
-        }
-        return context;
-    }
-    
+    private BlobStoreContext context;
+    private String testContainerName;
+
     @BeforeMethod(alwaysRun=true)
     public void setup() {
         testContainerName = CONTAINER_PREFIX+"-"+Identifiers.makeRandomId(8);
         mgmt = new LocalManagementContextForTests(BrooklynProperties.Factory.newDefault());
+        location = (JcloudsLocation) mgmt.getLocationRegistry().getLocationManaged(locationSpec);
+        identity = checkNotNull(location.getConfig(LocationConfigKeys.ACCESS_IDENTITY), "identity
must not be null");
+        credential = checkNotNull(location.getConfig(LocationConfigKeys.ACCESS_CREDENTIAL),
"credential must not be null");
+        endpoint = location.getConfig(CloudLocationConfig.CLOUD_ENDPOINT);
+        context = BlobStoreContextFactoryImpl.INSTANCE.newBlobStoreContext(location.config().getBag());
     }
     
     @AfterMethod(alwaysRun=true)
@@ -100,17 +152,7 @@ public class BlobStoreExpiryTest {
         context = null;
     }
 
-    /** FIXME
-     * see comment on this class
-     */
-    /*
     public void testRenewAuthSucceedsInSwiftObjectStore() throws Exception {
-        doTestRenewAuth();
-    }
-    
-    protected void doTestRenewAuth() throws Exception {
-        getSwiftBlobStoreContext();
-        
         injectShortLivedTokenForSwiftAuth();
         
         context.getBlobStore().createContainerInLocation(null, testContainerName);
@@ -126,42 +168,66 @@ public class BlobStoreExpiryTest {
         context.getBlobStore().deleteContainer(testContainerName);
     }
 
+    private void assertContainerFound() {
+        PageSet<? extends StorageMetadata> ps = context.getBlobStore().list();
+        BlobStoreTest.assertHasItemNamed(ps, testContainerName);
+    }
+    
+    /**
+     * Injects into the guts of jclouds' openstack-keystone a token that was requested, which

+     * should last for only 5 seconds. By sleeping for 10 seconds in the test, it should
mean
+     * the token subsequently used by jclouds will expire by the time the second half of
the 
+     * test executes.
+     */
     private void injectShortLivedTokenForSwiftAuth() throws Exception {
         URL endpointUrl = new URL(endpoint);
+        Credentials creds = new Credentials(identity, credential);
+        Set<Service> services = getServices(creds);
 
         HttpToolResponse tokenHttpResponse1 = requestTokenWithExplicitLifetime(endpointUrl,
             identity, credential, Duration.FIVE_SECONDS);
         
-        Builder<String, URI> servicesMapBuilder = ImmutableMap.builder();
-        for (Entry<String, List<String>> entry : tokenHttpResponse1.getHeaderLists().entrySet())
{
-            if (entry.getKey().toLowerCase().endsWith(URL_SUFFIX.toLowerCase()) ||
-                    entry.getKey().toLowerCase().endsWith("X-Auth-Token-Expires".toLowerCase())){
-                servicesMapBuilder.put(entry.getKey(), URI.create(entry.getValue().iterator().next()));
-            }
-        }
-        AuthenticationResponse authResponse = new AuthenticationResponse(tokenHttpResponse1.getHeaderLists().get(AuthHeaders.AUTH_TOKEN).get(0),
servicesMapBuilder.build());
-
-        getAuthCache().put(new Credentials(identity, credential), authResponse);
+        Access access = Access.builder()
+                .user(User.builder()
+                        .id(identity)
+                        .name(identity)
+                        .build())
+                .token(Token.builder()
+                        .id(tokenHttpResponse1.getHeaderLists().get(AuthHeaders.AUTH_TOKEN).get(0))
+                        .expires(new Date(System.currentTimeMillis() + 5000))
+                        .build())
+                .services(services)
+                .build();
+
+        getAuthCache(context).put(creds, access);
     }
 
-    private LoadingCache<Credentials, AuthenticationResponse> getAuthCache() {
+    private LoadingCache<Credentials, Access> getAuthCache(BlobStoreContext context)
{
         return context.utils().injector().getInstance(CachePeeker.class).authenticationResponseCache;
     }
     
+    private Set<Service> getServices(Credentials creds) throws Exception {
+        BlobStoreContext tmpContext = BlobStoreContextFactoryImpl.INSTANCE.newBlobStoreContext(location.config().getBag());
+        try {
+            tmpContext.getBlobStore().list();
+            LoadingCache<Credentials, Access> authCache = getAuthCache(tmpContext);
+            Access tmpAccess = authCache.get(creds);
+            return ImmutableSet.copyOf(tmpAccess);
+        } finally {
+            tmpContext.close();
+        }
+
+    }
+    
     public static class CachePeeker {
-        private final LoadingCache<Credentials, AuthenticationResponse> authenticationResponseCache;
+        private final LoadingCache<Credentials, Access> authenticationResponseCache;
 
         @Inject
-        protected CachePeeker(LoadingCache<Credentials, AuthenticationResponse> authenticationResponseCache)
{
+        protected CachePeeker(LoadingCache<Credentials, Access> authenticationResponseCache)
{
            this.authenticationResponseCache = authenticationResponseCache;
         }
     }
 
-    private void assertContainerFound() {
-        PageSet<? extends StorageMetadata> ps = context.getBlobStore().list();
-        BlobStoreTest.assertHasItemNamed(ps, testContainerName);
-    }
-    
     public static HttpToolResponse requestTokenWithExplicitLifetime(URL url, String user,
String key, Duration expiration) throws URISyntaxException {
         HttpClient client = HttpTool.httpClientBuilder().build();
         HttpToolResponse response = HttpTool.httpGet(client, url.toURI(), MutableMap.<String,String>of()
@@ -175,6 +241,4 @@ public class BlobStoreExpiryTest {
         log.info("Requested token with explicit lifetime: "+expiration+" at "+url+"\n"+response+"\n"+response.getHeaderLists());
         return response;
     }
-    */
-    
 }

http://git-wip-us.apache.org/repos/asf/brooklyn-server/blob/013b38bf/locations/jclouds/src/test/java/org/apache/brooklyn/core/mgmt/persist/jclouds/BlobStoreTest.java
----------------------------------------------------------------------
diff --git a/locations/jclouds/src/test/java/org/apache/brooklyn/core/mgmt/persist/jclouds/BlobStoreTest.java
b/locations/jclouds/src/test/java/org/apache/brooklyn/core/mgmt/persist/jclouds/BlobStoreTest.java
index 6033e2b..cd9ef0c 100644
--- a/locations/jclouds/src/test/java/org/apache/brooklyn/core/mgmt/persist/jclouds/BlobStoreTest.java
+++ b/locations/jclouds/src/test/java/org/apache/brooklyn/core/mgmt/persist/jclouds/BlobStoreTest.java
@@ -18,16 +18,14 @@
  */
 package org.apache.brooklyn.core.mgmt.persist.jclouds;
 
-import static com.google.common.base.Preconditions.checkNotNull;
-
 import java.io.IOException;
 
 import org.apache.brooklyn.api.mgmt.ManagementContext;
 import org.apache.brooklyn.core.entity.Entities;
 import org.apache.brooklyn.core.internal.BrooklynProperties;
-import org.apache.brooklyn.core.location.LocationConfigKeys;
-import org.apache.brooklyn.core.location.cloud.CloudLocationConfig;
 import org.apache.brooklyn.core.test.entity.LocalManagementContextForTests;
+import org.apache.brooklyn.location.jclouds.BlobStoreContextFactoryImpl;
+import org.apache.brooklyn.location.jclouds.JcloudsLocation;
 import org.apache.brooklyn.util.stream.Streams;
 import org.apache.brooklyn.util.text.Identifiers;
 import org.jclouds.blobstore.BlobStoreContext;
@@ -39,10 +37,6 @@ import org.testng.Assert;
 import org.testng.annotations.AfterMethod;
 import org.testng.annotations.BeforeMethod;
 import org.testng.annotations.Test;
-import org.apache.brooklyn.location.jclouds.JcloudsLocation;
-import org.apache.brooklyn.location.jclouds.JcloudsUtil;
-
-import com.google.common.base.Preconditions;
 
 @Test(groups={"Live", "Live-sanity"})
 public class BlobStoreTest {
@@ -51,9 +45,10 @@ public class BlobStoreTest {
      * Live tests as written require a location defined as follows:
      * 
      * <code>
-     * brooklyn.location.named.brooklyn-jclouds-objstore-test-1==jclouds:swift:https://ams01.objectstorage.softlayer.net/auth/v1.0
+     * brooklyn.location.named.brooklyn-jclouds-objstore-test-1==jclouds:openstack-swift:https://ams01.objectstorage.softlayer.net/auth/v1.0
      * brooklyn.location.named.brooklyn-jclouds-objstore-test-1.identity=IBMOS1234-5:yourname
      * brooklyn.location.named.brooklyn-jclouds-objstore-test-1.credential=0123abcd.......
+     * brooklyn.location.named.brooklyn-jclouds-objstore-test-1.jclouds.keystone.credential-type=tempAuthCredentials
      * </code>
      */
     public static final String PERSIST_TO_OBJECT_STORE_FOR_TEST_SPEC = "named:brooklyn-jclouds-objstore-test-1";
@@ -68,29 +63,12 @@ public class BlobStoreTest {
     private ManagementContext mgmt;
     private String testContainerName;
 
-    public synchronized BlobStoreContext getBlobStoreContext() {
-        if (context==null) {
-            if (location==null) {
-                Preconditions.checkNotNull(locationSpec, "locationSpec required for remote
object store when location is null");
-                Preconditions.checkNotNull(mgmt, "mgmt required for remote object store when
location is null");
-                location = (JcloudsLocation) mgmt.getLocationRegistry().getLocationManaged(locationSpec);
-            }
-            
-            String identity = checkNotNull(location.getConfig(LocationConfigKeys.ACCESS_IDENTITY),
"identity must not be null");
-            String credential = checkNotNull(location.getConfig(LocationConfigKeys.ACCESS_CREDENTIAL),
"credential must not be null");
-            String provider = checkNotNull(location.getConfig(LocationConfigKeys.CLOUD_PROVIDER),
"provider must not be null");
-            String endpoint = location.getConfig(CloudLocationConfig.CLOUD_ENDPOINT);
-
-            context = JcloudsUtil.newBlobstoreContext(provider, endpoint, identity, credential);
-        }
-        return context;
-    }
-    
     @BeforeMethod(alwaysRun=true)
     public void setup() {
         testContainerName = CONTAINER_PREFIX+"-"+Identifiers.makeRandomId(8);
         mgmt = new LocalManagementContextForTests(BrooklynProperties.Factory.newDefault());
-        getBlobStoreContext();
+        location = (JcloudsLocation) mgmt.getLocationRegistry().getLocationManaged(locationSpec);
+        context = BlobStoreContextFactoryImpl.INSTANCE.newBlobStoreContext(location.config().getBag());
     }
     
     @AfterMethod(alwaysRun=true)

http://git-wip-us.apache.org/repos/asf/brooklyn-server/blob/013b38bf/locations/jclouds/src/test/java/org/apache/brooklyn/core/mgmt/persist/jclouds/JcloudsExpect100ContinueTest.java
----------------------------------------------------------------------
diff --git a/locations/jclouds/src/test/java/org/apache/brooklyn/core/mgmt/persist/jclouds/JcloudsExpect100ContinueTest.java
b/locations/jclouds/src/test/java/org/apache/brooklyn/core/mgmt/persist/jclouds/JcloudsExpect100ContinueTest.java
index 02824f9..ebc3d24 100644
--- a/locations/jclouds/src/test/java/org/apache/brooklyn/core/mgmt/persist/jclouds/JcloudsExpect100ContinueTest.java
+++ b/locations/jclouds/src/test/java/org/apache/brooklyn/core/mgmt/persist/jclouds/JcloudsExpect100ContinueTest.java
@@ -22,6 +22,8 @@ import org.apache.brooklyn.api.mgmt.ManagementContext;
 import org.apache.brooklyn.core.entity.Entities;
 import org.apache.brooklyn.core.internal.BrooklynProperties;
 import org.apache.brooklyn.core.test.entity.LocalManagementContextForTests;
+import org.apache.brooklyn.location.jclouds.BlobStoreContextFactoryImpl;
+import org.apache.brooklyn.location.jclouds.JcloudsLocation;
 import org.apache.brooklyn.util.text.Identifiers;
 import org.jclouds.blobstore.BlobStore;
 import org.jclouds.blobstore.BlobStoreContext;
@@ -30,8 +32,6 @@ import org.slf4j.LoggerFactory;
 import org.testng.annotations.AfterMethod;
 import org.testng.annotations.BeforeMethod;
 import org.testng.annotations.Test;
-import org.apache.brooklyn.location.jclouds.JcloudsLocation;
-import org.apache.brooklyn.location.jclouds.JcloudsUtil;
 
 import ch.qos.logback.classic.Level;
 import ch.qos.logback.classic.Logger;
@@ -60,11 +60,9 @@ public class JcloudsExpect100ContinueTest {
 
         mgmt = new LocalManagementContextForTests(BrooklynProperties.Factory.newDefault());
         JcloudsLocation jcloudsLocation = (JcloudsLocation) mgmt.getLocationRegistry().getLocationManaged(LOCATION_SPEC);
-        context = JcloudsUtil.newBlobstoreContext(
-                jcloudsLocation.getProvider(),
-                jcloudsLocation.getEndpoint(),
-                jcloudsLocation.getIdentity(),
-                jcloudsLocation.getCredential());
+        
+        context = BlobStoreContextFactoryImpl.INSTANCE.newBlobStoreContext(jcloudsLocation.config().getBag());
+
         containerName = BlobStoreTest.CONTAINER_PREFIX+"-"+Identifiers.makeRandomId(8);
         context.getBlobStore().createContainerInLocation(null, containerName);
     }

http://git-wip-us.apache.org/repos/asf/brooklyn-server/blob/013b38bf/server-cli/src/test/java/org/apache/brooklyn/cli/CloudExplorerLiveTest.java
----------------------------------------------------------------------
diff --git a/server-cli/src/test/java/org/apache/brooklyn/cli/CloudExplorerLiveTest.java b/server-cli/src/test/java/org/apache/brooklyn/cli/CloudExplorerLiveTest.java
index cda96a5..1e99841 100644
--- a/server-cli/src/test/java/org/apache/brooklyn/cli/CloudExplorerLiveTest.java
+++ b/server-cli/src/test/java/org/apache/brooklyn/cli/CloudExplorerLiveTest.java
@@ -147,9 +147,10 @@ public class CloudExplorerLiveTest {
 
     /**
      * Expects in brooklyn.properties:
-     *     brooklyn.location.named.softlayer-swift-ams01=jclouds:swift:https://ams01.objectstorage.softlayer.net/auth/v1.0
+     *     brooklyn.location.named.softlayer-swift-ams01=jclouds:openstack-swift:https://ams01.objectstorage.softlayer.net/auth/v1.0
      *     brooklyn.location.named.softlayer-swift-ams01.identity=ABCDEFGH:myusername
      *     brooklyn.location.named.softlayer-swift-ams01.credential=1234567890...
+     *     brooklyn.location.named.softlayer-swift-ams01.jclouds.keystone.credential-type=tempAuthCredentials
      */
     @Test(groups={"Live", "Live-sanity"})
     public void testListContainers() throws Exception {


Mime
View raw message