brooklyn-dev mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From andreaturli <...@git.apache.org>
Subject [GitHub] brooklyn-server pull request #723: adds a new ComputeServiceRegistry to hand...
Date Thu, 08 Jun 2017 15:34:25 GMT
Github user andreaturli commented on a diff in the pull request:

    https://github.com/apache/brooklyn-server/pull/723#discussion_r120918534
  
    --- Diff: locations/jclouds/src/main/java/org/apache/brooklyn/location/jclouds/AbstractComputeServiceRegistry.java
---
    @@ -0,0 +1,283 @@
    +/*
    + * Licensed to the Apache Software Foundation (ASF) under one
    + * or more contributor license agreements.  See the NOTICE file
    + * distributed with this work for additional information
    + * regarding copyright ownership.  The ASF licenses this file
    + * to you under the Apache License, Version 2.0 (the
    + * "License"); you may not use this file except in compliance
    + * with the License.  You may obtain a copy of the License at
    + *
    + *     http://www.apache.org/licenses/LICENSE-2.0
    + *
    + * Unless required by applicable law or agreed to in writing,
    + * software distributed under the License is distributed on an
    + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
    + * KIND, either express or implied.  See the License for the
    + * specific language governing permissions and limitations
    + * under the License.
    + */
    +package org.apache.brooklyn.location.jclouds;
    +
    +import static com.google.common.base.Preconditions.checkNotNull;
    +import static org.apache.brooklyn.util.JavaGroovyEquivalents.groovyTruth;
    +import static org.jclouds.aws.ec2.reference.AWSEC2Constants.PROPERTY_EC2_AMI_QUERY;
    +import static org.jclouds.aws.ec2.reference.AWSEC2Constants.PROPERTY_EC2_CC_AMI_QUERY;
    +
    +import java.util.Map;
    +import java.util.Properties;
    +import java.util.concurrent.ConcurrentHashMap;
    +
    +import org.apache.brooklyn.core.config.Sanitizer;
    +import org.apache.brooklyn.core.location.cloud.CloudLocationConfig;
    +import org.apache.brooklyn.core.mgmt.persist.DeserializingJcloudsRenamesProvider;
    +import org.apache.brooklyn.util.collections.MutableMap;
    +import org.apache.brooklyn.util.core.config.ConfigBag;
    +import org.apache.brooklyn.util.text.Strings;
    +import org.apache.brooklyn.util.time.Duration;
    +import org.jclouds.Constants;
    +import org.jclouds.ContextBuilder;
    +import org.jclouds.azurecompute.arm.config.AzureComputeRateLimitModule;
    +import org.jclouds.compute.ComputeService;
    +import org.jclouds.compute.ComputeServiceContext;
    +import org.jclouds.domain.Credentials;
    +import org.jclouds.ec2.reference.EC2Constants;
    +import org.jclouds.encryption.bouncycastle.config.BouncyCastleCryptoModule;
    +import org.jclouds.location.reference.LocationConstants;
    +import org.jclouds.logging.slf4j.config.SLF4JLoggingModule;
    +import org.jclouds.sshj.config.SshjSshClientModule;
    +import org.slf4j.Logger;
    +import org.slf4j.LoggerFactory;
    +
    +import com.google.common.base.Predicates;
    +import com.google.common.base.Supplier;
    +import com.google.common.collect.ImmutableSet;
    +import com.google.common.collect.Maps;
    +import com.google.inject.Module;
    +
    +public abstract class AbstractComputeServiceRegistry implements ComputeServiceRegistry,
JcloudsLocationConfig {
    +
    +    private static final Logger LOG = LoggerFactory.getLogger(AbstractComputeServiceRegistry.class);
    +
    +    private final Map<Map<?, ?>, ComputeService> cachedComputeServices =
new ConcurrentHashMap<>();
    +
    +    @Override
    +    public ComputeService findComputeService(ConfigBag conf, boolean allowReuse) {
    +        PropertiesBuilder propertiesBuilder = new PropertiesBuilder(conf)
    +                .setCommonProperties();
    +
    +        Iterable<Module> modules = getCommonModules();
    +
    +        // Enable aws-ec2 lazy image fetching, if given a specific imageId; otherwise
customize for specific owners; or all as a last resort
    +        // See https://issues.apache.org/jira/browse/WHIRR-416
    +        String provider = getProviderFromConfig(conf);
    +        if ("aws-ec2".equals(provider)) {
    +            propertiesBuilder.setAWSEC2Properties();
    +        } else if ("azurecompute-arm".equals(provider)) {
    +            propertiesBuilder.setAzureComputeArmProperties();
    +            // jclouds 2.0.0 does not include the rate limit module for Azure ARM. This
quick fix enables this which will
    +            // avoid provisioning to fail due to rate limit exceeded
    +            // See https://issues.apache.org/jira/browse/JCLOUDS-1229
    +            modules = ImmutableSet.<Module>builder()
    +                    .addAll(modules)
    +                    .add(new AzureComputeRateLimitModule())
    +                    .build();
    +        }
    +
    +        Properties properties = propertiesBuilder
    +                .setJCloudsProperties()
    +                .setEndpointProperty()
    +                .build();
    +
    +        Supplier<ComputeService> computeServiceSupplier =  new ComputeServiceSupplier(conf,
modules, properties);
    +        if (allowReuse) {
    +            return cachedComputeServices.computeIfAbsent(makeCacheKey(conf, properties),
key -> computeServiceSupplier.get());
    +        }
    +        return computeServiceSupplier.get();
    +    }
    +
    +    private Map<?, ?> makeCacheKey(ConfigBag conf, Properties properties) {
    +        String provider = getProviderFromConfig(conf);
    +        String identity = checkNotNull(conf.get(CloudLocationConfig.ACCESS_IDENTITY),
"identity must not be null");
    +        String credential = checkNotNull(conf.get(CloudLocationConfig.ACCESS_CREDENTIAL),
"credential must not be null");
    +        String endpoint = properties.getProperty(Constants.PROPERTY_ENDPOINT);
    +        return MutableMap.builder()
    +                .putAll(properties)
    +                .put("provider", provider)
    +                .put("identity", identity)
    +                .put("credential", credential)
    +                .putIfNotNull("endpoint", endpoint)
    +                .build()
    +                .asUnmodifiable();
    +    }
    +
    +    public class PropertiesBuilder {
    +        private ConfigBag conf;
    +        private Properties properties = new Properties();
    +
    +        public PropertiesBuilder(ConfigBag conf) {
    +            this.conf = conf;
    +        }
    +
    +        public PropertiesBuilder setCommonProperties() {
    +            properties.setProperty(Constants.PROPERTY_TRUST_ALL_CERTS, Boolean.toString(true));
    +            properties.setProperty(Constants.PROPERTY_RELAX_HOSTNAME, Boolean.toString(true));
    +            properties.setProperty("jclouds.ssh.max-retries", conf.getStringKey("jclouds.ssh.max-retries")
!= null ?
    +                    conf.getStringKey("jclouds.ssh.max-retries").toString() : "50");
    +
    +            if (conf.get(OAUTH_ENDPOINT) != null)
    +                properties.setProperty(OAUTH_ENDPOINT.getName(), conf.get(OAUTH_ENDPOINT));
    +
    +            // See https://issues.apache.org/jira/browse/BROOKLYN-394
    +            // For retries, the backoff times are:
    +            //   Math.min(2^failureCount * retryDelayStart, retryDelayStart * 10) + random(10%)
    +            // Therefore the backoff times will be: 500ms, 1s, 2s, 4s, 5s, 5s.
    +            // The defaults (if not overridden here) are 50ms and 5 retires. This gives
backoff
    +            // times of 50ms, 100ms, 200ms, 400ms, 500ms (so a total backoff time of
1.25s),
    +            // which is not long when you're being rate-limited and there are multiple
thread all
    +            // retrying their API calls.
    +            properties.setProperty(Constants.PROPERTY_RETRY_DELAY_START, "500");
    +            properties.setProperty(Constants.PROPERTY_MAX_RETRIES, "6");
    +            return this;
    +        }
    +
    +        public PropertiesBuilder setAWSEC2Properties() {
    +            // TODO convert AWS-only flags to config keys
    +            if (groovyTruth(conf.get(IMAGE_ID))) {
    +                properties.setProperty(PROPERTY_EC2_AMI_QUERY, "");
    +                properties.setProperty(PROPERTY_EC2_CC_AMI_QUERY, "");
    +            } else if (groovyTruth(conf.getStringKey("imageOwner"))) {
    +                properties.setProperty(PROPERTY_EC2_AMI_QUERY, "owner-id=" + conf.getStringKey("imageOwner")
+ ";state=available;image-type=machine");
    +            } else if (groovyTruth(conf.getStringKey("anyOwner"))) {
    +                // set `anyOwner: true` to override the default query (which is restricted
to certain owners as per below),
    +                // allowing the AMI query to bind to any machine
    +                // (note however, we sometimes pick defaults in JcloudsLocationFactory);
    +                // (and be careful, this can give a LOT of data back, taking several
minutes,
    +                // and requiring extra memory allocated on the command-line)
    +                properties.setProperty(PROPERTY_EC2_AMI_QUERY, "state=available;image-type=machine");
    +                /*
    +                 * by default the following filters are applied:
    +                 * Filter.1.Name=owner-id&Filter.1.Value.1=137112412989&
    +                 * Filter.1.Value.2=063491364108&
    +                 * Filter.1.Value.3=099720109477&
    +                 * Filter.1.Value.4=411009282317&
    +                 * Filter.2.Name=state&Filter.2.Value.1=available&
    +                 * Filter.3.Name=image-type&Filter.3.Value.1=machine&
    +                 */
    +            }
    +
    +            // See https://issues.apache.org/jira/browse/BROOKLYN-399
    +            String region = conf.get(CLOUD_REGION_ID);
    +            if (Strings.isNonBlank(region)) {
    +                /*
    +                 * Drop availability zone suffixes. Without this deployments to regions
like us-east-1b fail
    +                 * because jclouds throws an IllegalStateException complaining that:
location id us-east-1b
    +                 * not found in: [{scope=PROVIDER, id=aws-ec2, description=https://ec2.us-east-1.amazonaws.com,
    +                 * iso3166Codes=[US-VA, US-CA, US-OR, BR-SP, IE, DE-HE, SG, AU-NSW, JP-13]}].
The exception is
    +                 * thrown by org.jclouds.compute.domain.internal.TemplateBuilderImpl#locationId(String).
    +                 */
    +                if (Character.isLetter(region.charAt(region.length() - 1))) {
    +                    region = region.substring(0, region.length() - 1);
    +                }
    +                properties.setProperty(LocationConstants.PROPERTY_REGIONS, region);
    +            }
    +
    +            // occasionally can get com.google.common.util.concurrent.UncheckedExecutionException:
java.lang.RuntimeException:
    +            //     security group eu-central-1/jclouds#brooklyn-bxza-alex-eu-central-shoul-u2jy-nginx-ielm
is not available after creating
    +            // the default timeout was 500ms so let's raise it in case that helps
    +            properties.setProperty(EC2Constants.PROPERTY_EC2_TIMEOUT_SECURITYGROUP_PRESENT,
"" + Duration.seconds(30).toMilliseconds());
    +            return this;
    +        }
    +
    +        private PropertiesBuilder setAzureComputeArmProperties() {
    +            String region = conf.get(CLOUD_REGION_ID);
    +            if (Strings.isNonBlank(region)) {
    +                properties.setProperty(LocationConstants.PROPERTY_REGIONS, region);
    +            }
    +            return this;
    +        }
    +
    +        private PropertiesBuilder setJCloudsProperties() {
    +            // Add extra jclouds-specific configuration
    +            Map<String, Object> extra = Maps.filterKeys(conf.getAllConfig(), Predicates.containsPattern("^jclouds\\."));
    +            if (extra.size() > 0) {
    +                String provider = getProviderFromConfig(conf);
    +                LOG.debug("Configuring custom jclouds property overrides for {}: {}",
provider, Sanitizer.sanitize(extra));
    +            }
    +            properties.putAll(Maps.filterValues(extra, Predicates.notNull()));
    +            return this;
    +        }
    +
    +        private PropertiesBuilder setEndpointProperty() {
    +            String endpoint = conf.get(CloudLocationConfig.CLOUD_ENDPOINT);
    +            if (!groovyTruth(endpoint)) endpoint = getDeprecatedProperty(conf, Constants.PROPERTY_ENDPOINT);
    +            if (groovyTruth(endpoint)) properties.setProperty(Constants.PROPERTY_ENDPOINT,
endpoint);
    +            return this;
    +        }
    +
    +        public Properties build() {
    +            return properties;
    +        }
    +    }
    +
    +    public class ComputeServiceSupplier implements Supplier<ComputeService> {
    +
    +        private final String provider;
    +        private final ConfigBag conf;
    +        private final Iterable<? extends Module> modules;
    +        private final Properties properties;
    +
    +        private final Object createComputeServicesMutex = new Object();
    +
    +        public ComputeServiceSupplier(ConfigBag conf, Iterable<? extends Module>
modules, Properties properties) {
    +            this.provider = getProviderFromConfig(conf);
    +            this.conf = conf;
    +            this.modules = modules;
    +            this.properties = properties;
    +        }
    +
    +        public ComputeService get() {
    +            // Synchronizing to avoid deadlock from sun.reflect.annotation.AnnotationType.
    +            // See https://github.com/brooklyncentral/brooklyn/issues/974
    +            synchronized (createComputeServicesMutex) {
    --- End diff --
    
    do we still need to synchronize the access to it?


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---

Mime
View raw message