hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ser...@apache.org
Subject [34/44] hive git commit: HIVE-14849: Support google-compute-engine provider on Hive ptest framework (Sergio Pena, reviewed by Prasanth Jayachandran)
Date Sat, 01 Oct 2016 01:09:16 GMT
HIVE-14849: Support google-compute-engine provider on Hive ptest framework (Sergio Pena, reviewed
by Prasanth Jayachandran)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/291f3d50
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/291f3d50
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/291f3d50

Branch: refs/heads/hive-14535
Commit: 291f3d503d5a8627f86ef5f7fdd7880d8da4760c
Parents: cf72a73
Author: Sergio Pena <sergio.pena@cloudera.com>
Authored: Wed Sep 28 21:33:00 2016 -0500
Committer: Sergio Pena <sergio.pena@cloudera.com>
Committed: Wed Sep 28 21:33:00 2016 -0500

----------------------------------------------------------------------
 .../ptest2/conf/cloudhost.properties.example    |  37 +++
 testutils/ptest2/pom.xml                        |   5 +
 .../execution/context/CloudComputeService.java  | 224 +++++++++++++++----
 .../context/CloudExecutionContextProvider.java  | 105 +++++++--
 4 files changed, 311 insertions(+), 60 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/291f3d50/testutils/ptest2/conf/cloudhost.properties.example
----------------------------------------------------------------------
diff --git a/testutils/ptest2/conf/cloudhost.properties.example b/testutils/ptest2/conf/cloudhost.properties.example
new file mode 100644
index 0000000..c336052
--- /dev/null
+++ b/testutils/ptest2/conf/cloudhost.properties.example
@@ -0,0 +1,37 @@
+#
+# This is just an example of different cloudhost providers
+#
+
+# This context provides configurations for AWS EC2 and GCE (google compute engine)
+executionContextProvider = org.apache.hive.ptest.execution.context.CloudExecutionContextProvider$Builder
+
+# Option: GCE
+cloudProvider = google-compute-engine
+gceJsonFile = # GCE JSON KEY FILE
+instanceType = https://www.googleapis.com/compute/v1/projects/<PROJECT_ID>/zones/us-central1-a/machineTypes/n1-standard-8
+imageId = https://www.googleapis.com/compute/v1/projects/<PROJECT_ID>/global/images/hive-ptest-debian-8-20160927
+# keyPair = # UNUSED
+securityGroup = hive-ptest
+
+# Option: AWS
+cloudProvider = aws-ec2
+apiKey =    # AWS ACCESS KEY
+accessKey = # AWS SECRET ACCESS KEY
+instanceType = c3.2xlarge
+imageId = us-west-1/ami-1fa1445b
+keyPair = hive-ptest
+securityGroup = hive-ptest
+
+# Generic options
+workingDirectory = /data/hive-ptest
+profileDirectory = /usr/local/hiveptest/etc/public/
+privateKey = /home/hiveptest/.ssh/hive-ptest-user-key
+dataDir = /data/hive-ptest/data/
+numHosts = 12
+groupName = hive-ptest-slaves
+localDirs = /home/hiveptest/
+user = hiveptest
+numThreads = 2
+maxLogDirectoriesPerProfile = 30
+userMetadata.owner = # USER
+maxHostsPerCreateRequest = 12
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hive/blob/291f3d50/testutils/ptest2/pom.xml
----------------------------------------------------------------------
diff --git a/testutils/ptest2/pom.xml b/testutils/ptest2/pom.xml
index cea29b6..97981fb 100644
--- a/testutils/ptest2/pom.xml
+++ b/testutils/ptest2/pom.xml
@@ -107,6 +107,11 @@ limitations under the License.
       <version>${jclouds.version}</version>
     </dependency>
     <dependency>
+      <groupId>org.apache.jclouds.labs</groupId>
+      <artifactId>google-compute-engine</artifactId>
+      <version>${jclouds.version}</version>
+    </dependency>
+    <dependency>
       <groupId>org.apache.jclouds.driver</groupId>
       <artifactId>jclouds-sshj</artifactId>
       <version>${jclouds.version}</version>

http://git-wip-us.apache.org/repos/asf/hive/blob/291f3d50/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/context/CloudComputeService.java
----------------------------------------------------------------------
diff --git a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/context/CloudComputeService.java
b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/context/CloudComputeService.java
index 64ee68e..e26c5ca 100644
--- a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/context/CloudComputeService.java
+++ b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/context/CloudComputeService.java
@@ -18,11 +18,13 @@
  */
 package org.apache.hive.ptest.execution.context;
 
-import java.util.Collections;
-import java.util.Properties;
-import java.util.Map;
-import java.util.Set;
+import java.io.File;
+import java.io.IOException;
+import java.util.*;
 
+import com.google.common.base.Supplier;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.io.Files;
 import org.jclouds.Constants;
 import org.jclouds.ContextBuilder;
 import org.jclouds.aws.ec2.compute.AWSEC2TemplateOptions;
@@ -34,7 +36,12 @@ import org.jclouds.compute.domain.ComputeMetadata;
 import org.jclouds.compute.domain.NodeMetadata;
 import org.jclouds.compute.domain.NodeMetadata.Status;
 import org.jclouds.compute.domain.Template;
+import org.jclouds.compute.options.TemplateOptions;
+import org.jclouds.domain.Credentials;
+import org.jclouds.googlecloud.GoogleCredentialsFromJson;
+import org.jclouds.googlecomputeengine.compute.options.GoogleComputeEngineTemplateOptions;
 import org.jclouds.logging.log4j.config.Log4JLoggingModule;
+import org.jclouds.sshj.config.SshjSshClientModule;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -43,55 +50,76 @@ import com.google.common.base.Strings;
 import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.Sets;
 
+import static com.google.common.base.Charsets.UTF_8;
+
 public class CloudComputeService {
   private static final Logger LOG = LoggerFactory
       .getLogger(CloudComputeService.class);
+  private final CloudComputeConfig mConfig;
   private final ComputeServiceContext mComputeServiceContext;
   private final ComputeService mComputeService;
-  private final String mInstanceType;
-  private final String mGroupName;
+  private final Template mTemplate;
   private final String mGroupTag;
-  private final String mImageId;
-  private final String mkeyPair;
-  private final String mSecurityGroup;
-  private final Map<String, String> mUserMetadata;
-  /**
-   * JClouds requests on-demand instances when null
-   */
-  private final Float mMaxBid;
-  public CloudComputeService(String apiKey, String accessKey, String instanceType, String
groupName,
-      String imageId, String keyPair, String securityGroup, Float maxBid, Map<String,String>
userMetadata) {
-    mInstanceType = instanceType;
-    mGroupName = groupName;
-    mImageId = imageId;
-    mkeyPair = keyPair;
-    mSecurityGroup = securityGroup;
-    mMaxBid = maxBid;
-    mGroupTag = "group=" + mGroupName;
-    mUserMetadata = userMetadata;
+
+  public CloudComputeService(final CloudComputeConfig config) {
+    mConfig = config;
+
+    mComputeServiceContext = initComputeServiceContext(config.getmProvider(), config.getIdentity(),
config.getCredential());
+    mComputeService = mComputeServiceContext.getComputeService();
+    mTemplate = mComputeService.templateBuilder().hardwareId(config.getInstanceType()).imageId(config.getImageId()).build();
+
+    TemplateOptions options = mTemplate.getOptions();
+
+    // Set generic options
+    options.blockOnPort(22, 60);
+    options.userMetadata(config.getUserMetaData());
+
+    // Set provider options
+    switch (config.getmProvider()) {
+      case AWS:
+        mGroupTag = String.format("group=%s", config.getGroupName());
+
+        options.as(AWSEC2TemplateOptions.class)
+                .keyPair(config.getKeyPairName())
+                .securityGroupIds(config.getSecurityGroup())
+                .spotPrice(config.getMaxBid())
+                .tags(Collections.singletonList(mGroupTag));
+        break;
+      case GCE:
+        mGroupTag = config.getGroupName();
+
+        options.as(GoogleComputeEngineTemplateOptions.class)
+                .tags(Arrays.asList(config.getGroupName(), config.getSecurityGroup())); //
GCE firewall is set through instance tags
+        break;
+      default:
+        mGroupTag = "";
+    }
+  }
+
+  private ComputeServiceContext initComputeServiceContext(CloudComputeConfig.CloudComputeProvider
provider, String identity, String credential) {
     Properties overrides = new Properties();
+
     overrides.put(ComputeServiceProperties.POLL_INITIAL_PERIOD, String.valueOf(60L * 1000L));
     overrides.put(ComputeServiceProperties.POLL_MAX_PERIOD, String.valueOf(600L * 1000L));
     overrides.put(Constants.PROPERTY_MAX_RETRIES, String.valueOf(60));
-    mComputeServiceContext = ContextBuilder.newBuilder("aws-ec2")
-        .credentials(apiKey, accessKey)
-        .modules(ImmutableSet.of(new Log4JLoggingModule()))
-        .overrides(overrides)
-        .buildView(ComputeServiceContext.class);
-    mComputeService = mComputeServiceContext.getComputeService();
+
+    return ContextBuilder.newBuilder(provider.getmJcloudsId())
+            .credentials(identity, credential)
+            .modules(ImmutableSet.of(
+                    new SshjSshClientModule(),
+                    new Log4JLoggingModule()
+            ))
+            .overrides(overrides)
+            .buildView(ComputeServiceContext.class);
   }
+
   public Set<NodeMetadata> createNodes(int count)
       throws RunNodesException {
     Set<NodeMetadata> result = Sets.newHashSet();
-    Template template = mComputeService.templateBuilder()
-        .hardwareId(mInstanceType).imageId(mImageId).build();
-    template.getOptions().as(AWSEC2TemplateOptions.class).keyPair(mkeyPair)
-    .securityGroupIds(mSecurityGroup).blockOnPort(22, 60)
-    .spotPrice(mMaxBid).tags(Collections.singletonList(mGroupTag))
-    .userMetadata(mUserMetadata);
-    result.addAll(mComputeService.createNodesInGroup(mGroupName, count, template));
+    result.addAll(mComputeService.createNodesInGroup(mConfig.getGroupName(), count, mTemplate));
     return result;
   }
+
   static Predicate<ComputeMetadata> createFilterPTestPredicate(final String groupName,
       final String groupTag) {
     return new Predicate<ComputeMetadata>() {
@@ -122,7 +150,7 @@ public class CloudComputeService {
   public Set<NodeMetadata> listRunningNodes(){
     Set<NodeMetadata> result = Sets.newHashSet();
     result.addAll(mComputeService.listNodesDetailsMatching(
-        createFilterPTestPredicate(mGroupName, mGroupTag)));
+        createFilterPTestPredicate(mConfig.getGroupName(), mGroupTag)));
     return result;
   }
   public void destroyNode(String nodeId) {
@@ -131,4 +159,124 @@ public class CloudComputeService {
   public void close() {
     mComputeServiceContext.close();
   }
+
+  public static class CloudComputeConfig {
+    public enum CloudComputeProvider {
+      AWS("aws-ec2"),
+      GCE("google-compute-engine");
+
+      private final String mJcloudsId;
+
+      CloudComputeProvider(String jcloudsId) {
+        mJcloudsId = jcloudsId;
+      }
+
+      public String getmJcloudsId() {
+        return mJcloudsId;
+      }
+    };
+
+    private final CloudComputeProvider mProvider;
+
+    private String mIdentity;
+    private String mCredential;
+    private String mInstanceType;
+    private String mImageId;
+    private String mGroupName;
+    private String mSecurityGroup;
+    private String mKeyPairName;
+    private Map<String, String> mUserMetaData;
+
+    /**
+     * JClouds requests on-demand instances when null
+     */
+    private Float mMaxBid;
+
+    public CloudComputeConfig(CloudComputeProvider provider) {
+      mProvider = provider;
+    }
+
+    public void setCredentials(String identity, String credential) {
+      mIdentity = identity;
+      mCredential = credential;
+    }
+
+    public void setInstanceType(String instanceType) {
+      mInstanceType = instanceType;
+    }
+
+    public void setImageId(String imageId) {
+      mImageId = imageId;
+    }
+
+    public void setGroupName(String groupName) {
+      mGroupName = groupName;
+    }
+
+    public void setSecurityGroup(String securityGroup) {
+      mSecurityGroup = securityGroup;
+    }
+
+    public void setMaxBid(Float maxBid) {
+      mMaxBid = maxBid;
+    }
+
+    public void setmKeyPairName(String keyPairName) {
+      mKeyPairName = keyPairName;
+    }
+
+    public void setUserMetaData(Map<String, String> userMetaData) {
+      mUserMetaData = userMetaData;
+    }
+
+    public CloudComputeProvider getmProvider() {
+      return mProvider;
+    }
+
+    public String getIdentity() {
+      return mIdentity;
+    }
+
+    public String getCredential() {
+      return mCredential;
+    }
+
+    public String getInstanceType() {
+      return mInstanceType;
+    }
+
+    public String getImageId() {
+      return mImageId;
+    }
+
+    public String getGroupName() {
+      return mGroupName;
+    }
+
+    public String getSecurityGroup() {
+      return mSecurityGroup;
+    }
+
+    public Float getMaxBid() {
+      return mMaxBid;
+    }
+
+    public String getKeyPairName() {
+      return mKeyPairName;
+    }
+
+    public Map<String, String> getUserMetaData() {
+      if (mUserMetaData == null) {
+        return ImmutableMap.of();
+      }
+
+      return mUserMetaData;
+    }
+  }
+
+  public static Credentials getCredentialsFromJsonKeyFile(String filename) throws IOException
{
+    String fileContents = Files.toString(new File(filename), UTF_8);
+    Supplier<Credentials> credentialSupplier = new GoogleCredentialsFromJson(fileContents);
+    return credentialSupplier.get();
+  }
 }

http://git-wip-us.apache.org/repos/asf/hive/blob/291f3d50/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/context/CloudExecutionContextProvider.java
----------------------------------------------------------------------
diff --git a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/context/CloudExecutionContextProvider.java
b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/context/CloudExecutionContextProvider.java
index 343efde..8b82497 100644
--- a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/context/CloudExecutionContextProvider.java
+++ b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/context/CloudExecutionContextProvider.java
@@ -34,12 +34,14 @@ import java.util.concurrent.TimeUnit;
 
 import org.apache.hive.ptest.execution.Constants;
 import org.apache.hive.ptest.execution.Dirs;
+import org.apache.hive.ptest.execution.LocalCommandFactory;
 import org.apache.hive.ptest.execution.conf.Context;
 import org.apache.hive.ptest.execution.conf.Host;
 import org.apache.hive.ptest.execution.ssh.SSHCommand;
 import org.apache.hive.ptest.execution.ssh.SSHCommandExecutor;
 import org.jclouds.compute.RunNodesException;
 import org.jclouds.compute.domain.NodeMetadata;
+import org.jclouds.domain.Credentials;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -55,16 +57,24 @@ import com.google.common.collect.Sets;
 public class CloudExecutionContextProvider implements ExecutionContextProvider {
   private static final Logger LOG = LoggerFactory
       .getLogger(CloudExecutionContextProvider.class);
-  public static final String DATA_DIR = "dataDir";
+  public static final String CLOUD_PROVIDER = "cloudProvider";
+
+  // GCE settings
+  public static final String GCE_JSON_CREDS_FILE = "gceJsonFile";
+  
+  // AWS settings
   public static final String API_KEY = "apiKey";
   public static final String ACCESS_KEY = "accessKey";
+  public static final String KEY_PAIR = "keyPair";
+  public static final String MAX_BID = "maxBid";
+  
+  // Generic settings
+  public static final String DATA_DIR = "dataDir";
   public static final String NUM_HOSTS = "numHosts";
   public static final String MAX_HOSTS_PER_CREATE_REQUEST = "maxHostsPerCreateRequest";
   public static final String GROUP_NAME = "groupName";
   public static final String IMAGE_ID = "imageId";
-  public static final String KEY_PAIR = "keyPair";
   public static final String SECURITY_GROUP = "securityGroup";
-  public static final String MAX_BID = "maxBid";
   public static final String SLAVE_LOCAL_DIRECTORIES = "localDirs";
   public static final String USERNAME = "user";
   public static final String INSTANCE_TYPE = "instanceType";
@@ -304,6 +314,7 @@ public class CloudExecutionContextProvider implements ExecutionContextProvider
{
           LOG.error("Verify command still executing on a host after 10 minutes");
         }
       } catch (InterruptedException e) {
+        terminateInternal(result);
         throw new CreateHostsFailedException("Interrupted while trying to create hosts",
e);
       } finally {
         if(!executorService.isShutdown()) {
@@ -418,29 +429,79 @@ public class CloudExecutionContextProvider implements ExecutionContextProvider
{
       return create(context, workingDirectory);
     }
   }
+  
+  private static CloudComputeService createAwsService(final Context context) {
+    String apiKey = Preconditions.checkNotNull(context.getString(API_KEY), API_KEY + " is
required");
+    String accessKey = Preconditions.checkNotNull(context.getString(ACCESS_KEY), ACCESS_KEY
+ " is required");
+    String imageId = Preconditions.checkNotNull(context.getString(IMAGE_ID), IMAGE_ID + "
is required");
+    String keyPair = Preconditions.checkNotNull(context.getString(KEY_PAIR), KEY_PAIR + "
is required");
+    String securityGroup = Preconditions.checkNotNull(context.getString(SECURITY_GROUP),
SECURITY_GROUP + " is required");
+
+    Float maxBid = context.getFloat(MAX_BID);
+    Preconditions.checkArgument(maxBid == null || maxBid > 0, MAX_BID + " must be null
or greater than zero");
+
+    String instanceType = context.getString(INSTANCE_TYPE, "c1.xlarge");
+    String groupName = context.getString(GROUP_NAME, "hive-ptest-slaves");
+
+    CloudComputeService.CloudComputeConfig config =
+            new CloudComputeService.CloudComputeConfig(CloudComputeService.CloudComputeConfig.CloudComputeProvider.AWS);
+
+    config.setCredentials(apiKey, accessKey);
+    config.setInstanceType(instanceType);
+    config.setGroupName(groupName);
+    config.setImageId(imageId);
+    config.setmKeyPairName(keyPair);
+    config.setSecurityGroup(securityGroup);
+    config.setMaxBid(maxBid);
+    config.setUserMetaData(context.getSubProperties(USER_METADATA + "."));
+
+    return new CloudComputeService(config);
+  }
+  
+  private static CloudComputeService createGceService(final Context context) throws IOException
{
+    String gceJsonFile = Preconditions.checkNotNull(context.getString(GCE_JSON_CREDS_FILE),
GCE_JSON_CREDS_FILE + " is required");
+    String imageId = Preconditions.checkNotNull(context.getString(IMAGE_ID), IMAGE_ID + "
is required");
+    String securityGroup = Preconditions.checkNotNull(context.getString(SECURITY_GROUP),
SECURITY_GROUP + " is required");
+    String instanceType = Preconditions.checkNotNull(context.getString(INSTANCE_TYPE, ""),
INSTANCE_TYPE + " is required");
+
+    String groupName = context.getString(GROUP_NAME, "hive-ptest-slaves");
+
+    CloudComputeService.CloudComputeConfig config =
+            new CloudComputeService.CloudComputeConfig(CloudComputeService.CloudComputeConfig.CloudComputeProvider.GCE);
+
+    Credentials creds = 
+            CloudComputeService.getCredentialsFromJsonKeyFile(gceJsonFile);
+
+    config.setCredentials(creds.identity, creds.credential);
+    config.setInstanceType(instanceType);
+    config.setGroupName(groupName);
+    config.setImageId(imageId);
+    config.setSecurityGroup(securityGroup);
+    config.setUserMetaData(context.getSubProperties(USER_METADATA + "."));
+
+    return new CloudComputeService(config);
+  }
+
+  private static CloudComputeService createService(final Context context) throws IOException
{
+    String cloudProvider = context.getString(CLOUD_PROVIDER, "aws-ec2");
+    
+    if (cloudProvider.equalsIgnoreCase("aws-ec2")) {
+      return createAwsService(context);
+    } else if (cloudProvider.equalsIgnoreCase("google-compute-engine")) {
+      return createGceService(context);
+    } else {
+      throw new IllegalArgumentException("Unknown cloud provider name: " + cloudProvider);
+    }
+  }
 
   private static CloudExecutionContextProvider create(Context context,
       String workingDirectory) throws IOException {
     String dataDir = Preconditions.checkNotNull(context.getString(DATA_DIR),
         DATA_DIR + " is required");
-    String apiKey = Preconditions.checkNotNull(context.getString(API_KEY),
-        API_KEY + " is required");
-    String accessKey = Preconditions.checkNotNull(
-        context.getString(ACCESS_KEY), ACCESS_KEY + " is required");
     int maxHostsPerCreateRequest = context.getInteger(MAX_HOSTS_PER_CREATE_REQUEST, 2);
     Integer numHosts = context.getInteger(NUM_HOSTS, 8);
     Preconditions.checkArgument(numHosts > 0, NUM_HOSTS
         + " must be greater than zero");
-    String groupName = context.getString(GROUP_NAME, "hive-ptest-slaves");
-    String imageId = Preconditions.checkNotNull(context.getString(IMAGE_ID),
-        IMAGE_ID + " is required");
-    String keyPair = Preconditions.checkNotNull(context.getString(KEY_PAIR),
-        KEY_PAIR + " is required");
-    String securityGroup = Preconditions.checkNotNull(
-        context.getString(SECURITY_GROUP), SECURITY_GROUP + " is required");
-    Float maxBid = context.getFloat(MAX_BID);
-    Preconditions.checkArgument(maxBid == null || maxBid > 0, MAX_BID
-        + " must be null or greater than zero");
     String privateKey = Preconditions.checkNotNull(
         context.getString(PRIVATE_KEY), PRIVATE_KEY + " is required");
     String user = context.getString(USERNAME, "hiveptest");
@@ -448,12 +509,12 @@ public class CloudExecutionContextProvider implements ExecutionContextProvider
{
         .split(context.getString(SLAVE_LOCAL_DIRECTORIES, "/home/hiveptest/")),
         String.class);
     Integer numThreads = context.getInteger(NUM_THREADS, 3);
-    String instanceType = context.getString(INSTANCE_TYPE, "c1.xlarge");
-    CloudComputeService cloudComputeService = new CloudComputeService(apiKey, accessKey,
-        instanceType, groupName, imageId, keyPair, securityGroup, maxBid, context.getSubProperties(USER_METADATA
+ "."));
+    
+    CloudComputeService cloudComputeService = createService(context);
     CloudExecutionContextProvider service = new CloudExecutionContextProvider(
-        dataDir, numHosts, cloudComputeService, new SSHCommandExecutor(LOG), workingDirectory,
-        privateKey, user, localDirs, numThreads, 60, maxHostsPerCreateRequest);
+        dataDir, numHosts, cloudComputeService,
+        new SSHCommandExecutor(LOG, new LocalCommandFactory(LOG), "-o UserKnownHostsFile=/dev/null
-o StrictHostKeyChecking=no"),
+        workingDirectory, privateKey, user, localDirs, numThreads, 60, maxHostsPerCreateRequest);
     return service;
   }
 }


Mime
View raw message