usergrid-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From sfeld...@apache.org
Subject [04/16] incubator-usergrid git commit: Updated NodeRegistry to use only tags instead of external simple db
Date Wed, 03 Dec 2014 17:58:19 GMT
Updated NodeRegistry to use only tags instead of external simple db

Upgraded AWS client

Removed unused registry list groovy script


Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/b12130f0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/b12130f0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/b12130f0

Branch: refs/heads/two-dot-o
Commit: b12130f06d503f8c5a3fc29d6049a4df89eb5a7e
Parents: 1f75549
Author: Todd Nine <tnine@apigee.com>
Authored: Tue Nov 25 17:38:42 2014 -0700
Committer: Todd Nine <tnine@apigee.com>
Committed: Tue Nov 25 17:38:42 2014 -0700

----------------------------------------------------------------------
 stack/awscluster/gatling-cluster-cf.json        |  86 +++++++---
 stack/awscluster/pom.xml                        |   2 +-
 .../src/main/groovy/NodeRegistry.groovy         | 165 ++++++++++++-------
 .../src/main/groovy/registry_list.groovy        |  36 ----
 4 files changed, 168 insertions(+), 121 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/b12130f0/stack/awscluster/gatling-cluster-cf.json
----------------------------------------------------------------------
diff --git a/stack/awscluster/gatling-cluster-cf.json b/stack/awscluster/gatling-cluster-cf.json
index ff41f14..b9f9752 100644
--- a/stack/awscluster/gatling-cluster-cf.json
+++ b/stack/awscluster/gatling-cluster-cf.json
@@ -1,53 +1,89 @@
 {
     "AWSTemplateFormatVersion" : "2010-09-09",
     "Description" : "Gatling cluster",
-
-    "Parameters" : {
+    "Parameters": {
         "InstanceCount": {
-            "Description" : "The number of gatling instances to create",
+            "Description": "The number of gatling instances to create",
             "Type": "Number",
             "Default": "1",
             "MinValue": "1"
         },
-
         "KeyName": {
-            "Description" : "Name of an existing EC2 KeyPair to enable SSH access to the
instance",
+            "Description": "Name of an existing EC2 KeyPair to enable SSH access to the instance",
             "Type": "AWS::EC2::KeyPair::KeyName",
-            "Default" : "jenkins-east",
-            "ConstraintDescription" : "must be the name of an existing EC2 KeyPair."
+            "Default": "jenkins-east",
+            "ConstraintDescription": "must be the name of an existing EC2 KeyPair."
         },
-
-        "InstanceType" : {
-            "Description" : "WebServer EC2 instance type",
-            "Type" : "String",
-            "Default" : "m1.xlarge",
-            "AllowedValues" : [ "t1.micro", "t2.micro", "t2.small", "t2.medium", "m1.small",
"m1.medium", "m1.large", "m1.xlarge", "m2.xlarge", "m2.2xlarge", "m2.4xlarge", "m3.medium",
"m3.large", "m3.xlarge", "m3.2xlarge", "c1.medium", "c1.xlarge", "c3.large", "c3.xlarge",
"c3.2xlarge", "c3.4xlarge", "c3.8xlarge", "g2.2xlarge", "r3.large", "r3.xlarge", "r3.2xlarge",
"r3.4xlarge", "r3.8xlarge", "i2.xlarge", "i2.2xlarge", "i2.4xlarge", "i2.8xlarge", "hi1.4xlarge",
"hs1.8xlarge", "cr1.8xlarge", "cc2.8xlarge", "cg1.4xlarge"]
-            ,
-            "ConstraintDescription" : "must be a valid EC2 instance type."
+        "InstanceType": {
+            "Description": "WebServer EC2 instance type",
+            "Type": "String",
+            "Default": "m1.xlarge",
+            "AllowedValues": [
+                "t1.micro",
+                "t2.micro",
+                "t2.small",
+                "t2.medium",
+                "m1.small",
+                "m1.medium",
+                "m1.large",
+                "m1.xlarge",
+                "m2.xlarge",
+                "m2.2xlarge",
+                "m2.4xlarge",
+                "m3.medium",
+                "m3.large",
+                "m3.xlarge",
+                "m3.2xlarge",
+                "c1.medium",
+                "c1.xlarge",
+                "c3.large",
+                "c3.xlarge",
+                "c3.2xlarge",
+                "c3.4xlarge",
+                "c3.8xlarge",
+                "g2.2xlarge",
+                "r3.large",
+                "r3.xlarge",
+                "r3.2xlarge",
+                "r3.4xlarge",
+                "r3.8xlarge",
+                "i2.xlarge",
+                "i2.2xlarge",
+                "i2.4xlarge",
+                "i2.8xlarge",
+                "hi1.4xlarge",
+                "hs1.8xlarge",
+                "cr1.8xlarge",
+                "cc2.8xlarge",
+                "cg1.4xlarge"
+            ],
+            "ConstraintDescription": "must be a valid EC2 instance type."
         },
-
         "OperatorEMail": {
             "Description": "EMail address to notify if there are any scaling operations",
             "Type": "String",
             "AllowedPattern": "([a-zA-Z0-9_\\-\\.]+)@((\\[[0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3}\\.)|(([a-zA-Z0-9\\-]+\\.)+))([a-zA-Z]{2,4}|[0-9]{1,3})(\\]?)",
             "ConstraintDescription": "must be a valid email address.",
-            "Default":"rbridges@apigee.com"
+            "Default": "rbridges@apigee.com"
         },
-
         "ReleaseBucket": {
             "Description": "S3 Bucket where Usergrid assembly is to be found.",
             "Type": "String",
             "Default": "usergrid-jenkins-builds"
         },
-
-        "SSHLocation" : {
-            "Description" : "The IP address range that can be used to SSH to the EC2 instances",
+        "SSHLocation": {
+            "Description": "The IP address range that can be used to SSH to the EC2 instances",
             "Type": "String",
             "MinLength": "9",
             "MaxLength": "18",
             "Default": "0.0.0.0/0",
             "AllowedPattern": "(\\d{1,3})\\.(\\d{1,3})\\.(\\d{1,3})\\.(\\d{1,3})/(\\d{1,2})",
             "ConstraintDescription": "must be a valid IP CIDR range of the form x.x.x.x/x."
+        },
+        "Branch": {
+            "Description": "The branch of usergrid to check out",
+            "Type": "String",
+            "Default": "two-dot-o"
         }
     },
 
@@ -231,15 +267,11 @@
                     "apt-add-repository ppa:awstools-dev/awstools --yes\n",
                     "apt-get update\n",
                     "echo oracle-java7-installer shared/accepted-oracle-license-v1-1 select
true | /usr/bin/debconf-set-selections\n",
-                    "apt-get install oracle-java7-installer oracle-java7-set-default expect
expect-dev git ec2-api-tools -y --force-yes\n",
+                    "apt-get install oracle-java7-installer oracle-java7-set-default expect
expect-dev git ec2-api-tools maven -y --force-yes\n",
                     "update-java-alternatives -s java-7-oracle\n",
 
-                    "mkdir -p /usr/local/apache-maven\n",
-                    "cd /usr/local/apache-maven\n",
-                    "wget http://apache.mirrors.timporter.net/maven/maven-3/3.1.1/binaries/apache-maven-3.1.1-bin.tar.gz\n",
-                    "tar -xzvf apache-maven-3.1.1-bin.tar.gz\n",
                     "cd ~ubuntu\n",
-                    "git clone https://github.com/apache/incubator-usergrid.git usergrid\n",
+                    "git clone -b ",{ "Ref":"Branch" } , " https://github.com/apache/incubator-usergrid.git
usergrid\n",
                     "chown -R ubuntu:ubuntu usergrid\n",
                     "ec2-create-tags $(curl -k http://169.254.169.254/latest/meta-data/instance-id)
 --aws-access-key ",{ "Ref":"GatlingKey" }, " --aws-secret-key ", { "Fn::GetAtt":[ "GatlingKey",
"SecretAccessKey" ] } ," --tag Name=", { "Ref":"AWS::StackName"  }, "-Gatling\n"
                 ]]}}

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/b12130f0/stack/awscluster/pom.xml
----------------------------------------------------------------------
diff --git a/stack/awscluster/pom.xml b/stack/awscluster/pom.xml
index 4a8f4d6..46157ab 100644
--- a/stack/awscluster/pom.xml
+++ b/stack/awscluster/pom.xml
@@ -215,7 +215,7 @@
     <dependency>
       <groupId>com.amazonaws</groupId>
       <artifactId>aws-java-sdk</artifactId>
-      <version>1.6.7</version>
+      <version>1.9.7</version>
     </dependency>
     <dependency>
       <artifactId>groovy-all</artifactId>

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/b12130f0/stack/awscluster/src/main/groovy/NodeRegistry.groovy
----------------------------------------------------------------------
diff --git a/stack/awscluster/src/main/groovy/NodeRegistry.groovy b/stack/awscluster/src/main/groovy/NodeRegistry.groovy
index 6291f93..277631b 100644
--- a/stack/awscluster/src/main/groovy/NodeRegistry.groovy
+++ b/stack/awscluster/src/main/groovy/NodeRegistry.groovy
@@ -20,34 +20,64 @@
  * A utility class that search simple db for the node type provided and returns a list of
hostnames as a string array
  */
 import com.amazonaws.auth.BasicAWSCredentials
-import com.amazonaws.services.simpledb.AmazonSimpleDBClient
-import com.amazonaws.services.simpledb.model.*
+import com.amazonaws.regions.Region
+import com.amazonaws.regions.Regions
+import com.amazonaws.services.ec2.model.DescribeInstancesRequest
+import com.amazonaws.services.ec2.model.DescribeInstancesResult
+import com.amazonaws.services.ec2.model.DescribeTagsRequest
+import com.amazonaws.services.ec2.AmazonEC2Client
+import com.amazonaws.services.ec2.model.CreateTagsRequest
+import com.amazonaws.services.ec2.model.Filter
+import com.amazonaws.services.ec2.model.Instance
+import com.amazonaws.services.ec2.model.Tag
 
 class NodeRegistry {
 
+
+    public static final String TAG_PREFIX = "tag:"
+    //taken from aws
+    public static final String STACK_NAME = "usergrid:stack-name";
+    public static final String NODE_TYPE = "usergrid:node_type";
+
     private String accessKey = (String) System.getenv().get("AWS_ACCESS_KEY")
     private String secretKey = (String) System.getenv().get("AWS_SECRET_KEY")
     private String stackName = (String) System.getenv().get("STACK_NAME")
-    private hostName = (String) System.getenv().get("PUBLIC_HOSTNAME")
+    private String instanceId = (String) System.getenv().get("EC2_INSTANCE_ID");
+    private String region = (String) System.getenv().get("EC2_REGION");
     private String domain = stackName
 
-    private def creds;
-    private def sdbClient;
+    private BasicAWSCredentials creds;
+    private AmazonEC2Client ec2Client;
 
 
     NodeRegistry() {
-        while ( true ) {
-            try {
-                // creates domain or no-op if it already exists
-                creds = new BasicAWSCredentials(accessKey, secretKey)
-                sdbClient = new AmazonSimpleDBClient(creds)
-                sdbClient.createDomain(new CreateDomainRequest(domain))
-
-            } catch ( Exception e ) {
-                continue
-            }
-            break
+
+        if (region == null) {
+            throw new IllegalArgumentException("EC2_REGION must be defined")
+        }
+
+        if (instanceId == null) {
+            throw new IllegalArgumentException("EC2_INSTANCE_ID must be defined")
+        }
+
+        if (stackName == null) {
+            throw new IllegalArgumentException("STACK_NAME must be defined")
+        }
+
+        if (accessKey == null) {
+            throw new IllegalArgumentException("AWS_ACCESS_KEY must be defined")
+        }
+
+        if (secretKey == null) {
+            throw new IllegalArgumentException("AWS_SECRET_KEY must be defined")
         }
+
+        creds = new BasicAWSCredentials(accessKey, secretKey)
+        ec2Client = new AmazonEC2Client(creds)
+        def regionEnum = Regions.fromName(region);
+        ec2Client.setRegion(Region.getRegion(regionEnum))
+
+
     }
 
     /**
@@ -55,64 +85,85 @@ class NodeRegistry {
      * @param defNodeType
      */
     def searchNode(def nodeType) {
-        //order by create time, if we have a conflict, then order by item name
-        def selectResult = sdbClient.select(new SelectRequest((String) \
-            "select * from `${domain}` where itemName() is not null AND createtime is not
null AND nodetype = '${nodeType}'  order by createtime"))
-        def result = []
-
-        for (item in selectResult.getItems()) {
-            def hostname = item.getName()
-            result.add(hostname)
+
+        def stackNameFilter = new Filter(TAG_PREFIX+STACK_NAME).withValues(stackName);
+        def nodeTypeFilter = new Filter(TAG_PREFIX+NODE_TYPE).withValues(nodeType);
+
+        def describeRequest = new DescribeInstancesRequest().withFilters(stackNameFilter,
nodeTypeFilter);
+
+
+        def nodes = ec2Client.describeInstances(describeRequest)
+
+        //sort by created date
+        def servers = [];
+
+        for(reservation in nodes.getReservations()){
+
+            //TODO, add these to a list then sort them by date, then name
+            for(instance in reservation.getInstances()){
+
+                servers.add(new ServerEntry(instance.launchTime, instance.publicDnsName))
+            }
+
         }
 
-        return result
 
+
+        return createResults(servers);
     }
 
-    /**
-     * Get the entire database back in raw form
-     * @return
-     */
-    def selectAll() {
-        def selectResult = sdbClient.select(new SelectRequest((String) "select * from `${domain}`")).getItems()
+    def createResults(def servers){
+
+
+        Collections.sort(servers);
+        def results = [];
 
-        return selectResult;
+        for(server in servers){
+            results.add(server.publicIp)
+        }
+
+        return results;
     }
 
+
+
     /**
      * Add the node to the database if it doesn't exist
      */
     def addNode(def nodeType) {
-        def gar = new GetAttributesRequest(domain, hostName);
-        def response = sdbClient.getAttributes(gar);
-        if (response.getAttributes().size() == 1) {
-            println "Already registered"
-            def attrs = response.getAttributes()
-            for (att in attrs) {
-                println("${hostName} -> ${att.getName()} : ${att.getValue()}")
-            }
 
-            return false;
-
-        } else {
-            println "Registering..."
-            def stackAtt = new ReplaceableAttribute("nodetype", nodeType, true)
-            def nowString = Calendar.getInstance(TimeZone.getTimeZone('UTC')).format("yyyy-MM-dd'T'HH:mm:ss.SSSZ")
-            def createTime = new ReplaceableAttribute("createtime", nowString, true)
-            def attrs = new ArrayList()
-            attrs.add(stackAtt)
-            attrs.add(createTime)
-            def par = new PutAttributesRequest(domain, hostName, attrs)
-            sdbClient.putAttributes(par);
-            println "Registration done."
-            return true;
-        }
+        //add the node type
+        def tagRequest = new CreateTagsRequest().withTags(new Tag(NODE_TYPE, nodeType), new
Tag(STACK_NAME, stackName)).withResources(instanceId)
+
+
+
+        ec2Client.createTags(tagRequest)
 
 
     }
 
-    def deleteRegistry(){
-        sdbClient.deleteDomain(new DeleteDomainRequest(domain))
+
+    class ServerEntry implements Comparable<ServerEntry>{
+        private final Date launchDate;
+        private final String publicIp;
+
+        ServerEntry(final Date launchDate, final String publicIp) {
+            this.launchDate = launchDate
+            this.publicIp = publicIp
+        }
+
+        @Override
+        int compareTo(final ServerEntry o) {
+            if(launchDate.before(o.launchDate)){
+                -1;
+            }else if (launchDate.after(o.launchDate)){
+                return 1;
+            }
+
+            return publicIp.compareTo(o.publicIp);
+
+
+        }
     }
 
 }

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/b12130f0/stack/awscluster/src/main/groovy/registry_list.groovy
----------------------------------------------------------------------
diff --git a/stack/awscluster/src/main/groovy/registry_list.groovy b/stack/awscluster/src/main/groovy/registry_list.groovy
deleted file mode 100644
index ca3bcda..0000000
--- a/stack/awscluster/src/main/groovy/registry_list.groovy
+++ /dev/null
@@ -1,36 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- *  contributor license agreements.  The ASF licenses this file to You
- * under the Apache License, Version 2.0 (the "License"); you may not
- * use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.  For additional information regarding
- * copyright in this work, please see the NOTICE file in the top level
- * directory of this distribution.
- */
-
-
-// 
-// registry_list.groovy 
-// 
-// List contents of registry as debugging aid. Not used in any other scripts. 
-//
-NodeRegistry registry = new NodeRegistry();
-
-
-
-
-for (item in registry.selectAll()) {
-    for(att in item.getAttributes()){
-        println "${item.getName()} -> ${att.getName()} : ${att.getValue()}"
-    }
-
-    println "\n"
-}


Mime
View raw message