gora-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From lewi...@apache.org
Subject [1/3] git commit: GORA-74 Implement Rackspace Cloud Orchestration in GoraCI
Date Sun, 28 Sep 2014 03:10:33 GMT
Repository: gora
Updated Branches:
  refs/heads/master 0c5eacf4e -> b712d9f9d


GORA-74 Implement Rackspace Cloud Orchestration in GoraCI


Project: http://git-wip-us.apache.org/repos/asf/gora/repo
Commit: http://git-wip-us.apache.org/repos/asf/gora/commit/3c5bd0ee
Tree: http://git-wip-us.apache.org/repos/asf/gora/tree/3c5bd0ee
Diff: http://git-wip-us.apache.org/repos/asf/gora/diff/3c5bd0ee

Branch: refs/heads/master
Commit: 3c5bd0ee53ab5d3651980d39c5a5b20002ababc9
Parents: a72c5d3
Author: Lewis John McGibbney <lewis.j.mcgibbney@jpl.nasa.gov>
Authored: Wed Sep 17 17:46:25 2014 -0400
Committer: Lewis John McGibbney <lewis.j.mcgibbney@jpl.nasa.gov>
Committed: Wed Sep 17 17:46:25 2014 -0400

----------------------------------------------------------------------
 bin/gora                                        |   5 +
 .../org/apache/gora/store/DataStoreFactory.java |   3 -
 .../libext/gora-core-tests-0.5-SNAPSHOT.jar     | Bin 0 -> 128135 bytes
 gora-goraci/pom.xml                             |  24 ++-
 .../rackspace/RackspaceOrchestration.java       | 151 +++++++++++++++++++
 .../gora/goraci/rackspace/package-info.java     |  22 +++
 gora-goraci/src/main/resources/gora.properties  |  55 +++++--
 pom.xml                                         |   2 +-
 8 files changed, 244 insertions(+), 18 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/gora/blob/3c5bd0ee/bin/gora
----------------------------------------------------------------------
diff --git a/bin/gora b/bin/gora
index d660099..7126972 100755
--- a/bin/gora
+++ b/bin/gora
@@ -46,6 +46,7 @@ if [ $# = 0 ]; then
   echo "  goracompiler               Run Compiler"
   echo "  specificcompiler           Run Avro Specific Compiler"
   echo "  dynamocompiler             Run Gora DynamoDB Compiler"
+  echo "  goracisetup                Run the GoraCI Rackspace orchestration setup"
   echo "  logmanager                 Run the tutorial log manager"
   echo "  loganalytics               Run the tutorial log analytics"
   echo "  junit         	     Run the given JUnit test"
@@ -121,6 +122,10 @@ elif [ "$COMMAND" = "dynamocompiler" ] ; then
   MODULE=gora-dynamodb
   CLASSPATH=$CLASSPATH:$GORA_HOME/$MODULE/target/classes/
   CLASS=org.apache.gora.dynamodb.compiler.GoraDynamoDBCompiler
+elif [ "$COMMAND" = "goracisetup" ] ; then
+  MODULE=gora-goraci
+  CLASSPATH=$CLASSPATH:$GORA_HOME/$MODULE/target/classes/
+  CLASS=org.apache.gora.goraci.rackspace.RackspaceOrchestration
 elif [ "$COMMAND" = "logmanager" ] ; then
   MODULE=gora-tutorial
   CLASSPATH=$CLASSPATH:$GORA_HOME/$MODULE/target/classes/

http://git-wip-us.apache.org/repos/asf/gora/blob/3c5bd0ee/gora-core/src/main/java/org/apache/gora/store/DataStoreFactory.java
----------------------------------------------------------------------
diff --git a/gora-core/src/main/java/org/apache/gora/store/DataStoreFactory.java b/gora-core/src/main/java/org/apache/gora/store/DataStoreFactory.java
index 4408e80..b1d9c75 100644
--- a/gora-core/src/main/java/org/apache/gora/store/DataStoreFactory.java
+++ b/gora-core/src/main/java/org/apache/gora/store/DataStoreFactory.java
@@ -20,9 +20,6 @@ package org.apache.gora.store;
 import java.io.IOException;
 import java.io.InputStream;
 import java.util.Properties;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.apache.gora.persistency.Persistent;

http://git-wip-us.apache.org/repos/asf/gora/blob/3c5bd0ee/gora-goraci/libext/gora-core-tests-0.5-SNAPSHOT.jar
----------------------------------------------------------------------
diff --git a/gora-goraci/libext/gora-core-tests-0.5-SNAPSHOT.jar b/gora-goraci/libext/gora-core-tests-0.5-SNAPSHOT.jar
new file mode 100755
index 0000000..48f0bb5
Binary files /dev/null and b/gora-goraci/libext/gora-core-tests-0.5-SNAPSHOT.jar differ

http://git-wip-us.apache.org/repos/asf/gora/blob/3c5bd0ee/gora-goraci/pom.xml
----------------------------------------------------------------------
diff --git a/gora-goraci/pom.xml b/gora-goraci/pom.xml
index d3d66c2..7011437 100644
--- a/gora-goraci/pom.xml
+++ b/gora-goraci/pom.xml
@@ -17,7 +17,8 @@
   specific language governing permissions and limitations
   under the License.
 -->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+  xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
 
   <modelVersion>4.0.0</modelVersion>
   <parent>
@@ -31,9 +32,9 @@
   <name>Apache Gora :: GoraCI</name>
   <url>http://gora.apache.org</url>
   <description>The GoraCI test runs many ingest clients that continually create 
-	linked lists containing 25 million nodes. At some point the clients are stopped 
-	and a map reduce job is run to ensure no linked list has a hole. A hole indicates 
-	data was lost.</description>
+  linked lists containing 25 million nodes. At some point the clients are stopped 
+  and a map reduce job is run to ensure no linked list has a hole. A hole indicates 
+  data was lost.</description>
   <organization>
     <name>The Apache Software Foundation</name>
     <url>http://www.apache.org/</url>
@@ -48,7 +49,7 @@
   </ciManagement>
 
   <properties>
-    <jclouds.version>1.7.3</jclouds.version>
+    <jclouds.version>1.8.0</jclouds.version>
   </properties>
 
   <build>
@@ -262,6 +263,13 @@
       <groupId>org.apache.gora</groupId>
       <artifactId>gora-core</artifactId>
     </dependency>
+    <dependency>
+      <groupId>org.apache.gora</groupId>
+      <artifactId>gora-core-tests</artifactId>
+      <version>0.5-SNAPSHOT</version>
+      <scope>system</scope>
+      <systemPath>${project.basedir}/libext/gora-core-tests-0.5-SNAPSHOT.jar</systemPath>
+    </dependency>
 
     <dependency>
       <groupId>org.apache.hadoop</groupId>
@@ -277,6 +285,12 @@
       <artifactId>commons-cli</artifactId>
     </dependency>
 
+    <!--  dependency>
+      <groupId>com.google.guava</groupId>
+      <artifactId>guava</artifactId>
+      <scope>runtime</scope>
+    </dependency-->
+
     <!-- jclouds dependencies -->
     <dependency>
       <groupId>org.apache.jclouds.driver</groupId>

http://git-wip-us.apache.org/repos/asf/gora/blob/3c5bd0ee/gora-goraci/src/main/java/org/apache/gora/goraci/rackspace/RackspaceOrchestration.java
----------------------------------------------------------------------
diff --git a/gora-goraci/src/main/java/org/apache/gora/goraci/rackspace/RackspaceOrchestration.java
b/gora-goraci/src/main/java/org/apache/gora/goraci/rackspace/RackspaceOrchestration.java
new file mode 100644
index 0000000..b27baaa
--- /dev/null
+++ b/gora-goraci/src/main/java/org/apache/gora/goraci/rackspace/RackspaceOrchestration.java
@@ -0,0 +1,151 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.gora.goraci.rackspace;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.NoSuchElementException;
+import java.util.Properties;
+
+import org.apache.gora.mock.store.MockDataStore;
+import org.apache.gora.store.DataStoreFactory;
+import org.jclouds.ContextBuilder;
+import org.jclouds.openstack.nova.v2_0.NovaApi;
+import org.jclouds.openstack.nova.v2_0.domain.Flavor;
+import org.jclouds.openstack.nova.v2_0.domain.Image;
+import org.jclouds.openstack.nova.v2_0.domain.KeyPair;
+import org.jclouds.openstack.nova.v2_0.domain.ServerCreated;
+import org.jclouds.openstack.nova.v2_0.extensions.KeyPairApi;
+import org.jclouds.openstack.nova.v2_0.features.FlavorApi;
+import org.jclouds.openstack.nova.v2_0.features.ImageApi;
+import org.jclouds.openstack.nova.v2_0.features.ServerApi;
+import org.jclouds.openstack.nova.v2_0.options.CreateServerOptions;
+import org.jclouds.openstack.nova.v2_0.predicates.ServerPredicates;
+
+import com.google.common.base.Charsets;
+import com.google.common.io.Files;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * This is the main class for initiating Rackspace cloud
+ * topography for use within the GoraCI job.
+ * @param <K>
+ *
+ */
+public class RackspaceOrchestration<K> {
+
+  private static final Logger LOG = LoggerFactory.getLogger(RackspaceOrchestration.class);
+
+  private static String RS_CONTINENT = "rackspace.continent";
+
+  private static String RS_USERNAME = "rackspace.username";
+
+  private static String RS_APIKEY = "rackspace.apikey";
+
+  private static String RS_REGION = "rackspace.region";
+
+  private static String RS_IMAGEID = "rackspace.imageid";
+
+  private static String RS_FLAVORID = "rackspace.flavorid";
+
+  private static String RS_SERVERNAME = "rackspace.servername";
+
+  private static String RS_NUM_SERVERS = "rackspace.num.servers";
+
+  /* we can optionally upload our own keys for security, otherwise 
+   * we can have Rackspace autocreate one for us*/
+  private static String RS_PUBKEY = "rackspace.uploadpubkey";
+
+  /**
+   * @param args
+   * @throws IllegalAccessException 
+   * @throws InstantiationException 
+   * @throws NoSuchElementException 
+   */
+  public static void main(String[] args) throws NoSuchElementException, InstantiationException,
IllegalAccessException {
+    Properties properties = DataStoreFactory.createProps();
+    String rsContinent = DataStoreFactory.findProperty(properties, MockDataStore.class.newInstance(),

+        RS_CONTINENT, "rackspace-cloudservers-us");
+    String rsUser = DataStoreFactory.findProperty(properties, MockDataStore.class.newInstance(),
RS_USERNAME, "asf-gora");
+    String rsApiKey = DataStoreFactory.findProperty(properties, MockDataStore.class.newInstance(),
RS_APIKEY, null);
+    String rs_region = DataStoreFactory.findProperty(properties, MockDataStore.class.newInstance(),
RS_REGION, "DFW");
+    String rs_flavourId = DataStoreFactory.findProperty(properties, MockDataStore.class.newInstance(),
RS_FLAVORID, null);
+    String rs_imageId = DataStoreFactory.findProperty(properties, MockDataStore.class.newInstance(),
RS_IMAGEID, null);
+    int num_servers = Integer.parseInt(DataStoreFactory.findProperty(properties, MockDataStore.class.newInstance(),
RS_NUM_SERVERS, "10"));
+    String serverName = DataStoreFactory.findProperty(properties, MockDataStore.class.newInstance(),
RS_SERVERNAME, "goraci_test_server");
+
+    NovaApi novaApi = ContextBuilder.newBuilder(rsContinent).credentials(rsUser, rsApiKey).buildApi(NovaApi.class);
+    LOG.info("Defining Rackspace cloudserver continent as: {}, and region: {}.", rsContinent,
rs_region);
+
+    //Choose operating system
+    ImageApi imageApi = novaApi.getImageApiForZone(rs_region);
+    Image image = imageApi.get(rs_imageId);
+    //Choose hardware configuration
+    FlavorApi flavorApi = novaApi.getFlavorApiForZone(rs_region);
+    Flavor flavor = flavorApi.get(rs_flavourId);
+    LOG.info("Defining Rackspace cloudserver flavors as: {}, with image id's {}", rs_flavourId,
rs_imageId);
+
+    //Manage keypairs
+    KeyPairApi keyPairApi = novaApi.getKeyPairExtensionForZone(rs_region).get();
+
+    File keyPairFile = null;
+    String publicKey = null;
+    //Use your own .pub key which should be on CP
+    if (DataStoreFactory.findBooleanProperty(properties, MockDataStore.class.newInstance(),
RS_PUBKEY, "true")) {
+      keyPairFile = new File("~/.ssh/id_rsa.pub");
+      LOG.info("Uploading local public key from ~/.ssh/id_rsa.pub to Rackspace...");
+    } else {
+      //obtain existing key from Rackspace
+      if (keyPairApi.get("goraci-keypair") != null) {
+        publicKey = keyPairApi.get("goraci-keypair").getPublicKey();
+        LOG.info("Obtained existing public key 'goraci-keypair' from Rackspace...");
+      } else {
+        //have Rackspace generate us a key
+        LOG.info("Generating local keypair 'goraci-keypair' and uploading to Rackspace...");
+        KeyPair keyPair = keyPairApi.create("goraci-keypair");
+        keyPairFile = new File("/target/goraci-keypair.pem");
+        try {
+          Files.write(keyPair.getPrivateKey(), keyPairFile, Charsets.UTF_8);
+        } catch (IOException e) {
+          e.printStackTrace();
+        }
+        try {
+          publicKey = Files.toString(keyPairFile, Charsets.UTF_8);
+        } catch (IOException e) {
+          e.printStackTrace();
+        }
+        keyPairApi.createWithPublicKey("goraci-keypair", publicKey);
+      }
+    }
+    
+    ServerApi serverApi = novaApi.getServerApiForZone(rs_region);
+    CreateServerOptions options = CreateServerOptions.Builder.keyPairName("goraci-keypair");
+    ServerCreated serverCreated = null;
+    for (int i = 0; i < num_servers; i++) {
+      if (serverName != null) {
+        serverCreated = serverApi.create(serverName + num_servers, rs_imageId, rs_flavourId,
options);
+
+      } else {
+        serverCreated = serverApi.create("goraci_test_server" + num_servers, image.getId(),
flavor.getId(), options);
+      }
+      ServerPredicates.awaitActive(serverApi).apply(serverCreated.getId());
+      LOG.info("Provisioned node " + i + 1 + " of " + num_servers + " with name: " + serverName
+ i);
+    }
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/gora/blob/3c5bd0ee/gora-goraci/src/main/java/org/apache/gora/goraci/rackspace/package-info.java
----------------------------------------------------------------------
diff --git a/gora-goraci/src/main/java/org/apache/gora/goraci/rackspace/package-info.java
b/gora-goraci/src/main/java/org/apache/gora/goraci/rackspace/package-info.java
new file mode 100644
index 0000000..5c526ee
--- /dev/null
+++ b/gora-goraci/src/main/java/org/apache/gora/goraci/rackspace/package-info.java
@@ -0,0 +1,22 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/**
+ * Main package for all Rackspace-enabling functionality for the 
+ * GoraCI test.
+ *
+ */
+package org.apache.gora.goraci.rackspace;
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/gora/blob/3c5bd0ee/gora-goraci/src/main/resources/gora.properties
----------------------------------------------------------------------
diff --git a/gora-goraci/src/main/resources/gora.properties b/gora-goraci/src/main/resources/gora.properties
index 7b4733b..bec6623 100644
--- a/gora-goraci/src/main/resources/gora.properties
+++ b/gora-goraci/src/main/resources/gora.properties
@@ -13,6 +13,52 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+
+# Available continents are cloudservers-us and cloudservers-uk
+gora.datastore.rackspace.continent=rackspace-cloudservers-us
+gora.datastore.rackspace.username=asf-gora
+gora.datastore.rackspace.apikey=
+#Currently you can pick from the following regions
+# * US Northern Virginia - IAD=Suppliers.ofInstance(https://iad.servers.api.rackspacecloud.com/v2/910883)
+# * US Chicago - ORD=Suppliers.ofInstance(https://ord.servers.api.rackspacecloud.com/v2/910883)
+# * US Dallas - DFW=Suppliers.ofInstance(https://dfw.servers.api.rackspacecloud.com/v2/910883)
+# * EU London - lon TBC
+# * Asia-Pacific Hong Kong - HKG=Suppliers.ofInstance(https://hkg.servers.api.rackspacecloud.com/v2/910883)
+# * Asia-Pacific Sydney - SYD=Suppliers.ofInstance(https://syd.servers.api.rackspacecloud.com/v2/910883)
+gora.datastore.rackspace.region=DFW
+# A list of images can be found at the link provided below
+# http://docs.rackspace.com/servers/api/v2/cs-devguide/content/List_Images-d1e4435.html
+# In this instance we pick Ubuntu 13.10 (Saucy Salamander) (PVHVM)
+gora.datastore.rackspace.imageid=6110edfe-8589-4bb1-aa27-385f12242627
+# Flavor Information can be found here
+#http://docs.rackspace.com/servers/api/v2/cs-devguide/content/List_Flavors-d1e4188.html
+# Can be one of the following
+# * 2 - 512MB Standard Instance
+# * 3 - 1GB Standard Instance
+# * 4 - 2GB Standard Instance
+# * 5 - 4GB Standard Instance
+# * 6 - 8GB Standard Instance
+# * 7 - 15GB Standard Instance
+# * 8 - 30GB Standard Instance
+# * onmetal-compute1 - OnMetal Compute v1
+# * onmetal-io1 - OnMetal I/O v1
+# * onmetal-memory1 - OnMetal Memory v1
+# * performance1-1 - 1 GB Performance
+# * performance1-2 - 2 GB Performance
+# * performance1-4 - 4 GB Performance
+# * performance1-8 - 8 GB Performance
+# * performance2-15 - 15 GB Performance
+# * performance2-30 - 30 GB Performance
+# * performance2-60 - 60 GB Performance
+# * performance2-90 - 90 GB Performance
+# * performance2-120 - 120 GB Performance
+gora.datastore.rackspace.flavorid=6 
+gora.datastore.rackspace.servername=goraci_test_server
+gora.datastore.rackspace.num.servers=20
+# we can optionally upload our own keys for security, otherwise we can have Rackspace
+# create one for us
+gora.datastore.rackspace.uploadpubkey=false
+
 ###############################
 # Default Accumulo properties #
 ###############################
@@ -22,15 +68,6 @@ gora.datastore.accumulo.zookeepers=localhost
 gora.datastore.accumulo.user=root
 gora.datastore.accumulo.password=secret
 
-###############################
-# Default SqlStore properties #
-###############################
-
-#gora.sqlstore.jdbc.driver=org.hsqldb.jdbcDriver
-#gora.sqlstore.jdbc.url=jdbc:hsqldb:hsql://localhost/nutchtest
-# gora.sqlstore.jdbc.user=
-# gora.sqlstore.jdbc.password=
-
 ################################
 # Default AvroStore properties #
 ################################

http://git-wip-us.apache.org/repos/asf/gora/blob/3c5bd0ee/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 4e1f160..825261f 100644
--- a/pom.xml
+++ b/pom.xml
@@ -635,7 +635,7 @@
     <restlet.version>2.2.1</restlet.version>
 
     <!-- Misc Dependencies -->
-    <guava.version>13.0</guava.version>
+    <guava.version>17.0</guava.version>
     <commons-lang.version>2.6</commons-lang.version>
     <jdom.version>1.1.2</jdom.version>
     <hsqldb.version>2.2.8</hsqldb.version>


Mime
View raw message