usergrid-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From snoopd...@apache.org
Subject [38/50] [abbrv] git commit: Fixes raid configuration to mount ephemeral drives in autoscaling instances
Date Mon, 20 Oct 2014 13:47:06 GMT
Fixes raid configuration to mount ephemeral drives in autoscaling instances


Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/d553e77e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/d553e77e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/d553e77e

Branch: refs/heads/two-dot-o-events
Commit: d553e77e403e1d9f15c4bd9afdab888770ed28f0
Parents: af29b4b
Author: Todd Nine <toddnine@apache.org>
Authored: Fri Oct 17 17:35:22 2014 -0600
Committer: Todd Nine <toddnine@apache.org>
Committed: Fri Oct 17 17:35:22 2014 -0600

----------------------------------------------------------------------
 .../src/main/dist/init_instance/create_raid0.sh | 19 +++----
 .../dist/init_instance/create_raid0_2disks.sh   | 54 --------------------
 .../main/dist/init_instance/init_db_server.sh   |  2 +-
 .../main/dist/init_instance/init_es_server.sh   |  2 +-
 .../dist/init_instance/init_graphite_server.sh  |  4 +-
 .../main/dist/init_instance/init_rest_server.sh |  2 +-
 .../dist/init_instance/install_elasticsearch.sh |  3 ++
 .../main/groovy/configure_elasticsearch.groovy  |  5 +-
 stack/awscluster/ugcluster-cf.json              | 28 ++++++++--
 9 files changed, 46 insertions(+), 73 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/d553e77e/stack/awscluster/src/main/dist/init_instance/create_raid0.sh
----------------------------------------------------------------------
diff --git a/stack/awscluster/src/main/dist/init_instance/create_raid0.sh b/stack/awscluster/src/main/dist/init_instance/create_raid0.sh
index 296cb9f..43d9f37 100644
--- a/stack/awscluster/src/main/dist/init_instance/create_raid0.sh
+++ b/stack/awscluster/src/main/dist/init_instance/create_raid0.sh
@@ -18,6 +18,12 @@
 #  directory of this distribution.
 #
 
+
+
+# WARNING: this does not work yet (and is not used by any other script)
+
+
+
 # install postfix first and prevent it from prompting the user
 debconf-set-selections <<< "postfix postfix/mailname string your.hostname.com"
 debconf-set-selections <<< "postfix postfix/main_mailer_type string 'Internet Site'"
@@ -30,15 +36,10 @@ apt-get -y --force-yes install mdadm
 # unmount /dev/xvdb so we can use it in our setup
 umount /mnt
 
-# create & format partition on each of our four disks
-(echo o; echo n; echo p; echo 1; echo ; echo; echo w) | fdisk /dev/xvdb
-(echo o; echo n; echo p; echo 1; echo ; echo; echo w) | fdisk /dev/xvdc
-(echo o; echo n; echo p; echo 1; echo ; echo; echo w) | fdisk /dev/xvdd
-(echo o; echo n; echo p; echo 1; echo ; echo; echo w) | fdisk /dev/xvde
 
+#We only support 2 ephemeral disks.  Most c3.x instances only have 2 disks and they're our
target
 # create striped RAID0 device with our four disks
-mdadm --create --verbose /dev/md0 --level=stripe \
---raid-devices=4 /dev/xvdb1 /dev/xvdc1 /dev/xvdd1 /dev/xvde1
+yes | mdadm --create --verbose /dev/md0 --level=stripe --raid-devices=2 -c 256  /dev/xvdb
/dev/xvdc
 
 # save config
 mdadm --detail --scan | tee /etc/mdadm/mdadm.conf
@@ -46,5 +47,5 @@ mdadm --detail --scan | tee /etc/mdadm/mdadm.conf
 # create, mount and save disk to stab
 mkfs.ext4 -b 4096 -E stride=32,stripe-width=128 -L Stripe /dev/md0
 mkdir -p /mnt
-echo "/dev/md0  /mnt  auto  defaults,nobootwait,noatime 0 2" | tee /etc/fstab
-mount /mnt 
+echo "/dev/md0  /mnt/  ext4    defaults,noatime,nofail 0 0" >> /etc/fstab
+mount /dev/md0 /mnt

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/d553e77e/stack/awscluster/src/main/dist/init_instance/create_raid0_2disks.sh
----------------------------------------------------------------------
diff --git a/stack/awscluster/src/main/dist/init_instance/create_raid0_2disks.sh b/stack/awscluster/src/main/dist/init_instance/create_raid0_2disks.sh
deleted file mode 100644
index 25733d3..0000000
--- a/stack/awscluster/src/main/dist/init_instance/create_raid0_2disks.sh
+++ /dev/null
@@ -1,54 +0,0 @@
-#!/bin/bash
-
-# 
-#  Licensed to the Apache Software Foundation (ASF) under one or more
-#   contributor license agreements.  The ASF licenses this file to You
-#  under the Apache License, Version 2.0 (the "License"); you may not
-#  use this file except in compliance with the License.
-#  You may obtain a copy of the License at
-# 
-#      http://www.apache.org/licenses/LICENSE-2.0
-# 
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License.  For additional information regarding
-#  copyright in this work, please see the NOTICE file in the top level
-#  directory of this distribution.
-#
-
-
-
-# WARNING: this does not work yet (and is not used by any other script)
-
-
-
-# install postfix first and prevent it from prompting the user
-debconf-set-selections <<< "postfix postfix/mailname string your.hostname.com"
-debconf-set-selections <<< "postfix postfix/main_mailer_type string 'Internet Site'"
-apt-get install -y postfix
-
-# install mdadm RAID controller
-apt-get -y --force-yes install mdadm
-
-# m1.xlarge has four 414GB disks, but only /dev/xvdb is mounted
-# unmount /dev/xvdb so we can use it in our setup
-umount /mnt
-
-# create & format partition on each of our four disks
-(echo o; echo n; echo p; echo 1; echo ; echo; echo w) | fdisk /dev/xvdb
-(echo o; echo n; echo p; echo 1; echo ; echo; echo w) | fdisk /dev/xvdc
-
-# create striped RAID0 device with our four disks
-mdadm --create --verbose /dev/md0 --level=stripe \
---raid-devices=4 /dev/xvdb1 /dev/xvdc1 /dev/xvdd1 /dev/xvde1
-
-# save config
-mdadm --detail --scan | tee /etc/mdadm/mdadm.conf
-
-# create, mount and save disk to stab
-mkfs.ext4 -b 4096 -E stride=32,stripe-width=128 -L Stripe /dev/md0
-mkdir -p /mnt
-echo "/dev/md0  /mnt  auto  defaults,nobootwait,noatime 0 2" | tee /etc/fstab
-mount /mnt 

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/d553e77e/stack/awscluster/src/main/dist/init_instance/init_db_server.sh
----------------------------------------------------------------------
diff --git a/stack/awscluster/src/main/dist/init_instance/init_db_server.sh b/stack/awscluster/src/main/dist/init_instance/init_db_server.sh
index a46e7f9..42a7ebf 100644
--- a/stack/awscluster/src/main/dist/init_instance/init_db_server.sh
+++ b/stack/awscluster/src/main/dist/init_instance/init_db_server.sh
@@ -54,7 +54,7 @@ ln -s /home/ubuntu/.groovy /root/.groovy
 
 # tag last so we can see in the console so that we know what's running
 cd /usr/share/usergrid/scripts
-groovy tag_instance.groovy BUILD-IN-PROGRESS
+groovy tag_instance.groovy -BUILD-IN-PROGRESS
 
 cd /usr/share/usergrid/init_instance
 ./install_oraclejdk.sh 

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/d553e77e/stack/awscluster/src/main/dist/init_instance/init_es_server.sh
----------------------------------------------------------------------
diff --git a/stack/awscluster/src/main/dist/init_instance/init_es_server.sh b/stack/awscluster/src/main/dist/init_instance/init_es_server.sh
index fbe41ab..4b6a7ec 100644
--- a/stack/awscluster/src/main/dist/init_instance/init_es_server.sh
+++ b/stack/awscluster/src/main/dist/init_instance/init_es_server.sh
@@ -53,7 +53,7 @@ ln -s /home/ubuntu/.groovy /root/.groovy
 
 # tag last so we can see in the console so that we know what's running
 cd /usr/share/usergrid/scripts
-groovy tag_instance.groovy BUILD-IN-PROGRESS
+groovy tag_instance.groovy -BUILD-IN-PROGRESS
 
 cd /usr/share/usergrid/init_instance
 ./install_oraclejdk.sh 

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/d553e77e/stack/awscluster/src/main/dist/init_instance/init_graphite_server.sh
----------------------------------------------------------------------
diff --git a/stack/awscluster/src/main/dist/init_instance/init_graphite_server.sh b/stack/awscluster/src/main/dist/init_instance/init_graphite_server.sh
index 6b1c81a..64d82bc 100644
--- a/stack/awscluster/src/main/dist/init_instance/init_graphite_server.sh
+++ b/stack/awscluster/src/main/dist/init_instance/init_graphite_server.sh
@@ -31,8 +31,6 @@ dpkg-reconfigure -f noninteractive tzdata
 . /etc/profile.d/aws-credentials.sh
 . /etc/profile.d/usergrid-env.sh
 
-cd /usr/share/usergrid/init_instance
-./create_raid0.sh
 
 # Install the easy stuff
 PKGS="ntp unzip groovy tomcat7 curl"
@@ -53,7 +51,7 @@ ln -s /home/ubuntu/.groovy /root/.groovy
 
 # tag last so we can see in the console so that we know what's running
 cd /usr/share/usergrid/scripts
-groovy tag_instance.groovy BUILD-IN-PROGRESS
+groovy tag_instance.groovy -BUILD-IN-PROGRESS
 
 cd /usr/share/usergrid/init_instance
 ./install_oraclejdk.sh

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/d553e77e/stack/awscluster/src/main/dist/init_instance/init_rest_server.sh
----------------------------------------------------------------------
diff --git a/stack/awscluster/src/main/dist/init_instance/init_rest_server.sh b/stack/awscluster/src/main/dist/init_instance/init_rest_server.sh
index dde1a00..6df1f59 100644
--- a/stack/awscluster/src/main/dist/init_instance/init_rest_server.sh
+++ b/stack/awscluster/src/main/dist/init_instance/init_rest_server.sh
@@ -43,7 +43,7 @@ ln -s /home/ubuntu/.groovy /root/.groovy
 
 # tag last so we can see in the console so that we know what's running
 cd /usr/share/usergrid/scripts
-groovy tag_instance.groovy BUILD-IN-PROGRESS
+groovy tag_instance.groovy -BUILD-IN-PROGRESS
 
 
 # Build environment for Groovy scripts

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/d553e77e/stack/awscluster/src/main/dist/init_instance/install_elasticsearch.sh
----------------------------------------------------------------------
diff --git a/stack/awscluster/src/main/dist/init_instance/install_elasticsearch.sh b/stack/awscluster/src/main/dist/init_instance/install_elasticsearch.sh
index a3c0b64..06ca60b 100644
--- a/stack/awscluster/src/main/dist/init_instance/install_elasticsearch.sh
+++ b/stack/awscluster/src/main/dist/init_instance/install_elasticsearch.sh
@@ -38,6 +38,9 @@ chown elasticsearch /mnt/log/elasticsearch
 # Configure ElasticSearch
 cd /usr/share/usergrid/scripts
 
+echo "vm.swappiness = 0" >> /etc/sysctl.conf
+sysctl -p
+
 # No need to do this, elasticsearch nodes are also cassandra nodes
 groovy registry_register.groovy elasticsearch
 groovy wait_for_instances.groovy elasticsearch ${ES_NUM_SERVERS}

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/d553e77e/stack/awscluster/src/main/groovy/configure_elasticsearch.groovy
----------------------------------------------------------------------
diff --git a/stack/awscluster/src/main/groovy/configure_elasticsearch.groovy b/stack/awscluster/src/main/groovy/configure_elasticsearch.groovy
index 549a23b..845f9d1 100644
--- a/stack/awscluster/src/main/groovy/configure_elasticsearch.groovy
+++ b/stack/awscluster/src/main/groovy/configure_elasticsearch.groovy
@@ -32,6 +32,9 @@ import com.amazonaws.services.simpledb.model.*
 String hostName  = (String)System.getenv().get("PUBLIC_HOSTNAME")
 def clusterName  = (String)System.getenv().get("ES_CLUSTER_NAME")
 
+int esNumServers = ((String)System.getenv().get("ES_NUM_SERVERS")).toInteger()
+int quorum = esNumServers/2+1;
+
 NodeRegistry registry = new NodeRegistry();
 
 // build seed list by listing all Elasticsearch nodes found in SimpleDB domain with our stackName
@@ -45,7 +48,7 @@ for (hostname in selectResult) {
 
 def elasticSearchConfig = """
 cluster.name: ${clusterName}
-discovery.zen.minimum_master_nodes: 4
+discovery.zen.minimum_master_nodes: ${quorum}
 discovery.zen.ping.multicast.enabled: false
 discovery.zen.ping.unicast.hosts: [${esnodes}]
 node:

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/d553e77e/stack/awscluster/ugcluster-cf.json
----------------------------------------------------------------------
diff --git a/stack/awscluster/ugcluster-cf.json b/stack/awscluster/ugcluster-cf.json
index b4d5a0b..dea9694 100644
--- a/stack/awscluster/ugcluster-cf.json
+++ b/stack/awscluster/ugcluster-cf.json
@@ -505,7 +505,7 @@
                         "export TYPE=cass\n",
                         "export STACK_NAME=", { "Ref":"AWS::StackName" }, "\n", "\n",
                         "export PUBLIC_HOSTNAME=`(curl -s http://169.254.169.254/latest/meta-data/public-hostname)`\n",
-                        "export INTERNAL_HOSTNAME=`(curl http://169.254.169.254/latest/meta-data/local-ipv4)`\n",

+                        "export INTERNAL_HOSTNAME=`(curl http://169.254.169.254/latest/meta-data/local-ipv4)`\n",
                         "\n",
                         "export EC2_INSTANCE_ID=`ec2metadata --instance-id`\n",
                         "export EC2_REGION=", { "Ref":"AWS::Region" },
@@ -595,7 +595,18 @@
                     {
                         "Ref": "CassSecurityGroup"
                     }
-                ]
+                ],
+
+                 "BlockDeviceMappings": [
+                     {
+                         "DeviceName": "/dev/sdb",
+                         "VirtualName": "ephemeral0"
+                     },
+                     {
+                         "DeviceName": "/dev/sdc",
+                         "VirtualName": "ephemeral1"
+                      }
+                    ]
             }
       },
         "CassAutoScalingGroup": {
@@ -745,7 +756,18 @@
                     {
                         "Ref": "ESSecurityGroup"
                     }
-                ]
+                ],
+
+                 "BlockDeviceMappings": [
+                     {
+                         "DeviceName": "/dev/sdb",
+                         "VirtualName": "ephemeral0"
+                     },
+                     {
+                         "DeviceName": "/dev/sdc",
+                         "VirtualName": "ephemeral1"
+                      }
+                    ]
             }
       },
         "ESAutoScalingGroup": {


Mime
View raw message