asterixdb-notifications mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From "Yingyi Bu (Code Review)" <do-not-re...@asterixdb.incubator.apache.org>
Subject Change in asterixdb[master]: Make aws scripts work with all latest instance types that ar...
Date Tue, 21 Mar 2017 01:33:46 GMT
Yingyi Bu has uploaded a new change for review.

  https://asterix-gerrit.ics.uci.edu/1603

Change subject: Make aws scripts work with all latest instance types that are equiped with
instance-stores.
......................................................................

Make aws scripts work with all latest instance types that are
equiped with instance-stores.

Change-Id: I16efff22a029e316c7a7dc0402c0131e0c0bdea4
---
M asterixdb/asterix-server/src/main/opt/ansible/yaml/instance_start.yml
M asterixdb/asterix-server/src/main/opt/ansible/yaml/start_cc.yml
A asterixdb/asterix-server/src/main/opt/aws/bin/gen_volumes.sh
M asterixdb/asterix-server/src/main/opt/aws/bin/start.sh
M asterixdb/asterix-server/src/main/opt/aws/conf/aws_types.yml
M asterixdb/asterix-server/src/main/opt/aws/yaml/aws_start.yml
M asterixdb/asterix-server/src/main/opt/aws/yaml/instance_start.yml
M asterixdb/asterix-server/src/main/opt/aws/yaml/mount.yml
8 files changed, 110 insertions(+), 49 deletions(-)


  git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb refs/changes/03/1603/1

diff --git a/asterixdb/asterix-server/src/main/opt/ansible/yaml/instance_start.yml b/asterixdb/asterix-server/src/main/opt/ansible/yaml/instance_start.yml
index 13de7cd..e1d005d 100644
--- a/asterixdb/asterix-server/src/main/opt/ansible/yaml/instance_start.yml
+++ b/asterixdb/asterix-server/src/main/opt/ansible/yaml/instance_start.yml
@@ -19,7 +19,7 @@
 
 - hosts: [localhost,]
   vars:
-       temp_dir: "/tmp/asterixdb"
+       temp_dir: "../conf/instance"
        inventory: "{{ temp_dir }}/inventory"
        ccconf: "{{ temp_dir }}/cc.conf"
   tasks:
diff --git a/asterixdb/asterix-server/src/main/opt/ansible/yaml/start_cc.yml b/asterixdb/asterix-server/src/main/opt/ansible/yaml/start_cc.yml
index 9d371d2..66fd79c 100644
--- a/asterixdb/asterix-server/src/main/opt/ansible/yaml/start_cc.yml
+++ b/asterixdb/asterix-server/src/main/opt/ansible/yaml/start_cc.yml
@@ -19,7 +19,7 @@
 
 - name: Copy cluster config to CC
   synchronize:
-    src: /tmp/asterixdb/cc.conf
+    src: ../conf/instance/cc.conf
     dest: "{{ binarydir }}/cc.conf"
 
 - name: Update cluster config
diff --git a/asterixdb/asterix-server/src/main/opt/aws/bin/gen_volumes.sh b/asterixdb/asterix-server/src/main/opt/aws/bin/gen_volumes.sh
new file mode 100755
index 0000000..dd22394
--- /dev/null
+++ b/asterixdb/asterix-server/src/main/opt/aws/bin/gen_volumes.sh
@@ -0,0 +1,37 @@
+#!/bin/bash
+# ------------------------------------------------------------
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+# ------------------------------------------------------------
+
+
+count=$1
+dest=$2
+
+if [ $count -le 0 ]
+then
+    printf "vols: []\n" >$dest
+    exit 0
+fi
+
+printf "vols:\n" >$dest
+
+for i in $(seq 1 $count)
+do
+  printf '  - { device_name: /dev/xvd%c, ephemeral: ephemeral%d }\n' `printf "\x$(printf
%x \`expr 97 + $i\`)"` \
+    $(($i-1)) >> $dest
+done
diff --git a/asterixdb/asterix-server/src/main/opt/aws/bin/start.sh b/asterixdb/asterix-server/src/main/opt/aws/bin/start.sh
index a836b4e..8bffaa4 100755
--- a/asterixdb/asterix-server/src/main/opt/aws/bin/start.sh
+++ b/asterixdb/asterix-server/src/main/opt/aws/bin/start.sh
@@ -30,7 +30,7 @@
 ansible-playbook -i "localhost," $AWS_PATH/yaml/aws_start.yml
 
 # Generates an Ansible inventory file and an AsterixDB configuration file.
-temp=/tmp/asterixdb
+temp=$AWS_PATH/conf/instance
 inventory=$temp/inventory
 
 # Installs asterixdb on all AWS instances.
diff --git a/asterixdb/asterix-server/src/main/opt/aws/conf/aws_types.yml b/asterixdb/asterix-server/src/main/opt/aws/conf/aws_types.yml
index e0a060e..72b3d63 100644
--- a/asterixdb/asterix-server/src/main/opt/aws/conf/aws_types.yml
+++ b/asterixdb/asterix-server/src/main/opt/aws/conf/aws_types.yml
@@ -1,37 +1,50 @@
-one_ephemeral:
-  - { device_name: /dev/xvdb, ephemeral: ephemeral0 }
-two_ephemeral:
-  - { device_name: /dev/xvdb, ephemeral: ephemeral0 }
-  - { device_name: /dev/xvdc, ephemeral: ephemeral1 }
-three_ephemeral:
-  - { device_name: /dev/xvdb, ephemeral: ephemeral0 }
-  - { device_name: /dev/xvdc, ephemeral: ephemeral1 }
-  - { device_name: /dev/xvdd, ephemeral: ephemeral2 }
-four_ephemeral:
-  - { device_name: /dev/xvdb, ephemeral: ephemeral0 }
-  - { device_name: /dev/xvdc, ephemeral: ephemeral1 }
-  - { device_name: /dev/xvdd, ephemeral: ephemeral2 }
-  - { device_name: /dev/xvde, ephemeral: ephemeral3 }
+# ------------------------------------------------------------
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+# ------------------------------------------------------------
 
 ephemeral_volumes:
-  c1.medium: "{{ one_ephemeral }}"
-  c1.xlarge: "{{ four_ephemeral }}"
-  c3.large: "{{ two_ephemeral }}"
-  c3.xlarge: "{{ two_ephemeral }}"
-  c3.2xlarge: "{{ two_ephemeral }}"
-  c3.4xlarge: "{{ two_ephemeral }}"
-  c3.8xlarge: "{{ two_ephemeral }}"
-  i2.xlarge: "{{ one_ephemeral }}"
-  i2.2xlarge: "{{ two_ephemeral }}"
-  i2.4xlarge: "{{ four_ephemeral }}"
-  m1.small: "{{ one_ephemeral }}"
-  m1.medium: "{{ one_ephemeral }}"
-  m1.large: "{{ two_ephemeral }}"
-  m1.xlarge: "{{ four_ephemeral }}"
-  m2.xlarge: "{{ one_ephemeral }}"
-  m2.2xlarge: "{{ one_ephemeral }}"
-  m2.4xlarge: "{{ two_ephemeral }}"
-  m3.medium: "{{ one_ephemeral }}"
-  m3.large: "{{ one_ephemeral }}"
-  m3.xlarge: "{{ two_ephemeral }}"
-  m3.2xlarge: "{{ two_ephemeral }}"
+  c3.large: 2
+  c3.xlarge: 2
+  c3.2xlarge: 2
+  c3.4xlarge: 2
+  c3.8xlarge: 2
+  d2.xlarge: 3
+  d2.2xlarge: 6
+  d2.4xlarge: 12
+  d2.8xlarge: 24
+  i3.large: 1
+  i3.xlarge: 1
+  i3.2xlarge: 1
+  i3.4xlarge: 2
+  i3.8xlarge: 4
+  i3.16xlarge: 8
+  f1.2xlarge: 1
+  f1.16xlarge: 4
+  g2.2xlarge: 1
+  g2.8xlarge: 2
+  m3.medium: 1
+  m3.large: 1
+  m3.xlarge: 2
+  m3.2xlarge: 2
+  r3.large: 1
+  r3.xlarge: 1
+  r3.2xlarge: 1
+  r3.4xlarge: 1
+  r3.8xlarge: 2
+  x1.16xlarge: 1
+  x1.32xlarge: 2
diff --git a/asterixdb/asterix-server/src/main/opt/aws/yaml/aws_start.yml b/asterixdb/asterix-server/src/main/opt/aws/yaml/aws_start.yml
index 344bfd5..2d6adcc 100644
--- a/asterixdb/asterix-server/src/main/opt/aws/yaml/aws_start.yml
+++ b/asterixdb/asterix-server/src/main/opt/aws/yaml/aws_start.yml
@@ -21,17 +21,31 @@
   hosts: localhost
   gather_facts: false
   vars:
-     temp_dir: "/tmp/asterixdb"
+     temp_dir: "{{ playbook_dir }}/../conf/instance"
+     bin_dir: "{{ playbook_dir }}/../bin"
      inventory: "{{ temp_dir }}/inventory"
      ccconf: "{{ temp_dir }}/cc.conf"
      home_dir: "/home/{{ user }}"
-     vols: "{{ ephemeral_volumes[instance_type]|default([]) }}"
+     num_vols: "{{ ephemeral_volumes[instance_type]|default(0) }}"
   tasks:
     - include_vars: ../conf/aws_settings.yml
     - include_vars: ../conf/aws_types.yml
 
     - name: Clean the temporary directory
-      shell: rm -rf "{{ temp_dir }}"
+      file:
+        path: "{{ temp_dir }}"
+        state: absent
+
+    - name: Create local temporary directory
+      file:
+        path: "{{ temp_dir }}"
+        state: directory
+
+    - name: Generate instance-store volumes for AWS instances
+      shell: "{{ playbook_dir }}/../bin/gen_volumes.sh {{ num_vols }} {{ temp_dir }}/volumes"
+
+    - name: Load generated volumes
+      include_vars: "{{ temp_dir }}/volumes"
 
     - name: Launch all instances
       ec2:
@@ -48,11 +62,6 @@
             Name: "{{ tag }}"
         volumes: "{{ vols }}"
       register: ec2
-
-    - name: Create local temporary directory
-      file:
-        path: "{{ temp_dir }}"
-        state: directory
 
     - name: Set CC node
       set_fact:
@@ -107,7 +116,7 @@
       when: vols|length == 0
 
     - name: Generates multiple iodevice paths
-      shell: printf "devices:\n" >> "{{ temp_dir }}/devices"
+      shell: printf "devices:\n" > "{{ temp_dir }}/devices"
       when: vols|length > 0
 
     - name: Populate multiple iodevices
@@ -126,3 +135,5 @@
 
     - name: Populate the NC command to the cluster configuration file {{ ccconf }}
       shell: printf "command=asterixnc\n" >> "{{ ccconf }}"
+
+
diff --git a/asterixdb/asterix-server/src/main/opt/aws/yaml/instance_start.yml b/asterixdb/asterix-server/src/main/opt/aws/yaml/instance_start.yml
index cf09b69..4a95ecc 100644
--- a/asterixdb/asterix-server/src/main/opt/aws/yaml/instance_start.yml
+++ b/asterixdb/asterix-server/src/main/opt/aws/yaml/instance_start.yml
@@ -25,7 +25,7 @@
 - hosts: all
   tasks:
     - include_vars: ../conf/aws_settings.yml
-    - include_vars: ../conf/aws_types.yml
+    - include_vars: ../conf/instance/volumes
 
     - name: Mount instance local stores
       include: mount.yml
diff --git a/asterixdb/asterix-server/src/main/opt/aws/yaml/mount.yml b/asterixdb/asterix-server/src/main/opt/aws/yaml/mount.yml
index 2e16476..03a740b 100644
--- a/asterixdb/asterix-server/src/main/opt/aws/yaml/mount.yml
+++ b/asterixdb/asterix-server/src/main/opt/aws/yaml/mount.yml
@@ -23,6 +23,6 @@
     sudo mkdir "/data{{ vol.0 + 1 }}"
     sudo mount "{{ vol.1.device_name }}" "/data{{ vol.0 + 1 }}"
     sudo chown -R $USER "/data{{ vol.0 + 1 }}"
-  with_indexed_items: "{{ ephemeral_volumes[instance_type]|default([]) }}"
+  with_indexed_items: "{{ vols }}"
   loop_control:
     loop_var: vol

-- 
To view, visit https://asterix-gerrit.ics.uci.edu/1603
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings

Gerrit-MessageType: newchange
Gerrit-Change-Id: I16efff22a029e316c7a7dc0402c0131e0c0bdea4
Gerrit-PatchSet: 1
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Yingyi Bu <buyingyi@gmail.com>

Mime
View raw message