myriad-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From javiro...@apache.org
Subject incubator-myriad git commit: MYRIAD-263 Add HDFS as system service in DC/OS vagrant testbed
Date Sat, 05 May 2018 07:44:17 GMT
Repository: incubator-myriad
Updated Branches:
  refs/heads/master f5e7406f8 -> 47c7be0de


MYRIAD-263 Add HDFS as system service in DC/OS vagrant testbed

The local development environment based on vagrant-libvirt (DC/OS)
needs deploy HDFS as foundation for Myriad. We have available HDFS
as Mesos framework at Mesosphere Universe, however running this
framework is not straightforward. So I propose deploy HDFS as systemd services,
this is easy and allow us focus the in our development (Myriad).

JIRA:
    [MYRIAD-263] https://issues.apache.org/jira/browse/MYRIAD-263

Pull Request:
    Closes #110

Author:    Javi Roman <jroman.espinar@gmail.com>
Date:      Sat Apr 21 13:11:48 2018 +0200


Project: http://git-wip-us.apache.org/repos/asf/incubator-myriad/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-myriad/commit/47c7be0d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-myriad/tree/47c7be0d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-myriad/diff/47c7be0d

Branch: refs/heads/master
Commit: 47c7be0de1c91470c766dda06c432c282b22bb01
Parents: f5e7406
Author: Javi Roman <jroman.espinar@gmail.com>
Authored: Sat Apr 21 13:11:48 2018 +0200
Committer: Javi Roman <javiroman@apache.org>
Committed: Sat May 5 09:40:51 2018 +0200

----------------------------------------------------------------------
 vagrant/libvirt/dcos/Vagrantfile                |  6 +-
 .../provisioning/group_vars/all/defaults.yml    | 26 ++++++++
 .../provisioning/roles/common/tasks/main.yml    |  3 +
 .../roles/common/templates/java.sh.j2           | 23 +++++++
 .../roles/dcos-setup/tasks/main.yml             |  1 -
 .../provisioning/roles/devel/tasks/main.yml     | 42 +++++++++++++
 .../roles/hadoop/files/core-site.xml            | 23 +++++++
 .../hadoop/files/hadoop-hdfs-datanode.service   | 16 +++++
 .../hadoop/files/hadoop-hdfs-namenode.service   | 15 +++++
 .../roles/hadoop/files/hdfs-site.xml            | 34 ++++++++++
 .../roles/hadoop/tasks/datanode.yml             | 42 +++++++++++++
 .../roles/hadoop/tasks/download.yml             | 51 +++++++++++++++
 .../provisioning/roles/hadoop/tasks/main.yml    | 56 +++++++++++++++++
 .../roles/hadoop/tasks/namenode.yml             | 65 ++++++++++++++++++++
 .../roles/jbod-disks/tasks/main.yml             | 64 +++++++++++++++++++
 .../roles/nfs-client/tasks/main.yml             | 30 +++++++++
 .../dcos/provisioning/roles/nfs/tasks/main.yml  | 41 ++++++++++++
 .../provisioning/roles/nfs/templates/exports.j2 | 34 ++++++++++
 vagrant/libvirt/dcos/provisioning/site.yml      | 19 +++++-
 19 files changed, 583 insertions(+), 8 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/47c7be0d/vagrant/libvirt/dcos/Vagrantfile
----------------------------------------------------------------------
diff --git a/vagrant/libvirt/dcos/Vagrantfile b/vagrant/libvirt/dcos/Vagrantfile
index 886c227..0f0022f 100644
--- a/vagrant/libvirt/dcos/Vagrantfile
+++ b/vagrant/libvirt/dcos/Vagrantfile
@@ -19,10 +19,8 @@
 # specific language governing permissions and limitations
 # under the License.
 #
-
-unless defined? VAGRANTFILE_API_VERSION 
-    VAGRANTFILE_API_VERSION = "2"
-end
+#
+VAGRANTFILE_API_VERSION = "2"
 
 # Tested with Vagrant version:
 Vagrant.require_version ">= 1.7.2"

http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/47c7be0d/vagrant/libvirt/dcos/provisioning/group_vars/all/defaults.yml
----------------------------------------------------------------------
diff --git a/vagrant/libvirt/dcos/provisioning/group_vars/all/defaults.yml b/vagrant/libvirt/dcos/provisioning/group_vars/all/defaults.yml
index 07e69e5..b8ce22e 100644
--- a/vagrant/libvirt/dcos/provisioning/group_vars/all/defaults.yml
+++ b/vagrant/libvirt/dcos/provisioning/group_vars/all/defaults.yml
@@ -33,6 +33,7 @@ system_packages:
   - xz 
   - unzip 
   - curl 
+  - nfs-utils
   - ipset
   - sysstat
 
@@ -42,3 +43,28 @@ system_epel_packages:
 
 with_dcos_version: "{{ lookup('env','DCOS_VERSION') }}"
 with_hadoop_version: "{{ lookup('env','HADOOP_VERSION') }}"
+
+# NFS enablement for sharing folders between nodes
+nfs_packages:
+  - rpcbind
+  - nfs-server
+  - nfs-lock
+  - nfs-idmap
+nfs_shared_folder: /opt/shared
+
+# Source code location for Mesos and Myriad
+myriad_cloned_folder: myriad
+myriad_repository: https://github.com/myriad-framework/myriad
+myriad_repo_branch: master
+
+# Hadoop configrations 
+hadoop_version: "{{ lookup('env','HADOOP_VERSION') }}"
+hadoop_binary_download_url: https://archive.apache.org/dist/hadoop/common/
+hadoop_binary_package: "{{ hadoop_binary_download_url }}/hadoop-{{ hadoop_version }}/hadoop-{{
hadoop_version }}.tar.gz"
+
+# Hadoop folders
+hadoop_data_root_folder: /cluster
+hadoop_namenode_folder: "{{ hadoop_data_root_folder }}/nn"
+hadoop_logs_folder: "{{ nfs_shared_folder }}/hadoop/logs"
+hadoop_etc_folder: "{{ nfs_shared_folder }}/hadoop/etc/hadoop"
+

http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/47c7be0d/vagrant/libvirt/dcos/provisioning/roles/common/tasks/main.yml
----------------------------------------------------------------------
diff --git a/vagrant/libvirt/dcos/provisioning/roles/common/tasks/main.yml b/vagrant/libvirt/dcos/provisioning/roles/common/tasks/main.yml
index f814085..9c772b2 100644
--- a/vagrant/libvirt/dcos/provisioning/roles/common/tasks/main.yml
+++ b/vagrant/libvirt/dcos/provisioning/roles/common/tasks/main.yml
@@ -67,6 +67,9 @@
 - name: Restarting SSH daemon
   service: name=sshd state=restarted
 
+- name: Setup wide JAVA_HOME at /etc/profile.d
+  template: src=java.sh.j2 dest=/etc/profile.d/java.sh
+
 - name: Configure /etc/host files
   template: src=hosts.j2 dest=/etc/hosts
 

http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/47c7be0d/vagrant/libvirt/dcos/provisioning/roles/common/templates/java.sh.j2
----------------------------------------------------------------------
diff --git a/vagrant/libvirt/dcos/provisioning/roles/common/templates/java.sh.j2 b/vagrant/libvirt/dcos/provisioning/roles/common/templates/java.sh.j2
new file mode 100644
index 0000000..dba8aaa
--- /dev/null
+++ b/vagrant/libvirt/dcos/provisioning/roles/common/templates/java.sh.j2
@@ -0,0 +1,23 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+{% if ansible_fqdn in groups['bootstrap'] %}
+export JAVA_HOME=/usr/lib/jvm/java
+{% else %}
+export JAVA_HOME=/opt/mesosphere/active/java/usr/java
+{% endif %}

http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/47c7be0d/vagrant/libvirt/dcos/provisioning/roles/dcos-setup/tasks/main.yml
----------------------------------------------------------------------
diff --git a/vagrant/libvirt/dcos/provisioning/roles/dcos-setup/tasks/main.yml b/vagrant/libvirt/dcos/provisioning/roles/dcos-setup/tasks/main.yml
index 35e8af7..a826dec 100644
--- a/vagrant/libvirt/dcos/provisioning/roles/dcos-setup/tasks/main.yml
+++ b/vagrant/libvirt/dcos/provisioning/roles/dcos-setup/tasks/main.yml
@@ -102,7 +102,6 @@
 - name: Downloading DC/OS CLI
   shell: >
     curl -fLsS --retry 20 -Y 100000 -y 60 
-    https://downloads.dcos.io/binaries/cli/linux/x86-64/dcos-1.11/dcos
     https://downloads.dcos.io/binaries/cli/linux/x86-64/{{ versions[with_dcos_version].cli
}}/dcos
     -o dcos && touch .cliskip
   args:

http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/47c7be0d/vagrant/libvirt/dcos/provisioning/roles/devel/tasks/main.yml
----------------------------------------------------------------------
diff --git a/vagrant/libvirt/dcos/provisioning/roles/devel/tasks/main.yml b/vagrant/libvirt/dcos/provisioning/roles/devel/tasks/main.yml
new file mode 100644
index 0000000..95b8581
--- /dev/null
+++ b/vagrant/libvirt/dcos/provisioning/roles/devel/tasks/main.yml
@@ -0,0 +1,42 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+---
+- name: Install the 'Development tools' package group
+  yum:
+    name: "@Development tools"
+    state: present
+
+- name: Install OpenJDK for Myriad Development
+  yum: name=java-1.8.0-openjdk-devel state=present
+
+- name: Setup shared folder permissions
+  file:
+    path: "{{ nfs_shared_folder }}"
+    state: directory
+    owner: vagrant
+    group: vagrant
+    mode: 0755
+
+- name: Checkout Apache Myriad repo
+  become_user: vagrant
+  git:
+    repo: "{{ myriad_repository }}"
+    dest: "{{ nfs_shared_folder }}/{{ myriad_cloned_folder }}"
+    version: "{{ myriad_repo_branch }}"
+    update: no

http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/47c7be0d/vagrant/libvirt/dcos/provisioning/roles/hadoop/files/core-site.xml
----------------------------------------------------------------------
diff --git a/vagrant/libvirt/dcos/provisioning/roles/hadoop/files/core-site.xml b/vagrant/libvirt/dcos/provisioning/roles/hadoop/files/core-site.xml
new file mode 100644
index 0000000..002d4e3
--- /dev/null
+++ b/vagrant/libvirt/dcos/provisioning/roles/hadoop/files/core-site.xml
@@ -0,0 +1,23 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License. See accompanying LICENSE file.
+-->
+
+<!-- Put site-specific property overrides in this file. -->
+<configuration>
+	<property>
+		<name>fs.default.name</name>
+		<value>hdfs://m1:9001</value>
+	</property>
+</configuration>

http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/47c7be0d/vagrant/libvirt/dcos/provisioning/roles/hadoop/files/hadoop-hdfs-datanode.service
----------------------------------------------------------------------
diff --git a/vagrant/libvirt/dcos/provisioning/roles/hadoop/files/hadoop-hdfs-datanode.service
b/vagrant/libvirt/dcos/provisioning/roles/hadoop/files/hadoop-hdfs-datanode.service
new file mode 100644
index 0000000..572e61d
--- /dev/null
+++ b/vagrant/libvirt/dcos/provisioning/roles/hadoop/files/hadoop-hdfs-datanode.service
@@ -0,0 +1,16 @@
+[Unit]
+Description=Apache Hadoop HDFS Datanode
+After=syslog.target network.target remote-fs.target nss-lookup.target
+
+[Service]
+Environment="JAVA_HOME=/opt/mesosphere/active/java/usr/java"
+Environment="HADOOP_PID_DIR=/var/run/hadoop/hdfs"
+Environment="HADOOP_LOG_DIR=/opt/shared/hadoop/logs/"
+ExecStart=/opt/shared/hadoop/sbin/hadoop-daemon.sh --config /opt/shared/hadoop/etc/hadoop
start datanode
+User=hdfs
+Group=hadoop
+Type=forking
+Type=forking
+
+[Install]
+WantedBy=multi-user.target

http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/47c7be0d/vagrant/libvirt/dcos/provisioning/roles/hadoop/files/hadoop-hdfs-namenode.service
----------------------------------------------------------------------
diff --git a/vagrant/libvirt/dcos/provisioning/roles/hadoop/files/hadoop-hdfs-namenode.service
b/vagrant/libvirt/dcos/provisioning/roles/hadoop/files/hadoop-hdfs-namenode.service
new file mode 100644
index 0000000..9395ad8
--- /dev/null
+++ b/vagrant/libvirt/dcos/provisioning/roles/hadoop/files/hadoop-hdfs-namenode.service
@@ -0,0 +1,15 @@
+[Unit]
+Description=Apache Hadoop HDFS Namenode
+After=syslog.target network.target remote-fs.target nss-lookup.target
+
+[Service]
+Environment="JAVA_HOME=/opt/mesosphere/active/java/usr/java"
+Environment="HADOOP_PID_DIR=/var/run/hadoop/hdfs"
+Environment="HADOOP_LOG_DIR=/opt/shared/hadoop/logs/"
+ExecStart=/opt/shared/hadoop/sbin/hadoop-daemon.sh --config /opt/shared/hadoop/etc/hadoop
start namenode
+User=hdfs
+Group=hadoop
+Type=forking
+
+[Install]
+WantedBy=multi-user.target

http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/47c7be0d/vagrant/libvirt/dcos/provisioning/roles/hadoop/files/hdfs-site.xml
----------------------------------------------------------------------
diff --git a/vagrant/libvirt/dcos/provisioning/roles/hadoop/files/hdfs-site.xml b/vagrant/libvirt/dcos/provisioning/roles/hadoop/files/hdfs-site.xml
new file mode 100644
index 0000000..aaa8f64
--- /dev/null
+++ b/vagrant/libvirt/dcos/provisioning/roles/hadoop/files/hdfs-site.xml
@@ -0,0 +1,34 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License. See accompanying LICENSE file.
+-->
+
+<!-- Put site-specific property overrides in this file. -->
+
+<configuration>
+    <property>
+            <name>dfs.namenode.name.dir</name>
+            <value>/cluster/nn</value>
+    </property>
+
+    <property>
+            <name>dfs.datanode.data.dir</name>
+            <value>/cluster/1/dn,/cluster/2/dn</value>
+    </property>
+
+    <property>
+            <name>dfs.replication</name>
+            <value>3</value>
+    </property>
+</configuration>

http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/47c7be0d/vagrant/libvirt/dcos/provisioning/roles/hadoop/tasks/datanode.yml
----------------------------------------------------------------------
diff --git a/vagrant/libvirt/dcos/provisioning/roles/hadoop/tasks/datanode.yml b/vagrant/libvirt/dcos/provisioning/roles/hadoop/tasks/datanode.yml
new file mode 100644
index 0000000..9b8543e
--- /dev/null
+++ b/vagrant/libvirt/dcos/provisioning/roles/hadoop/tasks/datanode.yml
@@ -0,0 +1,42 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+---
+- name: Set right permissions for data folders
+  file:
+    path: "{{ item }}"
+    state: directory
+    owner: hdfs
+    group: hadoop
+    mode: 0755
+  with_items:
+      - "{{ hadoop_data_root_folder }}/1/dn"
+      - "{{ hadoop_data_root_folder }}/2/dn"
+  tags: hadoop
+
+- name: Copy Systemd DataNode Unit File
+  copy:
+    src: hadoop-hdfs-datanode.service
+    dest: /etc/systemd/system/hadoop-hdfs-datanode.service
+  tags: hadoop
+
+- name: Starting DataNode System Daemon
+  service: name=hadoop-hdfs-datanode state=started enabled=yes
+
+
+

http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/47c7be0d/vagrant/libvirt/dcos/provisioning/roles/hadoop/tasks/download.yml
----------------------------------------------------------------------
diff --git a/vagrant/libvirt/dcos/provisioning/roles/hadoop/tasks/download.yml b/vagrant/libvirt/dcos/provisioning/roles/hadoop/tasks/download.yml
new file mode 100644
index 0000000..9491c8d
--- /dev/null
+++ b/vagrant/libvirt/dcos/provisioning/roles/hadoop/tasks/download.yml
@@ -0,0 +1,51 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+---
+- name: Check that you can connect (GET) to a page and it returns a status 200
+  uri:
+    url: "{{ hadoop_binary_download_url }}"
+  tags: download_hadoop
+
+- name: Download Hadoop binary from Apache releases
+  get_url:
+    url: "{{ hadoop_binary_package }}"
+    dest: "{{ nfs_shared_folder }}"
+  tags: download_hadoop
+
+- name: Check if already moved
+  stat: path="{{ nfs_shared_folder }}/hadoop"
+  register: hadoop_stat
+  tags: download_hadoop
+
+- name: Unarchive a file that is already on the remote machine
+  unarchive:
+    src: "{{ nfs_shared_folder }}/hadoop-{{ hadoop_version }}.tar.gz"
+    dest: "{{ nfs_shared_folder }}"
+    remote_src: yes
+    owner: vagrant
+    group: vagrant
+  when: hadoop_stat.stat.isdir is not defined
+  tags: download_hadoop
+
+- name: Rename folder name
+  command: mv "{{ nfs_shared_folder }}/hadoop-{{ hadoop_version }}" "{{ nfs_shared_folder
}}/hadoop"
+  when: hadoop_stat.stat.isdir is not defined
+  tags: download_hadoop
+
+

http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/47c7be0d/vagrant/libvirt/dcos/provisioning/roles/hadoop/tasks/main.yml
----------------------------------------------------------------------
diff --git a/vagrant/libvirt/dcos/provisioning/roles/hadoop/tasks/main.yml b/vagrant/libvirt/dcos/provisioning/roles/hadoop/tasks/main.yml
new file mode 100644
index 0000000..47bb5ac
--- /dev/null
+++ b/vagrant/libvirt/dcos/provisioning/roles/hadoop/tasks/main.yml
@@ -0,0 +1,56 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+---
+- include: download.yml
+  when: inventory_hostname == 'bt'
+  tags: hadoop
+
+- name: Create Hadoop group
+  group:
+    name: hadoop
+    state: present
+  tags: hadoop
+
+- name: Create Hadoop users
+  user: 
+    name: "{{ item }}"
+    password: "{{ 'vagrant' | password_hash('sha512') }}"
+    shell: /bin/bash 
+    group: hadoop
+  with_items:
+      - yarn
+      - hdfs
+      - mapred
+  tags: hadoop
+
+- name: Create Hadoop PID file folder
+  file: 
+    path: /var/run/hadoop/hdfs 
+    state: directory
+    mode: 0755
+    owner: hdfs
+    group: hadoop
+
+- include: namenode.yml
+  when: inventory_hostname in groups['dcos-masters']
+  tags: hadoop
+
+- include: datanode.yml
+  when: inventory_hostname in groups['dcos-agents-private'] and inventory_hostname != 'a1'
+  tags: hadoop

http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/47c7be0d/vagrant/libvirt/dcos/provisioning/roles/hadoop/tasks/namenode.yml
----------------------------------------------------------------------
diff --git a/vagrant/libvirt/dcos/provisioning/roles/hadoop/tasks/namenode.yml b/vagrant/libvirt/dcos/provisioning/roles/hadoop/tasks/namenode.yml
new file mode 100644
index 0000000..7ac742d
--- /dev/null
+++ b/vagrant/libvirt/dcos/provisioning/roles/hadoop/tasks/namenode.yml
@@ -0,0 +1,65 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+---
+- name: Create Hadoop metadata folder
+  file:
+    path: "{{ hadoop_namenode_folder }}"
+    state: directory
+    owner: hdfs
+    group: hadoop
+    mode: 0755
+  tags: hadoop
+
+- name: Create local logs folder for Hadoop daemons
+  file:
+    path: "{{ hadoop_logs_folder }}"
+    state: directory
+    owner: hdfs
+    group: hadoop
+    mode: 0775
+  tags: hadoop
+
+- name: Copy minimal (core,hdfs)-site.xml files
+  copy:
+    src: "{{ item }}"
+    dest: "{{ hadoop_etc_folder }}"
+  with_items:
+    - core-site.xml
+    - hdfs-site.xml
+  tags: hadoop
+
+- name: Format Hadoop filesystem
+  shell : "source /etc/profile.d/java.sh && {{ nfs_shared_folder }}/hadoop/bin/hdfs
namenode -format && touch .skipformat"
+  become: yes
+  become_user: hdfs
+  args:
+      creates: .skipformat
+  run_once: yes
+  tags: hadoop
+
+- name: Copy Systemd NameNode Unit File
+  copy:
+    src: hadoop-hdfs-namenode.service
+    dest: /etc/systemd/system/hadoop-hdfs-namenode.service
+  tags: hadoop
+
+- name: Starting NameNode System Daemon
+  service: name=hadoop-hdfs-namenode state=started enabled=yes
+
+

http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/47c7be0d/vagrant/libvirt/dcos/provisioning/roles/jbod-disks/tasks/main.yml
----------------------------------------------------------------------
diff --git a/vagrant/libvirt/dcos/provisioning/roles/jbod-disks/tasks/main.yml b/vagrant/libvirt/dcos/provisioning/roles/jbod-disks/tasks/main.yml
new file mode 100644
index 0000000..e13911e
--- /dev/null
+++ b/vagrant/libvirt/dcos/provisioning/roles/jbod-disks/tasks/main.yml
@@ -0,0 +1,64 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+---
+- name: Create partitions on JBOD disks
+  parted:
+    device: "{{ item }}"
+    number: 1
+    state: present
+  with_items:
+    - /dev/vdb
+    - /dev/vdc
+  tags: disk
+
+- name: Create a xfs filesystems on JBOD disk
+  filesystem:
+    fstype: xfs
+    dev: "{{ item }}"
+  with_items:
+    - /dev/vdb1
+    - /dev/vdc1
+  tags: disk
+
+- name: Create Hadoop data folders
+  file:
+    path: "{{ item }}"
+    state: directory
+    mode: 0755
+  with_items:
+      - "{{ hadoop_data_root_folder }}/1/dn"
+      - "{{ hadoop_data_root_folder }}/2/dn"
+  tags: disk
+
+- name: Mount up JBOD devices 1
+  mount:
+    path: "{{ hadoop_data_root_folder }}/1/dn"
+    src: /dev/vdb1
+    fstype: xfs
+    state: mounted
+  tags: disk
+
+- name: Mount up JBOD devices 2
+  mount:
+    path: "{{ hadoop_data_root_folder }}/2/dn"
+    src: /dev/vdc1
+    fstype: xfs
+    state: mounted
+  tags: disk
+

http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/47c7be0d/vagrant/libvirt/dcos/provisioning/roles/nfs-client/tasks/main.yml
----------------------------------------------------------------------
diff --git a/vagrant/libvirt/dcos/provisioning/roles/nfs-client/tasks/main.yml b/vagrant/libvirt/dcos/provisioning/roles/nfs-client/tasks/main.yml
new file mode 100644
index 0000000..e571220
--- /dev/null
+++ b/vagrant/libvirt/dcos/provisioning/roles/nfs-client/tasks/main.yml
@@ -0,0 +1,30 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+---
+- name: Create NFS mount point
+  file:
+    path: "{{ nfs_shared_folder }}"
+    state: directory
+
+- name: Mount NFS shared folder
+  mount:
+    path: "{{ nfs_shared_folder }}"
+    src: "bt:{{ nfs_shared_folder }}"
+    fstype: nfs
+    state: mounted

http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/47c7be0d/vagrant/libvirt/dcos/provisioning/roles/nfs/tasks/main.yml
----------------------------------------------------------------------
diff --git a/vagrant/libvirt/dcos/provisioning/roles/nfs/tasks/main.yml b/vagrant/libvirt/dcos/provisioning/roles/nfs/tasks/main.yml
new file mode 100644
index 0000000..9e8111d
--- /dev/null
+++ b/vagrant/libvirt/dcos/provisioning/roles/nfs/tasks/main.yml
@@ -0,0 +1,41 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+---
+- name: Creates directory
+  file:
+    path: /opt/shared
+    state: directory
+    owner: vagrant
+    group: vagrant
+    mode: 0775
+  when: inventory_hostname != 'a1'
+
+- name: Copy exports file.
+  template:
+    src: exports.j2
+    dest: /etc/exports
+    owner: root
+    group: root
+    mode: 0644
+  when: inventory_hostname == 'bt'
+
+- name: Enable NFS in all nodes
+  service: name={{ item }} state=started enabled=yes
+  with_items:
+    - "{{ nfs_packages }}"

http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/47c7be0d/vagrant/libvirt/dcos/provisioning/roles/nfs/templates/exports.j2
----------------------------------------------------------------------
diff --git a/vagrant/libvirt/dcos/provisioning/roles/nfs/templates/exports.j2 b/vagrant/libvirt/dcos/provisioning/roles/nfs/templates/exports.j2
new file mode 100644
index 0000000..aa2b5dd
--- /dev/null
+++ b/vagrant/libvirt/dcos/provisioning/roles/nfs/templates/exports.j2
@@ -0,0 +1,34 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+# /etc/exports: the access control list for filesystems which may be exported
+#   to NFS clients.  See exports(5).
+#
+# Example for NFSv2 and NFSv3:
+# /srv/homes       hostname1(rw,sync,no_subtree_check) hostname2(ro,sync,no_subtree_check)
+#
+# Example for NFSv4:
+# /srv/nfs4        gss/krb5i(rw,sync,fsid=0,crossmnt,no_subtree_check)
+# /srv/nfs4/homes  gss/krb5i(rw,sync,no_subtree_check)
+#
+{# % for item in play_hosts %}
+{{ nfs_shared_folder}} {{ hostvars[item]['ansible_host'] }}(rw,sync,no_root_squash)
+{% endfor % #}
+
+{{ nfs_shared_folder }} *(rw,sync,no_root_squash)

http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/47c7be0d/vagrant/libvirt/dcos/provisioning/site.yml
----------------------------------------------------------------------
diff --git a/vagrant/libvirt/dcos/provisioning/site.yml b/vagrant/libvirt/dcos/provisioning/site.yml
index 416aadf..530371d 100644
--- a/vagrant/libvirt/dcos/provisioning/site.yml
+++ b/vagrant/libvirt/dcos/provisioning/site.yml
@@ -27,34 +27,47 @@
 #   non in special order, you cannot control the order
 #   of roles within a play.
 
-# Play 0
+# Play 1
 - hosts: all
   roles:
     - common
     - docker
+    - { role: jbod-disks, 
+        when: "inventory_hostname in groups['dcos-agents-private'] and inventory_hostname
!= 'a1'" }
 
-# Play 1
+# Play 2
 - hosts: bootstrap 
   roles:
     - passwordless
+    - devel
+    - nfs
     - dcos-setup
     - registry
+    - hadoop
 
-# Play 2
+# Play 3
 - hosts: dcos-masters 
   strategy: free
   vars:
     node_type: master
   roles:
     - dcos-install
+    - nfs
+    - nfs-client
+    - hadoop
 
+# Play 4
 - hosts: dcos-agents-private 
   strategy: free
   vars:
     node_type: slave
   roles:
     - dcos-install
+    - nfs
+    - nfs-client
+    - hadoop
 
+# Play 4
 - hosts: dcos-agents-public
   strategy: free
   vars:


Mime
View raw message