ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From alejan...@apache.org
Subject [2/2] ambari git commit: AMBARI-12483. Simulate Ambari Agents to test 3k Node Cluster (Pengcheng Xu via alejandro)
Date Thu, 06 Aug 2015 18:30:17 GMT
AMBARI-12483. Simulate Ambari Agents to test 3k Node Cluster (Pengcheng Xu via alejandro)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/53dd3624
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/53dd3624
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/53dd3624

Branch: refs/heads/trunk
Commit: 53dd362457a32b68640ae1647ab8639afe3b432e
Parents: 5fd790f
Author: Alejandro Fernandez <afernandez@hortonworks.com>
Authored: Thu Aug 6 11:25:53 2015 -0700
Committer: Alejandro Fernandez <afernandez@hortonworks.com>
Committed: Thu Aug 6 11:30:05 2015 -0700

----------------------------------------------------------------------
 contrib/agent-simulator/Docker/Dockerfile       |  39 --
 contrib/agent-simulator/Docker/__init__.py      |  17 -
 .../Docker/ambari_agent_start.sh                |  18 -
 .../agent-simulator/Docker/launcher_agent.py    |  70 --
 .../Linux/CentOS7/docker_install.sh             |  10 +-
 .../Linux/CentOS7/set_docker_partition.sh       |  35 +
 .../Linux/CentOS7/weave_install.sh              |   6 +-
 .../Linux/Ubuntu12/docker_install.sh            |  17 -
 .../Linux/Ubuntu12/weave_install.sh             |  19 -
 contrib/agent-simulator/README.md               | 211 ++++++
 contrib/agent-simulator/cluster.py              | 680 +++++++++++++++----
 contrib/agent-simulator/config.py               |  45 +-
 contrib/agent-simulator/config/config.ini       |  49 --
 contrib/agent-simulator/data.py                 | 104 +++
 contrib/agent-simulator/docker.py               |  70 +-
 .../agent-simulator/docker_image/Yum_Dockerfile |  56 ++
 .../agent-simulator/docker_image/__init__.py    |  17 +
 .../docker_image/ambari_agent_install.sh        |  22 +
 .../docker_image/ambari_agent_start.sh          |  19 +
 .../docker_image/launcher_agent.py              |  86 +++
 .../docker_image/package_list.txt               | 235 +++++++
 .../docker_image/ssh_service/run.sh             |  40 ++
 .../docker_image/ssh_service/set_root_pw.sh     |  29 +
 contrib/agent-simulator/example/config.ini      |  99 +++
 .../agent-simulator/launcher_ambari_server.py   |  39 ++
 contrib/agent-simulator/launcher_cluster.py     | 244 +++++--
 contrib/agent-simulator/launcher_docker.py      |  45 +-
 .../agent-simulator/launcher_service_server.py  |  49 ++
 contrib/agent-simulator/log.py                  |  35 +
 contrib/agent-simulator/network/DNS_editor.py   |  53 ++
 .../network/set_ambari_server_network.sh        |  56 ++
 .../agent-simulator/network/set_host_network.sh |  68 ++
 .../server/ambari_server_install.sh             |  22 +
 .../server/ambari_server_reset_data.sh          |  28 +
 .../server/ambari_server_start.sh               |  20 +
 contrib/agent-simulator/server_setup.sh         |  42 --
 contrib/agent-simulator/tips.txt                |  18 -
 contrib/agent-simulator/vm.py                   | 266 +++++++-
 pom.xml                                         |   1 +
 39 files changed, 2431 insertions(+), 548 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/53dd3624/contrib/agent-simulator/Docker/Dockerfile
----------------------------------------------------------------------
diff --git a/contrib/agent-simulator/Docker/Dockerfile b/contrib/agent-simulator/Docker/Dockerfile
deleted file mode 100644
index d22435a..0000000
--- a/contrib/agent-simulator/Docker/Dockerfile
+++ /dev/null
@@ -1,39 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information rega4rding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-
-# Set the bas image to CentOS 6.x
-FROM centos:7
-
-# Author
-MAINTAINER Pengcheng_Xu
-
-# Copy the files into Docker: Agent package file
-ADD agent.rpm /agent.rpm
-# Copy the files into Docker: launcher_agent.py
-ADD launcher_agent.py /launcher_agent.py
-# Copy the files into Docker: ambari_agent_start.sh
-ADD ambari_agent_start.sh /ambari_agent_start.sh
-# Copy hostname file into Docker: hosts
-ADD hosts /hosts
-
-RUN chmod 755 /ambari_agent_start.sh
-
-# Install ambari-agent
-# RUN yum install -y /agent.rpm
-RUN yum install -y wget
-RUN wget -O /etc/yum.repos.d/ambari.repo http://s3.amazonaws.com/dev.hortonworks.com/ambari/centos7/2.x/latest/2.1.0/ambaribn.repo
-RUN yum install -y epel-release
-RUN yum install -y ambari-agent
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/ambari/blob/53dd3624/contrib/agent-simulator/Docker/__init__.py
----------------------------------------------------------------------
diff --git a/contrib/agent-simulator/Docker/__init__.py b/contrib/agent-simulator/Docker/__init__.py
deleted file mode 100644
index 62aeca1..0000000
--- a/contrib/agent-simulator/Docker/__init__.py
+++ /dev/null
@@ -1,17 +0,0 @@
-'''
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements.  See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership.  The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License.  You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-'''
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/ambari/blob/53dd3624/contrib/agent-simulator/Docker/ambari_agent_start.sh
----------------------------------------------------------------------
diff --git a/contrib/agent-simulator/Docker/ambari_agent_start.sh b/contrib/agent-simulator/Docker/ambari_agent_start.sh
deleted file mode 100644
index 4ba2361..0000000
--- a/contrib/agent-simulator/Docker/ambari_agent_start.sh
+++ /dev/null
@@ -1,18 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information rega4rding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-
-#!/bin/bash
-ambari-agent start
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/ambari/blob/53dd3624/contrib/agent-simulator/Docker/launcher_agent.py
----------------------------------------------------------------------
diff --git a/contrib/agent-simulator/Docker/launcher_agent.py b/contrib/agent-simulator/Docker/launcher_agent.py
deleted file mode 100644
index 5ff008d..0000000
--- a/contrib/agent-simulator/Docker/launcher_agent.py
+++ /dev/null
@@ -1,70 +0,0 @@
-'''
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements.  See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership.  The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License.  You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-'''
-
-import sys
-import subprocess
-import time
-
-def replace_conf(server_ip):
-    f = open("/etc/ambari-agent/conf/ambari-agent.ini")
-    lines = f.readlines()
-    f.close()
-
-    f = open("/etc/ambari-agent/conf/ambari-agent.ini", "w+")
-    for line in lines:
-        line = line.replace("hostname=localhost", "hostname=" + server_ip)
-        f.write(line)
-    f.close()
-
-def run_ambari_agent():
-    # command = ["sudo", "ambari-agent", "start"]
-    # subprocess.call(command)
-    subprocess.call("./ambari_agent_start.sh")
-
-# add all the hostnames of other containers to /etc/hosts
-def add_hostnames():
-    etc_hosts = open("/etc/hosts", "a")
-    etc_hosts.write("\n")
-
-    docker_hosts = open("/hosts")
-    for line in docker_hosts.readlines():
-        etc_hosts.write(line)
-    docker_hosts.close()
-
-    etc_hosts.close()
-
-def remove_default_hostname(hostname):
-    etc_hosts = open("/etc/hosts")
-    all_resolution = etc_hosts.readlines()
-    etc_hosts.close()
-
-    etc_hosts = open("/etc/hosts", "w")
-    for line in all_resolution:
-        if hostname not in line:
-            etc_hosts.write(line)
-        else:
-            etc_hosts.write("#")
-            etc_hosts.write(line)
-    etc_hosts.close()
-
-ambari_server_ip = sys.argv[1]
-my_hostname = sys.argv[2]
-replace_conf(ambari_server_ip)
-remove_default_hostname(my_hostname)
-add_hostnames()
-run_ambari_agent()
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/ambari/blob/53dd3624/contrib/agent-simulator/Linux/CentOS7/docker_install.sh
----------------------------------------------------------------------
diff --git a/contrib/agent-simulator/Linux/CentOS7/docker_install.sh b/contrib/agent-simulator/Linux/CentOS7/docker_install.sh
index 033c2b9..dce4ecb 100644
--- a/contrib/agent-simulator/Linux/CentOS7/docker_install.sh
+++ b/contrib/agent-simulator/Linux/CentOS7/docker_install.sh
@@ -1,3 +1,4 @@
+#!/bin/bash
 # Licensed to the Apache Software Foundation (ASF) under one or more
 # contributor license agreements.  See the NOTICE file distributed with
 # this work for additional information rega4rding copyright ownership.
@@ -13,8 +14,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-#!/bin/bash
-sudo curl -O -sSL https://get.docker.com/rpm/1.7.0/centos-7/RPMS/x86_64/docker-engine-1.7.0-1.el7.centos.x86_64.rpm
-sudo yum localinstall -y --nogpgcheck docker-engine-1.7.0-1.el7.centos.x86_64.rpm
-sudo yum update -y device-mapper
+# This script will install and start the Docker service on a CentOS 7 machine
+
+sudo curl -s -O -sSL https://get.docker.com/rpm/1.7.0/centos-7/RPMS/x86_64/docker-engine-1.7.0-1.el7.centos.x86_64.rpm
+sudo yum localinstall -y -q --nogpgcheck docker-engine-1.7.0-1.el7.centos.x86_64.rpm
+sudo yum update -y -q device-mapper
 sudo service docker start
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/ambari/blob/53dd3624/contrib/agent-simulator/Linux/CentOS7/set_docker_partition.sh
----------------------------------------------------------------------
diff --git a/contrib/agent-simulator/Linux/CentOS7/set_docker_partition.sh b/contrib/agent-simulator/Linux/CentOS7/set_docker_partition.sh
new file mode 100644
index 0000000..6c116d0
--- /dev/null
+++ b/contrib/agent-simulator/Linux/CentOS7/set_docker_partition.sh
@@ -0,0 +1,35 @@
+#!/bin/bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information rega4rding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This script will set the docker use a new partition as its storage
+
+# $1 mount point to other part
+
+mount_point=$1
+docker_dir=/var/lib/docker
+
+while [ ! -d "$docker_dir" ]; do
+    echo "$docker_dir does not exist, wait for docker service to create the directory"
+    sleep 5
+done
+
+echo "move $docker_dir to partition $mount_point"
+
+service docker stop
+cp -r ${docker_dir}/* $mount_point
+rm -rf $docker_dir
+ln -s $mount_point $docker_dir
+service docker start
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/ambari/blob/53dd3624/contrib/agent-simulator/Linux/CentOS7/weave_install.sh
----------------------------------------------------------------------
diff --git a/contrib/agent-simulator/Linux/CentOS7/weave_install.sh b/contrib/agent-simulator/Linux/CentOS7/weave_install.sh
index 6788184..4ce1ad1 100644
--- a/contrib/agent-simulator/Linux/CentOS7/weave_install.sh
+++ b/contrib/agent-simulator/Linux/CentOS7/weave_install.sh
@@ -1,3 +1,4 @@
+#!/bin/bash
 # Licensed to the Apache Software Foundation (ASF) under one or more
 # contributor license agreements.  See the NOTICE file distributed with
 # this work for additional information rega4rding copyright ownership.
@@ -13,6 +14,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-#!/bin/bash
-sudo curl -L git.io/weave -o /usr/bin/weave
+# This script will install Weave on a CentOS 7 machine
+
+sudo curl -s -L git.io/weave -o /usr/bin/weave
 sudo chmod a+x /usr/bin/weave
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/ambari/blob/53dd3624/contrib/agent-simulator/Linux/Ubuntu12/docker_install.sh
----------------------------------------------------------------------
diff --git a/contrib/agent-simulator/Linux/Ubuntu12/docker_install.sh b/contrib/agent-simulator/Linux/Ubuntu12/docker_install.sh
deleted file mode 100644
index 220f1a4..0000000
--- a/contrib/agent-simulator/Linux/Ubuntu12/docker_install.sh
+++ /dev/null
@@ -1,17 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information rega4rding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-#!/bin/bash
-wget -qO- https://get.docker.com/ | sh
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/ambari/blob/53dd3624/contrib/agent-simulator/Linux/Ubuntu12/weave_install.sh
----------------------------------------------------------------------
diff --git a/contrib/agent-simulator/Linux/Ubuntu12/weave_install.sh b/contrib/agent-simulator/Linux/Ubuntu12/weave_install.sh
deleted file mode 100644
index 332488d..0000000
--- a/contrib/agent-simulator/Linux/Ubuntu12/weave_install.sh
+++ /dev/null
@@ -1,19 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information rega4rding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-#!/bin/bash
-sudo apt-get install -y curl
-sudo curl -L git.io/weave -o /usr/bin/weave
-sudo chmod a+x /usr/bin/weave
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/ambari/blob/53dd3624/contrib/agent-simulator/README.md
----------------------------------------------------------------------
diff --git a/contrib/agent-simulator/README.md b/contrib/agent-simulator/README.md
new file mode 100644
index 0000000..b8f9d17
--- /dev/null
+++ b/contrib/agent-simulator/README.md
@@ -0,0 +1,211 @@
+<!---
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements.  See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License.  You may obtain a copy of the License at [http://www.apache.org/licenses/LICENSE-2.0](http://www.apache.org/licenses/LICENSE-2.0)
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+
+Ambari-agent Simulator
+============
+This project provides a tool to create a large Ambari-agent cluster in a convenient way.
+
+## Usage:
+Run python launcher_cluster.py to see usage
+
+python launcher_cluster.py request    
+
+    request a cluster from GCE, generate the configuration for the cluster. Parameters:
+	<the name of the cluster>, suggestion: {yourname}-group-a)
+	<number of VMs>
+	<number of dockers each VMs>
+	<number of service servers inside VM>
+	<number of ambari-server>, either 0 or 1
+		
+python launcher_cluster.py up
+        
+    run one cluster, and add to another cluster. Parameters:
+	<the name of the cluster>
+		
+python launcher_cluster.py merge    
+
+    run Docker containers with Ambari-agent in all VMs of the cluster. Parameters:
+	<the name of the cluster to be merged>
+	<the name of the cluster to be extended>
+
+python launcher_cluster.py merge 
+
+    run Docker containers with Ambari-agent in all VMs of the cluster. Parameters:
+    <the name of the cluster to be merged>
+    <Weave IP of the Ambari-server>
+    <External IP of the Ambari-server>
+
+python launcher_cluster.py list    
+        
+    list all the cluster
+    
+python launcher_cluster.py show
+    
+    show cluster information. Parameters:
+    <the name of the cluster>
+    
+python launcher_cluster.py help    
+        
+    show help info
+
+## Introduction to Weave Networking
+[Weave](https://github.com/weaveworks/weave) is a tool to connect Docker containers distributed across different hosts. 
+This project use Weave to assign each Docker container (with Ambari-agent) a unique internal IP and a domain name.
+In each VM, a special Docker container will be launched by Weave to act as a Router, 
+and to connect all the Docker containers in this VM with other Docker containers in other VMs.
+Also, another special Docker container will be launched by Weave to act as a DNS for this VM.
+Each Docker container can connect with each other by using the internal IP, host name or domain name.
+
+All the Weave internal IP should be configured by the user. 
+In this following document, we use subnet 192.168.#.#/16, and use the IP within this subnet to configure Weave. 
+Actually, You can use any IP as you wish, even public IP, in which case, 
+it will replace the connection to the real outside connection, and redirect the connection to the internal Docker container.
+
+## Quick Start
+With the following 6 steps, you can create a cluster with Ambari-agents, and connect them to your Ambari-server. 
+Among all the steps, the step 3 is to configure this program, the step 4 is the one which really matters, 
+and other steps act as a one-time configuration or suggestion for your Ambari-server.
+
+You can start with this guide by downloading the code to your own computer or any computers which can access the GCE controller.
+
+* Step 1: Mark down IP of the GCE VM which installed Ambari-server, Ambari_Server_IP=104.196.81.81
+* Step 2: Copy example/config.ini to config/config.ini
+* Step 3: Modify the following values in config/config.ini
+    * Output_folder
+    * GCE_controller_key_file
+    * VM_key_file
+    * cluster_info_file, use {yourname}-group-a
+* Step 4: Use the command line to run 3 VMs, each VM has 5 Ambari-agents.
+
+        python launcher_cluster.py all {yourname}-group-a 3 5 192.168.255.1 Ambari_Server_IP
+
+* Step 5: Log into your Ambari-server machine, run the following command line to set up Weave internal network 
+
+        cd agent-simulator/network
+        ./set_ambari_server_network.sh 192.168.255.1 192.168.255.2 16
+
+* Step 6: Operate on Ambari-server web GUI
+    * Add all agents: docker-[0-14]-{yourname}-group-a.weave.local
+    * Choose manual registration
+    * Choose to install HDFS, YARN, HBase, Zookeeper, Ambari-Metrics
+    
+
+#### More on Quick Start
+* Step 7: Add one more cluster to your Ambari-server
+    * Modify cluster_info_file in config/config.ini, use {yourname}-group-b
+    * Modify Docker_IP_base in config.config.ini, use 192.168.2.1
+    * Use the command line to run 2 VMs, each VM has 10 Ambari-agents:
+
+                python launcher_cluster.py all {yourname}-group-b 2 10 192.168.255.1 Ambari_Server_IP
+
+    * On Ambari-server web GUI, add all agents: docker-[0-19]-{yourname}-group-b.weave.local
+* Step 8: Add one VM with Ambari-agent installed to your Ambari-server
+    * Log into your VM, set Ambari_Server_IP as the value of server hostname in the file /etc/ambari-agent/conf/ambari-agent.ini
+    * On the VM, Run the following command set up Weave internal network
+
+                cd agent-simulator/network
+                ./set_host_network.sh 192.168.254.1 192.168.254.2 16 Ambari_Server_IP
+
+
+## Detail Work Flow:
+* Step 1: Install Ambari-server
+    * Use existing Ambari-server, or, install and launch a new one
+    * agent-simulator/server/ambari_server_install.sh: this shell might help you install the Ambari-server
+    * agent-simulator/server/ambari_server-_reset_data.sh: this shell might help you set Ambari-server to initial state
+        
+* Step 2: Decide IP in your mind
+    * Mark down the IP of the Ambari-server, say {IP of Ambari-server = 104.196.81.81}
+    * Come up a subnet say, {subnet = 192.168.#.#/16} {Docker_IP_mask = 16}
+    * Pick one address closer to the END of the subnet as the Weave INTERNAL IP of Ambari-server, 
+    and another one as the Weave DNS IP of Ambari-server, 
+    say {Weave IP of Ambari-server = 192.168.255.1} {Weave DNS IP of Ambari-server = 192.168.255.2}
+    * Pick one address closer to the START of the subnet as the Weave INTERNAL IP of the FIRST Ambari-agent, 
+    say {Docker_IP_base = 192.168.1.1}
+    * Other Weave INTERNAL IP of Amari-agent will be automatically assigned based on the FIRST one (increasingly).
+    
+* Step 3: First time set up Ambari-server       
+    * Copy all the agent-simulator code base to Ambari-server
+    * cd agent-simulator/network
+    * Run set_ambari_server_network.sh
+    * In this example, use parameters: {Weave IP of Ambari-server = 192.168.255.1} {Weave DNS IP of Ambari-server = 192.168.255.2} {Docker_IP_mask = 16}
+    
+* Step 4: Modify config.ini
+    * Modify attributes: Output_folder, GCE_controller_key_file, GCE_VM_key_file, cluster_info_file
+    * Change Docker_IP_base and Docker_IP_mask, in this example {Docker_IP_base = 192.168.1.1} {Docker_IP_mask = 16}
+    
+* Step 5: Request Ambari-agent cluster
+    * Run python launcher_cluster.py request
+    * Use {your name}-group-a as the cluster name. In case you wanna add more cluster to your Ambari-server, change the last letter
+    
+* Step 6: Modify Cluster Information File
+    * A TXT file will appear under directory ./config within 1 minutes, which has the information about the cluster
+    * Typically, you would like NameNode, RegionServer, ResourceManager, etc.. to dominate one VM. 
+    * In this example, change the configuration of the first and the second VM, make each of them only have one Docker. 
+    You can install different server services only into these two Docker containers later on the Ambari-server web GUI.
+
+* Step 7: Run Ambari-agent Cluster
+    Run python launcher_cluster.py run
+    In this exmaple, use parameters: {Weave IP of Ambari-server = 192.168.255.1} {IP of Ambari-server = 104.196.81.81}
+    
+* Step 8: Operate on Ambari-server web GUI
+
+
+## Expand Cluster With This Script
+Be careful if you wanna use this script to add more Ambari-agents AGAIN to your Ambari-server
+
+* Use different Cluster Name when providing parameters to launcher_cluster.py
+* In config.ini, use the same Docker_IP_mask, make sure the same subnet
+* Change config.ini to use different Docker_IP_base, make sure that all new IPs never overlap with the existing IPs
+* Change config.ini to use different cluster_info_file, make sure the existing cluster information is not overwritten
+   
+## Expand Cluster By Adding other Hosts/VMs
+   
+## Naming Convention
+Cluster Name: the name of the cluster must be unique to make sure every VM on GCE has its unique name. The suggestion is using {your name}-group-{a-z}
+
+VM Name: each VM has a domain name assigned by GCE, its host name is {cluster name}-{index}
+
+Docker Container Name: the domain name of Docker container is docker-{index}-{cluster name}.weave.local, 
+the prefix "docker" can be configured by value Container_hostname_fix in config/config.ini, 
+you can find out which VM has which Docker container in the cluster information file.
+
+
+## Image for Docker Container
+
+## Use Different Partition for Docker Container
+
+## The IP assign mechanism.
+Basically, you don't have to worry about IP. The maximum number of IP is limited by weave_ip_base and weave_ip_mask.
+By fault, the subnet is 192.168.#.#/16. Once you have already created 256*256 agents (the real number is smaller, 
+since the DNS on the VM also uses IP address), some address might fall out side of subnet and some fall inside,
+which causes a connection issue. The function related is cluster._increase_ip(). You might want to wrap the IP around, 
+but corner cases are always there, there is no silver bullet.
+
+
+## Issues
+* This tool do NOT support parallel usage
+* If GCE has no enough resource, the cluster returned to you will have a smaller number of VM
+* Don't merge your cluster into someone else's cluster. Actually you can do it, but you have to dig into the network, and
+make sure the IP configuration is right.
+
+## Suggestions:
+* Make sure your cluster name is unique, or you might cause trouble to other people's VM
+* Use CTRL + P, then CTRL + Q to exit Docker container. Use "exit" will terminate the container.
+* Remove ~/.ssh/know_hosts files, especially if you run a large cluster. 
+You might get a warning from SSH, because the new GCE VM assigned to you might have the same IP with the VMs you saved in know_hosts file. 
+Remove .ssh/know_hosts before run this script.
+* Ambari-agent and Ambari-server have to be the same version to successfully register. 
+The command used to install Ambari-agent is in the Dockerfile
+    
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/ambari/blob/53dd3624/contrib/agent-simulator/cluster.py
----------------------------------------------------------------------
diff --git a/contrib/agent-simulator/cluster.py b/contrib/agent-simulator/cluster.py
index 55c3c44..dbef555 100644
--- a/contrib/agent-simulator/cluster.py
+++ b/contrib/agent-simulator/cluster.py
@@ -1,4 +1,4 @@
-'''
+"""
 Licensed to the Apache Software Foundation (ASF) under one
 or more contributor license agreements.  See the NOTICE file
 distributed with this work for additional information
@@ -14,162 +14,572 @@ distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
-'''
+"""
 
 import subprocess
-import time
+import datetime
 from config import Config
 from docker import Docker
 from vm import VM
-
+import os
+import time
+from log import Log
+from data import Data
 
 
 class Cluster:
+    """
+    The Cluster instance holds a list of VMs,
+    it has the methods to request cluster, generate information and run all Ambari-agent and Ambari-server
+    """
+
+    # The constants represent the state of the cluster
+    # A newly requested cluster is in FREE state
+    STATE_FREE = "FREE"
+    # A cluster with running Ambari-server and Ambari-agents is in RUNNING state
+    STATE_RUNNING = "RUNNING"
+    # A cluster is merged into another cluster and running, is in MERGE state
+    # the name of the extended cluster is directly following the state String in JSON
+    STATE_MERGE = "MERGE"
+
     def __init__(self):
         self.cluster_name = ""
-        self.VMs_num = 0
-        self.VM_list = []
-
-    # read cluster info from a file
-    def load_cluster_info(self, filename):
-        file = open(filename)
-
-        self.cluster_name = file.next().split()[1]
-        self.VMs_num = int(file.next().split()[1])
-        for VM_index in range(0, self.VMs_num):
-            vm = VM(file.next().split()[1])
-            docker_num = int(file.next().split()[1])
-            for Docker_index in range(0, docker_num):
-                line = file.next()
-                IP = line.split()[0].split("/")[0]
-                mask = line.split()[0].split("/")[1]
-                hostname = line.split()[1]
-                docker = Docker(IP, mask, hostname)
-                vm.add_docker(docker)
-            self.VM_list.append(vm)
+        self.state = ""
+        self.create_time = ""
+        # The list should only has one or zero VM, which holds the Ambari-server
+        self.ambari_server_vm = []
+        # The list of VMs, with Ambari-agents directly inside (not in Docker)
+        self.service_server_vm_list = []
+        # The list of VMs, each will hold multiple Docker containers with Ambari-agent inside
+        self.ambari_agent_vm_list = []
+
+    def _get_int_interval(self, int_list):
+        """
+        get the interval of the integer list
+        example: input[4,5,6,1,2,3], output [(1,3),(4,6)]
+        example: input[4,5,6,100,2,3], output [(2,3),(4,6),(100,100)]
+        :param int_list: the list of integer
+        :return: a tuple, each tuple has 2 integer, representing one interval
+        """
+        interval_list = []
+        int_list.sort()
+
+        begin = None
+        end = None
+        for integer in int_list:
+            if begin is None:
+                begin = integer
+                end = integer
+            else:
+                if integer == end + 1:
+                    end = integer
+                else:
+                    interval_list.append((begin, end))
+                    begin = integer
+                    end = integer
+
+        if begin is not None:
+            interval_list.append((begin, end))
+
+        return interval_list
+
+    def print_description(self):
+        print "cluster name: ", self.cluster_name
+        print "create time: ", self.create_time
+        print "state: ", self.state
+        print
+
+        print "Ambari Server: "
+        ambari_server_vm = self.get_ambari_server_vm()
+        if ambari_server_vm is None:
+            print "None"
+        else:
+            print ambari_server_vm.domain_name, " ", ambari_server_vm.external_ip, " ",\
+                ambari_server_vm.weave_internal_ip
+        print
+
+        print "Service Server with Ambari Agent directly installed: "
+        if len(self.service_server_vm_list) == 0:
+            print "None"
+        for vm in self.service_server_vm_list:
+            print vm.weave_domain_name, " ", vm.external_ip, " ", vm.weave_internal_ip
+        print
+
+        print "Ambari Agent in Docker Container: "
+        int_list = []
+        for vm in self.ambari_agent_vm_list:
+            for docker in vm.docker_list:
+                int_list.append(int(docker.get_index()))
+        interval_list = self._get_int_interval(int_list)
+        for interval in interval_list:
+            interval_str = ""
+            if interval[0] == interval[1]:
+                interval_str = str(interval(0))
+            else:
+                interval_str = "[{0}-{1}]".format(interval[0], interval[1])
+            print Docker.get_pattern_presentation(self.cluster_name, interval_str)
+        print
+
+    def get_agent_vm(self, vm_ip):
+        """
+        get the VM instance which holds Docker containers from the cluster instance
+        :param vm_ip: the external IP of the target VM
+        :return: the VM instance with the specified iP
+        """
+        for vm in self.ambari_agent_vm_list:
+            if vm.external_ip == vm_ip:
+                return vm
+
+    def get_ambari_server_vm(self):
+        """
+        get the VM instance which hold the Ambari-server
+        :return: the VM instance hold the Ambari-server, or None if no Ambari-server in this cluster
+        """
+        for vm in self.ambari_server_vm:
+            return vm
+        return None
+
+    def get_service_server_vm(self, vm_ip):
+        """
+        get the VM instance which directly hold the Ambari-agent
+        :param vm_ip: the external IP of the target VM
+        :return:
+        """
+        for vm in self.service_server_vm_list:
+            if vm.external_ip == vm_ip:
+                return vm
+
+    def to_json(self):
+        """
+        create a map to hold the information of the Cluster instance
+        :return: A map, which is JSON format object.
+        """
+        cluster = {}
+        cluster["cluster_name"] = self.cluster_name
+        cluster["create_time"] = self.create_time
+        cluster["state"] = self.state
+
+        cluster["ambari_server_vm"] = []
+        for vm in self.ambari_server_vm:
+            cluster["ambari_server_vm"].append(vm.to_json())
 
-        file.close()
+        cluster["service_server_vm_list"] = []
+        for vm in self.service_server_vm_list:
+            cluster["service_server_vm_list"].append(vm.to_json())
 
-    def __extract_VM_IP__(self, GCE_info_file_name):
-        f = open(GCE_info_file_name)
-        lines = f.readlines()
-        f.close()
+        cluster["ambari_agent_vm_list"] = []
+        for vm in self.ambari_agent_vm_list:
+            cluster["ambari_agent_vm_list"].append(vm.to_json())
 
-        ip_list = []
-        for line in lines:
+        return cluster
+
+    @staticmethod
+    def load_from_json(cluster_name):
+        """
+        load the cluster information from json file
+        :param cluster_name: the name of the cluster
+        :return: a Cluster instance or None if no such cluster
+        """
+        data = Data()
+        json_data = data.read_cluster_json(cluster_name)
+        if json_data is None:
+            return None
+
+        ambari_server_vm = []
+        service_server_vm_list = []
+        ambari_agent_vm_list = []
+
+        for vm_json in json_data["ambari_server_vm"]:
+            ambari_server_vm.append(VM.load_from_json(vm_json))
+
+        for vm_json in json_data["service_server_vm_list"]:
+            service_server_vm_list.append(VM.load_from_json(vm_json))
+
+        for vm_json in json_data["ambari_agent_vm_list"]:
+            ambari_agent_vm_list.append(VM.load_from_json(vm_json))
+
+        cluster = Cluster()
+        cluster.cluster_name = cluster_name
+        cluster.state = json_data["state"]
+        cluster.create_time = json_data["create_time"]
+        cluster.ambari_server_vm = ambari_server_vm
+        cluster.service_server_vm_list = service_server_vm_list
+        cluster.ambari_agent_vm_list = ambari_agent_vm_list
+        return cluster
+
+    def _extract_vm_fqdn_ip(self, gce_info_file_name):
+        """
+        exatract domain name and IP address of VMs from the output file of GCE
+        :param gce_info_file_name: output file of "GCE info" command
+        :return: A list of tuple, each tuple has domain name and IP of a VM
+        """
+        lines = []
+        with open(gce_info_file_name) as f:
+            lines = f.readlines()
+
+        vm_list = []
+        # the first line in the output file is title
+        for line in lines[1:]:
             tokens = line.split()
-            ip_list.append(tokens[1])
-        return ip_list[1:]
-
-    # request a new cluster
-    def request_GCE_cluster(self, vms_num, docker_num, cluster_name):
-        # reload configuration file
-        config = Config()
-        config.load()
-        # request cluster
-        gce_key = config.ATTRIBUTES["GCE_controller_key_file"]
-        gce_login = config.ATTRIBUTES["GCE_controller_user"] + "@" + config.ATTRIBUTES["GCE_controller_IP"]
-        gce_up_cmd = "gce up " + cluster_name + " " + str(vms_num) + " " + config.ATTRIBUTES["GCE_VM_type"] + \
-            " " + config.ATTRIBUTES["GCE_VM_OS"]
+            fqdn_ip = (tokens[0], tokens[1])
+            vm_list.append(fqdn_ip)
+        return vm_list
+
+    def request_vm(self, name, vm_num, gce_vm_type, gce_vm_os, gce_extra_cmd):
+        """
+        Request VMs from GCE
+        :param name: the name prefix of all requesting VMs
+        :param vm_num: the number of VM
+        :param gce_vm_type: the type of VM
+        :param gce_vm_os: the OS of VM
+        :param gce_extra_cmd: extra command for requesting the VMs
+        :return: A list of tuple, each tuple has domain name and IP of a VM
+        """
+        gce_key = Config.ATTRIBUTES["gce_controller_key_file"]
+        gce_login = "{0}@{1}".format(Config.ATTRIBUTES["gce_controller_user"], Config.ATTRIBUTES["gce_controller_ip"])
+        gce_up_cmd = "gce up {0} {1} {2} {3} {4}".format(name, vm_num, gce_vm_type, gce_vm_os, gce_extra_cmd)
         subprocess.call(["ssh", "-o", "StrictHostKeyChecking=no", "-i", gce_key, gce_login, gce_up_cmd])
 
-        print "cluster launched successufully, wait 5 seconds for cluster info ... ..."
-        time.sleep(5)
+        Log.write("cluster launched, wait for cluster info ... ...")
+
+        fqdn_ip_pairs = []
+        # wait for long enough. the more VM, more time it takes.
+        for retry in range(max(6, vm_num)):
+            time.sleep(10)
+
+            # request cluster info
+            with open(Config.ATTRIBUTES["gce_info_output"], "w") as gce_info_output_file:
+                gce_info_cmd = "gce info {0}".format(name)
+                subprocess.call(["ssh", "-o", "StrictHostKeyChecking=no", "-i", gce_key, gce_login, gce_info_cmd],
+                                stdout=gce_info_output_file)
+
+            fqdn_ip_pairs = self._extract_vm_fqdn_ip(Config.ATTRIBUTES["gce_info_output"])
+
+            if len(fqdn_ip_pairs) == vm_num:
+                Log.write("Get info for all ", str(len(fqdn_ip_pairs)), " VMs successfully")
+                break
+            Log.write("Only get info for ", str(len(fqdn_ip_pairs)), " VMs, retry ... ...")
+        return fqdn_ip_pairs
+
+    def request_ambari_server_vm(self, name):
+        """
+        request a VM for holding Ambari-server
+        :param name: the name prefix of all requesting VMs
+        :return: A list of tuple, each tuple has domain name and IP of a VM
+        """
+        # only 1 ambari server
+        vm_num = 1
+        gce_vm_type = Config.ATTRIBUTES["ambari_server_vm_type"]
+        gce_vm_os = Config.ATTRIBUTES["ambari_server_vm_os"]
+
+        gce_extra_cmd = ""
+        if "ambari_server_vm_extra" in Config.ATTRIBUTES:
+            gce_extra_cmd = Config.ATTRIBUTES["ambari_server_vm_extra"]
+
+        fqdn_ip_pairs = self.request_vm(name, vm_num, gce_vm_type, gce_vm_os, gce_extra_cmd)
+        return fqdn_ip_pairs
+
+    def reqeust_service_server_vm(self, vm_num, name):
+        """
+        Request VMs to directly hold Ambari-agent (not inside Docker)
+        :param vm_num: the number of VM to request
+        :param name: the name prefix of all requesting VMs
+        :return: A list of tuple, each tuple has domain name and IP of a VM
+        """
+        gce_vm_type = Config.ATTRIBUTES["service_server_vm_type"]
+        gce_vm_os = Config.ATTRIBUTES["service_server_vm_os"]
+
+        gce_extra_cmd = ""
+        if "service_server_vm_extra" in Config.ATTRIBUTES:
+            gce_extra_cmd = Config.ATTRIBUTES["service_server_vm_extra"]
+
+        fqdn_ip_pairs = self.request_vm(name, vm_num, gce_vm_type, gce_vm_os, gce_extra_cmd)
+        return fqdn_ip_pairs
 
-        # request cluster info
-        gce_info_output_file = open(config.ATTRIBUTES["GCE_info_output"], "w")
-        gce_info_cmd = "gce info " + cluster_name
-        subprocess.call(["ssh", "-o", "StrictHostKeyChecking=no", "-i", gce_key, gce_login, gce_info_cmd], \
-                        stdout=gce_info_output_file)
-        gce_info_output_file.close()
-        print "cluster info is saved to file " + config.ATTRIBUTES["GCE_info_output"]
+    def request_agent_vm(self, vm_num, name):
+        """
+        Request VMs to hold Docker containers, each with Ambari-agent inside
+        :param vm_num: the number of VM to request
+        :param name: the name prefix of all requesting VMs
+        :return: A list of tuple, each tuple has domain name and IP of a VM
+        """
+        gce_vm_type = Config.ATTRIBUTES["ambari_agent_vm_type"]
+        gce_vm_os = Config.ATTRIBUTES["ambari_agent_vm_os"]
+        gce_extra_disk = ""
+        if "ambari_agent_vm_extra_disk" in Config.ATTRIBUTES:
+            gce_extra_disk = Config.ATTRIBUTES["ambari_agent_vm_extra_disk"]
+
+        fqdn_ip_pairs = self.request_vm(name, vm_num, gce_vm_type, gce_vm_os, gce_extra_disk)
+        return fqdn_ip_pairs
+
+    def request_gce_cluster(self, ambari_agent_vm_num, docker_num,
+                            service_server_num, with_ambari_server, cluster_name):
+        """
+        Request a cluster from GCE
+        :param ambari_agent_vm_num: number of VMs to hold Docker containers
+        :param docker_num: number of Docker containers inside each VM
+        :param service_server_num: number of VMs which has Ambari-agent directly installed (not in Docker)
+        :param with_ambari_server: True or False, whether to request a VM to hold Ambari-server
+        :param cluster_name: the name of the cluster
+        :return: None
+        """
+        ambari_server_fqdn_ip_pairs = []
+        if with_ambari_server is True:
+            ambari_server_fqdn_ip_pairs = self.request_ambari_server_vm(VM.get_ambari_server_vm_name(cluster_name))
+        service_server_fqdn_ip_pairs = self.reqeust_service_server_vm(service_server_num,
+                                                                      VM.get_service_server_vm_name(cluster_name))
+        ambari_agent_fqdn_ip_pairs = self.request_agent_vm(ambari_agent_vm_num,
+                                                           VM.get_ambari_agent_vm_name(cluster_name))
 
         # prepare all attributes of the cluster, write to a file
-        VM_IP_list = self.__extract_VM_IP__(config.ATTRIBUTES["GCE_info_output"])
-        self.generate_cluster_info(VM_IP_list, cluster_name, docker_num)
-        self.overwrite_to_file(config.ATTRIBUTES["cluster_info_file"])
-        # server need this file to resolve the host names of the agents
-        self.export_hostnames(config.ATTRIBUTES["Docker_hostname_info"])
-
-    # save info to file
-    def overwrite_to_file(self, filename):
-        file = open(filename, "w")
-        file.write("cluster_name: " + self.cluster_name + "\n")
-        file.write("VMs_num: " + str(self.VMs_num) + "\n")
-
-        for vm in self.VM_list:
-            file.write("\t\t")
-            file.write("VM_IP: " + vm.external_ip + "\n")
-            file.write("\t\t")
-            file.write("Docker_num: " + str(len(vm.docker_list)) + "\n")
-            for docker in vm.docker_list:
-                file.write("\t\t\t\t")
-                file.write(docker.IP + "/" + docker.mask + " " + docker.hostname + "\n")
+        self.generate_cluster_info(cluster_name, ambari_server_fqdn_ip_pairs, service_server_fqdn_ip_pairs,
+                                   ambari_agent_fqdn_ip_pairs, docker_num)
 
-        file.close()
+    def generate_cluster_info(self, cluster_name, ambari_server_fqdn_ip_pairs, service_server_fqdn_ip_pairs,
+                              ambari_agent_fqdn_ip_pairs, docker_num):
+        """
+        generate VM and docker info for this cluster
+        set up parameter of the class instance as this info
+        :param cluster_name: the name of the cluster
+        :param ambari_server_fqdn_ip_pairs: the domain name and IP pairs for Ambari-server
+        :param service_server_fqdn_ip_pairs: the domain name and IP pairs for VMs with Ambari-agent installed
+        :param ambari_agent_fqdn_ip_pairs: the domain name and IP pairs for VM with Docker containers
+        :param docker_num: the number of Dockers inside each VMs
+        :return: None
+        """
+        weave_ip_base = Config.ATTRIBUTES["weave_ip_base"]
+        weave_ip_mask = Config.ATTRIBUTES["weave_ip_mask"]
+        current_ip = weave_ip_base
 
-    def __increase_IP__(self, base_IP, increase):
-        IP = [int(base_IP[0]), int(base_IP[1]), int(base_IP[2]), int(base_IP[3])]
-        IP[3] = IP[3] + increase
-        for index in reversed(range(0, 4)):
-            if IP[index] > 255:
-                IP[index - 1] = IP[index - 1] + IP[index] / 256
-                IP[index] = IP[index] % 256
-        return IP
-
-    # generate VM and docker info for this cluster
-    # set up parameter as this info
-    def generate_cluster_info(self, VM_IP_list, cluster_name, docker_num):
-        config = Config()
-        config.load()
-        Docker_IP_base = config.ATTRIBUTES["Docker_IP_base"].split(".")
-        Docker_IP_mask = config.ATTRIBUTES["Docker_IP_mask"]
-
-        VM_index = 0
-        for VM_IP in VM_IP_list:
-            vm = VM(VM_IP)
-
-            for Docker_index in range(0, docker_num):
-                total_Docker_index = VM_index * docker_num + Docker_index
-                docker_IP = self.__increase_IP__(Docker_IP_base, total_Docker_index)
-                docker_IP_str = str(docker_IP[0]) + "." + str(docker_IP[1]) + "." + \
-                                str(docker_IP[2]) + "." + str(docker_IP[3])
-                docker_hostname = cluster_name + "-" + str(VM_index) + "-" + str(Docker_index)
-                docker = Docker(docker_IP_str, str(Docker_IP_mask), docker_hostname)
-                # print docker
+        for vm_domain_name, vm_ip in ambari_server_fqdn_ip_pairs:
+            current_ip = self._increase_ip(current_ip, 1)
+            weave_dns_ip = current_ip
+            vm = VM(vm_ip, vm_domain_name, weave_dns_ip, weave_ip_mask)
+            current_ip = self._increase_ip(current_ip, 1)
+            vm.weave_internal_ip = current_ip
+            self.ambari_server_vm.append(vm)
+
+        for vm_domain_name, vm_ip in service_server_fqdn_ip_pairs:
+            current_ip = self._increase_ip(current_ip, 1)
+            weave_dns_ip = current_ip
+            vm = VM(vm_ip, vm_domain_name, weave_dns_ip, weave_ip_mask)
+            current_ip = self._increase_ip(current_ip, 1)
+            vm.weave_internal_ip = current_ip
+            self.service_server_vm_list.append(vm)
+
+        vm_index = 0
+        for vm_domain_name, vm_ip in ambari_agent_fqdn_ip_pairs:
+            current_ip = self._increase_ip(current_ip, 1)
+            weave_dns_ip = current_ip
+            vm = VM(vm_ip, vm_domain_name, weave_dns_ip, weave_ip_mask)
+
+            for docker_index in range(0, docker_num):
+                current_ip = self._increase_ip(current_ip, 1)
+                docker_ip_str = current_ip
+
+                total_docker_index = vm_index * docker_num + docker_index
+                docker_domain_name = Docker.get_weave_domain_name(cluster_name, total_docker_index)
+
+                docker = Docker(docker_ip_str, str(weave_ip_mask), docker_domain_name)
                 vm.add_docker(docker)
-            VM_index = VM_index + 1
-            self.VM_list.append(vm)
 
-        self.VMs_num = len(VM_IP_list)
+            vm_index += 1
+            self.ambari_agent_vm_list.append(vm)
+
         self.cluster_name = cluster_name
+        self.create_time = str(datetime.datetime.now())
+        self.state = Cluster.STATE_FREE
 
-    # run all dockers for all the VMs in the cluster
-    # upload necessary file to each machine in cluster, run launcher_docker.py in each machine with parameter
-    def run_docker_on_cluster(self, server_external_IP, server_Weave_IP):
-        config = Config()
-        config.load()
-
-        for vm in self.VM_list:
-            # upload necessary file to each machine in cluster
-            VM_external_IP = vm.external_ip
-            VM_directory = "root@" + VM_external_IP + ":" + config.ATTRIBUTES["VM_code_directory"]
-            VM_key = config.ATTRIBUTES["GCE_VM_key_file"]
-            subprocess.call(["scp", "-o", "StrictHostKeyChecking=no", "-i", VM_key, "-r", ".", VM_directory])
-
-            # run launcher_docker.py in each machine with parameters
-            subprocess.call(["ssh", "-o", "StrictHostKeyChecking=no", "-t", "-i", VM_key, \
-                             "root@" + VM_external_IP, \
-                             "cd " + config.ATTRIBUTES["VM_code_directory"] + "; python launcher_docker.py" + \
-                             " " + VM_external_IP + " " + server_Weave_IP + " " + server_external_IP])
-
-    # export host names to a file
-    def export_hostnames(self, filename):
-        hostname_file = open(filename, "w")
-        for vm in self.VM_list:
-            for docker in vm.docker_list:
-                hostname_file.write(docker.IP)
-                hostname_file.write(" ")
-                hostname_file.write(docker.hostname)
-                hostname_file.write("\n")
-        hostname_file.close()
+        # update config file.
+        # This step makes the user avoid reconfiguring the IP for next cluster creation
+        Config.update("weave", "weave_ip_base", current_ip)
+
+    def _increase_ip(self, base_ip_str, increase):
+        """
+        increase the IP address.
+        example: 192.168.1.1, increased by 1: 192.168.1.2
+        example: 192.168.1.254, increased by 2: 192.168.2.1
+        :param base_ip_str: the IP to be increased
+        :param increase: the amount of increase
+        :return: the new IP address, in String
+        """
+        base_ip = base_ip_str.split(".")
+        new_ip = [int(base_ip[0]), int(base_ip[1]), int(base_ip[2]), int(base_ip[3])]
+        new_ip[3] = new_ip[3] + increase
+        for index in reversed(range(0, 4)):
+            if new_ip[index] > 255:
+                new_ip[index - 1] += (new_ip[index] / 256)
+                new_ip[index] %= 256
+        return "{0}.{1}.{2}.{3}".format(new_ip[0], new_ip[1], new_ip[2], new_ip[3])
+
+    def _scp_upload(self, vm_external_ip):
+        """
+        upload all the code in a VM
+        :param vm_external_ip: the external IP of the VM
+        :return: None
+        """
+        # upload necessary file to VM
+        vm_directory = "{0}@{1}:{2}".format(Config.ATTRIBUTES["vm_user"], vm_external_ip,
+                                            Config.ATTRIBUTES["vm_code_directory"])
+        vm_key = Config.ATTRIBUTES["vm_key_file"]
+
+        upload_return_code = 0
+        with open(os.devnull, 'w') as shutup:
+            upload_return_code = subprocess.call(["scp", "-o", "StrictHostKeyChecking=no", "-i",
+                                                  vm_key, "-r", ".", vm_directory],
+                                                 stdout=shutup, stderr=shutup)
+        if upload_return_code == 0:
+            Log.write("VM ", vm_external_ip, " file upload succeed")
+        else:
+            Log.write("VM ", vm_external_ip, " file upload fail")
+
+    def run_cluster(self, server_weave_ip, server_external_ip):
+        """
+        Run all Ambari-agents and Ambari-server in the cluster in parallel
+        Wait until all processes finish
+        :param server_weave_ip: the Weave IP of Ambari-server
+        :param server_external_ip: the external IP of Ambari-server
+        :return: None
+        """
+        process_list = {}
+        process_list.update(self.run_ambari_server_asyn())
+        process_list.update(self.run_service_server_asyn(server_weave_ip, server_external_ip))
+        process_list.update(self.run_docker_on_cluster_asyn(server_weave_ip, server_external_ip))
+
+        terminate_state_list = {}
+        for hostname in process_list:
+            terminate_state_list[hostname] = False
+
+        Log.write("Wait for all VMs to finish configuration ... ...")
+
+        # Wait for all configuration subprocesses
+        while True:
+            all_finished = True
+            for hostname in process_list:
+                output_file, output_file_path, process = process_list[hostname]
+                if terminate_state_list[hostname] is False:
+                    all_finished = False
+                    returncode = process.poll()
+                    if returncode is None:
+                        continue
+                    else:
+                        Log.write("VM ", hostname, " configuration completed, return code: ", str(returncode) \
+                                  , ", output file path: ", output_file_path)
+                        terminate_state_list[hostname] = True
+                        output_file.close()
+                else:
+                    pass
+            if all_finished:
+                break
+            time.sleep(5)
+
+        Log.write("All VM configuration completed.")
+
+    def run_ambari_server_asyn(self):
+        """
+        Run configuration for Ambari-server in this cluster
+        Set up Ambari-server and Weave network
+        The method is NON-BLOCK
+        :return: a map of tuple, the key of the map is the host name of the VM,
+                the tuple has 3 elements: the file handler of the output of the VM,
+                the file path of the output of the VM,
+                and the process object of configuration for the VM
+        """
+        process_list = {}
+
+        for vm in self.ambari_server_vm:
+            vm_external_ip = vm.external_ip
+            self._scp_upload(vm_external_ip)
+
+            vm_output_file_path = vm.get_ssh_output_file_path()
+            vm_output_file = open(vm_output_file_path, "w")
+
+            # ssh install server
+            vm_ssh_login = "{0}@{1}".format(Config.ATTRIBUTES["vm_user"], vm_external_ip)
+            vm_ssh_cd_cmd = "cd {0}".format(Config.ATTRIBUTES["vm_code_directory"])
+            vm_ssh_python_cmd = "python launcher_ambari_server.py {0}".format(self.cluster_name)
+            vm_ssh_cmd = "{0};{1}".format(vm_ssh_cd_cmd, vm_ssh_python_cmd)
+            vm_key = Config.ATTRIBUTES["vm_key_file"]
+            Log.write(vm_ssh_python_cmd)
+
+            process = subprocess.Popen(["ssh", "-o", "StrictHostKeyChecking=no", "-t", "-i", vm_key,
+                                       vm_ssh_login, vm_ssh_cmd],
+                                       stdout=vm_output_file, stderr=vm_output_file)
+            process_list[vm.hostname] = (vm_output_file, vm_output_file_path, process)
+            Log.write("Configuring VM ", vm.hostname, " ... ...")
+        return process_list
+
+    def run_service_server_asyn(self, server_weave_ip, server_external_ip):
+        """
+        Run configuration, set up Ambari-agent in this VM, and the Weave network
+        :param server_weave_ip: the Weave IP of the Ambari-server
+        :param server_external_ip: the external IP of the Ambari-server
+        The method is NON-BLOCK
+        :return: a map of tuple, the key of the map is the host name of the VM,
+                the tuple has 3 elements: the file handler of the output of the VM,
+                the file path of the output of the VM,
+                and the process object of configuration for the VM
+        """
+        process_list = {}
+
+        for vm in self.service_server_vm_list:
+            vm_external_ip = vm.external_ip
+            self._scp_upload(vm_external_ip)
+
+            vm_output_file_path = vm.get_ssh_output_file_path()
+            vm_output_file = open(vm_output_file_path, "w")
+
+            # ssh install server
+            vm_ssh_login = "{0}@{1}".format(Config.ATTRIBUTES["vm_user"], vm_external_ip)
+            vm_ssh_cd_cmd = "cd {0}".format(Config.ATTRIBUTES["vm_code_directory"])
+            vm_ssh_python_cmd = "python launcher_service_server.py {0} {1} {2} {3}".format(
+                vm_external_ip, server_weave_ip, server_external_ip, self.cluster_name)
+            vm_ssh_cmd = "{0};{1}".format(vm_ssh_cd_cmd, vm_ssh_python_cmd)
+            vm_key = Config.ATTRIBUTES["vm_key_file"]
+            Log.write(vm_ssh_python_cmd)
+
+            process = subprocess.Popen(["ssh", "-o", "StrictHostKeyChecking=no", "-t", "-i", vm_key,
+                                        vm_ssh_login, vm_ssh_cmd],
+                                       stdout=vm_output_file, stderr=vm_output_file)
+
+            process_list[vm.hostname] = (vm_output_file, vm_output_file_path, process)
+            Log.write("Configuring VM ", vm.hostname, " ... ...")
+        return process_list
+
+    def run_docker_on_cluster_asyn(self, server_weave_ip, server_external_ip):
+        """
+        Run configuration, set up Docker and Weave network
+        run all containers, each with Ambari-agent inside
+        :param server_weave_ip: the Weave IP of the Ambari-server
+        :param server_external_ip: the external IP of the Ambari-server
+        The method is NON-BLOCK
+        :return: a map of tuple, the key of the map is the host name of the VM,
+                the tuple has 3 elements: the file handler of the output of the VM,
+                the file path of the output of the VM,
+                and the process object of configuration for the VM
+        """
+        process_list = {}
+
+        for vm in self.ambari_agent_vm_list:
+            vm_external_ip = vm.external_ip
+            self._scp_upload(vm_external_ip)
+
+            vm_output_file_path = vm.get_ssh_output_file_path()
+            vm_output_file = open(vm_output_file_path, "w")
+
+            vm_ssh_login = "{0}@{1}".format(Config.ATTRIBUTES["vm_user"], vm_external_ip)
+            vm_ssh_cd_cmd = "cd {0}".format(Config.ATTRIBUTES["vm_code_directory"])
+            vm_ssh_python_cmd = "python launcher_docker.py {0} {1} {2} {3}".format(
+                vm_external_ip, server_weave_ip, server_external_ip, self.cluster_name)
+            vm_ssh_cmd = "{0};{1}".format(vm_ssh_cd_cmd, vm_ssh_python_cmd)
+            vm_key = Config.ATTRIBUTES["vm_key_file"]
+            Log.write(vm_ssh_python_cmd)
+
+            process = subprocess.Popen(["ssh", "-o", "StrictHostKeyChecking=no", "-t", "-i", vm_key,
+                                        vm_ssh_login, vm_ssh_cmd],
+                                       stdout=vm_output_file, stderr=vm_output_file)
+
+            process_list[vm.hostname] = (vm_output_file, vm_output_file_path, process)
+            Log.write("Configuring VM ", vm.hostname, " ... ...")
+
+        return process_list

http://git-wip-us.apache.org/repos/asf/ambari/blob/53dd3624/contrib/agent-simulator/config.py
----------------------------------------------------------------------
diff --git a/contrib/agent-simulator/config.py b/contrib/agent-simulator/config.py
index 20cc8de..8d67340 100644
--- a/contrib/agent-simulator/config.py
+++ b/contrib/agent-simulator/config.py
@@ -1,4 +1,4 @@
-'''
+"""
 Licensed to the Apache Software Foundation (ASF) under one
 or more contributor license agreements.  See the NOTICE file
 distributed with this work for additional information
@@ -14,23 +14,58 @@ distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
-'''
+"""
 
 import ConfigParser
+import os
+
 
 class Config:
     ATTRIBUTES = {}
 
+    AMBARI_AGENT_VM = "ambari_agent_vm"
+    AMBARI_SERVER_VM = "ambari_server_vm"
+    SERVICE_SERVER_VM = "service_server_vm"
+
+    RELATIVE_CONFIG_FILE_PATH = "config/config.ini"
+
+    def __init__(self):
+        pass
+
     @staticmethod
     def load():
+        """
+        load configuration from file, add all configuration to the map ATTRIBUTES
+        :return: None
+        """
         config = ConfigParser.RawConfigParser()
         # keep file case sensitive
         config.optionxform = str
-        config.read("config/config.ini")
+        config.read(Config.RELATIVE_CONFIG_FILE_PATH)
         for section in config.sections():
             for key in config.options(section):
                 Config.ATTRIBUTES[key] = config.get(section, key)
 
+        # set output file path
+        for key in config.options("output"):
+            if key == "output_folder":
+                # create the folder
+                if not os.path.exists(Config.ATTRIBUTES["output_folder"]):
+                    os.makedirs(Config.ATTRIBUTES["output_folder"])
+            else:
+                Config.ATTRIBUTES[key] = Config.ATTRIBUTES["output_folder"] + "/" + Config.ATTRIBUTES[key]
 
-# Config.load()
-# print Config.ATTRIBUTES
\ No newline at end of file
+    @staticmethod
+    def update(section, key, value):
+        """
+        Update the key value of the configuration file
+        :param section: section is inside []
+        :param key: the key
+        :param value: the value
+        :return: None
+        """
+        config = ConfigParser.RawConfigParser()
+        config.read(Config.RELATIVE_CONFIG_FILE_PATH)
+        config.set(section, key, value)
+        with open(Config.RELATIVE_CONFIG_FILE_PATH, 'wb') as configfile:
+                config.write(configfile)

http://git-wip-us.apache.org/repos/asf/ambari/blob/53dd3624/contrib/agent-simulator/config/config.ini
----------------------------------------------------------------------
diff --git a/contrib/agent-simulator/config/config.ini b/contrib/agent-simulator/config/config.ini
deleted file mode 100644
index a3afa33..0000000
--- a/contrib/agent-simulator/config/config.ini
+++ /dev/null
@@ -1,49 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information rega4rding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# This configuration file is case SENSITIVE
-# For more information about this configuration file, check https://docs.python.org/2/library/configparser.html
-
-[GCE]
-GCE_controller_IP = 104.196.89.197
-GCE_controller_user = root
-GCE_controller_key_file = ~/gce-key
-GCE_VM_type = --xlarge
-# only support centos 7
-GCE_VM_OS = --centos7
-# the result of command "gce info clustername" on the GCE controller
-GCE_info_output = output/gce_info_output.txt
-# the same as the GCE controller for GCE
-GCE_VM_key_file = ~/gce-key
-
-
-[Cluster]
-cluster_info_file = config/cluster.txt
-VM_code_directory = /simulator-script
-VM_hostname_file = /etc/hosts
-Docker_hostname_info = config/hosts.txt
-
-[Weave]
-# Docker_IP_base is the starting point of IP address. with mask 16, ideally:
-# the IP of docker will be 192.168.[1-255].[1-255]/16
-# if there are more dockers, which means there is not enough IPs within this mask for all dockers,
-# the IP will continue to increase to be 192.169.*.*, dockers might not be able to talk to each other,
-# because the mask is 16
-Docker_IP_base = 192.168.1.2
-Docker_IP_mask = 16
-
-
-[Docker]
-Docker_image_name = ambari/agent-sim

http://git-wip-us.apache.org/repos/asf/ambari/blob/53dd3624/contrib/agent-simulator/data.py
----------------------------------------------------------------------
diff --git a/contrib/agent-simulator/data.py b/contrib/agent-simulator/data.py
new file mode 100644
index 0000000..b469f33
--- /dev/null
+++ b/contrib/agent-simulator/data.py
@@ -0,0 +1,104 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+
+import os.path
+import json
+from config import Config
+
+
+class Data:
+    def __init__(self):
+        self.data_filename = Config.ATTRIBUTES["cluster_info_file"]
+
+    def _load_data(self):
+        """
+        load all data from JSON file
+        :return: a map, which is a JSON format object
+        """
+        json_data = {"clusters": []}
+        if os.path.isfile(self.data_filename):
+            with open(self.data_filename) as f:
+                json_data = json.load(f)
+        return json_data
+
+    def _save_data(self, json_data):
+        """
+        save the JSON object into a file
+        :param json_data: a map, which is a JSON format object
+        :return: None
+        """
+        with open(self.data_filename, "w") as f:
+            json.dump(json_data, f, indent=4, separators=(',', ': '))
+
+    def add_new_cluster(self, cluster):
+        """
+        add a new cluster into the JSON file
+        :param cluster: the cluster instance
+        :return: None
+        """
+        json_data = self._load_data()
+        new_cluster_json = cluster.to_json()
+        json_data["clusters"].insert(0, new_cluster_json)
+        self._save_data(json_data)
+
+    def set_cluster_state(self, cluster_name, state):
+        """
+        set the state of a cluster into JSON file
+        :param cluster_name: the name of the cluster
+        :param state: the name of the state
+        :return: None
+        """
+        json_data = self._load_data()
+        for cluster in json_data["clusters"]:
+            if cluster["cluster_name"] == cluster_name:
+                cluster["state"] = state
+                break
+        self._save_data(json_data)
+
+    def read_cluster_json(self, cluster_name):
+        """
+        get the JSON object for the cluster
+        :param cluster_name: the name of cluster
+        :return: a map which is a JSON object or None if the cluster is not found
+        """
+        json_data = self._load_data()
+        for cluster_json in json_data["clusters"]:
+            if cluster_json["cluster_name"] == cluster_name:
+                return cluster_json
+        return None
+
+    def print_cluster_summary_list(self):
+        """
+        get a brief description of all the cluster from the JSON file
+        :return: a list of tuple. The elements of the tuple are:
+                 cluster_name, state, agent_number,
+                 service_server_num, ambari_server_num, create_time
+        """
+        print "(cluster_name, state, agent_number, service_server_num, ambari_server_num, create_time)"
+
+        json_data = self._load_data()
+        for cluster in json_data["clusters"]:
+            cluster_name = cluster["cluster_name"]
+            state = cluster["state"]
+            create_time = cluster["create_time"]
+            agent_number = 0
+            for agent_vm in cluster["ambari_agent_vm_list"]:
+                agent_number += len(agent_vm["docker_list"])
+            service_server_num = len(cluster["service_server_vm_list"])
+            ambari_server_num = len(cluster["ambari_server_vm"])
+            print cluster_name, state, agent_number, service_server_num, ambari_server_num, create_time

http://git-wip-us.apache.org/repos/asf/ambari/blob/53dd3624/contrib/agent-simulator/docker.py
----------------------------------------------------------------------
diff --git a/contrib/agent-simulator/docker.py b/contrib/agent-simulator/docker.py
index e397c4b..0dd38c0 100644
--- a/contrib/agent-simulator/docker.py
+++ b/contrib/agent-simulator/docker.py
@@ -1,4 +1,4 @@
-'''
+"""
 Licensed to the Apache Software Foundation (ASF) under one
 or more contributor license agreements.  See the NOTICE file
 distributed with this work for additional information
@@ -14,14 +14,72 @@ distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
-'''
+"""
+
+from config import Config
 
 
 class Docker:
-    def __init__(self, IP, mask, hostname):
-        self.IP = IP
+    """
+    Docker represents a Docker container, each with its IP and domain name
+    """
+    def __init__(self, ip, mask, weave_domain_name):
+        self.ip = ip
         self.mask = mask
-        self.hostname = hostname
+        self.weave_domain_name = weave_domain_name
+
+    def to_json(self):
+        """
+        create a map to hold the information of the Docker instance
+        :return: A map, which is JSON format object.
+        """
+        docker_json = {}
+        docker_json["weave_ip"] = "{0}/{1}".format(self.ip, self.mask)
+        docker_json["weave_domain_name"] = self.weave_domain_name
+        return docker_json
+
+    @staticmethod
+    def load_from_json(json_data):
+        """
+        load the docker information from a JSON object
+        :param json_data: a map, which is a JSON object
+        :return: a Docker object
+        """
+        ip = json_data["weave_ip"].split("/")[0]
+        mask = json_data["weave_ip"].split("/")[1]
+        weave_domain_name = json_data["weave_domain_name"]
+        return Docker(ip, mask, weave_domain_name)
 
     def __str__(self):
-        return str(self.IP) + "/" + str(self.mask) + " " + self.hostname
\ No newline at end of file
+        return str(self.ip) + "/" + str(self.mask) + " " + self.weave_domain_name
+
+    @staticmethod
+    def get_weave_domain_name(cluster_name, index):
+        """
+        given the index and the name of cluster, generate the  Weave domain name for the docker
+        :param cluster_name: the name of the cluster
+        :param index: a number
+        :return: Weave domain name of the docker container
+        """
+        return "{0}-{1}-{2}.{3}".format(Config.ATTRIBUTES["container_hostname_fix"],
+                                        index, cluster_name, "weave.local")
+
+    @staticmethod
+    def get_pattern_presentation(cluster_name, range_str):
+        return Docker.get_weave_domain_name(cluster_name, range_str)
+
+    def get_index(self):
+        """
+        extract the index of the docker within the cluster
+        :return: the index
+        """
+        return self.weave_domain_name.split("-")[1]
+
+    def get_container_name(self):
+        """
+        :return: the name of the container
+        """
+        return self.get_hostname()
+
+    def get_hostname(self):
+        return self.weave_domain_name.split(".")[0]

http://git-wip-us.apache.org/repos/asf/ambari/blob/53dd3624/contrib/agent-simulator/docker_image/Yum_Dockerfile
----------------------------------------------------------------------
diff --git a/contrib/agent-simulator/docker_image/Yum_Dockerfile b/contrib/agent-simulator/docker_image/Yum_Dockerfile
new file mode 100644
index 0000000..774c19a
--- /dev/null
+++ b/contrib/agent-simulator/docker_image/Yum_Dockerfile
@@ -0,0 +1,56 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information rega4rding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Set the base image to CentOS 7
+FROM centos:7
+
+#====================================set SSH service====================================================================
+RUN yum -y -q install openssh-server epel-release openssh-clients&& \
+    yum -y -q install pwgen && \
+    rm -f /etc/ssh/ssh_host_ecdsa_key /etc/ssh/ssh_host_rsa_key && \
+    ssh-keygen -q -N "" -t dsa -f /etc/ssh/ssh_host_ecdsa_key && \
+    ssh-keygen -q -N "" -t rsa -f /etc/ssh/ssh_host_rsa_key && \
+    sed -i "s/#UsePrivilegeSeparation.*/UsePrivilegeSeparation no/g" /etc/ssh/sshd_config
+
+ADD ssh_service/set_root_pw.sh /set_root_pw.sh
+ADD ssh_service/run.sh /run_ssh.sh
+RUN chmod +x /*.sh
+
+ENV AUTHORIZED_KEYS **None**
+ENV ROOT_PASS ambariagent
+#====================================set SSH service====================================================================
+
+
+# Copy the files into Docker
+ADD launcher_agent.py /launcher_agent.py
+ADD ambari_agent_start.sh /ambari_agent_start.sh
+ADD package_list.txt /package_list.txt
+ADD ambari_agent_install.sh /ambari_agent_install.sh
+
+RUN chmod 755 /ambari_agent_start.sh && \
+    chmod 755 /ambari_agent_install.sh
+
+# Install ambari-agent
+RUN /ambari_agent_install.sh
+
+# Set Hadoop Repo
+RUN wget -O /etc/yum.repos.d/hdp.repo http://s3.amazonaws.com/dev.hortonworks.com/HDP/centos7/2.x/updates/2.3.0.0/hdp.repo
+
+# Install all package
+RUN yum install -y -q $(cat package_list.txt)
+RUN yum update -y -q
+
+# Clean yum download cache
+RUN yum clean -y all
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/ambari/blob/53dd3624/contrib/agent-simulator/docker_image/__init__.py
----------------------------------------------------------------------
diff --git a/contrib/agent-simulator/docker_image/__init__.py b/contrib/agent-simulator/docker_image/__init__.py
new file mode 100644
index 0000000..67bb99d
--- /dev/null
+++ b/contrib/agent-simulator/docker_image/__init__.py
@@ -0,0 +1,17 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/ambari/blob/53dd3624/contrib/agent-simulator/docker_image/ambari_agent_install.sh
----------------------------------------------------------------------
diff --git a/contrib/agent-simulator/docker_image/ambari_agent_install.sh b/contrib/agent-simulator/docker_image/ambari_agent_install.sh
new file mode 100644
index 0000000..d9c5cd6
--- /dev/null
+++ b/contrib/agent-simulator/docker_image/ambari_agent_install.sh
@@ -0,0 +1,22 @@
+#!/bin/bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information rega4rding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# this script will install Ambari agent on Centos 7
+
+yum install -y -q wget
+wget -q -O /etc/yum.repos.d/ambari.repo http://s3.amazonaws.com/dev.hortonworks.com/ambari/centos7/2.x/latest/2.1.1/ambaribn.repo
+yum install -y -q epel-release
+yum install -y -q ambari-agent
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/ambari/blob/53dd3624/contrib/agent-simulator/docker_image/ambari_agent_start.sh
----------------------------------------------------------------------
diff --git a/contrib/agent-simulator/docker_image/ambari_agent_start.sh b/contrib/agent-simulator/docker_image/ambari_agent_start.sh
new file mode 100644
index 0000000..1e0cb0f
--- /dev/null
+++ b/contrib/agent-simulator/docker_image/ambari_agent_start.sh
@@ -0,0 +1,19 @@
+#!/bin/bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information rega4rding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# this script will start Ambari agent on Centos 7
+
+ambari-agent start
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/ambari/blob/53dd3624/contrib/agent-simulator/docker_image/launcher_agent.py
----------------------------------------------------------------------
diff --git a/contrib/agent-simulator/docker_image/launcher_agent.py b/contrib/agent-simulator/docker_image/launcher_agent.py
new file mode 100644
index 0000000..1d3c87d
--- /dev/null
+++ b/contrib/agent-simulator/docker_image/launcher_agent.py
@@ -0,0 +1,86 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+
+
+import sys
+import subprocess
+
+
+def replace_conf(server_ip):
+    """
+    replace the server host IP in the Ambari-agent configuration file
+    :param server_ip: internal Weave IP address of Ambari-server
+    :return: None
+    """
+    lines = []
+    with open("/etc/ambari-agent/conf/ambari-agent.ini") as f:
+        lines = f.readlines()
+
+    with open("/etc/ambari-agent/conf/ambari-agent.ini", "w+") as f:
+        for line in lines:
+            line = line.replace("hostname=localhost", "hostname=" + server_ip)
+            f.write(line)
+
+
+def run_ssh():
+    """
+    run SSH service on this Docker Container
+    :return: None
+    """
+    subprocess.call("/run_ssh.sh")
+
+
+def run_ambari_agent():
+    """
+    command line to run Ambari-agent
+    :return: None
+    """
+    subprocess.call("/ambari_agent_start.sh")
+
+
+def set_weave_ip(weave_ip):
+    """
+    set the IP and hostname mapping for this Container
+    Docker will assign an IP to each Container, and map it to hostname, which is not we want
+    We want our Weave IP to be mapped to hostname
+    :param weave_ip:
+    :return: None
+    """
+    with open("/etc/hosts") as etc_hosts:
+        all_resolution = etc_hosts.readlines()
+
+    with open("/etc/hosts", "w") as etc_hosts:
+        for index in range(len(all_resolution)):
+            if index == 0:
+                token = all_resolution[index].split()
+                etc_hosts.write("{0} {1} {2}\n".format(weave_ip, token[1], token[2]))
+            else:
+                etc_hosts.write(all_resolution[index])
+
+
+def main():
+    ambari_server_ip = sys.argv[1]
+    my_weave_ip = sys.argv[2]
+    replace_conf(ambari_server_ip)
+    set_weave_ip(my_weave_ip)
+    run_ambari_agent()
+    run_ssh()
+
+
+if __name__ == "__main__":
+    main()

http://git-wip-us.apache.org/repos/asf/ambari/blob/53dd3624/contrib/agent-simulator/docker_image/package_list.txt
----------------------------------------------------------------------
diff --git a/contrib/agent-simulator/docker_image/package_list.txt b/contrib/agent-simulator/docker_image/package_list.txt
new file mode 100644
index 0000000..857cd28
--- /dev/null
+++ b/contrib/agent-simulator/docker_image/package_list.txt
@@ -0,0 +1,235 @@
+alsa-lib.x86_64
+at.x86_64
+atk.x86_64
+avahi-libs.x86_64
+bc.x86_64
+cairo.x86_64
+cpp.x86_64
+cronie.x86_64
+cronie-anacron.x86_64
+crontabs.noarch
+desktop-file-utils.x86_64
+ed.x86_64
+emacs-filesystem.noarch
+fontconfig.x86_64
+fontpackages-filesystem.noarch
+foomatic-filters.x86_64
+fuse.x86_64
+fuse-libs.x86_64
+gcc.x86_64
+gdbm-devel.x86_64
+gdk-pixbuf2.x86_64
+gettext.x86_64
+gettext-libs.x86_64
+ghostscript.x86_64
+ghostscript-fonts.noarch
+glibc-devel.x86_64
+glibc-headers.x86_64
+graphite2.x86_64
+gtk2.x86_64
+harfbuzz.x86_64
+hicolor-icon-theme.noarch
+initscripts.x86_64
+jasper-libs.x86_64
+jbigkit-libs.x86_64
+kmod.x86_64
+kmod-libs.x86_64
+lcms2.x86_64
+libICE.x86_64
+libSM.x86_64
+libX11.x86_64
+libX11-common.noarch
+libXau.x86_64
+libXcomposite.x86_64
+libXcursor.x86_64
+libXdamage.x86_64
+libXext.x86_64
+libXfixes.x86_64
+libXfont.x86_64
+libXft.x86_64
+libXi.x86_64
+libXinerama.x86_64
+libXrandr.x86_64
+libXrender.x86_64
+libXt.x86_64
+libXtst.x86_64
+libXxf86vm.x86_64
+libdb-devel.x86_64
+libdrm.x86_64
+libfontenc.x86_64
+libjpeg-turbo.x86_64
+libmng.x86_64
+libmpc.x86_64
+libpciaccess.x86_64
+libpipeline.x86_64
+libpng.x86_64
+libpng12.x86_64
+libthai.x86_64
+libtiff.x86_64
+libtirpc.x86_64
+libxcb.x86_64
+m4.x86_64
+mailcap.noarch
+mailx.x86_64
+make.x86_64
+man-db.x86_64
+mesa-libEGL.x86_64
+mesa-libGL.x86_64
+mesa-libGLU.x86_64
+mesa-libgbm.x86_64
+mesa-libglapi.x86_64
+mpfr.x86_64
+nmap-ncat.x86_64
+pango.x86_64
+passwd.x86_64
+patch.x86_64
+pciutils.x86_64
+pciutils-libs.x86_64
+perl.x86_64
+perl-B-Lint.noarch
+perl-Business-ISBN.noarch
+perl-Business-ISBN-Data.noarch
+perl-CGI.noarch
+perl-CPAN.noarch
+perl-Carp.noarch
+perl-Class-ISA.noarch
+perl-Compress-Raw-Bzip2.x86_64
+perl-Compress-Raw-Zlib.x86_64
+perl-Data-Dumper.x86_64
+perl-Digest.noarch
+perl-Digest-MD5.x86_64
+perl-Digest-SHA.x86_64
+perl-Encode.x86_64
+perl-Encode-Locale.noarch
+perl-Env.noarch
+perl-Exporter.noarch
+perl-ExtUtils-Install.noarch
+perl-ExtUtils-MakeMaker.noarch
+perl-ExtUtils-Manifest.noarch
+perl-ExtUtils-ParseXS.noarch
+perl-FCGI.x86_64
+perl-File-CheckTree.noarch
+perl-File-Listing.noarch
+perl-File-Path.noarch
+perl-File-Temp.noarch
+perl-Filter.x86_64
+perl-Getopt-Long.noarch
+perl-HTML-Parser.x86_64
+perl-HTML-Tagset.noarch
+perl-HTTP-Cookies.noarch
+perl-HTTP-Daemon.noarch
+perl-HTTP-Date.noarch
+perl-HTTP-Message.noarch
+perl-HTTP-Negotiate.noarch
+perl-HTTP-Tiny.noarch
+perl-IO-Compress.noarch
+perl-IO-HTML.noarch
+perl-IO-Socket-IP.noarch
+perl-IO-Socket-SSL.noarch
+perl-LWP-MediaTypes.noarch
+perl-Locale-Codes.noarch
+perl-Locale-Maketext.noarch
+perl-Module-Pluggable.noarch
+perl-Net-HTTP.noarch
+perl-Net-LibIDN.x86_64
+perl-Net-SSLeay.x86_64
+perl-PathTools.x86_64
+perl-Pod-Checker.noarch
+perl-Pod-Escapes.noarch
+perl-Pod-LaTeX.noarch
+perl-Pod-Parser.noarch
+perl-Pod-Perldoc.noarch
+perl-Pod-Plainer.noarch
+perl-Pod-Simple.noarch
+perl-Pod-Usage.noarch
+perl-Scalar-List-Utils.x86_64
+perl-Socket.x86_64
+perl-Storable.x86_64
+perl-Sys-Syslog.x86_64
+perl-Test-Harness.noarch
+perl-Test-Simple.noarch
+perl-Text-ParseWords.noarch
+perl-Text-Soundex.x86_64
+perl-Text-Unidecode.noarch
+perl-Time-HiRes.x86_64
+perl-Time-Local.noarch
+perl-TimeDate.noarch
+perl-URI.noarch
+perl-WWW-RobotRules.noarch
+perl-XML-LibXML.x86_64
+perl-XML-NamespaceSupport.noarch
+perl-XML-SAX.noarch
+perl-XML-SAX-Base.noarch
+perl-autodie.noarch
+perl-constant.noarch
+perl-devel.x86_64
+perl-libs.x86_64
+perl-libwww-perl.noarch
+perl-local-lib.noarch
+perl-macros.x86_64
+perl-parent.noarch
+perl-podlators.noarch
+perl-threads.x86_64
+perl-threads-shared.x86_64
+pixman.x86_64
+poppler-data.noarch
+psmisc.x86_64
+pyparsing.noarch
+qt.x86_64
+qt-settings.noarch
+qt-x11.x86_64
+qt3.x86_64
+redhat-lsb.x86_64
+redhat-lsb-core.x86_64
+redhat-lsb-cxx.x86_64
+redhat-lsb-desktop.x86_64
+redhat-lsb-languages.x86_64
+redhat-lsb-printing.x86_64
+redhat-lsb-submod-multimedia.x86_64
+redhat-lsb-submod-security.x86_64
+rpcbind.x86_64
+snappy-devel.x86_64
+spax.x86_64
+sysvinit-tools.x86_64
+tcp_wrappers-libs.x86_64
+time.x86_64
+urw-fonts.noarch
+wget.x86_64
+xorg-x11-font-utils.x86_64
+bigtop-jsvc.x86_64
+bigtop-tomcat.noarch
+hadoop_2_3_0_0_2557.x86_64
+hadoop_2_3_0_0_2557-client.x86_64
+hadoop_2_3_0_0_2557-conf-pseudo.x86_64
+hadoop_2_3_0_0_2557-doc.x86_64
+hadoop_2_3_0_0_2557-hdfs.x86_64
+hadoop_2_3_0_0_2557-hdfs-datanode.x86_64
+hadoop_2_3_0_0_2557-hdfs-fuse.x86_64
+hadoop_2_3_0_0_2557-hdfs-journalnode.x86_64
+hadoop_2_3_0_0_2557-hdfs-namenode.x86_64
+hadoop_2_3_0_0_2557-hdfs-zkfc.x86_64
+hadoop_2_3_0_0_2557-httpfs.x86_64
+hadoop_2_3_0_0_2557-httpfs-server.x86_64
+hadoop_2_3_0_0_2557-libhdfs.x86_64
+hadoop_2_3_0_0_2557-mapreduce.x86_64
+hadoop_2_3_0_0_2557-source.x86_64
+hadoop_2_3_0_0_2557-yarn.x86_64
+hadoop_2_3_0_0_2557-yarn-nodemanager.x86_64
+hadoop_2_3_0_0_2557-yarn-proxyserver.x86_64
+hadoop_2_3_0_0_2557-yarn-resourcemanager.x86_64
+hadoop_2_3_0_0_2557-yarn-timelineserver.x86_64
+hbase_2_3_0_0_2557.noarch
+hbase_2_3_0_0_2557-doc.noarch
+hbase_2_3_0_0_2557-master.noarch
+hbase_2_3_0_0_2557-regionserver.noarch
+hbase_2_3_0_0_2557-rest.noarch
+hbase_2_3_0_0_2557-thrift.noarch
+hbase_2_3_0_0_2557-thrift2.noarch
+hdp-select.noarch
+phoenix_2_3_0_0_2557.noarch
+ranger_2_3_0_0_2557-hbase-plugin.x86_64
+ranger_2_3_0_0_2557-hdfs-plugin.x86_64
+ranger_2_3_0_0_2557-yarn-plugin.x86_64
+zookeeper_2_3_0_0_2557.noarch
+ambari-metrics-monitor.x86_64
+ambari-metrics-hadoop-sink.x86_64
\ No newline at end of file


Mime
View raw message