ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From smoha...@apache.org
Subject ambari git commit: AMBARI-12637. Ensure Flume can be deployed without any additional services (smohanty)
Date Wed, 05 Aug 2015 21:26:46 GMT
Repository: ambari
Updated Branches:
  refs/heads/trunk b32bf6e42 -> 314ae6c04


AMBARI-12637. Ensure Flume can be deployed without any additional services (smohanty)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/314ae6c0
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/314ae6c0
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/314ae6c0

Branch: refs/heads/trunk
Commit: 314ae6c046a3031a9fd8f9c3a3eabdd26a112a86
Parents: b32bf6e
Author: Sumit Mohanty <smohanty@hortonworks.com>
Authored: Wed Aug 5 14:25:32 2015 -0700
Committer: Sumit Mohanty <smohanty@hortonworks.com>
Committed: Wed Aug 5 14:25:32 2015 -0700

----------------------------------------------------------------------
 .../2.0.6/hooks/before-START/scripts/params.py  |  1 +
 .../before-START/scripts/rack_awareness.py      |  6 +-
 .../python/stacks/2.0.6/FLUME/test_flume.py     | 46 +++++++++--
 .../python/stacks/2.0.6/configs/flume_only.json | 81 ++++++++++++++++++++
 4 files changed, 127 insertions(+), 7 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/314ae6c0/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/scripts/params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/scripts/params.py
b/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/scripts/params.py
index 3ddefe3..1b42d44 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/scripts/params.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/scripts/params.py
@@ -56,6 +56,7 @@ current_service = config['serviceName']
 security_enabled = config['configurations']['cluster-env']['security_enabled']
 
 #users and groups
+has_hadoop_env = 'hadoop-env' in config['configurations']
 mapred_user = config['configurations']['mapred-env']['mapred_user']
 hdfs_user = config['configurations']['hadoop-env']['hdfs_user']
 yarn_user = config['configurations']['yarn-env']['yarn_user']

http://git-wip-us.apache.org/repos/asf/ambari/blob/314ae6c0/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/scripts/rack_awareness.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/scripts/rack_awareness.py
b/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/scripts/rack_awareness.py
index d24eb49..0b18ecb 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/scripts/rack_awareness.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/scripts/rack_awareness.py
@@ -43,5 +43,7 @@ def create_topology_script():
   )
 
 def create_topology_script_and_mapping():
-  create_topology_mapping()
-  create_topology_script()
\ No newline at end of file
+  import params
+  if params.has_hadoop_env:
+    create_topology_mapping()
+    create_topology_script()
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/ambari/blob/314ae6c0/ambari-server/src/test/python/stacks/2.0.6/FLUME/test_flume.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/FLUME/test_flume.py b/ambari-server/src/test/python/stacks/2.0.6/FLUME/test_flume.py
index 8143005..c3722c6 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/FLUME/test_flume.py
+++ b/ambari-server/src/test/python/stacks/2.0.6/FLUME/test_flume.py
@@ -73,6 +73,40 @@ class TestFlumeHandler(RMFTestCase):
 
     self.assertNoMoreResources()
 
+  @patch("os.path.isfile")
+  @patch("flume.cmd_target_names")
+  @patch("flume._set_desired_state")
+  def test_start_flume_only(self, set_desired_mock, cmd_target_names_mock, os_path_isfile_mock):
+    # 1st call is to check if the conf file is there - that should be True
+    # 2nd call is to check if the process is live - that should be False
+    os_path_isfile_mock.side_effect = [True, False]
+    cmd_target_names_mock.return_value = ["a1"]
+
+    self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/flume_handler.py",
+                       classname = "FlumeHandler",
+                       command = "start",
+                       config_file="flume_only.json",
+                       hdp_stack_version = self.STACK_VERSION,
+                       target = RMFTestCase.TARGET_COMMON_SERVICES)
+
+    self.assert_configure_default(check_mc=False)
+
+    self.assertTrue(set_desired_mock.called)
+    self.assertTrue(set_desired_mock.call_args[0][0] == 'STARTED')
+
+
+    self.assertResourceCalled('Execute', "ambari-sudo.sh su flume -l -s /bin/bash -c 'export
 PATH=/bin JAVA_HOME=/usr/jdk64/jdk1.7.0_45 ; /usr/bin/flume-ng agent --name a1 --conf /etc/flume/conf/a1
--conf-file /etc/flume/conf/a1/flume.conf  > /var/log/flume/a1.out 2>&1' &",
+                              environment = {'JAVA_HOME': u'/usr/jdk64/jdk1.7.0_45'},
+                              wait_for_finish = False,
+                              )
+    self.assertResourceCalled('Execute', "ambari-sudo.sh [RMF_ENV_PLACEHOLDER] -H -E pgrep
-o -u flume -f '^/usr/jdk64/jdk1.7.0_45.*a1.*' | ambari-sudo.sh [RMF_ENV_PLACEHOLDER] -H -E
tee /var/run/flume/a1.pid  && test ${PIPESTATUS[0]} -eq 0",
+                              logoutput = True,
+                              tries = 20,
+                              try_sleep = 10,
+                              )
+
+    self.assertNoMoreResources()
+
   @patch("glob.glob")
   @patch("flume._set_desired_state")
   @patch("flume.await_flume_process_termination")
@@ -179,7 +213,7 @@ class TestFlumeHandler(RMFTestCase):
     self.assertTrue(struct_out.has_key('processes'))
     self.assertNoMoreResources()
 
-  def assert_configure_default(self):
+  def assert_configure_default(self, check_mc=True):
     self.assertResourceCalled('Directory', '/var/run/flume',)
     self.assertResourceCalled('Directory',
                               '/etc/flume/conf',
@@ -217,10 +251,12 @@ class TestFlumeHandler(RMFTestCase):
                               owner="flume",
                               content=InlineTemplate(self.getConfig()['configurations']['flume-env']['content'])
     )
-    self.assertResourceCalled('File', "/etc/flume/conf/a1/flume-metrics2.properties",
-                              owner="flume",
-                              content=Template("flume-metrics2.properties.j2")
-    )
+
+    if check_mc:
+      self.assertResourceCalled('File', "/etc/flume/conf/a1/flume-metrics2.properties",
+                                owner="flume",
+                                content=Template("flume-metrics2.properties.j2")
+      )
 
   def assert_configure_many(self):
     self.assertResourceCalled('Directory', '/var/run/flume')

http://git-wip-us.apache.org/repos/asf/ambari/blob/314ae6c0/ambari-server/src/test/python/stacks/2.0.6/configs/flume_only.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/configs/flume_only.json b/ambari-server/src/test/python/stacks/2.0.6/configs/flume_only.json
new file mode 100644
index 0000000..4f81af8
--- /dev/null
+++ b/ambari-server/src/test/python/stacks/2.0.6/configs/flume_only.json
@@ -0,0 +1,81 @@
+{
+    "roleCommand": "SERVICE_CHECK", 
+    "clusterName": "c1", 
+    "hostname": "c6401.ambari.apache.org", 
+    "hostLevelParams": {
+        "jdk_location": "http://c6401.ambari.apache.org:8080/resources/", 
+        "ambari_db_rca_password": "mapred", 
+        "ambari_db_rca_url": "jdbc:postgresql://c6401.ambari.apache.org/ambarirca",
+        "repo_info": "[{\"baseUrl\":\"http://public-repo-1.hortonworks.com/HDP/centos6/2.x/updates/2.0.6.0\",\"osType\":\"centos6\",\"repoId\":\"HDP-2.0._\",\"repoName\":\"HDP\",\"defaultBaseUrl\":\"http://public-repo-1.hortonworks.com/HDP/centos6/2.x/updates/2.0.6.0\"}]",
+        "jce_name": "UnlimitedJCEPolicyJDK7.zip", 
+        "stack_version": "2.0",
+        "stack_name": "HDP", 
+        "ambari_db_rca_driver": "org.postgresql.Driver", 
+        "jdk_name": "jdk-7u67-linux-x64.tar.gz",
+        "ambari_db_rca_username": "mapred", 
+        "java_home": "/usr/jdk64/jdk1.7.0_45",
+        "java_version": "8",
+        "db_name": "ambari",
+        "group_list": "[\"hadoop\",\"nobody\",\"users\"]",
+        "user_list": "[\"hive\",\"oozie\",\"nobody\",\"ambari-qa\",\"flume\",\"hdfs\",\"storm\",\"mapred\",\"hbase\",\"tez\",\"zookeeper\",\"falcon\",\"sqoop\",\"yarn\",\"hcat\"]"
+    }, 
+    "commandType": "EXECUTION_COMMAND", 
+    "roleParams": {}, 
+    "serviceName": "HIVE",
+    "role": "HIVE_SERVER",
+    "commandParams": {
+        "command_timeout": "300", 
+        "service_package_folder": "OOZIE",
+        "script_type": "PYTHON",
+        "script": "scripts/service_check.py",
+        "excluded_hosts": "host1,host2",
+        "mark_draining_only" : "false",
+        "update_exclude_file_only" : "false",
+        "xml_configs_list":[{"hdfs-site.xml":"hdfs-site"}],
+        "env_configs_list":[{"hadoop-env.sh":"hadoop-env"},{"log4j.properties":"hdfs-log4j,yarn-log4j"}],
+        "properties_configs_list":[{"runtime.properties":"falcon-runtime.properties"},{"startup.properties":"falcon-startup.properties"}],
+        "output_file":"HDFS_CLIENT-configs.tar.gz",
+        "refresh_topology": "True"
+    },
+    "taskId": 152, 
+    "public_hostname": "c6401.ambari.apache.org", 
+    "configurations": {
+        "flume-env": {
+             "content": "export JAVA_HOME={{java64_home}}\nexport HIVE_HOME={{flume_hive_home}}",
+             "flume_user": "flume",
+             "flume_log_dir": "/var/log/flume",
+             "flume_run_dir": "/var/run/flume"
+        },
+        "flume-conf": {
+          "content": "a1.sources = r1\n a1.sinks = k1\n a1.channels = c1\n # Describe/configure
the source\n a1.sources.r1.type = netcat\n a1.sources.r1.bind = localhost\n a1.sources.r1.port
= 44444\n \n # Describe the sink\n a1.sinks.k1.type = logger\n \n # Use a channel which buffers
events in memory\n a1.channels.c1.type = memory\n a1.channels.c1.capacity = 1000\n a1.channels.c1.transactionCapacity
= 100\n \n # Bind the source and sink to the channel\n a1.sources.r1.channels = c1\n a1.sinks.k1.channel
= c1\n"
+        },
+        "flume-log4j": {
+          "content": "log4jproperties\nline2"
+        }
+    },
+    "configuration_attributes": {
+    },
+    "configurationTags": {
+    },
+    "commandId": "7-1", 
+    "clusterHostInfo": {
+      "all_ping_ports": [
+        "8670"
+      ],
+      "flume_hosts": [
+        "c6401.ambari.apache.org"
+      ],
+      "all_hosts": [
+        "c6401.ambari.apache.org"
+      ],
+      "all_racks": [
+        "/default-rack"
+      ],
+      "all_ipv4_ips": [
+        "192.168.64.101"
+      ],
+      "ambari_server_host": [
+        "c6401.ambari.apache.org"
+      ]
+    }
+}


Mime
View raw message