ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jonathanhur...@apache.org
Subject [20/30] ambari git commit: Integrate Druid with Ambari (Nishant Bangarwa, Slim Bouguerra via Swapan Shridhar).
Date Fri, 21 Oct 2016 17:20:08 GMT
http://git-wip-us.apache.org/repos/asf/ambari/blob/685e926d/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/scripts/status_params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/scripts/status_params.py b/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/scripts/status_params.py
new file mode 100644
index 0000000..ee1d61c
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/scripts/status_params.py
@@ -0,0 +1,24 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+from resource_management.libraries.script.script import Script
+
+config = Script.get_config()
+
+druid_pid_dir = config['configurations']['druid-env']['druid_pid_dir']

http://git-wip-us.apache.org/repos/asf/ambari/blob/685e926d/ambari-server/src/main/resources/common-services/DRUID/0.9.2/quicklinks/quicklinks.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/DRUID/0.9.2/quicklinks/quicklinks.json b/ambari-server/src/main/resources/common-services/DRUID/0.9.2/quicklinks/quicklinks.json
new file mode 100644
index 0000000..c68b9b9
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/DRUID/0.9.2/quicklinks/quicklinks.json
@@ -0,0 +1,37 @@
+{
+  "name": "default",
+  "description": "default quick links configuration",
+  "configuration": {
+    "protocol": {
+      "type": "HTTP_ONLY"
+    },
+    "links": [
+      {
+        "name": "coordinator_console",
+        "label": "Druid Coordinator Console",
+        "component_name": "DRUID_COORDINATOR",
+        "requires_user_name": "false",
+        "url": "%@://%@:%@",
+        "port": {
+          "http_property": "druid.port",
+          "http_default_port": "8081",
+          "regex": "^(\\d+)$",
+          "site": "druid-coordinator"
+        }
+      },
+      {
+        "name": "overlord_console",
+        "label": "Druid Overlord Console",
+        "component_name": "DRUID_OVERLORD",
+        "requires_user_name": "false",
+        "url": "%@://%@:%@",
+        "port": {
+          "http_property": "druid.port",
+          "http_default_port": "8090",
+          "regex": "^(\\d+)$",
+          "site": "druid-overlord"
+        }
+      }
+    ]
+  }
+}

http://git-wip-us.apache.org/repos/asf/ambari/blob/685e926d/ambari-server/src/main/resources/common-services/DRUID/0.9.2/themes/theme.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/DRUID/0.9.2/themes/theme.json b/ambari-server/src/main/resources/common-services/DRUID/0.9.2/themes/theme.json
new file mode 100644
index 0000000..f494594
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/DRUID/0.9.2/themes/theme.json
@@ -0,0 +1,120 @@
+{
+  "name": "default",
+  "description": "Default theme for Druid service",
+  "configuration": {
+    "layouts": [
+      {
+        "name": "default",
+        "tabs": [
+          {
+            "name": "metadata_storage",
+            "display-name": "META DATA STORAGE CONFIG",
+            "layout": {
+              "tab-columns": "1",
+              "tab-rows": "1",
+              "sections": [
+                {
+                  "name": "section-metadata-storage",
+                  "display-name": "",
+                  "row-index": "0",
+                  "column-index": "0",
+                  "row-span": "2",
+                  "column-span": "1",
+                  "section-columns": "1",
+                  "section-rows": "1",
+                  "subsections": [
+                    {
+                      "name": "subsection-metadata-storage-row1-col1",
+                      "display-name": "META DATA STORAGE",
+                      "row-index": "0",
+                      "column-index": "0",
+                      "row-span": "1",
+                      "column-span": "1"
+                    }
+                  ]
+                }
+              ]
+            }
+          }
+        ]
+      }
+    ],
+    "placement": {
+      "configuration-layout": "default",
+      "configs": [
+        {
+          "config": "druid-common/database_name",
+          "subsection-name": "subsection-metadata-storage-row1-col1"
+        },
+        {
+          "config": "druid-common/druid.metadata.storage.type",
+          "subsection-name": "subsection-metadata-storage-row1-col1"
+        },
+        {
+          "config": "druid-common/druid.metadata.storage.connector.user",
+          "subsection-name": "subsection-metadata-storage-row1-col1"
+        },
+        {
+          "config": "druid-common/druid.metadata.storage.connector.password",
+          "subsection-name": "subsection-metadata-storage-row1-col1"
+        },
+        {
+          "config": "druid-common/metastore_hostname",
+          "subsection-name": "subsection-metadata-storage-row1-col1"
+        },
+        {
+          "config": "druid-common/druid.metadata.storage.connector.port",
+          "subsection-name": "subsection-metadata-storage-row1-col1"
+        },
+        {
+          "config": "druid-common/druid.metadata.storage.connector.connectURI",
+          "subsection-name": "subsection-metadata-storage-row1-col1"
+        }
+      ]
+    },
+    "widgets": [
+      {
+        "config": "druid-common/database_name",
+        "widget": {
+          "type": "text-field"
+        }
+      },
+      {
+        "config": "druid-common/druid.metadata.storage.type",
+        "widget": {
+          "type": "combo"
+        }
+      },
+      {
+        "config": "druid-common/druid.metadata.storage.connector.user",
+        "widget": {
+          "type": "text-field"
+        }
+      },
+      {
+        "config": "druid-common/druid.metadata.storage.connector.password",
+        "widget": {
+          "type": "password"
+        }
+      },
+      {
+        "config": "druid-common/metastore_hostname",
+        "widget": {
+          "type": "text-field"
+        }
+      },
+      {
+        "config": "druid-common/druid.metadata.storage.connector.port",
+        "widget": {
+          "type": "text-field"
+        }
+      },
+      {
+        "config": "druid-common/druid.metadata.storage.connector.connectURI",
+        "widget": {
+          "type": "text-field"
+        }
+      }
+    ]
+  }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/ambari/blob/685e926d/ambari-server/src/main/resources/stacks/HDP/2.6/role_command_order.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/role_command_order.json b/ambari-server/src/main/resources/stacks/HDP/2.6/role_command_order.json
new file mode 100644
index 0000000..66b6ed2
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/HDP/2.6/role_command_order.json
@@ -0,0 +1,19 @@
+{
+  "_comment" : "Record format:",
+  "_comment" : "blockedRole-blockedCommand: [blockerRole1-blockerCommand1, blockerRole2-blockerCommand2, ...]",
+  "general_deps" : {
+    "_comment" : "dependencies for all cases",
+    "DRUID_HISTORICAL-START" : ["ZOOKEEPER_SERVER-START", "NAMENODE-START", "DATANODE-START", "RESOURCEMANAGER-START", "NODEMANAGER-START"],
+    "DRUID_OVERLORD-START" : ["ZOOKEEPER_SERVER-START", "NAMENODE-START", "DATANODE-START", "RESOURCEMANAGER-START", "NODEMANAGER-START"],
+    "DRUID_MIDDLEMANAGER-START" : ["ZOOKEEPER_SERVER-START", "NAMENODE-START", "DATANODE-START", "RESOURCEMANAGER-START", "NODEMANAGER-START"],
+    "DRUID_BROKER-START" : ["ZOOKEEPER_SERVER-START", "NAMENODE-START", "DATANODE-START", "RESOURCEMANAGER-START", "NODEMANAGER-START"],
+    "DRUID_ROUTER-START" : ["ZOOKEEPER_SERVER-START", "NAMENODE-START", "DATANODE-START", "RESOURCEMANAGER-START", "NODEMANAGER-START"],
+    "DRUID_COORDINATOR-START" : ["ZOOKEEPER_SERVER-START", "NAMENODE-START", "DATANODE-START", "RESOURCEMANAGER-START", "NODEMANAGER-START"],
+    "DRUID_OVERLORD-RESTART" : ["DRUID_HISTORICAL-RESTART"],
+    "DRUID_MIDDLEMANAGER-RESTART" : ["DRUID_OVERLORD-RESTART"],
+    "DRUID_BROKER-RESTART" : ["DRUID_MIDDLEMANAGER-RESTART"],
+    "DRUID_ROUTER-RESTART" : ["DRUID_BROKER-RESTART"],
+    "DRUID_COORDINATOR-RESTART" : ["DRUID_ROUTER-RESTART"],
+    "DRUID_SERVICE_CHECK-SERVICE_CHECK" : ["DRUID_HISTORICAL-START", "DRUID_COORDINATOR-START", "DRUID_OVERLORD-START", "DRUID_MIDDLEMANAGER-START", "DRUID_BROKER-START", "DRUID_ROUTER-START"]
+  }
+}

http://git-wip-us.apache.org/repos/asf/ambari/blob/685e926d/ambari-server/src/main/resources/stacks/HDP/2.6/services/DRUID/kerberos.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/services/DRUID/kerberos.json b/ambari-server/src/main/resources/stacks/HDP/2.6/services/DRUID/kerberos.json
new file mode 100644
index 0000000..1661285
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/HDP/2.6/services/DRUID/kerberos.json
@@ -0,0 +1,78 @@
+{
+  "services": [
+    {
+      "name": "DRUID",
+      "identities": [
+        {
+          "name": "/spnego"
+        },
+        {
+          "name": "druid",
+          "principal": {
+            "value": "${druid-env/druid_user}@${realm}",
+            "type": "user",
+            "configuration": "druid-common/druid.hadoop.security.kerberos.principal",
+            "local_username": "${druid-env/druid_user}"
+          },
+          "keytab": {
+            "file": "${keytab_dir}/druid.headless.keytab",
+            "owner": {
+              "name": "${druid-env/druid_user}",
+              "access": "r"
+            },
+            "group": {
+              "name": "${cluster-env/user_group}",
+              "access": "r"
+            },
+            "configuration": "druid-common/druid.hadoop.security.kerberos.keytab"
+          }
+        },
+        {
+          "name": "/smokeuser"
+        }
+      ],
+      "components": [
+        {
+          "name": "DRUID_HISTORICAL",
+          "identities": [
+            {
+              "name": "/druid"
+            }
+          ]
+        },
+        {
+          "name": "DRUID_BROKER",
+          "identities": [
+            {
+              "name": "/druid"
+            }
+          ]
+        },
+        {
+          "name": "DRUID_OVERLORD",
+          "identities": [
+            {
+              "name": "/druid"
+            }
+          ]
+        },
+        {
+          "name": "DRUID_COORDINATOR",
+          "identities": [
+            {
+              "name": "/druid"
+            }
+          ]
+        },
+        {
+          "name": "DRUID_MIDDLEMANAGER",
+          "identities": [
+            {
+              "name": "/druid"
+            }
+          ]
+        }
+      ]
+    }
+  ]
+}

http://git-wip-us.apache.org/repos/asf/ambari/blob/685e926d/ambari-server/src/main/resources/stacks/HDP/2.6/services/DRUID/metainfo.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/services/DRUID/metainfo.xml b/ambari-server/src/main/resources/stacks/HDP/2.6/services/DRUID/metainfo.xml
new file mode 100644
index 0000000..49d09b1
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/HDP/2.6/services/DRUID/metainfo.xml
@@ -0,0 +1,27 @@
+<?xml version="1.0"?>
+<!--
+   Licensed to the Apache Software Foundation (ASF) under one or more
+   contributor license agreements.  See the NOTICE file distributed with
+   this work for additional information regarding copyright ownership.
+   The ASF licenses this file to You under the Apache License, Version 2.0
+   (the "License"); you may not use this file except in compliance with
+   the License.  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+-->
+<metainfo>
+  <schemaVersion>2.0</schemaVersion>
+  <services>
+    <service>
+      <name>DRUID</name>
+      <version>0.9.2.2.6</version>
+      <extends>common-services/DRUID/0.9.2</extends>
+    </service>
+  </services>
+</metainfo>

http://git-wip-us.apache.org/repos/asf/ambari/blob/685e926d/ambari-server/src/main/resources/stacks/HDP/2.6/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.6/services/stack_advisor.py
index 1f722dc..49dd086 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.6/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.6/services/stack_advisor.py
@@ -16,6 +16,191 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 """
+from resource_management.core.logger import Logger
+import json
+from resource_management.libraries.functions import format
+
 
 class HDP26StackAdvisor(HDP25StackAdvisor):
-  pass
\ No newline at end of file
+    def __init__(self):
+        super(HDP26StackAdvisor, self).__init__()
+        Logger.initialize_logger()
+
+    def getServiceConfigurationRecommenderDict(self):
+        parentRecommendConfDict = super(HDP26StackAdvisor, self).getServiceConfigurationRecommenderDict()
+        childRecommendConfDict = {
+            "DRUID": self.recommendDruidConfigurations
+        }
+        parentRecommendConfDict.update(childRecommendConfDict)
+        return parentRecommendConfDict
+
+    def recommendDruidConfigurations(self, configurations, clusterData, services, hosts):
+
+        componentsListList = [service["components"] for service in services["services"]]
+        componentsList = [item["StackServiceComponents"] for sublist in componentsListList for item in sublist]
+        servicesList = [service["StackServices"]["service_name"] for service in services["services"]]
+        putCommonProperty = self.putProperty(configurations, "druid-common", services)
+
+        putCommonProperty('druid.zk.service.host', self.getZKHostPortString(services))
+        self.recommendDruidMaxMemoryLimitConfigurations(configurations, clusterData, services, hosts)
+
+        # recommending the metadata storage uri
+        database_name = services['configurations']["druid-common"]["properties"]["database_name"]
+        metastore_hostname = services['configurations']["druid-common"]["properties"]["metastore_hostname"]
+        database_type = services['configurations']["druid-common"]["properties"]["druid.metadata.storage.type"]
+        metadata_storage_port = "1527"
+        mysql_extension_name = "io.druid.extensions:mysql-metadata-storage"
+        mysql_module_name = "mysql-metadata-storage"
+        postgres_module_name = "postgresql-metadata-storage"
+        extensions_load_list = services['configurations']['druid-common']['properties']['druid.extensions.loadList']
+        extensions_pull_list = services['configurations']['druid-common']['properties']['druid.extensions.pullList']
+        putDruidCommonProperty = self.putProperty(configurations, "druid-common", services)
+
+        extensions_pull_list = self.removeFromList(extensions_pull_list, mysql_extension_name)
+        extensions_load_list = self.removeFromList(extensions_load_list, mysql_module_name)
+        extensions_load_list = self.removeFromList(extensions_load_list, postgres_module_name)
+
+        if database_type == 'mysql':
+            metadata_storage_port = "3306"
+            extensions_pull_list = self.addToList(extensions_pull_list, mysql_extension_name)
+            extensions_load_list = self.addToList(extensions_load_list, mysql_module_name)
+
+        if database_type == 'postgres':
+            extensions_load_list = self.addToList(extensions_load_list, postgres_module_name)
+            metadata_storage_port = "5432"
+
+        putDruidCommonProperty('druid.metadata.storage.connector.port', metadata_storage_port)
+        putDruidCommonProperty('druid.metadata.storage.connector.connectURI',
+                               self.getMetadataConnectionString(database_type).format(metastore_hostname, database_name,
+                                                                                      metadata_storage_port))
+        # HDFS is installed
+        if "HDFS" in servicesList and "hdfs-site" in services["configurations"]:
+            # recommend HDFS as default deep storage
+            extensions_load_list = self.addToList(extensions_load_list, "druid-hdfs-storage")
+            putCommonProperty("druid.storage.type", "hdfs")
+            putCommonProperty("druid.storage.storageDirectory", "/user/druid/data")
+            # configure indexer logs configs
+            putCommonProperty("druid.indexer.logs.type", "hdfs")
+            putCommonProperty("druid.indexer.logs.directory", "/user/druid/logs")
+
+        if "KAFKA" in servicesList:
+            extensions_load_list = self.addToList(extensions_load_list, "druid-kafka-indexing-service")
+
+        putCommonProperty('druid.extensions.loadList', extensions_load_list)
+        putCommonProperty('druid.extensions.pullList', extensions_pull_list)
+
+        # JVM Configs go to env properties
+        putEnvProperty = self.putProperty(configurations, "druid-env", services)
+
+        # processing thread pool Config
+        for component in ['DRUID_HISTORICAL', 'DRUID_BROKER']:
+            component_hosts = self.getHostsWithComponent("DRUID", component, services, hosts)
+            nodeType = self.DRUID_COMPONENT_NODE_TYPE_MAP[component]
+            putComponentProperty = self.putProperty(configurations, format("druid-{nodeType}"), services)
+            if (component_hosts is not None and len(component_hosts) > 0):
+                totalAvailableCpu = self.getMinCpu(component_hosts)
+                processingThreads = 1
+                if totalAvailableCpu > 1:
+                    processingThreads = totalAvailableCpu - 1
+                putComponentProperty('druid.processing.numThreads', processingThreads)
+                putComponentProperty('druid.server.http.numThreads', max(10, (totalAvailableCpu * 17) / 16 + 2) + 30)
+
+    def getMetadataConnectionString(self, database_type):
+        driverDict = {
+            'mysql': 'jdbc:mysql://{0}:{2}/{1}?createDatabaseIfNotExist=true',
+            'derby': 'jdbc:derby://{0}:{2}/{1};create=true',
+            'postgres': 'jdbc:postgresql://{0}:{2}/{1}'
+        }
+        return driverDict.get(database_type.lower())
+
+    def addToList(self, json_list, word):
+        desr_list = json.loads(json_list)
+        if word not in desr_list:
+            desr_list.append(word)
+        return json.dumps(desr_list)
+
+    def removeFromList(self, json_list, word):
+        desr_list = json.loads(json_list)
+        if word in desr_list:
+            desr_list.remove(word)
+        return json.dumps(desr_list)
+
+    def recommendDruidMaxMemoryLimitConfigurations(self, configurations, clusterData, services, hosts):
+        putEnvPropertyAttribute = self.putPropertyAttribute(configurations, "druid-env")
+        for component in ["DRUID_HISTORICAL", "DRUID_MIDDLEMANAGER", "DRUID_BROKER", "DRUID_OVERLORD",
+                          "DRUID_COORDINATOR"]:
+            component_hosts = self.getHostsWithComponent("DRUID", component, services, hosts)
+            if component_hosts is not None and len(component_hosts) > 0:
+                totalAvailableMem = self.getMinMemory(component_hosts) / 1024  # In MB
+                nodeType = self.DRUID_COMPONENT_NODE_TYPE_MAP[component]
+                putEnvPropertyAttribute(format('druid.{nodeType}.jvm.heap.memory'), 'maximum',
+                                        max(totalAvailableMem, 1024))
+
+    DRUID_COMPONENT_NODE_TYPE_MAP = {
+        'DRUID_BROKER': 'broker',
+        'DRUID_COORDINATOR': 'coordinator',
+        'DRUID_HISTORICAL': 'historical',
+        'DRUID_MIDDLEMANAGER': 'middlemanager',
+        'DRUID_OVERLORD': 'overlord',
+        'DRUID_ROUTER': 'router'
+    }
+
+    def getMinMemory(self, component_hosts):
+        min_ram_kb = 1073741824  # 1 TB
+        for host in component_hosts:
+            ram_kb = host['Hosts']['total_mem']
+            min_ram_kb = min(min_ram_kb, ram_kb)
+        return min_ram_kb
+
+    def getMinCpu(self, component_hosts):
+        min_cpu = 256
+        for host in component_hosts:
+            cpu_count = host['Hosts']['cpu_count']
+            min_cpu = min(min_cpu, cpu_count)
+        return min_cpu
+
+    def getServiceConfigurationValidators(self):
+        parentValidators = super(HDP26StackAdvisor, self).getServiceConfigurationValidators()
+        childValidators = {
+            "DRUID": {"druid-env": self.validateDruidEnvConfigurations,
+                      "druid-historical": self.validateDruidHistoricalConfigurations,
+                      "druid-broker": self.validateDruidBrokerConfigurations}
+        }
+        self.mergeValidators(parentValidators, childValidators)
+        return parentValidators
+
+    def validateDruidEnvConfigurations(self, properties, recommendedDefaults, configurations, services, hosts):
+        validationItems = []
+        #  Minimum Direct memory Validation
+        envProperties = services['configurations']['druid-env']['properties']
+        for nodeType in ['broker', 'historical']:
+            properties = services['configurations'][format('druid-{nodeType}')]['properties']
+            intermediateBufferSize = int(properties['druid.processing.buffer.sizeBytes']) / (1024 * 1024)  # In MBs
+            processingThreads = int(properties['druid.processing.numThreads'])
+            directMemory = int(envProperties[format('druid.{nodeType}.jvm.direct.memory')])
+            if directMemory < (processingThreads + 1) * intermediateBufferSize:
+                validationItems.extend(
+                    {"config-name": format("druid.{nodeType}.jvm.direct.memory"), "item": self.getErrorItem(
+                        format(
+                            "Not enough direct memory available for {nodeType} Node."
+                            "Please adjust druid.{nodeType}.jvm.direct.memory, druid.processing.buffer.sizeBytes, druid.processing.numThreads"
+                        )
+                    )
+                     })
+        return self.toConfigurationValidationProblems(validationItems, "druid-env")
+
+    def validateDruidHistoricalConfigurations(self, properties, recommendedDefaults, configurations, services, hosts):
+        validationItems = [
+            {"config-name": "druid.processing.numThreads",
+             "item": self.validatorEqualsToRecommendedItem(properties, recommendedDefaults,
+                                                           "druid.processing.numThreads")}
+        ]
+        return self.toConfigurationValidationProblems(validationItems, "druid-historical")
+
+    def validateDruidBrokerConfigurations(self, properties, recommendedDefaults, configurations, services, hosts):
+        validationItems = [
+            {"config-name": "druid.processing.numThreads",
+             "item": self.validatorEqualsToRecommendedItem(properties, recommendedDefaults,
+                                                           "druid.processing.numThreads")}
+        ]
+        return self.toConfigurationValidationProblems(validationItems, "druid-broker")

http://git-wip-us.apache.org/repos/asf/ambari/blob/685e926d/ambari-server/src/test/python/stacks/2.6/DRUID/test_druid.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.6/DRUID/test_druid.py b/ambari-server/src/test/python/stacks/2.6/DRUID/test_druid.py
new file mode 100644
index 0000000..2988169
--- /dev/null
+++ b/ambari-server/src/test/python/stacks/2.6/DRUID/test_druid.py
@@ -0,0 +1,647 @@
+#!/usr/bin/env python
+
+'''
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+'''
+
+import os
+
+from stacks.utils.RMFTestCase import *
+
+from mock.mock import MagicMock, patch
+from resource_management.libraries import functions
+from resource_management.libraries.functions import format
+from resource_management.core.logger import Logger
+
+@patch("resource_management.libraries.Script.get_tmp_dir", new=MagicMock(return_value=('/var/lib/ambari-agent/tmp')))
+@patch.object(functions, "get_stack_version", new=MagicMock(return_value="2.0.0.0-1234"))
+class TestDruid(RMFTestCase):
+  COMMON_SERVICES_PACKAGE_DIR = "DRUID/0.9.2/package"
+  STACK_VERSION = "2.6"
+  DEFAULT_IMMUTABLE_PATHS = ['/apps/hive/warehouse', '/apps/falcon', '/mr-history/done', '/app-logs', '/tmp']
+
+  def setUp(self):
+    Logger.logger = MagicMock()
+    self.testDirectory = os.path.dirname(os.path.abspath(__file__))
+    self.num_times_to_iterate = 3
+    self.wait_time = 1
+
+  def test_configure_overlord(self):
+    self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/overlord.py",
+                       classname="DruidOverlord",
+                       command="configure",
+                       config_file=self.get_src_folder() + "/test/python/stacks/2.6/configs/default.json",
+                       config_overrides = { 'role' : 'DRUID_OVERLORD' },
+                       stack_version=self.STACK_VERSION,
+                       target=RMFTestCase.TARGET_COMMON_SERVICES
+                       )
+    self.assert_configure_default('druid-overlord')
+    self.assertNoMoreResources()
+
+  def test_start_overlord(self):
+    self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/overlord.py",
+                       classname="DruidOverlord",
+                       command="start",
+                       config_file=self.get_src_folder() + "/test/python/stacks/2.6/configs/default.json",
+                       stack_version=self.STACK_VERSION,
+                       config_overrides = { 'role' : 'DRUID_OVERLORD' },
+                       target=RMFTestCase.TARGET_COMMON_SERVICES
+                       )
+    self.assert_configure_default('druid-overlord')
+    self.assertResourceCalled('Execute', format('source /usr/hdp/current/druid-overlord/conf/druid-env.sh ; /usr/hdp/current/druid-overlord/bin/node.sh overlord start'),
+                              user='druid'
+                              )
+    self.assertNoMoreResources()
+
+  def test_stop_overlord(self):
+    self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/overlord.py",
+                       classname="DruidOverlord",
+                       command="stop",
+                       config_file=self.get_src_folder() + "/test/python/stacks/2.6/configs/default.json",
+                       stack_version=self.STACK_VERSION,
+                       config_overrides = { 'role' : 'DRUID_OVERLORD' },
+                       target=RMFTestCase.TARGET_COMMON_SERVICES
+                       )
+    self.assertResourceCalled('Execute', format('source /usr/hdp/current/druid-overlord/conf/druid-env.sh ; /usr/hdp/current/druid-overlord/bin/node.sh overlord stop'),
+                              user='druid'
+                              )
+    self.assertNoMoreResources()
+
+  def test_configure_coordinator(self):
+    self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/coordinator.py",
+                       classname="DruidCoordinator",
+                       command="configure",
+                       config_file=self.get_src_folder() + "/test/python/stacks/2.6/configs/default.json",
+                       config_overrides = { 'role' : 'DRUID_COORDINATOR' },
+                       stack_version=self.STACK_VERSION,
+                       target=RMFTestCase.TARGET_COMMON_SERVICES
+                       )
+    self.assert_configure_default('druid-coordinator')
+    self.assertNoMoreResources()
+
+  def test_start_coordinator(self):
+    self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/coordinator.py",
+                       classname="DruidCoordinator",
+                       command="start",
+                       config_file=self.get_src_folder() + "/test/python/stacks/2.6/configs/default.json",
+                       stack_version=self.STACK_VERSION,
+                       config_overrides = { 'role' : 'DRUID_COORDINATOR' },
+                       target=RMFTestCase.TARGET_COMMON_SERVICES
+                       )
+    self.assert_configure_default('druid-coordinator')
+    self.assertResourceCalled('Execute', format('source /usr/hdp/current/druid-coordinator/conf/druid-env.sh ; /usr/hdp/current/druid-coordinator/bin/node.sh coordinator start'),
+                              user='druid'
+                              )
+    self.assertNoMoreResources()
+
+  def test_stop_coordinator(self):
+    self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/coordinator.py",
+                       classname="DruidCoordinator",
+                       command="stop",
+                       config_file=self.get_src_folder() + "/test/python/stacks/2.6/configs/default.json",
+                       stack_version=self.STACK_VERSION,
+                       config_overrides = { 'role' : 'DRUID_COORDINATOR' },
+                       target=RMFTestCase.TARGET_COMMON_SERVICES
+                       )
+    self.assertResourceCalled('Execute', format('source /usr/hdp/current/druid-coordinator/conf/druid-env.sh ; /usr/hdp/current/druid-coordinator/bin/node.sh coordinator stop'),
+                              user='druid'
+                              )
+    self.assertNoMoreResources()
+
+  def test_configure_broker(self):
+    self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/broker.py",
+                       classname="DruidBroker",
+                       command="configure",
+                       config_file=self.get_src_folder() + "/test/python/stacks/2.6/configs/default.json",
+                       config_overrides = { 'role' : 'DRUID_BROKER' },
+                       stack_version=self.STACK_VERSION,
+                       target=RMFTestCase.TARGET_COMMON_SERVICES
+                       )
+    self.assert_configure_default('druid-broker')
+    self.assertNoMoreResources()
+
+  def test_start_broker(self):
+    self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/broker.py",
+                       classname="DruidBroker",
+                       command="start",
+                       config_file=self.get_src_folder() + "/test/python/stacks/2.6/configs/default.json",
+                       stack_version=self.STACK_VERSION,
+                       config_overrides = { 'role' : 'DRUID_BROKER' },
+                       target=RMFTestCase.TARGET_COMMON_SERVICES
+                       )
+    self.assert_configure_default('druid-broker')
+    self.assertResourceCalled('Execute', format('source /usr/hdp/current/druid-broker/conf/druid-env.sh ; /usr/hdp/current/druid-broker/bin/node.sh broker start'),
+                              user='druid'
+                              )
+    self.assertNoMoreResources()
+
+  def test_stop_broker(self):
+    self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/broker.py",
+                       classname="DruidBroker",
+                       command="stop",
+                       config_file=self.get_src_folder() + "/test/python/stacks/2.6/configs/default.json",
+                       stack_version=self.STACK_VERSION,
+                       config_overrides = { 'role' : 'DRUID_BROKER' },
+                       target=RMFTestCase.TARGET_COMMON_SERVICES
+                       )
+    self.assertResourceCalled('Execute', format('source /usr/hdp/current/druid-broker/conf/druid-env.sh ; /usr/hdp/current/druid-broker/bin/node.sh broker stop'),
+                              user='druid'
+                              )
+    self.assertNoMoreResources()
+
+  def test_configure_router(self):
+    self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/router.py",
+                       classname="DruidRouter",
+                       command="configure",
+                       config_file=self.get_src_folder() + "/test/python/stacks/2.6/configs/default.json",
+                       config_overrides = { 'role' : 'DRUID_ROUTER' },
+                       stack_version=self.STACK_VERSION,
+                       target=RMFTestCase.TARGET_COMMON_SERVICES
+                       )
+    self.assert_configure_default('druid-router')
+    self.assertNoMoreResources()
+
+  def test_start_router(self):
+    self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/router.py",
+                       classname="DruidRouter",
+                       command="start",
+                       config_file=self.get_src_folder() + "/test/python/stacks/2.6/configs/default.json",
+                       stack_version=self.STACK_VERSION,
+                       config_overrides = { 'role' : 'DRUID_ROUTER' },
+                       target=RMFTestCase.TARGET_COMMON_SERVICES
+                       )
+    self.assert_configure_default('druid-router')
+    self.assertResourceCalled('Execute', format('source /usr/hdp/current/druid-router/conf/druid-env.sh ; /usr/hdp/current/druid-router/bin/node.sh router start'),
+                              user='druid'
+                              )
+    self.assertNoMoreResources()
+
+  def test_stop_router(self):
+    self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/router.py",
+                       classname="DruidRouter",
+                       command="stop",
+                       config_file=self.get_src_folder() + "/test/python/stacks/2.6/configs/default.json",
+                       stack_version=self.STACK_VERSION,
+                       config_overrides = { 'role' : 'DRUID_ROUTER' },
+                       target=RMFTestCase.TARGET_COMMON_SERVICES
+                       )
+    self.assertResourceCalled('Execute', format('source /usr/hdp/current/druid-router/conf/druid-env.sh ; /usr/hdp/current/druid-router/bin/node.sh router stop'),
+                              user='druid'
+                              )
+    self.assertNoMoreResources()
+
+  def test_configure_historical(self):
+    self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/historical.py",
+                       classname="DruidHistorical",
+                       command="configure",
+                       config_file=self.get_src_folder() + "/test/python/stacks/2.6/configs/default.json",
+                       config_overrides = { 'role' : 'DRUID_HISTORICAL' },
+                       stack_version=self.STACK_VERSION,
+                       target=RMFTestCase.TARGET_COMMON_SERVICES
+                       )
+    self.assert_configure_default('druid-historical')
+    self.assertNoMoreResources()
+
+  def test_start_historical(self):
+    self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/historical.py",
+                       classname="DruidHistorical",
+                       command="start",
+                       config_file=self.get_src_folder() + "/test/python/stacks/2.6/configs/default.json",
+                       stack_version=self.STACK_VERSION,
+                       config_overrides = { 'role' : 'DRUID_HISTORICAL' },
+                       target=RMFTestCase.TARGET_COMMON_SERVICES
+                       )
+    self.assert_configure_default('druid-historical')
+    self.assertResourceCalled('Execute', format('source /usr/hdp/current/druid-historical/conf/druid-env.sh ; /usr/hdp/current/druid-historical/bin/node.sh historical start'),
+                              user='druid'
+                              )
+    self.assertNoMoreResources()
+
+  def test_stop_historical(self):
+    self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/historical.py",
+                       classname="DruidHistorical",
+                       command="stop",
+                       config_file=self.get_src_folder() + "/test/python/stacks/2.6/configs/default.json",
+                       stack_version=self.STACK_VERSION,
+                       config_overrides = { 'role' : 'DRUID_HISTORICAL' },
+                       target=RMFTestCase.TARGET_COMMON_SERVICES
+                       )
+    self.assertResourceCalled('Execute', format('source /usr/hdp/current/druid-historical/conf/druid-env.sh ; /usr/hdp/current/druid-historical/bin/node.sh historical stop'),
+                              user='druid'
+                              )
+    self.assertNoMoreResources()
+
+  def test_configure_middleManager(self):
+    self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/middlemanager.py",
+                       classname="DruidMiddleManager",
+                       command="configure",
+                       config_file=self.get_src_folder() + "/test/python/stacks/2.6/configs/default.json",
+                       config_overrides = { 'role' : 'DRUID_MIDDLEMANAGER' },
+                       stack_version=self.STACK_VERSION,
+                       target=RMFTestCase.TARGET_COMMON_SERVICES
+                       )
+    self.assert_configure_default('druid-middlemanager')
+    self.assertNoMoreResources()
+
+  def test_start_middleManager(self):
+    self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/middlemanager.py",
+                       classname="DruidMiddleManager",
+                       command="start",
+                       config_file=self.get_src_folder() + "/test/python/stacks/2.6/configs/default.json",
+                       stack_version=self.STACK_VERSION,
+                       config_overrides = { 'role' : 'DRUID_MIDDLEMANAGER' },
+                       target=RMFTestCase.TARGET_COMMON_SERVICES
+                       )
+    self.assert_configure_default('druid-middlemanager')
+    self.assertResourceCalled('Execute', format('source /usr/hdp/current/druid-middlemanager/conf/druid-env.sh ; /usr/hdp/current/druid-middlemanager/bin/node.sh middleManager start'),
+                              user='druid'
+                              )
+    self.assertNoMoreResources()
+
+  def test_stop_middleManager(self):
+    self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/middlemanager.py",
+                       classname="DruidMiddleManager",
+                       command="stop",
+                       config_file=self.get_src_folder() + "/test/python/stacks/2.6/configs/default.json",
+                       stack_version=self.STACK_VERSION,
+                       config_overrides = { 'role' : 'DRUID_MIDDLEMANAGER' },
+                       target=RMFTestCase.TARGET_COMMON_SERVICES
+                       )
+    self.assertResourceCalled('Execute', format('source /usr/hdp/current/druid-middlemanager/conf/druid-env.sh ; /usr/hdp/current/druid-middlemanager/bin/node.sh middleManager stop'),
+                              user='druid'
+                              )
+    self.assertNoMoreResources()
+
+  def assert_configure_default(self, role):
+
+    self.assertResourceCalled('Directory', '/var/log/druid',
+                              mode=0755,
+                              cd_access='a',
+                              owner='druid',
+                              group='hadoop',
+                              create_parents=True,
+                              recursive_ownership=True
+                              )
+
+    self.assertResourceCalled('Directory', '/var/run/druid',
+                              mode=0755,
+                              cd_access='a',
+                              owner='druid',
+                              group='hadoop',
+                              create_parents=True,
+                              recursive_ownership=True
+                              )
+
+    self.assertResourceCalled('Directory', format('/usr/hdp/current/{role}/conf/_common'),
+                              mode=0755,
+                              cd_access='a',
+                              owner='druid',
+                              group='hadoop',
+                              create_parents=True,
+                              recursive_ownership=True
+                              )
+
+    self.assertResourceCalled('Directory', format('/usr/hdp/current/{role}/conf/coordinator'),
+                              mode=0755,
+                              cd_access='a',
+                              owner='druid',
+                              group='hadoop',
+                              create_parents=True,
+                              recursive_ownership=True
+                              )
+
+    self.assertResourceCalled('Directory', format('/usr/hdp/current/{role}/conf/broker'),
+                              mode=0755,
+                              cd_access='a',
+                              owner='druid',
+                              group='hadoop',
+                              create_parents=True,
+                              recursive_ownership=True
+                              )
+
+    self.assertResourceCalled('Directory', format('/usr/hdp/current/{role}/conf/middleManager'),
+                              mode=0755,
+                              cd_access='a',
+                              owner='druid',
+                              group='hadoop',
+                              create_parents=True,
+                              recursive_ownership=True
+                              )
+
+    self.assertResourceCalled('Directory', format('/usr/hdp/current/{role}/conf/historical'),
+                              mode=0755,
+                              cd_access='a',
+                              owner='druid',
+                              group='hadoop',
+                              create_parents=True,
+                              recursive_ownership=True
+                              )
+
+    self.assertResourceCalled('Directory', format('/usr/hdp/current/{role}/conf/overlord'),
+                              mode=0755,
+                              cd_access='a',
+                              owner='druid',
+                              group='hadoop',
+                              create_parents=True,
+                              recursive_ownership=True
+                              )
+
+    self.assertResourceCalled('Directory', format('/usr/hdp/current/{role}/conf/router'),
+                              mode=0755,
+                              cd_access='a',
+                              owner='druid',
+                              group='hadoop',
+                              create_parents=True,
+                              recursive_ownership=True
+                              )
+
+    self.assertResourceCalled('Directory', '/apps/druid/segmentCache',
+                              mode=0755,
+                              cd_access='a',
+                              owner='druid',
+                              group='hadoop',
+                              create_parents=True,
+                              recursive_ownership=True
+                              )
+
+    self.assertResourceCalled('File', format('/usr/hdp/current/{role}/conf/druid-env.sh'),
+                              owner = 'druid',
+                              content = InlineTemplate(self.getConfig()['configurations']['druid-env']['content'])
+                              )
+    druid_common_config = mutable_config_dict(self.getConfig()['configurations']['druid-common'])
+    druid_common_config['druid.host'] = 'c6401.ambari.apache.org'
+    druid_common_config['druid.extensions.directory'] = format('/usr/hdp/current/{role}/extensions')
+    druid_common_config['druid.extensions.hadoopDependenciesDir'] = format('/usr/hdp/current/{role}/hadoop-dependencies')
+    druid_common_config['druid.selectors.indexing.serviceName'] = 'druid/overlord'
+    druid_common_config['druid.selectors.coordinator.serviceName'] = 'druid/coordinator'
+
+    self.assertResourceCalled('PropertiesFile', 'common.runtime.properties',
+                              dir=format("/usr/hdp/current/{role}/conf/_common"),
+                              properties=druid_common_config,
+                              owner='druid',
+                              group='hadoop'
+                              )
+
+    self.assertResourceCalled('File', format('/usr/hdp/current/{role}/conf/_common/druid-log4j.xml'),
+                              mode=0644,
+                              owner = 'druid',
+                              group = 'hadoop',
+                              content = InlineTemplate(self.getConfig()['configurations']['druid-log4j']['content'])
+                              )
+
+    self.assertResourceCalled('File', '/etc/logrotate.d/druid',
+                              mode=0644,
+                              owner = 'root',
+                              group = 'root',
+                              content = InlineTemplate(self.getConfig()['configurations']['druid-logrotate']['content'])
+                              )
+
+    self.assertResourceCalled('XmlConfig', "core-site.xml",
+                              conf_dir=format('/usr/hdp/current/{role}/conf/_common'),
+                              configurations=self.getConfig()['configurations']['core-site'],
+                              configuration_attributes=self.getConfig()['configuration_attributes']['core-site'],
+                              owner='druid',
+                              group='hadoop'
+                              )
+
+    self.assertResourceCalled('XmlConfig', "yarn-site.xml",
+                              conf_dir=format('/usr/hdp/current/{role}/conf/_common'),
+                              configurations=self.getConfig()['configurations']['yarn-site'],
+                              configuration_attributes=self.getConfig()['configuration_attributes']['yarn-site'],
+                              owner='druid',
+                              group='hadoop'
+                              )
+
+    self.assertResourceCalled('XmlConfig', "hdfs-site.xml",
+                              conf_dir=format('/usr/hdp/current/{role}/conf/_common'),
+                              configurations=self.getConfig()['configurations']['hdfs-site'],
+                              configuration_attributes=self.getConfig()['configuration_attributes']['hdfs-site'],
+                              owner='druid',
+                              group='hadoop'
+                              )
+
+    self.assertResourceCalled('PropertiesFile', "runtime.properties",
+                              dir=format('/usr/hdp/current/{role}/conf/coordinator'),
+                              properties=self.getConfig()['configurations']['druid-coordinator'],
+                              owner='druid',
+                              group='hadoop'
+                              )
+
+    self.assertResourceCalled('File', format("/usr/hdp/current/{role}/conf/coordinator/jvm.config"),
+                              owner='druid',
+                              group='hadoop',
+                              content=InlineTemplate("-server \n-Xms{{node_heap_memory}}m \n-Xmx{{node_heap_memory}}m \n-XX:MaxDirectMemorySize={{node_direct_memory}}m \n-Dlog4j.configurationFile={{log4j_config_file}} \n-Dlog4j.debug \n{{node_jvm_opts}}",
+                                                     node_heap_memory=1024,
+                                                     node_direct_memory=2048,
+                                                     node_jvm_opts='-Duser.timezone=UTC -Dfile.encoding=UTF-8',
+                                                     log4j_config_file=format('/usr/hdp/current/{role}/conf/_common/druid-log4j.xml')
+                                                     )
+                              )
+
+    self.assertResourceCalled('PropertiesFile', "runtime.properties",
+                              dir=format('/usr/hdp/current/{role}/conf/overlord'),
+                              properties=self.getConfig()['configurations']['druid-overlord'],
+                              owner='druid',
+                              group='hadoop'
+                              )
+
+    self.assertResourceCalled('File', format("/usr/hdp/current/{role}/conf/overlord/jvm.config"),
+                              owner='druid',
+                              group='hadoop',
+                              content=InlineTemplate("-server \n-Xms{{node_heap_memory}}m \n-Xmx{{node_heap_memory}}m \n-XX:MaxDirectMemorySize={{node_direct_memory}}m \n-Dlog4j.configurationFile={{log4j_config_file}} \n-Dlog4j.debug \n{{node_jvm_opts}}",
+                                                     node_heap_memory=1024,
+                                                     node_direct_memory=2048,
+                                                     node_jvm_opts='-Duser.timezone=UTC -Dfile.encoding=UTF-8',
+                                                     log4j_config_file=format('/usr/hdp/current/{role}/conf/_common/druid-log4j.xml')
+                                                     )
+                              )
+
+    self.assertResourceCalled('PropertiesFile', "runtime.properties",
+                              dir=format('/usr/hdp/current/{role}/conf/historical'),
+                              properties=self.getConfig()['configurations']['druid-historical'],
+                              owner='druid',
+                              group='hadoop'
+                              )
+
+    self.assertResourceCalled('File', format("/usr/hdp/current/{role}/conf/historical/jvm.config"),
+                            owner='druid',
+                            group='hadoop',
+                            content=InlineTemplate("-server \n-Xms{{node_heap_memory}}m \n-Xmx{{node_heap_memory}}m \n-XX:MaxDirectMemorySize={{node_direct_memory}}m \n-Dlog4j.configurationFile={{log4j_config_file}} \n-Dlog4j.debug \n{{node_jvm_opts}}",
+                                                   node_heap_memory=1024,
+                                                   node_direct_memory=2048,
+                                                   node_jvm_opts='-Duser.timezone=UTC -Dfile.encoding=UTF-8',
+                                                   log4j_config_file=format('/usr/hdp/current/{role}/conf/_common/druid-log4j.xml')
+                                                   )
+                            )
+
+
+    self.assertResourceCalled('PropertiesFile', "runtime.properties",
+                          dir=format('/usr/hdp/current/{role}/conf/broker'),
+                          properties=self.getConfig()['configurations']['druid-broker'],
+                          owner='druid',
+                          group='hadoop'
+                          )
+
+    self.assertResourceCalled('File', format("/usr/hdp/current/{role}/conf/broker/jvm.config"),
+                          owner='druid',
+                          group='hadoop',
+                          content=InlineTemplate("-server \n-Xms{{node_heap_memory}}m \n-Xmx{{node_heap_memory}}m \n-XX:MaxDirectMemorySize={{node_direct_memory}}m \n-Dlog4j.configurationFile={{log4j_config_file}} \n-Dlog4j.debug \n{{node_jvm_opts}}",
+                                                 node_heap_memory=1024,
+                                                 node_direct_memory=2048,
+                                                 node_jvm_opts='-Duser.timezone=UTC -Dfile.encoding=UTF-8',
+                                                 log4j_config_file=format('/usr/hdp/current/{role}/conf/_common/druid-log4j.xml')
+                                                 )
+                          )
+
+
+    self.assertResourceCalled('PropertiesFile', "runtime.properties",
+                          dir=format('/usr/hdp/current/{role}/conf/middleManager'),
+                          properties=self.getConfig()['configurations']['druid-middlemanager'],
+                          owner='druid',
+                          group='hadoop'
+                          )
+
+    self.assertResourceCalled('File', format("/usr/hdp/current/{role}/conf/middleManager/jvm.config"),
+                          owner='druid',
+                          group='hadoop',
+                          content=InlineTemplate("-server \n-Xms{{node_heap_memory}}m \n-Xmx{{node_heap_memory}}m \n-XX:MaxDirectMemorySize={{node_direct_memory}}m \n-Dlog4j.configurationFile={{log4j_config_file}} \n-Dlog4j.debug \n{{node_jvm_opts}}",
+                                                 node_heap_memory=1024,
+                                                 node_direct_memory=2048,
+                                                 node_jvm_opts='-Duser.timezone=UTC -Dfile.encoding=UTF-8',
+                                                 log4j_config_file=format('/usr/hdp/current/{role}/conf/_common/druid-log4j.xml')
+                                                 )
+                          )
+
+    self.assertResourceCalled('PropertiesFile', "runtime.properties",
+                              dir=format('/usr/hdp/current/{role}/conf/router'),
+                              properties=self.getConfig()['configurations']['druid-router'],
+                              owner='druid',
+                              group='hadoop'
+                              )
+
+    self.assertResourceCalled('File', format("/usr/hdp/current/{role}/conf/router/jvm.config"),
+                              owner='druid',
+                              group='hadoop',
+                              content=InlineTemplate("-server \n-Xms{{node_heap_memory}}m \n-Xmx{{node_heap_memory}}m \n-XX:MaxDirectMemorySize={{node_direct_memory}}m \n-Dlog4j.configurationFile={{log4j_config_file}} \n-Dlog4j.debug \n{{node_jvm_opts}}",
+                                                     node_heap_memory=1024,
+                                                     node_direct_memory=2048,
+                                                     node_jvm_opts='-Duser.timezone=UTC -Dfile.encoding=UTF-8',
+                                                     log4j_config_file=format('/usr/hdp/current/{role}/conf/_common/druid-log4j.xml')
+                                                     )
+                              )
+
+    self.assertResourceCalled('HdfsResource', '/user/druid',
+                              immutable_paths = self.DEFAULT_IMMUTABLE_PATHS,
+                              security_enabled = False,
+                              hadoop_bin_dir = '/usr/hdp/current/hadoop-client/bin',
+                              keytab = UnknownConfigurationMock(),
+                              default_fs = 'hdfs://c6401.ambari.apache.org:8020',
+                              hdfs_site = {u'a': u'b'},
+                              kinit_path_local = '/usr/bin/kinit',
+                              principal_name = 'missing_principal',
+                              user = 'hdfs',
+                              owner = 'druid',
+                              hadoop_conf_dir = '/usr/hdp/current/hadoop-client/conf',
+                              type = 'directory',
+                              action = ['create_on_execute'], hdfs_resource_ignore_file='/var/lib/ambari-agent/data/.hdfs_resource_ignore',
+                              dfs_type = '',
+                              recursive_chown=True,
+                              recursive_chmod=True
+                              )
+
+    self.assertResourceCalled('HdfsResource', '/user/druid/data',
+                              immutable_paths = self.DEFAULT_IMMUTABLE_PATHS,
+                              security_enabled = False,
+                              hadoop_bin_dir = '/usr/hdp/current/hadoop-client/bin',
+                              keytab = UnknownConfigurationMock(),
+                              default_fs = 'hdfs://c6401.ambari.apache.org:8020',
+                              hdfs_site = {u'a': u'b'},
+                              kinit_path_local = '/usr/bin/kinit',
+                              principal_name = 'missing_principal',
+                              user = 'hdfs',
+                              owner = 'druid',
+                              hadoop_conf_dir = '/usr/hdp/current/hadoop-client/conf',
+                              type = 'directory',
+                              action = ['create_on_execute'], hdfs_resource_ignore_file='/var/lib/ambari-agent/data/.hdfs_resource_ignore',
+                              dfs_type = '',
+                              mode=0755
+                              )
+
+    self.assertResourceCalled('HdfsResource', '/tmp/druid-indexing',
+                              immutable_paths = self.DEFAULT_IMMUTABLE_PATHS,
+                              security_enabled = False,
+                              hadoop_bin_dir = '/usr/hdp/current/hadoop-client/bin',
+                              keytab = UnknownConfigurationMock(),
+                              default_fs = 'hdfs://c6401.ambari.apache.org:8020',
+                              hdfs_site = {u'a': u'b'},
+                              kinit_path_local = '/usr/bin/kinit',
+                              principal_name = 'missing_principal',
+                              user = 'hdfs',
+                              owner = 'druid',
+                              hadoop_conf_dir = '/usr/hdp/current/hadoop-client/conf',
+                              type = 'directory',
+                              action = ['create_on_execute'], hdfs_resource_ignore_file='/var/lib/ambari-agent/data/.hdfs_resource_ignore',
+                              dfs_type = '',
+                              mode=0755
+                              )
+
+    self.assertResourceCalled('HdfsResource', '/user/druid/logs',
+                              immutable_paths = self.DEFAULT_IMMUTABLE_PATHS,
+                              security_enabled = False,
+                              hadoop_bin_dir = '/usr/hdp/current/hadoop-client/bin',
+                              keytab = UnknownConfigurationMock(),
+                              default_fs = 'hdfs://c6401.ambari.apache.org:8020',
+                              hdfs_site = {u'a': u'b'},
+                              kinit_path_local = '/usr/bin/kinit',
+                              principal_name = 'missing_principal',
+                              user = 'hdfs',
+                              owner = 'druid',
+                              hadoop_conf_dir = '/usr/hdp/current/hadoop-client/conf',
+                              type = 'directory',
+                              action = ['create_on_execute'], hdfs_resource_ignore_file='/var/lib/ambari-agent/data/.hdfs_resource_ignore',
+                              dfs_type = '',
+                              mode=0755
+                              )
+
+    self.assertResourceCalled('Directory', format('/usr/hdp/current/{role}/extensions'),
+                              mode=0755,
+                              cd_access='a',
+                              owner='druid',
+                              group='hadoop',
+                              create_parents=True,
+                              recursive_ownership=True
+                              )
+
+    self.assertResourceCalled('Directory', format('/usr/hdp/current/{role}/hadoop-dependencies'),
+                              mode=0755,
+                              cd_access='a',
+                              owner='druid',
+                              group='hadoop',
+                              create_parents=True,
+                              recursive_ownership=True
+                              )
+
+    self.assertResourceCalled('Execute', format("source /usr/hdp/current/{role}/conf/druid-env.sh ; java -classpath '/usr/hdp/current/{role}/lib/*' -Ddruid.extensions.loadList=[] -Ddruid.extensions.directory=/usr/hdp/current/{role}/extensions -Ddruid.extensions.hadoopDependenciesDir=/usr/hdp/current/{role}/hadoop-dependencies io.druid.cli.Main tools pull-deps -c mysql-metadata-storage --no-default-hadoop"),
+                              user='druid'
+                              )
+
+
+def mutable_config_dict(config):
+  rv = {}
+  for key, value in config.iteritems():
+    rv[key] = value
+  return rv

http://git-wip-us.apache.org/repos/asf/ambari/blob/685e926d/ambari-server/src/test/python/stacks/2.6/common/test_stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.6/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.6/common/test_stack_advisor.py
new file mode 100644
index 0000000..8227d69
--- /dev/null
+++ b/ambari-server/src/test/python/stacks/2.6/common/test_stack_advisor.py
@@ -0,0 +1,575 @@
+'''
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+'''
+
+import json
+import os
+from unittest import TestCase
+from mock.mock import patch
+
+
+class TestHDP26StackAdvisor(TestCase):
+  def setUp(self):
+    import imp
+    self.maxDiff = None
+    self.testDirectory = os.path.dirname(os.path.abspath(__file__))
+    stackAdvisorPath = os.path.join(self.testDirectory, '../../../../../main/resources/stacks/stack_advisor.py')
+    hdp206StackAdvisorPath = os.path.join(self.testDirectory,
+                                          '../../../../../main/resources/stacks/HDP/2.0.6/services/stack_advisor.py')
+    hdp21StackAdvisorPath = os.path.join(self.testDirectory,
+                                         '../../../../../main/resources/stacks/HDP/2.1/services/stack_advisor.py')
+    hdp22StackAdvisorPath = os.path.join(self.testDirectory,
+                                         '../../../../../main/resources/stacks/HDP/2.2/services/stack_advisor.py')
+    hdp23StackAdvisorPath = os.path.join(self.testDirectory,
+                                         '../../../../../main/resources/stacks/HDP/2.3/services/stack_advisor.py')
+    hdp24StackAdvisorPath = os.path.join(self.testDirectory,
+                                         '../../../../../main/resources/stacks/HDP/2.4/services/stack_advisor.py')
+    hdp25StackAdvisorPath = os.path.join(self.testDirectory,
+                                         '../../../../../main/resources/stacks/HDP/2.5/services/stack_advisor.py')
+    hdp26StackAdvisorPath = os.path.join(self.testDirectory,
+                                         '../../../../../main/resources/stacks/HDP/2.6/services/stack_advisor.py')
+    hdp26StackAdvisorClassName = 'HDP26StackAdvisor'
+
+    with open(stackAdvisorPath, 'rb') as fp:
+      imp.load_module('stack_advisor', fp, stackAdvisorPath, ('.py', 'rb', imp.PY_SOURCE))
+    with open(hdp206StackAdvisorPath, 'rb') as fp:
+      imp.load_module('stack_advisor_impl', fp, hdp206StackAdvisorPath, ('.py', 'rb', imp.PY_SOURCE))
+    with open(hdp21StackAdvisorPath, 'rb') as fp:
+      imp.load_module('stack_advisor_impl', fp, hdp21StackAdvisorPath, ('.py', 'rb', imp.PY_SOURCE))
+    with open(hdp22StackAdvisorPath, 'rb') as fp:
+      imp.load_module('stack_advisor_impl', fp, hdp22StackAdvisorPath, ('.py', 'rb', imp.PY_SOURCE))
+    with open(hdp23StackAdvisorPath, 'rb') as fp:
+      imp.load_module('stack_advisor_impl', fp, hdp23StackAdvisorPath, ('.py', 'rb', imp.PY_SOURCE))
+    with open(hdp24StackAdvisorPath, 'rb') as fp:
+      imp.load_module('stack_advisor_impl', fp, hdp24StackAdvisorPath, ('.py', 'rb', imp.PY_SOURCE))
+    with open(hdp25StackAdvisorPath, 'rb') as fp:
+      imp.load_module('stack_advisor_impl', fp, hdp25StackAdvisorPath, ('.py', 'rb', imp.PY_SOURCE))
+    with open(hdp26StackAdvisorPath, 'rb') as fp:
+      stack_advisor_impl = imp.load_module('stack_advisor_impl', fp, hdp26StackAdvisorPath,
+                                           ('.py', 'rb', imp.PY_SOURCE))
+    clazz = getattr(stack_advisor_impl, hdp26StackAdvisorClassName)
+    self.stackAdvisor = clazz()
+
+    # substitute method in the instance
+    self.get_system_min_uid_real = self.stackAdvisor.get_system_min_uid
+    self.stackAdvisor.get_system_min_uid = self.get_system_min_uid_magic
+
+  @patch('__builtin__.open')
+  @patch('os.path.exists')
+  def get_system_min_uid_magic(self, exists_mock, open_mock):
+    class MagicFile(object):
+      def read(self):
+        return """
+          #test line UID_MIN 200
+          UID_MIN 500
+          """
+
+      def __exit__(self, exc_type, exc_val, exc_tb):
+        pass
+
+      def __enter__(self):
+        return self
+
+    exists_mock.return_value = True
+    open_mock.return_value = MagicFile()
+    return self.get_system_min_uid_real()
+
+  def test_recommendDruidConfigurations_withMysql(self):
+    hosts = {
+      "items": [
+        {
+          "Hosts": {
+            "cpu_count": 4,
+            "total_mem": 50331648,
+            "disk_info": [
+              {"mountpoint": "/"},
+              {"mountpoint": "/dev/shm"},
+              {"mountpoint": "/vagrant"},
+              {"mountpoint": "/"},
+              {"mountpoint": "/dev/shm"},
+              {"mountpoint": "/vagrant"}
+            ],
+            "public_host_name": "c6401.ambari.apache.org",
+            "host_name": "c6401.ambari.apache.org"
+          },
+        }
+      ]
+    }
+
+    services = {
+      "Versions": {
+        "parent_stack_version": "2.5",
+        "stack_name": "HDP",
+        "stack_version": "2.6",
+        "stack_hierarchy": {
+          "stack_name": "HDP",
+          "stack_versions": ["2.5", "2.4", "2.3", "2.2", "2.1", "2.0.6"]
+        }
+      },
+      "services": [{
+        "StackServices": {
+          "service_name": "DRUID",
+        },
+        "components": [
+          {
+            "StackServiceComponents": {
+              "component_name": "DRUID_COORDINATOR",
+              "hostnames": ["c6401.ambari.apache.org"]
+            },
+          },
+          {
+            "StackServiceComponents": {
+              "component_name": "DRUID_OVERLORD",
+              "hostnames": ["c6401.ambari.apache.org"]
+            },
+          },
+          {
+            "StackServiceComponents": {
+              "component_name": "DRUID_BROKER",
+              "hostnames": ["c6401.ambari.apache.org"]
+            },
+          },
+          {
+            "StackServiceComponents": {
+              "component_name": "DRUID_HISTORICAL",
+              "hostnames": ["c6401.ambari.apache.org"]
+            },
+          },
+          {
+            "StackServiceComponents": {
+              "component_name": "DRUID_MIDDLEMANAGER",
+              "hostnames": ["c6401.ambari.apache.org"]
+            },
+          }
+        ]
+      }
+      ],
+      "configurations": {
+        "druid-common": {
+          "properties": {
+            "database_name": "druid",
+            "metastore_hostname": "c6401.ambari.apache.org",
+            "druid.metadata.storage.type": "mysql",
+            "druid.extensions.loadList": "[\"postgresql-metadata-storage\"]",
+            "druid.extensions.pullList": "[]"
+          }
+        }
+      }
+    }
+
+    clusterData = {
+      "cpu": 4,
+      "mapMemory": 30000,
+      "amMemory": 20000,
+      "reduceMemory": 20560,
+      "containers": 30,
+      "ramPerContainer": 512,
+      "referenceNodeManagerHost": {
+        "total_mem": 10240 * 1024
+      }
+    }
+
+    configurations = {
+    }
+
+    self.stackAdvisor.recommendDruidConfigurations(configurations, clusterData, services, hosts)
+    self.assertEquals(configurations,
+                      {'druid-historical': {
+                        'properties': {'druid.processing.numThreads': '3', 'druid.server.http.numThreads': '40'}},
+                        'druid-broker': {
+                          'properties': {'druid.processing.numThreads': '3', 'druid.server.http.numThreads': '40'}},
+                        'druid-common': {'properties': {'druid.extensions.loadList': '["mysql-metadata-storage"]',
+                                                        'druid.metadata.storage.connector.port': '3306',
+                                                        'druid.metadata.storage.connector.connectURI': 'jdbc:mysql://c6401.ambari.apache.org:3306/druid?createDatabaseIfNotExist=true',
+                                                        'druid.zk.service.host': '',
+                                                        'druid.extensions.pullList': '["io.druid.extensions:mysql-metadata-storage"]'}},
+                        'druid-env': {'properties': {},
+                                      'property_attributes': {'druid.coordinator.jvm.heap.memory': {'maximum': '49152'},
+                                                              'druid.overlord.jvm.heap.memory': {'maximum': '49152'},
+                                                              'druid.middlemanager.jvm.heap.memory': {
+                                                                'maximum': '49152'},
+                                                              'druid.historical.jvm.heap.memory': {'maximum': '49152'},
+                                                              'druid.broker.jvm.heap.memory': {'maximum': '49152'}}}}
+                      )
+
+  def test_recommendDruidConfigurations_WithPostgresql(self):
+    hosts = {
+      "items": [
+        {
+          "Hosts": {
+            "cpu_count": 4,
+            "total_mem": 50331648,
+            "disk_info": [
+              {"mountpoint": "/"},
+              {"mountpoint": "/dev/shm"},
+              {"mountpoint": "/vagrant"},
+              {"mountpoint": "/"},
+              {"mountpoint": "/dev/shm"},
+              {"mountpoint": "/vagrant"}
+            ],
+            "public_host_name": "c6401.ambari.apache.org",
+            "host_name": "c6401.ambari.apache.org"
+          }
+        }
+      ]
+    }
+
+    services = {
+      "Versions": {
+        "parent_stack_version": "2.5",
+        "stack_name": "HDP",
+        "stack_version": "2.6",
+        "stack_hierarchy": {
+          "stack_name": "HDP",
+          "stack_versions": ["2.5", "2.4", "2.3", "2.2", "2.1", "2.0.6"]
+        }
+      },
+      "services": [{
+        "StackServices": {
+          "service_name": "DRUID",
+        },
+        "components": [
+          {
+            "StackServiceComponents": {
+              "component_name": "DRUID_COORDINATOR",
+              "hostnames": ["c6401.ambari.apache.org"]
+            },
+          },
+          {
+            "StackServiceComponents": {
+              "component_name": "DRUID_OVERLORD",
+              "hostnames": ["c6401.ambari.apache.org"]
+            },
+          },
+          {
+            "StackServiceComponents": {
+              "component_name": "DRUID_BROKER",
+              "hostnames": ["c6401.ambari.apache.org"]
+            },
+          },
+          {
+            "StackServiceComponents": {
+              "component_name": "DRUID_HISTORICAL",
+              "hostnames": ["c6401.ambari.apache.org"]
+            },
+          },
+          {
+            "StackServiceComponents": {
+              "component_name": "DRUID_MIDDLEMANAGER",
+              "hostnames": ["c6401.ambari.apache.org"]
+            },
+          }
+        ]
+      }
+      ],
+      "configurations": {
+        "druid-common": {
+          "properties": {
+            "database_name": "druid",
+            "metastore_hostname": "c6401.ambari.apache.org",
+            "druid.metadata.storage.type": "postgres",
+            "druid.extensions.loadList": "[\"mysql-metadata-storage\"]",
+            "druid.extensions.pullList": "[]"
+          }
+        }
+      }
+    }
+
+    clusterData = {
+    }
+
+    configurations = {
+    }
+
+    self.stackAdvisor.recommendDruidConfigurations(configurations, clusterData, services, hosts)
+    self.assertEquals(configurations,
+                      {'druid-historical': {
+                        'properties': {'druid.processing.numThreads': '3', 'druid.server.http.numThreads': '40'}},
+                        'druid-broker': {
+                          'properties': {'druid.processing.numThreads': '3', 'druid.server.http.numThreads': '40'}},
+                        'druid-common': {'properties': {'druid.extensions.loadList': '["postgresql-metadata-storage"]',
+                                                        'druid.metadata.storage.connector.port': '5432',
+                                                        'druid.metadata.storage.connector.connectURI': 'jdbc:postgresql://c6401.ambari.apache.org:5432/druid',
+                                                        'druid.zk.service.host': '',
+                                                        'druid.extensions.pullList': '[]'}},
+                        'druid-env': {'properties': {},
+                                      'property_attributes': {'druid.coordinator.jvm.heap.memory': {'maximum': '49152'},
+                                                              'druid.overlord.jvm.heap.memory': {'maximum': '49152'},
+                                                              'druid.middlemanager.jvm.heap.memory': {
+                                                                'maximum': '49152'},
+                                                              'druid.historical.jvm.heap.memory': {'maximum': '49152'},
+                                                              'druid.broker.jvm.heap.memory': {'maximum': '49152'}}}}
+                      )
+
+  def test_recommendDruidConfigurations_WithDerby(self):
+    hosts = {
+      "items": [
+        {
+          "Hosts": {
+            "cpu_count": 4,
+            "total_mem": 50331648,
+            "disk_info": [
+              {"mountpoint": "/"},
+              {"mountpoint": "/dev/shm"},
+              {"mountpoint": "/vagrant"},
+              {"mountpoint": "/"},
+              {"mountpoint": "/dev/shm"},
+              {"mountpoint": "/vagrant"}
+            ],
+            "public_host_name": "c6401.ambari.apache.org",
+            "host_name": "c6401.ambari.apache.org"
+          }
+        }
+      ]
+    }
+
+    services = {
+      "Versions": {
+        "parent_stack_version": "2.5",
+        "stack_name": "HDP",
+        "stack_version": "2.6",
+        "stack_hierarchy": {
+          "stack_name": "HDP",
+          "stack_versions": ["2.5", "2.4", "2.3", "2.2", "2.1", "2.0.6"]
+        }
+      },
+      "services": [{
+        "StackServices": {
+          "service_name": "DRUID",
+        },
+        "components": [
+          {
+            "StackServiceComponents": {
+              "component_name": "DRUID_COORDINATOR",
+              "hostnames": ["c6401.ambari.apache.org"]
+            },
+          },
+          {
+            "StackServiceComponents": {
+              "component_name": "DRUID_OVERLORD",
+              "hostnames": ["c6401.ambari.apache.org"]
+            },
+          },
+          {
+            "StackServiceComponents": {
+              "component_name": "DRUID_BROKER",
+              "hostnames": ["c6401.ambari.apache.org"]
+            },
+          },
+          {
+            "StackServiceComponents": {
+              "component_name": "DRUID_HISTORICAL",
+              "hostnames": ["c6401.ambari.apache.org"]
+            },
+          },
+          {
+            "StackServiceComponents": {
+              "component_name": "DRUID_MIDDLEMANAGER",
+              "hostnames": ["c6401.ambari.apache.org"]
+            },
+          }
+        ]
+      }
+      ],
+      "configurations": {
+        "druid-common": {
+          "properties": {
+            "database_name": "druid",
+            "metastore_hostname": "c6401.ambari.apache.org",
+            "druid.metadata.storage.type": "derby",
+            "druid.extensions.loadList": "[\"mysql-metadata-storage\"]",
+            "druid.extensions.pullList": "[]"
+          }
+        }
+      }
+    }
+
+    clusterData = {
+    }
+
+    configurations = {
+    }
+
+    self.stackAdvisor.recommendDruidConfigurations(configurations, clusterData, services, hosts)
+    self.assertEquals(configurations,
+                      {'druid-historical': {
+                        'properties': {'druid.processing.numThreads': '3', 'druid.server.http.numThreads': '40'}},
+                        'druid-broker': {
+                          'properties': {'druid.processing.numThreads': '3', 'druid.server.http.numThreads': '40'}},
+                        'druid-common': {'properties': {'druid.extensions.loadList': '[]',
+                                                        'druid.metadata.storage.connector.port': '1527',
+                                                        'druid.metadata.storage.connector.connectURI': 'jdbc:derby://c6401.ambari.apache.org:1527/druid;create=true',
+                                                        'druid.zk.service.host': '',
+                                                        'druid.extensions.pullList': '[]'}},
+                        'druid-env': {'properties': {},
+                                      'property_attributes': {'druid.coordinator.jvm.heap.memory': {'maximum': '49152'},
+                                                              'druid.overlord.jvm.heap.memory': {'maximum': '49152'},
+                                                              'druid.middlemanager.jvm.heap.memory': {
+                                                                'maximum': '49152'},
+                                                              'druid.historical.jvm.heap.memory': {'maximum': '49152'},
+                                                              'druid.broker.jvm.heap.memory': {'maximum': '49152'}}}}
+                      )
+
+  def test_recommendDruidConfigurations_heterogeneous_hosts(self):
+    hosts = {
+      "items": [
+        {
+          "href": "/api/v1/hosts/c6401.ambari.apache.org",
+          "Hosts": {
+            "cpu_count": 4,
+            "total_mem": 50331648,
+            "disk_info": [
+              {"mountpoint": "/"},
+              {"mountpoint": "/dev/shm"},
+              {"mountpoint": "/vagrant"},
+              {"mountpoint": "/"},
+              {"mountpoint": "/dev/shm"},
+              {"mountpoint": "/vagrant"}
+            ],
+            "public_host_name": "c6401.ambari.apache.org",
+            "host_name": "c6401.ambari.apache.org"
+          }
+        }, {
+          "href": "/api/v1/hosts/c6402.ambari.apache.org",
+          "Hosts": {
+            "cpu_count": 1,
+            "total_mem": 1922680,
+            "disk_info": [
+              {"mountpoint": "/"},
+              {"mountpoint": "/dev/shm"},
+              {"mountpoint": "/vagrant"},
+              {"mountpoint": "/"},
+              {"mountpoint": "/dev/shm"},
+              {"mountpoint": "/vagrant"}
+            ],
+            "public_host_name": "c6402.ambari.apache.org",
+            "host_name": "c6402.ambari.apache.org"
+          }
+        },
+        {
+          "href": "/api/v1/hosts/c6403.ambari.apache.org",
+          "Hosts": {
+            "cpu_count": 3,
+            "total_mem": 3845360,
+            "disk_info": [
+              {"mountpoint": "/"},
+              {"mountpoint": "/dev/shm"},
+              {"mountpoint": "/vagrant"},
+              {"mountpoint": "/"},
+              {"mountpoint": "/dev/shm"},
+              {"mountpoint": "/vagrant"}
+            ],
+            "public_host_name": "c6403.ambari.apache.org",
+            "host_name": "c6403.ambari.apache.org"
+          }
+        }
+      ]
+    }
+
+    services = {
+      "Versions": {
+        "parent_stack_version": "2.5",
+        "stack_name": "HDP",
+        "stack_version": "2.6",
+        "stack_hierarchy": {
+          "stack_name": "HDP",
+          "stack_versions": ["2.5", "2.4", "2.3", "2.2", "2.1", "2.0.6"]
+        }
+      },
+      "services": [{
+        "StackServices": {
+          "service_name": "DRUID",
+        },
+        "components": [
+          {
+            "StackServiceComponents": {
+              "component_name": "DRUID_COORDINATOR",
+              "hostnames": ["c6401.ambari.apache.org"]
+            },
+          },
+          {
+            "StackServiceComponents": {
+              "component_name": "DRUID_OVERLORD",
+              "hostnames": ["c6401.ambari.apache.org"]
+            },
+          },
+          {
+            "StackServiceComponents": {
+              "component_name": "DRUID_BROKER",
+              "hostnames": ["c6402.ambari.apache.org", "c6403.ambari.apache.org"]
+            },
+          },
+          {
+            "StackServiceComponents": {
+              "component_name": "DRUID_HISTORICAL",
+              "hostnames": ["c6401.ambari.apache.org", "c6403.ambari.apache.org"]
+            },
+          },
+          {
+            "StackServiceComponents": {
+              "component_name": "DRUID_MIDDLEMANAGER",
+              "hostnames": ["c6401.ambari.apache.org"]
+            },
+          }
+        ]
+      }
+      ],
+      "configurations": {
+        "druid-common": {
+          "properties": {
+            "database_name": "druid",
+            "metastore_hostname": "c6401.ambari.apache.org",
+            "druid.metadata.storage.type": "derby",
+            "druid.extensions.loadList": "[\"mysql-metadata-storage\"]",
+            "druid.extensions.pullList": "[]"
+          }
+        }
+      }
+    }
+
+    clusterData = {
+    }
+
+    configurations = {
+    }
+
+    self.stackAdvisor.recommendDruidConfigurations(configurations, clusterData, services, hosts)
+    self.assertEquals(configurations,
+                      {'druid-historical': {
+                        'properties': {'druid.processing.numThreads': '2', 'druid.server.http.numThreads': '40'}},
+                        'druid-broker': {
+                          'properties': {'druid.processing.numThreads': '1', 'druid.server.http.numThreads': '40'}},
+                        'druid-common': {'properties': {'druid.extensions.loadList': '[]',
+                                                        'druid.metadata.storage.connector.port': '1527',
+                                                        'druid.metadata.storage.connector.connectURI': 'jdbc:derby://c6401.ambari.apache.org:1527/druid;create=true',
+                                                        'druid.zk.service.host': '',
+                                                        'druid.extensions.pullList': '[]'}},
+                        'druid-env': {'properties': {},
+                                      'property_attributes': {'druid.coordinator.jvm.heap.memory': {'maximum': '49152'},
+                                                              'druid.overlord.jvm.heap.memory': {'maximum': '49152'},
+                                                              'druid.middlemanager.jvm.heap.memory': {
+                                                                'maximum': '49152'},
+                                                              'druid.historical.jvm.heap.memory': {'maximum': '3755'},
+                                                              'druid.broker.jvm.heap.memory': {'maximum': '1877'}}}}
+                      )
+
+
+def load_json(self, filename):
+  file = os.path.join(self.testDirectory, filename)
+  with open(file, 'rb') as f:
+    data = json.load(f)
+  return data


Mime
View raw message