Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id 74EDA200AE3 for ; Wed, 4 May 2016 23:41:12 +0200 (CEST) Received: by cust-asf.ponee.io (Postfix) id 739081609FF; Wed, 4 May 2016 21:41:12 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id 6E4861609A1 for ; Wed, 4 May 2016 23:41:11 +0200 (CEST) Received: (qmail 69799 invoked by uid 500); 4 May 2016 21:41:10 -0000 Mailing-List: contact commits-help@ambari.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: ambari-dev@ambari.apache.org Delivered-To: mailing list commits@ambari.apache.org Received: (qmail 69790 invoked by uid 99); 4 May 2016 21:41:10 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 04 May 2016 21:41:10 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 82D3CDFC55; Wed, 4 May 2016 21:41:10 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: oleewere@apache.org To: commits@ambari.apache.org Message-Id: X-Mailer: ASF-Git Admin Mailer Subject: ambari git commit: AMBARI-16255. Basic stack advisor recommendation (shard/replication) (oleewere) Date: Wed, 4 May 2016 21:41:10 +0000 (UTC) archived-at: Wed, 04 May 2016 21:41:12 -0000 Repository: ambari Updated Branches: refs/heads/trunk 3e11675be -> 8a2a9fdc1 AMBARI-16255. Basic stack advisor recommendation (shard/replication) (oleewere) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/8a2a9fdc Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/8a2a9fdc Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/8a2a9fdc Branch: refs/heads/trunk Commit: 8a2a9fdc153eee316c49e68067741d0d59339e7b Parents: 3e11675 Author: oleewere Authored: Wed May 4 23:38:57 2016 +0200 Committer: oleewere Committed: Wed May 4 23:40:31 2016 +0200 ---------------------------------------------------------------------- .../configuration/logsearch-properties.xml | 2 +- .../stacks/HDP/2.3/services/stack_advisor.py | 21 ++++- .../stacks/2.3/common/test_stack_advisor.py | 94 ++++++++++++++++++++ 3 files changed, 115 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/8a2a9fdc/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logsearch-properties.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logsearch-properties.xml b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logsearch-properties.xml index dc5ebb6..172ed95 100644 --- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logsearch-properties.xml +++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logsearch-properties.xml @@ -43,7 +43,7 @@ int 1 - 20 + 10 int 1 http://git-wip-us.apache.org/repos/asf/ambari/blob/8a2a9fdc/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py index 7b792b9..cf0990d 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py @@ -19,6 +19,7 @@ limitations under the License. import os import re import fnmatch +import math import socket DB_TYPE_DEFAULT_PORT_MAP = {"MYSQL":"3306", "ORACLE":"1521", "POSTGRES":"5432", "MSSQL":"1433", "SQLA":"2638"} @@ -171,7 +172,8 @@ class HDP23StackAdvisor(HDP22StackAdvisor): "HAWQ": self.recommendHAWQConfigurations, "FALCON": self.recommendFalconConfigurations, "STORM": self.recommendStormConfigurations, - "SQOOP": self.recommendSqoopConfigurations + "SQOOP": self.recommendSqoopConfigurations, + "LOGSEARCH" : self.recommendLogsearchConfigurations } parentRecommendConfDict.update(childRecommendConfDict) return parentRecommendConfDict @@ -834,6 +836,23 @@ class HDP23StackAdvisor(HDP22StackAdvisor): application_services_value = " " putFalconStartupProperty(application_services_property, application_services_value) + def recommendLogsearchConfigurations(self, configurations, clusterData, services, hosts): + putLogsearchProperty = self.putProperty(configurations, "logsearch-properties", services) + logsearchSolrHosts = self.getComponentHostNames(services, "LOGSEARCH", "LOGSEARCH_SOLR") + + if logsearchSolrHosts is not None and len(logsearchSolrHosts) > 0 \ + and "logsearch-properties" in services["configurations"]: + # recommend number of shard + putLogsearchAttribute = self.putPropertyAttribute(configurations, "logsearch-properties") + putLogsearchAttribute('logsearch.collection.numshards', 'minimum', len(logsearchSolrHosts)) + putLogsearchAttribute('logsearch.collection.numshards', 'maximum', 3 * len(logsearchSolrHosts)) + putLogsearchProperty("logsearch.collection.numshards", 2 * len(logsearchSolrHosts)) + # recommend replication factor + replicationReccomendFloat = math.log(len(logsearchSolrHosts), 5) + recommendedReplicationFactor = int(1 + math.floor(replicationReccomendFloat)) + putLogsearchProperty("logsearch.collection.replication.factor", recommendedReplicationFactor) + + def getServiceConfigurationValidators(self): parentValidators = super(HDP23StackAdvisor, self).getServiceConfigurationValidators() childValidators = { http://git-wip-us.apache.org/repos/asf/ambari/blob/8a2a9fdc/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py index 2080c52..fcb5407 100644 --- a/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py +++ b/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py @@ -2881,3 +2881,97 @@ class TestHDP23StackAdvisor(TestCase): services['ambari-server-properties'] = {'java.home': '/usr/jdk64/jdk1.7.3_23'} self.stackAdvisor.recommendSqoopConfigurations(configurations, clusterData, services, hosts) self.assertEquals(configurations, expected) + + def test_recommendLogsearchConfiguration(self): + configurations = { + "logsearch-properties": { + "properties": { + "logsearch.collection.numshards" : "5", + "logsearch.collection.replication.factor": "0" + } + } + } + + clusterData = { + "cpu": 4, + "mapMemory": 3000, + "amMemory": 2000, + "reduceMemory": 2056, + "containers": 3, + "ramPerContainer": 256 + } + expected = { + 'logsearch-properties': { + 'properties': { + "logsearch.collection.numshards" : "2", + "logsearch.collection.replication.factor": "1" + }, + "property_attributes": { + "logsearch.collection.numshards": { + "minimum": "1", + "maximum": "3" + } + } + } + } + services = { + "services": [ + { + "href": "/api/v1/stacks/HDP/versions/2.3/services/LOGSEARCH", + "StackServices": { + "service_name": "LOGSEARCH", + "service_version": "2.6.0.2.2", + "stack_name": "HDP", + "stack_version": "2.3" + }, + "components": [ + { + "StackServiceComponents": { + "advertise_version": "false", + "cardinality": "1", + "component_category": "MASTER", + "component_name": "LOGSEARCH_SOLR", + "display_name": "LogSearch Solr Instance", + "is_client": "false", + "is_master": "true", + "hostnames": [] + }, + "dependencies": [] + } + ] + }, + ], + "configurations": { + "logsearch-properties": { + "properties": { + "logsearch.collection.numshards" : "5", + "logsearch.collection.replication.factor": "0" + } + } + }, + "changed-configurations": [ ] + + } + + hosts = { + "items" : [ + { + "href" : "/api/v1/hosts/c6401.ambari.apache.org", + "Hosts" : { + "cpu_count" : 1, + "host_name" : "c6401.ambari.apache.org", + "os_arch" : "x86_64", + "os_type" : "centos6", + "ph_cpu_count" : 1, + "public_host_name" : "c6401.ambari.apache.org", + "rack_info" : "/default-rack", + "total_mem" : 1922680 + } + } + ] + } + def return_c6401_hostname(services, service_name, component_name): + return ["c6401.ambari.apache.org"] + self.stackAdvisor.getComponentHostNames = return_c6401_hostname + self.stackAdvisor.recommendLogsearchConfigurations(configurations, clusterData, services, hosts) + self.assertEquals(configurations, expected)