ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From oleew...@apache.org
Subject ambari git commit: AMBARI-16255. Basic stack advisor recommendation (shard/replication) (oleewere)
Date Wed, 04 May 2016 21:41:10 GMT
Repository: ambari
Updated Branches:
  refs/heads/trunk 3e11675be -> 8a2a9fdc1


AMBARI-16255. Basic stack advisor recommendation (shard/replication) (oleewere)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/8a2a9fdc
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/8a2a9fdc
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/8a2a9fdc

Branch: refs/heads/trunk
Commit: 8a2a9fdc153eee316c49e68067741d0d59339e7b
Parents: 3e11675
Author: oleewere <oleewere@gmail.com>
Authored: Wed May 4 23:38:57 2016 +0200
Committer: oleewere <oleewere@gmail.com>
Committed: Wed May 4 23:40:31 2016 +0200

----------------------------------------------------------------------
 .../configuration/logsearch-properties.xml      |  2 +-
 .../stacks/HDP/2.3/services/stack_advisor.py    | 21 ++++-
 .../stacks/2.3/common/test_stack_advisor.py     | 94 ++++++++++++++++++++
 3 files changed, 115 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/8a2a9fdc/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logsearch-properties.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logsearch-properties.xml
b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logsearch-properties.xml
index dc5ebb6..172ed95 100644
--- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logsearch-properties.xml
+++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logsearch-properties.xml
@@ -43,7 +43,7 @@
     <value-attributes>
       <type>int</type>
       <minimum>1</minimum>
-      <maximum>20</maximum>
+      <maximum>10</maximum>
       <unit>int</unit>
       <increment-step>1</increment-step>
     </value-attributes>

http://git-wip-us.apache.org/repos/asf/ambari/blob/8a2a9fdc/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py
index 7b792b9..cf0990d 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py
@@ -19,6 +19,7 @@ limitations under the License.
 import os
 import re
 import fnmatch
+import math
 import socket
 
 DB_TYPE_DEFAULT_PORT_MAP = {"MYSQL":"3306", "ORACLE":"1521", "POSTGRES":"5432", "MSSQL":"1433",
"SQLA":"2638"}
@@ -171,7 +172,8 @@ class HDP23StackAdvisor(HDP22StackAdvisor):
       "HAWQ": self.recommendHAWQConfigurations,
       "FALCON": self.recommendFalconConfigurations,
       "STORM": self.recommendStormConfigurations,
-      "SQOOP": self.recommendSqoopConfigurations
+      "SQOOP": self.recommendSqoopConfigurations,
+      "LOGSEARCH" : self.recommendLogsearchConfigurations
     }
     parentRecommendConfDict.update(childRecommendConfDict)
     return parentRecommendConfDict
@@ -834,6 +836,23 @@ class HDP23StackAdvisor(HDP22StackAdvisor):
         application_services_value = " "
     putFalconStartupProperty(application_services_property, application_services_value)
 
+  def recommendLogsearchConfigurations(self, configurations, clusterData, services, hosts):
+    putLogsearchProperty = self.putProperty(configurations, "logsearch-properties", services)
+    logsearchSolrHosts = self.getComponentHostNames(services, "LOGSEARCH", "LOGSEARCH_SOLR")
+
+    if logsearchSolrHosts is not None and len(logsearchSolrHosts) > 0 \
+      and "logsearch-properties" in services["configurations"]:
+      # recommend number of shard
+      putLogsearchAttribute = self.putPropertyAttribute(configurations, "logsearch-properties")
+      putLogsearchAttribute('logsearch.collection.numshards', 'minimum', len(logsearchSolrHosts))
+      putLogsearchAttribute('logsearch.collection.numshards', 'maximum', 3 * len(logsearchSolrHosts))
+      putLogsearchProperty("logsearch.collection.numshards", 2 * len(logsearchSolrHosts))
+      # recommend replication factor
+      replicationReccomendFloat = math.log(len(logsearchSolrHosts), 5)
+      recommendedReplicationFactor = int(1 + math.floor(replicationReccomendFloat))
+      putLogsearchProperty("logsearch.collection.replication.factor", recommendedReplicationFactor)
+
+
   def getServiceConfigurationValidators(self):
     parentValidators = super(HDP23StackAdvisor, self).getServiceConfigurationValidators()
     childValidators = {

http://git-wip-us.apache.org/repos/asf/ambari/blob/8a2a9fdc/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py
index 2080c52..fcb5407 100644
--- a/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py
+++ b/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py
@@ -2881,3 +2881,97 @@ class TestHDP23StackAdvisor(TestCase):
     services['ambari-server-properties'] = {'java.home': '/usr/jdk64/jdk1.7.3_23'}
     self.stackAdvisor.recommendSqoopConfigurations(configurations, clusterData, services,
hosts)
     self.assertEquals(configurations, expected)
+
+  def test_recommendLogsearchConfiguration(self):
+    configurations = {
+      "logsearch-properties": {
+        "properties": {
+          "logsearch.collection.numshards" : "5",
+          "logsearch.collection.replication.factor": "0"
+        }
+      }
+    }
+
+    clusterData = {
+      "cpu": 4,
+      "mapMemory": 3000,
+      "amMemory": 2000,
+      "reduceMemory": 2056,
+      "containers": 3,
+      "ramPerContainer": 256
+    }
+    expected = {
+      'logsearch-properties': {
+        'properties': {
+          "logsearch.collection.numshards" : "2",
+          "logsearch.collection.replication.factor": "1"
+        },
+        "property_attributes": {
+          "logsearch.collection.numshards": {
+            "minimum": "1",
+            "maximum": "3"
+          }
+        }
+      }
+    }
+    services = {
+      "services": [
+        {
+          "href": "/api/v1/stacks/HDP/versions/2.3/services/LOGSEARCH",
+          "StackServices": {
+            "service_name": "LOGSEARCH",
+            "service_version": "2.6.0.2.2",
+            "stack_name": "HDP",
+            "stack_version": "2.3"
+          },
+          "components": [
+            {
+              "StackServiceComponents": {
+                "advertise_version": "false",
+                "cardinality": "1",
+                "component_category": "MASTER",
+                "component_name": "LOGSEARCH_SOLR",
+                "display_name": "LogSearch Solr Instance",
+                "is_client": "false",
+                "is_master": "true",
+                "hostnames": []
+              },
+              "dependencies": []
+            }
+          ]
+        },
+      ],
+      "configurations": {
+        "logsearch-properties": {
+          "properties": {
+            "logsearch.collection.numshards" : "5",
+            "logsearch.collection.replication.factor": "0"
+          }
+        }
+      },
+      "changed-configurations": [ ]
+
+    }
+
+    hosts = {
+      "items" : [
+        {
+          "href" : "/api/v1/hosts/c6401.ambari.apache.org",
+          "Hosts" : {
+            "cpu_count" : 1,
+            "host_name" : "c6401.ambari.apache.org",
+            "os_arch" : "x86_64",
+            "os_type" : "centos6",
+            "ph_cpu_count" : 1,
+            "public_host_name" : "c6401.ambari.apache.org",
+            "rack_info" : "/default-rack",
+            "total_mem" : 1922680
+          }
+        }
+      ]
+    }
+    def return_c6401_hostname(services, service_name, component_name):
+      return ["c6401.ambari.apache.org"]
+    self.stackAdvisor.getComponentHostNames = return_c6401_hostname
+    self.stackAdvisor.recommendLogsearchConfigurations(configurations, clusterData, services,
hosts)
+    self.assertEquals(configurations, expected)


Mime
View raw message