Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id 7669E200CD2 for ; Thu, 27 Jul 2017 16:53:47 +0200 (CEST) Received: by cust-asf.ponee.io (Postfix) id 74CDF16AE7A; Thu, 27 Jul 2017 14:53:47 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id 6B8D316AE75 for ; Thu, 27 Jul 2017 16:53:46 +0200 (CEST) Received: (qmail 89389 invoked by uid 500); 27 Jul 2017 14:53:45 -0000 Mailing-List: contact commits-help@ambari.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: ambari-dev@ambari.apache.org Delivered-To: mailing list commits@ambari.apache.org Received: (qmail 89379 invoked by uid 99); 27 Jul 2017 14:53:45 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 27 Jul 2017 14:53:45 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 99A09E967F; Thu, 27 Jul 2017 14:53:44 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: mgergely@apache.org To: commits@ambari.apache.org Date: Thu, 27 Jul 2017 14:53:44 -0000 Message-Id: <196f213b847340788625f28bf4605559@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [1/2] ambari git commit: AMBARI-21507 Log Search Solr output properties should be provided by the Config API (mgergely) archived-at: Thu, 27 Jul 2017 14:53:47 -0000 Repository: ambari Updated Branches: refs/heads/trunk e3a50d946 -> dc85e67d7 http://git-wip-us.apache.org/repos/asf/ambari/blob/dc85e67d/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/service_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/service_advisor.py b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/service_advisor.py index fc49824..6b29dbc 100644 --- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/service_advisor.py +++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/service_advisor.py @@ -122,24 +122,19 @@ class LogSearchServiceAdvisor(service_advisor.ServiceAdvisor): logSearchServerHosts = self.getComponentHostNames(services, "LOGSEARCH", "LOGSEARCH_SERVER") # if there is no Log Search server on the cluster, i.e. there is an external server if logSearchServerHosts is None or len(logSearchServerHosts) == 0: - # hide logsearch specific attributes, except for a few which are used by the logfeeders too + # hide logsearch specific attributes for key in services['configurations']['logsearch-env']['properties']: putLogSearchEnvAttribute(key, 'visible', 'false') for key in services['configurations']['logsearch-properties']['properties']: - if key not in ['logsearch.collection.service.logs.numshards', 'logsearch.collection.audit.logs.numshards', - 'logsearch.solr.collection.service.logs', 'logsearch.solr.collection.audit.logs', - 'logsearch.service.logs.split.interval.mins', 'logsearch.audit.logs.split.interval.mins']: - putLogSearchAttribute(key, 'visible', 'false') + putLogSearchAttribute(key, 'visible', 'false') for key in services['configurations']['logsearch-audit_logs-solrconfig']['properties']: self.putPropertyAttribute(configurations, "logsearch-audit_logs-solrconfig")(key, 'visible', 'false') for key in services['configurations']['logsearch-service_logs-solrconfig']['properties']: self.putPropertyAttribute(configurations, "logsearch-service_logs-solrconfig")(key, 'visible', 'false') for key in services['configurations']['logsearch-log4j']['properties']: self.putPropertyAttribute(configurations, "logsearch-log4j")(key, 'visible', 'false') - - # in the abscence of a server we can't provide a good estimation for the number of shards - putLogSearchProperty("logsearch.collection.service.logs.numshards", 2) - putLogSearchProperty("logsearch.collection.audit.logs.numshards", 2) + for key in services['configurations']['logsearch-admin-json']['properties']: + self.putPropertyAttribute(configurations, "logsearch-admin-json")(key, 'visible', 'false') # if there is a Log Search server on the cluster else: infraSolrHosts = self.getComponentHostNames(services, "AMBARI_INFRA", "INFRA_SOLR") @@ -161,17 +156,17 @@ class LogSearchServiceAdvisor(service_advisor.ServiceAdvisor): putLogSearchCommonEnvProperty('logsearch_use_external_solr', 'true') - # recommend number of shard - putLogSearchAttribute('logsearch.collection.service.logs.numshards', 'minimum', recommendedMinShards) - putLogSearchAttribute('logsearch.collection.service.logs.numshards', 'maximum', recommendedMaxShards) - putLogSearchProperty("logsearch.collection.service.logs.numshards", recommendedShards) + # recommend number of shard + putLogSearchAttribute('logsearch.collection.service.logs.numshards', 'minimum', recommendedMinShards) + putLogSearchAttribute('logsearch.collection.service.logs.numshards', 'maximum', recommendedMaxShards) + putLogSearchProperty("logsearch.collection.service.logs.numshards", recommendedShards) - putLogSearchAttribute('logsearch.collection.audit.logs.numshards', 'minimum', recommendedMinShards) - putLogSearchAttribute('logsearch.collection.audit.logs.numshards', 'maximum', recommendedMaxShards) - putLogSearchProperty("logsearch.collection.audit.logs.numshards", recommendedShards) - # recommend replication factor - putLogSearchProperty("logsearch.collection.service.logs.replication.factor", recommendedReplicationFactor) - putLogSearchProperty("logsearch.collection.audit.logs.replication.factor", recommendedReplicationFactor) + putLogSearchAttribute('logsearch.collection.audit.logs.numshards', 'minimum', recommendedMinShards) + putLogSearchAttribute('logsearch.collection.audit.logs.numshards', 'maximum', recommendedMaxShards) + putLogSearchProperty("logsearch.collection.audit.logs.numshards", recommendedShards) + # recommend replication factor + putLogSearchProperty("logsearch.collection.service.logs.replication.factor", recommendedReplicationFactor) + putLogSearchProperty("logsearch.collection.audit.logs.replication.factor", recommendedReplicationFactor) kerberos_authentication_enabled = self.isSecurityEnabled(services) # if there is no kerberos enabled hide kerberor related properties http://git-wip-us.apache.org/repos/asf/ambari/blob/dc85e67d/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog300Test.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog300Test.java b/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog300Test.java index d7bdf75..a7417d4 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog300Test.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog300Test.java @@ -331,9 +331,9 @@ public class UpgradeCatalog300Test { Map expectedLogFeederLog4j = ImmutableMap.of( "content", ""); - Config mockLogFeederLog4j = easyMockSupport.createNiceMock(Config.class); - expect(cluster.getDesiredConfigByType("logfeeder-log4j")).andReturn(mockLogFeederLog4j).atLeastOnce(); - expect(mockLogFeederLog4j.getProperties()).andReturn(oldLogFeederLog4j).anyTimes(); + Config logFeederLog4jConf = easyMockSupport.createNiceMock(Config.class); + expect(cluster.getDesiredConfigByType("logfeeder-log4j")).andReturn(logFeederLog4jConf).atLeastOnce(); + expect(logFeederLog4jConf.getProperties()).andReturn(oldLogFeederLog4j).anyTimes(); Capture> logFeederLog4jCapture = EasyMock.newCapture(); expect(controller.createConfig(anyObject(Cluster.class), anyObject(StackId.class), anyString(), capture(logFeederLog4jCapture), anyString(), EasyMock.>>anyObject())).andReturn(config).once(); @@ -344,45 +344,75 @@ public class UpgradeCatalog300Test { Map expectedLogSearchLog4j = ImmutableMap.of( "content", ""); - Config mockLogSearchLog4j = easyMockSupport.createNiceMock(Config.class); - expect(cluster.getDesiredConfigByType("logsearch-log4j")).andReturn(mockLogSearchLog4j).atLeastOnce(); - expect(mockLogSearchLog4j.getProperties()).andReturn(oldLogSearchLog4j).anyTimes(); + Config logSearchLog4jConf = easyMockSupport.createNiceMock(Config.class); + expect(cluster.getDesiredConfigByType("logsearch-log4j")).andReturn(logSearchLog4jConf).atLeastOnce(); + expect(logSearchLog4jConf.getProperties()).andReturn(oldLogSearchLog4j).anyTimes(); Capture> logSearchLog4jCapture = EasyMock.newCapture(); expect(controller.createConfig(anyObject(Cluster.class), anyObject(StackId.class), anyString(), capture(logSearchLog4jCapture), anyString(), EasyMock.>>anyObject())).andReturn(config).once(); Map oldLogSearchServiceLogsConf = ImmutableMap.of( - "content", ""); + "content", ""); Map expectedLogSearchServiceLogsConf = ImmutableMap.of( - "content", ""); + "content", ""); - Config confLogSearchServiceLogsConf = easyMockSupport.createNiceMock(Config.class); - expect(cluster.getDesiredConfigByType("logsearch-service_logs-solrconfig")).andReturn(confLogSearchServiceLogsConf).atLeastOnce(); - expect(confLogSearchServiceLogsConf.getProperties()).andReturn(oldLogSearchServiceLogsConf).anyTimes(); + Config logSearchServiceLogsConf = easyMockSupport.createNiceMock(Config.class); + expect(cluster.getDesiredConfigByType("logsearch-service_logs-solrconfig")).andReturn(logSearchServiceLogsConf).atLeastOnce(); + expect(logSearchServiceLogsConf.getProperties()).andReturn(oldLogSearchServiceLogsConf).anyTimes(); Capture> logSearchServiceLogsConfCapture = EasyMock.newCapture(); expect(controller.createConfig(anyObject(Cluster.class), anyObject(StackId.class), anyString(), capture(logSearchServiceLogsConfCapture), anyString(), - EasyMock.>>anyObject())).andReturn(config).once(); + EasyMock.>>anyObject())).andReturn(config).once(); Map oldLogSearchAuditLogsConf = ImmutableMap.of( - "content", ""); + "content", ""); Map expectedLogSearchAuditLogsConf = ImmutableMap.of( - "content", ""); + "content", ""); - Config confLogSearchAuditLogsConf = easyMockSupport.createNiceMock(Config.class); - expect(cluster.getDesiredConfigByType("logsearch-audit_logs-solrconfig")).andReturn(confLogSearchAuditLogsConf).atLeastOnce(); - expect(confLogSearchAuditLogsConf.getProperties()).andReturn(oldLogSearchAuditLogsConf).anyTimes(); + Config logSearchAuditLogsConf = easyMockSupport.createNiceMock(Config.class); + expect(cluster.getDesiredConfigByType("logsearch-audit_logs-solrconfig")).andReturn(logSearchAuditLogsConf).atLeastOnce(); + expect(logSearchAuditLogsConf.getProperties()).andReturn(oldLogSearchAuditLogsConf).anyTimes(); Capture> logSearchAuditLogsConfCapture = EasyMock.newCapture(); expect(controller.createConfig(anyObject(Cluster.class), anyObject(StackId.class), anyString(), capture(logSearchAuditLogsConfCapture), anyString(), - EasyMock.>>anyObject())).andReturn(config).once(); + EasyMock.>>anyObject())).andReturn(config).once(); + + Map oldLogFeederOutputConf = ImmutableMap.of( + "content", + " \"zk_connect_string\":\"{{logsearch_solr_zk_quorum}}{{logsearch_solr_zk_znode}}\",\n" + + " \"collection\":\"{{logsearch_solr_collection_service_logs}}\",\n" + + " \"number_of_shards\": \"{{logsearch_collection_service_logs_numshards}}\",\n" + + " \"splits_interval_mins\": \"{{logsearch_service_logs_split_interval_mins}}\",\n" + + "\n" + + " \"zk_connect_string\":\"{{logsearch_solr_zk_quorum}}{{logsearch_solr_zk_znode}}\",\n" + + " \"collection\":\"{{logsearch_solr_collection_audit_logs}}\",\n" + + " \"number_of_shards\": \"{{logsearch_collection_audit_logs_numshards}}\",\n" + + " \"splits_interval_mins\": \"{{logsearch_audit_logs_split_interval_mins}}\",\n" + ); + + Map expectedLogFeederOutputConf = ImmutableMap.of( + "content", + " \"zk_connect_string\":\"{{logsearch_solr_zk_quorum}}{{logsearch_solr_zk_znode}}\",\n" + + " \"type\": \"service\",\n" + + "\n" + + " \"zk_connect_string\":\"{{logsearch_solr_zk_quorum}}{{logsearch_solr_zk_znode}}\",\n" + + " \"type\": \"audit\",\n" + ); + + Config logFeederOutputConf = easyMockSupport.createNiceMock(Config.class); + expect(cluster.getDesiredConfigByType("logfeeder-output-config")).andReturn(logFeederOutputConf).atLeastOnce(); + expect(logFeederOutputConf.getProperties()).andReturn(oldLogFeederOutputConf).anyTimes(); + Capture> logFeederOutputConfCapture = EasyMock.newCapture(); + expect(controller.createConfig(anyObject(Cluster.class), anyObject(StackId.class), anyString(), capture(logFeederOutputConfCapture), anyString(), + EasyMock.>>anyObject())).andReturn(config).once(); replay(clusters, cluster); replay(controller, injector2); replay(confSomethingElse1, confSomethingElse2, confLogSearchConf1, confLogSearchConf2); replay(logSearchPropertiesConf, logFeederPropertiesConf); - replay(mockLogFeederLog4j, mockLogSearchLog4j); - replay(confLogSearchServiceLogsConf, confLogSearchAuditLogsConf); + replay(logFeederLog4jConf, logSearchLog4jConf); + replay(logSearchServiceLogsConf, logSearchAuditLogsConf); + replay(logFeederOutputConf); new UpgradeCatalog300(injector2).updateLogSearchConfigs(); easyMockSupport.verifyAll(); @@ -409,5 +439,8 @@ public class UpgradeCatalog300Test { Map updatedAuditLogsConf = logSearchAuditLogsConfCapture.getValue(); assertTrue(Maps.difference(expectedLogSearchAuditLogsConf, updatedAuditLogsConf).areEqual()); + + Map updatedLogFeederOutputConf = logFeederOutputConfCapture.getValue(); + assertTrue(Maps.difference(expectedLogFeederOutputConf, updatedLogFeederOutputConf).areEqual()); } }