ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mgerg...@apache.org
Subject [1/2] ambari git commit: AMBARI-21507 Log Search Solr output properties should be provided by the Config API (mgergely)
Date Thu, 27 Jul 2017 14:53:44 GMT
Repository: ambari
Updated Branches:
  refs/heads/trunk e3a50d946 -> dc85e67d7


http://git-wip-us.apache.org/repos/asf/ambari/blob/dc85e67d/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/service_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/service_advisor.py
b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/service_advisor.py
index fc49824..6b29dbc 100644
--- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/service_advisor.py
+++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/service_advisor.py
@@ -122,24 +122,19 @@ class LogSearchServiceAdvisor(service_advisor.ServiceAdvisor):
     logSearchServerHosts = self.getComponentHostNames(services, "LOGSEARCH", "LOGSEARCH_SERVER")
     # if there is no Log Search server on the cluster, i.e. there is an external server
     if logSearchServerHosts is None or len(logSearchServerHosts) == 0:
-      # hide logsearch specific attributes, except for a few which are used by the logfeeders
too
+      # hide logsearch specific attributes
       for key in services['configurations']['logsearch-env']['properties']:
         putLogSearchEnvAttribute(key, 'visible', 'false')
       for key in services['configurations']['logsearch-properties']['properties']:
-        if key not in ['logsearch.collection.service.logs.numshards', 'logsearch.collection.audit.logs.numshards',
-                       'logsearch.solr.collection.service.logs', 'logsearch.solr.collection.audit.logs',
-                       'logsearch.service.logs.split.interval.mins', 'logsearch.audit.logs.split.interval.mins']:
-          putLogSearchAttribute(key, 'visible', 'false')
+        putLogSearchAttribute(key, 'visible', 'false')
       for key in services['configurations']['logsearch-audit_logs-solrconfig']['properties']:
         self.putPropertyAttribute(configurations, "logsearch-audit_logs-solrconfig")(key,
'visible', 'false')
       for key in services['configurations']['logsearch-service_logs-solrconfig']['properties']:
         self.putPropertyAttribute(configurations, "logsearch-service_logs-solrconfig")(key,
'visible', 'false')
       for key in services['configurations']['logsearch-log4j']['properties']:
         self.putPropertyAttribute(configurations, "logsearch-log4j")(key, 'visible', 'false')
-      
-      # in the abscence of a server we can't provide a good estimation for the number of
shards
-      putLogSearchProperty("logsearch.collection.service.logs.numshards", 2)
-      putLogSearchProperty("logsearch.collection.audit.logs.numshards", 2)
+      for key in services['configurations']['logsearch-admin-json']['properties']:
+        self.putPropertyAttribute(configurations, "logsearch-admin-json")(key, 'visible',
'false')
     # if there is a Log Search server on the cluster
     else:
       infraSolrHosts = self.getComponentHostNames(services, "AMBARI_INFRA", "INFRA_SOLR")
@@ -161,17 +156,17 @@ class LogSearchServiceAdvisor(service_advisor.ServiceAdvisor):
         
         putLogSearchCommonEnvProperty('logsearch_use_external_solr', 'true')
         
-      # recommend number of shard
-      putLogSearchAttribute('logsearch.collection.service.logs.numshards', 'minimum', recommendedMinShards)
-      putLogSearchAttribute('logsearch.collection.service.logs.numshards', 'maximum', recommendedMaxShards)
-      putLogSearchProperty("logsearch.collection.service.logs.numshards", recommendedShards)
+        # recommend number of shard
+        putLogSearchAttribute('logsearch.collection.service.logs.numshards', 'minimum', recommendedMinShards)
+        putLogSearchAttribute('logsearch.collection.service.logs.numshards', 'maximum', recommendedMaxShards)
+        putLogSearchProperty("logsearch.collection.service.logs.numshards", recommendedShards)
       
-      putLogSearchAttribute('logsearch.collection.audit.logs.numshards', 'minimum', recommendedMinShards)
-      putLogSearchAttribute('logsearch.collection.audit.logs.numshards', 'maximum', recommendedMaxShards)
-      putLogSearchProperty("logsearch.collection.audit.logs.numshards", recommendedShards)
-      # recommend replication factor
-      putLogSearchProperty("logsearch.collection.service.logs.replication.factor", recommendedReplicationFactor)
-      putLogSearchProperty("logsearch.collection.audit.logs.replication.factor", recommendedReplicationFactor)
+        putLogSearchAttribute('logsearch.collection.audit.logs.numshards', 'minimum', recommendedMinShards)
+        putLogSearchAttribute('logsearch.collection.audit.logs.numshards', 'maximum', recommendedMaxShards)
+        putLogSearchProperty("logsearch.collection.audit.logs.numshards", recommendedShards)
+        # recommend replication factor
+        putLogSearchProperty("logsearch.collection.service.logs.replication.factor", recommendedReplicationFactor)
+        putLogSearchProperty("logsearch.collection.audit.logs.replication.factor", recommendedReplicationFactor)
       
     kerberos_authentication_enabled = self.isSecurityEnabled(services)
     # if there is no kerberos enabled hide kerberor related properties

http://git-wip-us.apache.org/repos/asf/ambari/blob/dc85e67d/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog300Test.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog300Test.java
b/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog300Test.java
index d7bdf75..a7417d4 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog300Test.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog300Test.java
@@ -331,9 +331,9 @@ public class UpgradeCatalog300Test {
     Map<String, String> expectedLogFeederLog4j = ImmutableMap.of(
         "content", "<!DOCTYPE log4j:configuration SYSTEM \"http://logging.apache.org/log4j/1.2/apidocs/org/apache/log4j/xml/doc-files/log4j.dtd\">");
 
-    Config mockLogFeederLog4j = easyMockSupport.createNiceMock(Config.class);
-    expect(cluster.getDesiredConfigByType("logfeeder-log4j")).andReturn(mockLogFeederLog4j).atLeastOnce();
-    expect(mockLogFeederLog4j.getProperties()).andReturn(oldLogFeederLog4j).anyTimes();
+    Config logFeederLog4jConf = easyMockSupport.createNiceMock(Config.class);
+    expect(cluster.getDesiredConfigByType("logfeeder-log4j")).andReturn(logFeederLog4jConf).atLeastOnce();
+    expect(logFeederLog4jConf.getProperties()).andReturn(oldLogFeederLog4j).anyTimes();
     Capture<Map<String, String>> logFeederLog4jCapture = EasyMock.newCapture();
     expect(controller.createConfig(anyObject(Cluster.class), anyObject(StackId.class), anyString(),
capture(logFeederLog4jCapture), anyString(),
         EasyMock.<Map<String, Map<String, String>>>anyObject())).andReturn(config).once();
@@ -344,45 +344,75 @@ public class UpgradeCatalog300Test {
     Map<String, String> expectedLogSearchLog4j = ImmutableMap.of(
         "content", "<!DOCTYPE log4j:configuration SYSTEM \"http://logging.apache.org/log4j/1.2/apidocs/org/apache/log4j/xml/doc-files/log4j.dtd\">");
 
-    Config mockLogSearchLog4j = easyMockSupport.createNiceMock(Config.class);
-    expect(cluster.getDesiredConfigByType("logsearch-log4j")).andReturn(mockLogSearchLog4j).atLeastOnce();
-    expect(mockLogSearchLog4j.getProperties()).andReturn(oldLogSearchLog4j).anyTimes();
+    Config logSearchLog4jConf = easyMockSupport.createNiceMock(Config.class);
+    expect(cluster.getDesiredConfigByType("logsearch-log4j")).andReturn(logSearchLog4jConf).atLeastOnce();
+    expect(logSearchLog4jConf.getProperties()).andReturn(oldLogSearchLog4j).anyTimes();
     Capture<Map<String, String>> logSearchLog4jCapture = EasyMock.newCapture();
     expect(controller.createConfig(anyObject(Cluster.class), anyObject(StackId.class), anyString(),
capture(logSearchLog4jCapture), anyString(),
         EasyMock.<Map<String, Map<String, String>>>anyObject())).andReturn(config).once();
 
     Map<String, String> oldLogSearchServiceLogsConf = ImmutableMap.of(
-      "content", "<before/><requestHandler name=\"/admin/\"   class=\"solr.admin.AdminHandlers\"
/><after/>");
+        "content", "<before/><requestHandler name=\"/admin/\"   class=\"solr.admin.AdminHandlers\"
/><after/>");
 
     Map<String, String> expectedLogSearchServiceLogsConf = ImmutableMap.of(
-      "content", "<before/><after/>");
+        "content", "<before/><after/>");
 
-    Config confLogSearchServiceLogsConf = easyMockSupport.createNiceMock(Config.class);
-    expect(cluster.getDesiredConfigByType("logsearch-service_logs-solrconfig")).andReturn(confLogSearchServiceLogsConf).atLeastOnce();
-    expect(confLogSearchServiceLogsConf.getProperties()).andReturn(oldLogSearchServiceLogsConf).anyTimes();
+    Config logSearchServiceLogsConf = easyMockSupport.createNiceMock(Config.class);
+    expect(cluster.getDesiredConfigByType("logsearch-service_logs-solrconfig")).andReturn(logSearchServiceLogsConf).atLeastOnce();
+    expect(logSearchServiceLogsConf.getProperties()).andReturn(oldLogSearchServiceLogsConf).anyTimes();
     Capture<Map<String, String>> logSearchServiceLogsConfCapture = EasyMock.newCapture();
     expect(controller.createConfig(anyObject(Cluster.class), anyObject(StackId.class), anyString(),
capture(logSearchServiceLogsConfCapture), anyString(),
-      EasyMock.<Map<String, Map<String, String>>>anyObject())).andReturn(config).once();
+        EasyMock.<Map<String, Map<String, String>>>anyObject())).andReturn(config).once();
 
     Map<String, String> oldLogSearchAuditLogsConf = ImmutableMap.of(
-      "content", "<before/><requestHandler name=\"/admin/\"   class=\"solr.admin.AdminHandlers\"
/><after/>");
+        "content", "<before/><requestHandler name=\"/admin/\"   class=\"solr.admin.AdminHandlers\"
/><after/>");
 
     Map<String, String> expectedLogSearchAuditLogsConf = ImmutableMap.of(
-      "content", "<before/><after/>");
+        "content", "<before/><after/>");
 
-    Config confLogSearchAuditLogsConf = easyMockSupport.createNiceMock(Config.class);
-    expect(cluster.getDesiredConfigByType("logsearch-audit_logs-solrconfig")).andReturn(confLogSearchAuditLogsConf).atLeastOnce();
-    expect(confLogSearchAuditLogsConf.getProperties()).andReturn(oldLogSearchAuditLogsConf).anyTimes();
+    Config logSearchAuditLogsConf = easyMockSupport.createNiceMock(Config.class);
+    expect(cluster.getDesiredConfigByType("logsearch-audit_logs-solrconfig")).andReturn(logSearchAuditLogsConf).atLeastOnce();
+    expect(logSearchAuditLogsConf.getProperties()).andReturn(oldLogSearchAuditLogsConf).anyTimes();
     Capture<Map<String, String>> logSearchAuditLogsConfCapture = EasyMock.newCapture();
     expect(controller.createConfig(anyObject(Cluster.class), anyObject(StackId.class), anyString(),
capture(logSearchAuditLogsConfCapture), anyString(),
-      EasyMock.<Map<String, Map<String, String>>>anyObject())).andReturn(config).once();
+        EasyMock.<Map<String, Map<String, String>>>anyObject())).andReturn(config).once();
+
+    Map<String, String> oldLogFeederOutputConf = ImmutableMap.of(
+        "content",
+        "      \"zk_connect_string\":\"{{logsearch_solr_zk_quorum}}{{logsearch_solr_zk_znode}}\",\n"
+
+        "      \"collection\":\"{{logsearch_solr_collection_service_logs}}\",\n" +
+        "      \"number_of_shards\": \"{{logsearch_collection_service_logs_numshards}}\",\n"
+
+        "      \"splits_interval_mins\": \"{{logsearch_service_logs_split_interval_mins}}\",\n"
+
+        "\n" +
+        "      \"zk_connect_string\":\"{{logsearch_solr_zk_quorum}}{{logsearch_solr_zk_znode}}\",\n"
+
+        "      \"collection\":\"{{logsearch_solr_collection_audit_logs}}\",\n" +
+        "      \"number_of_shards\": \"{{logsearch_collection_audit_logs_numshards}}\",\n"
+
+        "      \"splits_interval_mins\": \"{{logsearch_audit_logs_split_interval_mins}}\",\n"
+    );
+
+    Map<String, String> expectedLogFeederOutputConf = ImmutableMap.of(
+        "content",
+        "      \"zk_connect_string\":\"{{logsearch_solr_zk_quorum}}{{logsearch_solr_zk_znode}}\",\n"
+
+        "      \"type\": \"service\",\n" +
+        "\n" +
+        "      \"zk_connect_string\":\"{{logsearch_solr_zk_quorum}}{{logsearch_solr_zk_znode}}\",\n"
+
+        "      \"type\": \"audit\",\n"
+    );
+
+    Config logFeederOutputConf = easyMockSupport.createNiceMock(Config.class);
+    expect(cluster.getDesiredConfigByType("logfeeder-output-config")).andReturn(logFeederOutputConf).atLeastOnce();
+    expect(logFeederOutputConf.getProperties()).andReturn(oldLogFeederOutputConf).anyTimes();
+    Capture<Map<String, String>> logFeederOutputConfCapture = EasyMock.newCapture();
+    expect(controller.createConfig(anyObject(Cluster.class), anyObject(StackId.class), anyString(),
capture(logFeederOutputConfCapture), anyString(),
+        EasyMock.<Map<String, Map<String, String>>>anyObject())).andReturn(config).once();
 
     replay(clusters, cluster);
     replay(controller, injector2);
     replay(confSomethingElse1, confSomethingElse2, confLogSearchConf1, confLogSearchConf2);
     replay(logSearchPropertiesConf, logFeederPropertiesConf);
-    replay(mockLogFeederLog4j, mockLogSearchLog4j);
-    replay(confLogSearchServiceLogsConf, confLogSearchAuditLogsConf);
+    replay(logFeederLog4jConf, logSearchLog4jConf);
+    replay(logSearchServiceLogsConf, logSearchAuditLogsConf);
+    replay(logFeederOutputConf);
     new UpgradeCatalog300(injector2).updateLogSearchConfigs();
     easyMockSupport.verifyAll();
 
@@ -409,5 +439,8 @@ public class UpgradeCatalog300Test {
 
     Map<String, String> updatedAuditLogsConf = logSearchAuditLogsConfCapture.getValue();
     assertTrue(Maps.difference(expectedLogSearchAuditLogsConf, updatedAuditLogsConf).areEqual());
+
+    Map<String, String> updatedLogFeederOutputConf = logFeederOutputConfCapture.getValue();
+    assertTrue(Maps.difference(expectedLogFeederOutputConf, updatedLogFeederOutputConf).areEqual());
   }
 }


Mime
View raw message