ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From lpus...@apache.org
Subject [24/50] [abbrv] ambari git commit: AMBARI-21623 Log Search Config should be separated into a Server and Log Feeder interface (mgergely)
Date Tue, 08 Aug 2017 16:02:40 GMT
http://git-wip-us.apache.org/repos/asf/ambari/blob/2d1ac668/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/input/InputConfigUploader.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/input/InputConfigUploader.java
b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/input/InputConfigUploader.java
index 5fdfded..8f8c4fd 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/input/InputConfigUploader.java
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/input/InputConfigUploader.java
@@ -27,8 +27,8 @@ import java.util.Set;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
-import org.apache.ambari.logsearch.config.api.LogSearchConfig;
 import org.apache.ambari.logfeeder.util.LogFeederPropertiesUtil;
+import org.apache.ambari.logsearch.config.api.LogSearchConfigLogFeeder;
 import org.apache.log4j.Logger;
 
 import com.google.common.io.Files;
@@ -47,13 +47,13 @@ public class InputConfigUploader extends Thread {
   };
   private final Set<String> filesHandled = new HashSet<>();
   private final Pattern serviceNamePattern = Pattern.compile("input.config-(.+).json");
-  private final LogSearchConfig config;
+  private final LogSearchConfigLogFeeder config;
   
-  public static void load(LogSearchConfig config) {
+  public static void load(LogSearchConfigLogFeeder config) {
     new InputConfigUploader(config).start();
   }
   
-  private InputConfigUploader(LogSearchConfig config) {
+  private InputConfigUploader(LogSearchConfigLogFeeder config) {
     super("Input Config Loader");
     setDaemon(true);
     
@@ -73,7 +73,7 @@ public class InputConfigUploader extends Thread {
             String serviceName = m.group(1);
             String inputConfig = Files.toString(inputConfigFile, Charset.defaultCharset());
             
-            if (!config.inputConfigExistsLogFeeder(serviceName)) {
+            if (!config.inputConfigExists(serviceName)) {
               config.createInputConfig(LogFeederPropertiesUtil.getClusterName(), serviceName,
inputConfig);
             }
             filesHandled.add(inputConfigFile.getAbsolutePath());

http://git-wip-us.apache.org/repos/asf/ambari/blob/2d1ac668/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/Output.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/Output.java
b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/Output.java
index b370e58..c0075b5 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/Output.java
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/Output.java
@@ -28,7 +28,7 @@ import org.apache.ambari.logfeeder.common.ConfigBlock;
 import org.apache.ambari.logfeeder.input.InputMarker;
 import org.apache.ambari.logfeeder.metrics.MetricData;
 import org.apache.ambari.logfeeder.util.LogFeederUtil;
-import org.apache.ambari.logsearch.config.api.LogSearchConfig;
+import org.apache.ambari.logsearch.config.api.LogSearchConfigLogFeeder;
 import org.apache.ambari.logsearch.config.api.OutputConfigMonitor;
 import org.apache.ambari.logsearch.config.api.model.outputconfig.OutputProperties;
 
@@ -104,9 +104,9 @@ public abstract class Output extends ConfigBlock implements OutputConfigMonitor
     this.destination = destination;
   }
 
-  protected LogSearchConfig logSearchConfig;
+  protected LogSearchConfigLogFeeder logSearchConfig;
 
-  public void setLogSearchConfig(LogSearchConfig logSearchConfig) {
+  public void setLogSearchConfig(LogSearchConfigLogFeeder logSearchConfig) {
     this.logSearchConfig = logSearchConfig;
   }
 

http://git-wip-us.apache.org/repos/asf/ambari/blob/2d1ac668/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/output/OutputSolrTest.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/output/OutputSolrTest.java
b/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/output/OutputSolrTest.java
index ce040f9..5ab271a 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/output/OutputSolrTest.java
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/output/OutputSolrTest.java
@@ -28,7 +28,7 @@ import java.util.concurrent.ConcurrentHashMap;
 
 import org.apache.ambari.logfeeder.input.Input;
 import org.apache.ambari.logfeeder.input.InputMarker;
-import org.apache.ambari.logsearch.config.api.LogSearchConfig;
+import org.apache.ambari.logsearch.config.api.LogSearchConfigLogFeeder;
 import org.apache.ambari.logsearch.config.api.model.outputconfig.OutputSolrProperties;
 import org.apache.ambari.logsearch.config.zookeeper.model.outputconfig.impl.OutputSolrPropertiesImpl;
 import org.apache.log4j.Logger;
@@ -50,7 +50,7 @@ public class OutputSolrTest {
   private static final Logger LOG = Logger.getLogger(OutputSolrTest.class);
 
   private OutputSolr outputSolr;
-  private LogSearchConfig logSearchConfigMock;
+  private LogSearchConfigLogFeeder logSearchConfigMock;
   private Map<Integer, SolrInputDocument> receivedDocs = new ConcurrentHashMap<>();
 
   @Rule
@@ -80,7 +80,7 @@ public class OutputSolrTest {
     };
     
     OutputSolrProperties outputSolrProperties = new OutputSolrPropertiesImpl("hadoop_logs",
"none");
-    logSearchConfigMock = EasyMock.createNiceMock(LogSearchConfig.class);
+    logSearchConfigMock = EasyMock.createNiceMock(LogSearchConfigLogFeeder.class);
     EasyMock.expect(logSearchConfigMock.getOutputSolrProperties("service")).andReturn(outputSolrProperties);
     EasyMock.replay(logSearchConfigMock);
     

http://git-wip-us.apache.org/repos/asf/ambari/blob/2d1ac668/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/configurer/LogSearchConfigConfigurer.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/configurer/LogSearchConfigConfigurer.java
b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/configurer/LogSearchConfigConfigurer.java
index 3f6df75..23b7430 100644
--- a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/configurer/LogSearchConfigConfigurer.java
+++ b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/configurer/LogSearchConfigConfigurer.java
@@ -25,10 +25,9 @@ import javax.inject.Named;
 
 import org.apache.ambari.logsearch.common.PropertiesHelper;
 import org.apache.ambari.logsearch.conf.global.LogSearchConfigState;
-import org.apache.ambari.logsearch.config.api.LogSearchConfig;
 import org.apache.ambari.logsearch.config.api.LogSearchConfigFactory;
-import org.apache.ambari.logsearch.config.api.LogSearchConfig.Component;
-import org.apache.ambari.logsearch.config.zookeeper.LogSearchConfigZK;
+import org.apache.ambari.logsearch.config.api.LogSearchConfigServer;
+import org.apache.ambari.logsearch.config.zookeeper.LogSearchConfigServerZK;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -38,8 +37,8 @@ public class LogSearchConfigConfigurer implements Configurer {
   
   private static final int RETRY_INTERVAL_SECONDS = 10;
   
-  private static LogSearchConfig logSearchConfig;
-  public static LogSearchConfig getConfig() {
+  private LogSearchConfigServer logSearchConfig;
+  public LogSearchConfigServer getConfig() {
     return logSearchConfig;
   }
   
@@ -55,8 +54,8 @@ public class LogSearchConfigConfigurer implements Configurer {
         logger.info("Started thread to set up log search config");
         while (true) {
           try {
-            logSearchConfig = LogSearchConfigFactory.createLogSearchConfig(Component.SERVER,
PropertiesHelper.getProperties(),
-                null, LogSearchConfigZK.class);
+            logSearchConfig = LogSearchConfigFactory.createLogSearchConfigServer(PropertiesHelper.getProperties(),
+                LogSearchConfigServerZK.class);
             logSearchConfigState.setLogSearchConfigAvailable(true);
             break;
           } catch (Exception e) {

http://git-wip-us.apache.org/repos/asf/ambari/blob/2d1ac668/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/configurer/SolrCollectionConfigurer.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/configurer/SolrCollectionConfigurer.java
b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/configurer/SolrCollectionConfigurer.java
index 6de34ce..ff828a3 100644
--- a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/configurer/SolrCollectionConfigurer.java
+++ b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/configurer/SolrCollectionConfigurer.java
@@ -219,7 +219,8 @@ public class SolrCollectionConfigurer implements Configurer {
       boolean reloadCollectionNeeded) {
     try {
       List<String> allCollectionList = new ListCollectionHandler().handle(solrClient,
null);
-      boolean collectionCreated = new CreateCollectionHandler(allCollectionList).handle(solrClient,
solrPropsConfig);
+      CreateCollectionHandler handler = new CreateCollectionHandler(solrDaoBase.getLogSearchConfig(),
allCollectionList);
+      boolean collectionCreated = handler.handle(solrClient, solrPropsConfig);
       boolean collectionReloaded = true;
       if (reloadCollectionNeeded) {
         collectionReloaded = new ReloadCollectionHandler().handle(solrClient, solrPropsConfig);

http://git-wip-us.apache.org/repos/asf/ambari/blob/2d1ac668/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/dao/SolrDaoBase.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/dao/SolrDaoBase.java
b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/dao/SolrDaoBase.java
index 15f59e4..ef5e697 100644
--- a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/dao/SolrDaoBase.java
+++ b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/dao/SolrDaoBase.java
@@ -26,6 +26,8 @@ import org.apache.ambari.logsearch.conf.SolrKerberosConfig;
 import org.apache.ambari.logsearch.conf.SolrPropsConfig;
 import org.apache.ambari.logsearch.conf.global.LogSearchConfigState;
 import org.apache.ambari.logsearch.conf.global.SolrCollectionState;
+import org.apache.ambari.logsearch.config.api.LogSearchConfigServer;
+import org.apache.ambari.logsearch.configurer.LogSearchConfigConfigurer;
 import org.apache.ambari.logsearch.util.RESTErrorUtil;
 import org.apache.ambari.logsearch.util.SolrUtil;
 import org.apache.log4j.Logger;
@@ -56,7 +58,10 @@ public abstract class SolrDaoBase {
   private SolrKerberosConfig solrKerberosConfig;
 
   @Inject
-  protected LogSearchConfigState logSearchConfigState;
+  private LogSearchConfigState logSearchConfigState;
+
+  @Inject
+  private LogSearchConfigConfigurer logSearchConfigConfigurer;
 
   protected SolrDaoBase(LogType logType) {
     this.logType = logType;
@@ -152,6 +157,10 @@ public abstract class SolrDaoBase {
     return (CloudSolrClient) getSolrTemplate().getSolrClient();
   }
 
+  public LogSearchConfigServer getLogSearchConfig() {
+    return logSearchConfigConfigurer.getConfig();
+  }
+
   public abstract SolrTemplate getSolrTemplate();
 
   public abstract void setSolrTemplate(SolrTemplate solrTemplate);

http://git-wip-us.apache.org/repos/asf/ambari/blob/2d1ac668/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/handler/CreateCollectionHandler.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/handler/CreateCollectionHandler.java
b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/handler/CreateCollectionHandler.java
index 8b4c7bf..a988856 100644
--- a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/handler/CreateCollectionHandler.java
+++ b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/handler/CreateCollectionHandler.java
@@ -19,6 +19,7 @@
 package org.apache.ambari.logsearch.handler;
 
 import org.apache.ambari.logsearch.conf.SolrPropsConfig;
+import org.apache.ambari.logsearch.config.api.LogSearchConfigServer;
 import org.apache.ambari.logsearch.config.api.model.outputconfig.OutputSolrProperties;
 import org.apache.ambari.logsearch.config.zookeeper.model.outputconfig.impl.OutputSolrPropertiesImpl;
 import org.apache.ambari.logsearch.configurer.LogSearchConfigConfigurer;
@@ -54,9 +55,11 @@ public class CreateCollectionHandler implements SolrZkRequestHandler<Boolean>
{
   private static final String MODIFY_COLLECTION_QUERY = "/admin/collections?action=MODIFYCOLLECTION&collection=%s&%s=%d";
   private static final String MAX_SHARDS_PER_NODE = "maxShardsPerNode";
 
+  private final LogSearchConfigServer logSearchConfig;
   private final List<String> allCollectionList;
 
-  public CreateCollectionHandler(List<String> allCollectionList) {
+  public CreateCollectionHandler(LogSearchConfigServer logSearchConfig, List<String>
allCollectionList) {
+    this.logSearchConfig = logSearchConfig;
     this.allCollectionList = allCollectionList;
   }
 
@@ -65,7 +68,7 @@ public class CreateCollectionHandler implements SolrZkRequestHandler<Boolean>
{
     if (solrPropsConfig.getLogType() != null) {
       OutputSolrProperties outputSolrProperties = new OutputSolrPropertiesImpl(solrPropsConfig.getCollection(),
           solrPropsConfig.getSplitInterval());
-      LogSearchConfigConfigurer.getConfig().saveOutputSolrProperties(solrPropsConfig.getLogType(),
outputSolrProperties);
+      logSearchConfig.saveOutputSolrProperties(solrPropsConfig.getLogType(), outputSolrProperties);
     }
 
     boolean result;

http://git-wip-us.apache.org/repos/asf/ambari/blob/2d1ac668/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/manager/ShipperConfigManager.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/manager/ShipperConfigManager.java
b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/manager/ShipperConfigManager.java
index a1181b4..b19db72 100644
--- a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/manager/ShipperConfigManager.java
+++ b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/manager/ShipperConfigManager.java
@@ -50,24 +50,24 @@ public class ShipperConfigManager extends JsonManagerBase {
   private LogSearchConfigConfigurer logSearchConfigConfigurer;
   
   public List<String> getServices(String clusterName) {
-    return LogSearchConfigConfigurer.getConfig().getServices(clusterName);
+    return logSearchConfigConfigurer.getConfig().getServices(clusterName);
   }
 
   public LSServerInputConfig getInputConfig(String clusterName, String serviceName) {
-    InputConfig inputConfig = LogSearchConfigConfigurer.getConfig().getInputConfig(clusterName,
serviceName);
+    InputConfig inputConfig = logSearchConfigConfigurer.getConfig().getInputConfig(clusterName,
serviceName);
     return new LSServerInputConfig(inputConfig);
   }
 
   public Response createInputConfig(String clusterName, String serviceName, LSServerInputConfig
inputConfig) {
     try {
-      if (LogSearchConfigConfigurer.getConfig().inputConfigExistsServer(clusterName, serviceName))
{
+      if (logSearchConfigConfigurer.getConfig().inputConfigExists(clusterName, serviceName))
{
         return Response.serverError()
             .type(MediaType.APPLICATION_JSON)
             .entity(ImmutableMap.of("errorMessage", "Input config already exists for service
" + serviceName))
             .build();
       }
       
-      LogSearchConfigConfigurer.getConfig().createInputConfig(clusterName, serviceName, new
ObjectMapper().writeValueAsString(inputConfig));
+      logSearchConfigConfigurer.getConfig().createInputConfig(clusterName, serviceName, new
ObjectMapper().writeValueAsString(inputConfig));
       return Response.ok().build();
     } catch (Exception e) {
       logger.warn("Could not create input config", e);
@@ -77,14 +77,14 @@ public class ShipperConfigManager extends JsonManagerBase {
 
   public Response setInputConfig(String clusterName, String serviceName, LSServerInputConfig
inputConfig) {
     try {
-      if (!LogSearchConfigConfigurer.getConfig().inputConfigExistsServer(clusterName, serviceName))
{
+      if (!logSearchConfigConfigurer.getConfig().inputConfigExists(clusterName, serviceName))
{
         return Response.serverError()
             .type(MediaType.APPLICATION_JSON)
             .entity(ImmutableMap.of("errorMessage", "Input config doesn't exist for service
" + serviceName))
             .build();
       }
       
-      LogSearchConfigConfigurer.getConfig().setInputConfig(clusterName, serviceName, new
ObjectMapper().writeValueAsString(inputConfig));
+      logSearchConfigConfigurer.getConfig().setInputConfig(clusterName, serviceName, new
ObjectMapper().writeValueAsString(inputConfig));
       return Response.ok().build();
     } catch (Exception e) {
       logger.warn("Could not update input config", e);
@@ -101,7 +101,7 @@ public class ShipperConfigManager extends JsonManagerBase {
         throw new IllegalArgumentException("Error validating shipper config:\n" + violations);
       }
       
-      String globalConfigs = LogSearchConfigConfigurer.getConfig().getGlobalConfigs(clusterName);
+      String globalConfigs = logSearchConfigConfigurer.getConfig().getGlobalConfigs(clusterName);
       LogEntryParseTester tester = new LogEntryParseTester(testEntry, shipperConfig, globalConfigs,
logId);
       return tester.parse();
     } catch (Exception e) {
@@ -111,12 +111,12 @@ public class ShipperConfigManager extends JsonManagerBase {
   }
 
   public LSServerLogLevelFilterMap getLogLevelFilters(String clusterName) {
-    return new LSServerLogLevelFilterMap(LogSearchConfigConfigurer.getConfig().getLogLevelFilters(clusterName));
+    return new LSServerLogLevelFilterMap(logSearchConfigConfigurer.getConfig().getLogLevelFilters(clusterName));
   }
 
   public Response setLogLevelFilters(String clusterName, LSServerLogLevelFilterMap request)
{
     try {
-      LogSearchConfigConfigurer.getConfig().setLogLevelFilters(clusterName, request.convertToApi());
+      logSearchConfigConfigurer.getConfig().setLogLevelFilters(clusterName, request.convertToApi());
       return Response.ok().build();
     } catch (Exception e) {
       logger.warn("Could not update log level filters", e);

http://git-wip-us.apache.org/repos/asf/ambari/blob/2d1ac668/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/model/common/LSServerFilter.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/model/common/LSServerFilter.java
b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/model/common/LSServerFilter.java
index 84bddb8..72cc72b 100644
--- a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/model/common/LSServerFilter.java
+++ b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/model/common/LSServerFilter.java
@@ -70,10 +70,12 @@ public abstract class LSServerFilter {
     this.sourceField = filterDescriptor.getSourceField();
     this.removeSourceField = filterDescriptor.isRemoveSourceField();
     
-    this.postMapValues = new HashMap<String, LSServerPostMapValuesList>();
-    for (Map.Entry<String, ? extends List<? extends PostMapValues>> e : filterDescriptor.getPostMapValues().entrySet())
{
-      LSServerPostMapValuesList lsServerPostMapValuesList = new LSServerPostMapValuesList(e.getValue());
-      postMapValues.put(e.getKey(), lsServerPostMapValuesList);
+    if (filterDescriptor.getPostMapValues() != null) {
+      this.postMapValues = new HashMap<String, LSServerPostMapValuesList>();
+      for (Map.Entry<String, ? extends List<? extends PostMapValues>> e : filterDescriptor.getPostMapValues().entrySet())
{
+        LSServerPostMapValuesList lsServerPostMapValuesList = new LSServerPostMapValuesList(e.getValue());
+        postMapValues.put(e.getKey(), lsServerPostMapValuesList);
+      }
     }
     
     this.isEnabled = filterDescriptor.isEnabled();


Mime
View raw message