ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From lpus...@apache.org
Subject [21/26] ambari git commit: AMBARI-21399 Create property descriptions for internal Log Feeder configs (mgergely)
Date Thu, 06 Jul 2017 16:17:43 GMT
http://git-wip-us.apache.org/repos/asf/ambari/blob/5c9bdbfe/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/util/LogFeederUtil.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/util/LogFeederUtil.java
b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/util/LogFeederUtil.java
index 0fdc21f..a244a4e 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/util/LogFeederUtil.java
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/util/LogFeederUtil.java
@@ -32,6 +32,7 @@ import java.util.Properties;
 
 import org.apache.ambari.logfeeder.LogFeeder;
 import org.apache.ambari.logfeeder.metrics.MetricData;
+import org.apache.ambari.logsearch.config.api.LogSearchPropertyDescription;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.log4j.Level;
 import org.apache.log4j.Logger;
@@ -46,6 +47,26 @@ import com.google.gson.reflect.TypeToken;
 public class LogFeederUtil {
   private static final Logger LOG = Logger.getLogger(LogFeederUtil.class);
 
+  public static final String LOGFEEDER_PROPERTIES_FILE = "logfeeder.properties";
+
+  @LogSearchPropertyDescription(
+    name = "cluster.name",
+    description = "The name of the cluster the Log Feeder program runs in.",
+    examples = {"cl1"},
+    sources = {LOGFEEDER_PROPERTIES_FILE}
+  )
+  private static final String CLUSTER_NAME_PROPERTY = "cluster.name";
+
+  private static final String DEFAULT_TMP_DIR = "/tmp/$username/logfeeder/";
+  @LogSearchPropertyDescription(
+    name = "logfeeder.tmp.dir",
+    description = "The tmp dir used for creating temporary files.",
+    examples = {"/tmp/"},
+    defaultValue = DEFAULT_TMP_DIR,
+    sources = {LOGFEEDER_PROPERTIES_FILE}
+   )
+  private static final String TMP_DIR_PROPERTY = "logfeeder.tmp.dir";
+
   private final static String GSON_DATE_FORMAT = "yyyy-MM-dd HH:mm:ss.SSS";
   private static Gson gson = new GsonBuilder().setDateFormat(GSON_DATE_FORMAT).create();
   
@@ -81,6 +102,10 @@ public class LogFeederUtil {
     return props;
   }
 
+  public static void loadProperties() throws Exception {
+    loadProperties(LOGFEEDER_PROPERTIES_FILE);
+  }
+  
   /**
    * This method will read the properties from System, followed by propFile and finally from
the map
    */
@@ -236,16 +261,20 @@ public class LogFeederUtil {
     }
   }
   
-  private static String logfeederTempDir = null;
+  public static String getClusterName() {
+    return getStringProperty(CLUSTER_NAME_PROPERTY);
+  }
+  
+  private static String logFeederTempDir = null;
   
-  public synchronized static String getLogfeederTempDir() {
-    if (logfeederTempDir == null) {
-      String tempDirValue = getStringProperty("logfeeder.tmp.dir", "/tmp/$username/logfeeder/");
+  public synchronized static String getLogFeederTempDir() {
+    if (logFeederTempDir == null) {
+      String tempDirValue = getStringProperty(TMP_DIR_PROPERTY, DEFAULT_TMP_DIR);
       HashMap<String, String> contextParam = new HashMap<String, String>();
       String username = System.getProperty("user.name");
       contextParam.put("username", username);
-      logfeederTempDir = PlaceholderUtil.replaceVariables(tempDirValue, contextParam);
+      logFeederTempDir = PlaceholderUtil.replaceVariables(tempDirValue, contextParam);
     }
-    return logfeederTempDir;
+    return logFeederTempDir;
   }
 }

http://git-wip-us.apache.org/repos/asf/ambari/blob/5c9bdbfe/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/util/LogfeederHDFSUtil.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/util/LogfeederHDFSUtil.java
b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/util/LogfeederHDFSUtil.java
deleted file mode 100644
index c975b99..0000000
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/util/LogfeederHDFSUtil.java
+++ /dev/null
@@ -1,77 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- * 
- * http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.ambari.logfeeder.util;
-
-import java.io.IOException;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.log4j.Logger;
-
-public class LogfeederHDFSUtil {
-  private static final Logger LOG = Logger.getLogger(LogfeederHDFSUtil.class);
-
-  private LogfeederHDFSUtil() {
-    throw new UnsupportedOperationException();
-  }
-  
-  public static boolean copyFromLocal(String sourceFilepath, String destFilePath, FileSystem
fileSystem, boolean overwrite,
-      boolean delSrc) {
-    Path src = new Path(sourceFilepath);
-    Path dst = new Path(destFilePath);
-    boolean isCopied = false;
-    try {
-      LOG.info("copying localfile := " + sourceFilepath + " to hdfsPath := " + destFilePath);
-      fileSystem.copyFromLocalFile(delSrc, overwrite, src, dst);
-      isCopied = true;
-    } catch (Exception e) {
-      LOG.error("Error copying local file :" + sourceFilepath + " to hdfs location : " +
destFilePath, e);
-    }
-    return isCopied;
-  }
-
-  public static FileSystem buildFileSystem(String hdfsHost, String hdfsPort) {
-    try {
-      Configuration configuration = buildHdfsConfiguration(hdfsHost, hdfsPort);
-      FileSystem fs = FileSystem.get(configuration);
-      return fs;
-    } catch (Exception e) {
-      LOG.error("Exception is buildFileSystem :", e);
-    }
-    return null;
-  }
-
-  private static Configuration buildHdfsConfiguration(String hdfsHost, String hdfsPort) {
-    String url = "hdfs://" + hdfsHost + ":" + hdfsPort + "/";
-    Configuration configuration = new Configuration();
-    configuration.set("fs.default.name", url);
-    return configuration;
-  }
-
-  public static void closeFileSystem(FileSystem fileSystem) {
-    if (fileSystem != null) {
-      try {
-        fileSystem.close();
-      } catch (IOException e) {
-        LOG.error(e.getLocalizedMessage(), e.getCause());
-      }
-    }
-  }
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/ambari/blob/5c9bdbfe/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/util/SSLUtil.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/util/SSLUtil.java
b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/util/SSLUtil.java
index d963de3..573bb1c 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/util/SSLUtil.java
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/util/SSLUtil.java
@@ -19,6 +19,7 @@
 
 package org.apache.ambari.logfeeder.util;
 
+import org.apache.ambari.logsearch.config.api.LogSearchPropertyDescription;
 import org.apache.commons.io.FileUtils;
 import org.apache.commons.lang.StringUtils;
 import org.apache.commons.lang3.ArrayUtils;
@@ -28,6 +29,8 @@ import org.apache.log4j.Logger;
 import java.io.File;
 import java.nio.charset.Charset;
 
+import static org.apache.ambari.logfeeder.util.LogFeederUtil.LOGFEEDER_PROPERTIES_FILE;
+
 public class SSLUtil {
   private static final Logger LOG = Logger.getLogger(SSLUtil.class);
 
@@ -42,6 +45,12 @@ public class SSLUtil {
   private static final String KEYSTORE_PASSWORD_FILE = "ks_pass.txt";
   private static final String TRUSTSTORE_PASSWORD_FILE = "ts_pass.txt";
 
+  @LogSearchPropertyDescription(
+    name = "hadoop.security.credential.provider.path",
+    description = "The jceks file that provides passwords.",
+    examples = {"jceks://file/etc/ambari-logsearch-logfeeder/conf/logfeeder.jceks"},
+    sources = {LOGFEEDER_PROPERTIES_FILE}
+  )
   private static final String CREDENTIAL_STORE_PROVIDER_PATH = "hadoop.security.credential.provider.path";
   private static final String LOGFEEDER_CERT_DEFAULT_FOLDER = "/etc/ambari-logsearch-portal/conf/keys";
   private static final String LOGFEEDER_STORE_DEFAULT_PASSWORD = "bigdata";

http://git-wip-us.apache.org/repos/asf/ambari/blob/5c9bdbfe/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/filter/FilterJSONTest.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/filter/FilterJSONTest.java
b/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/filter/FilterJSONTest.java
index 8f75c3a..7abf177 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/filter/FilterJSONTest.java
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/filter/FilterJSONTest.java
@@ -25,7 +25,7 @@ import java.util.Map;
 import java.util.TimeZone;
 
 import org.apache.ambari.logfeeder.common.LogFeederConstants;
-import org.apache.ambari.logfeeder.common.LogfeederException;
+import org.apache.ambari.logfeeder.common.LogFeederException;
 import org.apache.ambari.logfeeder.input.InputMarker;
 import org.apache.ambari.logfeeder.output.OutputManager;
 import org.apache.ambari.logsearch.config.zookeeper.model.inputconfig.impl.FilterJsonDescriptorImpl;
@@ -135,9 +135,9 @@ public class FilterJSONTest {
     String inputStr="invalid json";
     try{
     filterJson.apply(inputStr,new InputMarker(null, null, 0));
-    fail("Expected LogfeederException was not occured");
-    }catch(LogfeederException logfeederException){
-      assertEquals("Json parsing failed for inputstr = "+inputStr, logfeederException.getLocalizedMessage());
+    fail("Expected LogFeederException was not occured");
+    }catch(LogFeederException logFeederException){
+      assertEquals("Json parsing failed for inputstr = "+inputStr, logFeederException.getLocalizedMessage());
     }
   }
 

http://git-wip-us.apache.org/repos/asf/ambari/blob/5c9bdbfe/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/input/InputManagerTest.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/input/InputManagerTest.java
b/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/input/InputManagerTest.java
index 625e362..46fbc3b 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/input/InputManagerTest.java
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/input/InputManagerTest.java
@@ -143,32 +143,6 @@ public class InputManagerTest {
   }
 
   @Test
-  public void testInputManagr_waitOnAllInputs() throws Exception {
-    Input input1 = strictMock(Input.class);
-    Input input2 = strictMock(Input.class);
-    Input input3 = strictMock(Input.class);
-    
-    Thread mockThread = strictMock(Thread.class);
-    
-    expect(input1.getThread()).andReturn(null);
-    expect(input2.getThread()).andReturn(null);
-    expect(input3.getThread()).andReturn(mockThread);
-    
-    mockThread.join(); expectLastCall();
-    
-    replay(input1, input2, input3);
-    
-    InputManager manager = new InputManager();
-    manager.add("serviceName", input1);
-    manager.add("serviceName", input2);
-    manager.add("serviceName", input3);
-    
-    manager.waitOnAllInputs();
-    
-    verify(input1, input2, input3);
-  }
-
-  @Test
   public void testInputManager_checkInAll() throws Exception {
     Input input1 = strictMock(Input.class);
     Input input2 = strictMock(Input.class);

http://git-wip-us.apache.org/repos/asf/ambari/blob/5c9bdbfe/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/metrics/MetricsManagerTest.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/metrics/MetricsManagerTest.java
b/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/metrics/MetricsManagerTest.java
index 1461352..4b4067c 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/metrics/MetricsManagerTest.java
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/metrics/MetricsManagerTest.java
@@ -80,7 +80,6 @@ public class MetricsManagerTest {
     assertEquals(metricList.size(), 1);
     
     TimelineMetric metric = metricList.get(0);
-    assertEquals(metric.getHostName(), "test_host_name");
     assertEquals(metric.getAppId(), "logfeeder");
     assertEquals(metric.getMetricName(), "metric1");
     assertEquals(metric.getType(), "Long");
@@ -116,7 +115,6 @@ public class MetricsManagerTest {
     assertEquals(metricList.size(), 1);
     
     TimelineMetric metric = metricList.get(0);
-    assertEquals(metric.getHostName(), "test_host_name");
     assertEquals(metric.getAppId(), "logfeeder");
     assertEquals(metric.getMetricName(), "metric1");
     assertEquals(metric.getType(), "Long");

http://git-wip-us.apache.org/repos/asf/ambari/blob/5c9bdbfe/ambari-logsearch/ambari-logsearch-logfeeder/src/test/resources/logfeeder.properties
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/test/resources/logfeeder.properties
b/ambari-logsearch/ambari-logsearch-logfeeder/src/test/resources/logfeeder.properties
index 19027d1..5476c96 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/test/resources/logfeeder.properties
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/test/resources/logfeeder.properties
@@ -17,5 +17,4 @@ logfeeder.log.filter.enable=true
 logfeeder.solr.config.interval=5
 logfeeder.solr.zk_connect_string=some_connect_string
 logfeeder.metrics.collector.hosts=some_collector_host
-node.hostname=test_host_name
 logfeeder.include.default.level=FATAL,ERROR,WARN
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/ambari/blob/5c9bdbfe/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/configurer/LogSearchConfigConfigurer.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/configurer/LogSearchConfigConfigurer.java
b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/configurer/LogSearchConfigConfigurer.java
index 978e91a..c34dce6 100644
--- a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/configurer/LogSearchConfigConfigurer.java
+++ b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/configurer/LogSearchConfigConfigurer.java
@@ -53,7 +53,7 @@ public class LogSearchConfigConfigurer implements Configurer {
         while (true) {
           try {
             logSearchConfig = LogSearchConfigFactory.createLogSearchConfig(Component.SERVER,
PropertiesHelper.getProperties(),
-                LogSearchConfigZK.class);
+                null, LogSearchConfigZK.class);
             logSearchConfigState.setLogSearchConfigAvailable(true);
             break;
           } catch (Exception e) {


Mime
View raw message