ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From rnettle...@apache.org
Subject ambari git commit: AMBARI-19658. LogSearch Integration Cache Timeout should be configurable. (rnettleton)
Date Tue, 24 Jan 2017 22:15:44 GMT
Repository: ambari
Updated Branches:
  refs/heads/trunk 78fefdd4c -> 604620021


AMBARI-19658. LogSearch Integration Cache Timeout should be configurable. (rnettleton)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/60462002
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/60462002
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/60462002

Branch: refs/heads/trunk
Commit: 604620021908995d7b4581176ceecf6c44ffea26
Parents: 78fefdd
Author: Bob Nettleton <rnettleton@hortonworks.com>
Authored: Tue Jan 24 17:15:07 2017 -0500
Committer: Bob Nettleton <rnettleton@hortonworks.com>
Committed: Tue Jan 24 17:15:29 2017 -0500

----------------------------------------------------------------------
 .../server/configuration/Configuration.java     | 26 +++++++++++++++++-
 .../logging/LogSearchDataRetrievalService.java  | 28 +++++++++++++++++---
 .../LogSearchDataRetrievalServiceTest.java      | 27 ++++++++++++++++++-
 3 files changed, 76 insertions(+), 5 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/60462002/ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java
b/ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java
index df1b627..73c70dc 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java
@@ -2601,7 +2601,6 @@ public class Configuration {
   public static final ConfigurationProperty<Integer> LOGSEARCH_PORTAL_READ_TIMEOUT
= new ConfigurationProperty<>(
     "logsearch.portal.read.timeout", 5000);
 
-
   /**
    * Global disable flag for AmbariServer Metrics.
    */
@@ -2609,6 +2608,18 @@ public class Configuration {
   public static final ConfigurationProperty<Boolean> AMBARISERVER_METRICS_DISABLE =
new ConfigurationProperty<>(
     "ambariserver.metrics.disable", false);
 
+  /**
+   * The time, in hours, that the Ambari Server will hold Log File metadata in its internal
cache before making
+   *   a request to the LogSearch Portal to get the latest metadata.
+   *
+   * The logging metadata (in this case, log file names) is generally quite static, so the
default should
+   *   generally be quite long.
+   *
+   */
+  @Markdown(description = "The time, in hours, that the Ambari Server will hold Log File
metadata in its internal cache before making a request to the LogSearch Portal to get the
latest metadata.")
+  public static final ConfigurationProperty<Integer> LOGSEARCH_METADATA_CACHE_EXPIRE_TIMEOUT
= new ConfigurationProperty<>(
+    "logsearch.metadata.cache.expire.timeout", 24);
+
   private static final Logger LOG = LoggerFactory.getLogger(
     Configuration.class);
 
@@ -5406,6 +5417,19 @@ public class Configuration {
     return NumberUtils.toInt(getProperty(LOGSEARCH_PORTAL_READ_TIMEOUT));
   }
 
+
+  /**
+   *
+   * Get the max time, in hours, to hold data in the LogSearch
+   *   metadata cache prior to expiring the cache and re-loading
+   *   the data from the LogSearch Portal service.
+   *
+   * @return max number of hours that the LogSearch metadata is cached
+   */
+  public int getLogSearchMetadataCacheExpireTimeout() {
+    return NumberUtils.toInt(getProperty(LOGSEARCH_METADATA_CACHE_EXPIRE_TIMEOUT));
+  }
+
   /**
    * Generates a markdown table which includes:
    * <ul>

http://git-wip-us.apache.org/repos/asf/ambari/blob/60462002/ambari-server/src/main/java/org/apache/ambari/server/controller/logging/LogSearchDataRetrievalService.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/logging/LogSearchDataRetrievalService.java
b/ambari-server/src/main/java/org/apache/ambari/server/controller/logging/LogSearchDataRetrievalService.java
index ce6094c..6b484a4 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/logging/LogSearchDataRetrievalService.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/logging/LogSearchDataRetrievalService.java
@@ -23,6 +23,7 @@ import java.util.concurrent.Executors;
 import java.util.concurrent.TimeUnit;
 
 import org.apache.ambari.server.AmbariService;
+import org.apache.ambari.server.configuration.Configuration;
 import org.apache.ambari.server.controller.AmbariManagementController;
 import org.apache.ambari.server.controller.AmbariServer;
 import org.apache.commons.collections.CollectionUtils;
@@ -77,6 +78,11 @@ public class LogSearchDataRetrievalService extends AbstractService {
   @Inject
   private Injector injector;
 
+  @Inject
+  private Configuration ambariServerConfiguration;
+
+
+
   /**
    * A Cache of host+component names to a set of log files associated with
    *  that Host/Component combination.  This data is retrieved from the
@@ -114,10 +120,17 @@ public class LogSearchDataRetrievalService extends AbstractService {
   protected void doStart() {
 
     LOG.debug("Initializing caches");
+
+    // obtain the max cache expire time from the ambari configuration
+    final int maxTimeoutForCacheInHours =
+      ambariServerConfiguration.getLogSearchMetadataCacheExpireTimeout();
+
+    LOG.debug("Caches configured with a max expire timeout of " + maxTimeoutForCacheInHours
+ " hours.");
+
     // initialize the log file name cache
-    logFileNameCache = CacheBuilder.newBuilder().expireAfterWrite(1, TimeUnit.HOURS).build();
+    logFileNameCache = CacheBuilder.newBuilder().expireAfterWrite(maxTimeoutForCacheInHours,
TimeUnit.HOURS).build();
     // initialize the log file tail URI cache
-    logFileTailURICache = CacheBuilder.newBuilder().expireAfterWrite(1, TimeUnit.HOURS).build();
+    logFileTailURICache = CacheBuilder.newBuilder().expireAfterWrite(maxTimeoutForCacheInHours,
TimeUnit.HOURS).build();
 
     // initialize the Executor
     executor = Executors.newSingleThreadExecutor();
@@ -126,7 +139,7 @@ public class LogSearchDataRetrievalService extends AbstractService {
   @Override
   protected void doStop() {
     LOG.debug("Invalidating LogSearch caches");
-    // invalidate the cache
+    // invalidate the caches
     logFileNameCache.invalidateAll();
 
     logFileTailURICache.invalidateAll();
@@ -230,6 +243,15 @@ public class LogSearchDataRetrievalService extends AbstractService {
   }
 
   /**
+   * Package-level setter to facilitate simpler unit testing
+   *
+   * @param ambariServerConfiguration
+   */
+  void setConfiguration(Configuration ambariServerConfiguration) {
+    this.ambariServerConfiguration = ambariServerConfiguration;
+  }
+
+  /**
    * This protected method allows for simpler unit tests.
    *
    * @return the Set of current Requests that are not yet completed

http://git-wip-us.apache.org/repos/asf/ambari/blob/60462002/ambari-server/src/test/java/org/apache/ambari/server/controller/logging/LogSearchDataRetrievalServiceTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/logging/LogSearchDataRetrievalServiceTest.java
b/ambari-server/src/test/java/org/apache/ambari/server/controller/logging/LogSearchDataRetrievalServiceTest.java
index 033d698..4296004 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/controller/logging/LogSearchDataRetrievalServiceTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/logging/LogSearchDataRetrievalServiceTest.java
@@ -29,6 +29,7 @@ import java.util.Set;
 import java.util.concurrent.Executor;
 
 
+import org.apache.ambari.server.configuration.Configuration;
 import org.apache.ambari.server.controller.AmbariManagementController;
 import org.easymock.EasyMockSupport;
 import org.junit.Test;
@@ -60,13 +61,18 @@ public class LogSearchDataRetrievalServiceTest {
     LoggingRequestHelper helperMock =
       mockSupport.createMock(LoggingRequestHelper.class);
 
+    Configuration configurationMock =
+      mockSupport.createMock(Configuration.class);
+
     expect(helperFactoryMock.getHelper(null, expectedClusterName)).andReturn(helperMock);
     expect(helperMock.createLogFileTailURI("http://localhost", expectedComponentName, expectedHostName)).andReturn(expectedResultURI);
+    expect(configurationMock.getLogSearchMetadataCacheExpireTimeout()).andReturn(1).atLeastOnce();
 
     mockSupport.replayAll();
 
     LogSearchDataRetrievalService retrievalService = new LogSearchDataRetrievalService();
     retrievalService.setLoggingRequestHelperFactory(helperFactoryMock);
+    retrievalService.setConfiguration(configurationMock);
     // call the initialization routine called by the Google framework
     retrievalService.doStart();
 
@@ -86,16 +92,23 @@ public class LogSearchDataRetrievalServiceTest {
 
     EasyMockSupport mockSupport = new EasyMockSupport();
 
-    LoggingRequestHelperFactory helperFactoryMock = mockSupport.createMock(LoggingRequestHelperFactory.class);
+    LoggingRequestHelperFactory helperFactoryMock =
+      mockSupport.createMock(LoggingRequestHelperFactory.class);
+
+    Configuration configurationMock =
+      mockSupport.createMock(Configuration.class);
 
     // return null, to simulate the case where LogSearch Server is
     // not available for some reason
     expect(helperFactoryMock.getHelper(null, expectedClusterName)).andReturn(null);
 
+    expect(configurationMock.getLogSearchMetadataCacheExpireTimeout()).andReturn(1).atLeastOnce();
+
     mockSupport.replayAll();
 
     LogSearchDataRetrievalService retrievalService = new LogSearchDataRetrievalService();
     retrievalService.setLoggingRequestHelperFactory(helperFactoryMock);
+    retrievalService.setConfiguration(configurationMock);
     // call the initialization routine called by the Google framework
     retrievalService.doStart();
 
@@ -122,6 +135,9 @@ public class LogSearchDataRetrievalServiceTest {
     Injector injectorMock =
       mockSupport.createMock(Injector.class);
 
+    Configuration configurationMock =
+      mockSupport.createMock(Configuration.class);
+
     // expect the executor to be called to execute the LogSearch request
     executorMock.execute(isA(LogSearchDataRetrievalService.LogSearchFileNameRequestRunnable.class));
     // executor should only be called once
@@ -129,11 +145,14 @@ public class LogSearchDataRetrievalServiceTest {
 
     expect(injectorMock.getInstance(LoggingRequestHelperFactory.class)).andReturn(helperFactoryMock);
 
+    expect(configurationMock.getLogSearchMetadataCacheExpireTimeout()).andReturn(1).atLeastOnce();
+
     mockSupport.replayAll();
 
     LogSearchDataRetrievalService retrievalService = new LogSearchDataRetrievalService();
     retrievalService.setLoggingRequestHelperFactory(helperFactoryMock);
     retrievalService.setInjector(injectorMock);
+    retrievalService.setConfiguration(configurationMock);
     // call the initialization routine called by the Google framework
     retrievalService.doStart();
     retrievalService.setExecutor(executorMock);
@@ -164,10 +183,16 @@ public class LogSearchDataRetrievalServiceTest {
 
     Executor executorMock = mockSupport.createMock(Executor.class);
 
+    Configuration configurationMock =
+      mockSupport.createMock(Configuration.class);
+
+    expect(configurationMock.getLogSearchMetadataCacheExpireTimeout()).andReturn(1).atLeastOnce();
+
     mockSupport.replayAll();
 
     LogSearchDataRetrievalService retrievalService = new LogSearchDataRetrievalService();
     retrievalService.setLoggingRequestHelperFactory(helperFactoryMock);
+    retrievalService.setConfiguration(configurationMock);
     // call the initialization routine called by the Google framework
     retrievalService.doStart();
     // there should be no expectations set on this mock


Mime
View raw message