AMBARI-19658. LogSearch Integration Cache Timeout should be configurable. (rnettleton)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/60462002 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/60462002 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/60462002 Branch: refs/heads/branch-dev-patch-upgrade Commit: 604620021908995d7b4581176ceecf6c44ffea26 Parents: 78fefdd Author: Bob Nettleton <rnettle...@hortonworks.com> Authored: Tue Jan 24 17:15:07 2017 -0500 Committer: Bob Nettleton <rnettle...@hortonworks.com> Committed: Tue Jan 24 17:15:29 2017 -0500 ---------------------------------------------------------------------- .../server/configuration/Configuration.java | 26 +++++++++++++++++- .../logging/LogSearchDataRetrievalService.java | 28 +++++++++++++++++--- .../LogSearchDataRetrievalServiceTest.java | 27 ++++++++++++++++++- 3 files changed, 76 insertions(+), 5 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/60462002/ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java b/ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java index df1b627..73c70dc 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java @@ -2601,7 +2601,6 @@ public class Configuration { public static final ConfigurationProperty<Integer> LOGSEARCH_PORTAL_READ_TIMEOUT = new ConfigurationProperty<>( "logsearch.portal.read.timeout", 5000); - /** * Global disable flag for AmbariServer Metrics. */ @@ -2609,6 +2608,18 @@ public class Configuration { public static final ConfigurationProperty<Boolean> AMBARISERVER_METRICS_DISABLE = new ConfigurationProperty<>( "ambariserver.metrics.disable", false); + /** + * The time, in hours, that the Ambari Server will hold Log File metadata in its internal cache before making + * a request to the LogSearch Portal to get the latest metadata. + * + * The logging metadata (in this case, log file names) is generally quite static, so the default should + * generally be quite long. + * + */ + @Markdown(description = "The time, in hours, that the Ambari Server will hold Log File metadata in its internal cache before making a request to the LogSearch Portal to get the latest metadata.") + public static final ConfigurationProperty<Integer> LOGSEARCH_METADATA_CACHE_EXPIRE_TIMEOUT = new ConfigurationProperty<>( + "logsearch.metadata.cache.expire.timeout", 24); + private static final Logger LOG = LoggerFactory.getLogger( Configuration.class); @@ -5406,6 +5417,19 @@ public class Configuration { return NumberUtils.toInt(getProperty(LOGSEARCH_PORTAL_READ_TIMEOUT)); } + + /** + * + * Get the max time, in hours, to hold data in the LogSearch + * metadata cache prior to expiring the cache and re-loading + * the data from the LogSearch Portal service. + * + * @return max number of hours that the LogSearch metadata is cached + */ + public int getLogSearchMetadataCacheExpireTimeout() { + return NumberUtils.toInt(getProperty(LOGSEARCH_METADATA_CACHE_EXPIRE_TIMEOUT)); + } + /** * Generates a markdown table which includes: * <ul> http://git-wip-us.apache.org/repos/asf/ambari/blob/60462002/ambari-server/src/main/java/org/apache/ambari/server/controller/logging/LogSearchDataRetrievalService.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/logging/LogSearchDataRetrievalService.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/logging/LogSearchDataRetrievalService.java index ce6094c..6b484a4 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/logging/LogSearchDataRetrievalService.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/logging/LogSearchDataRetrievalService.java @@ -23,6 +23,7 @@ import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; import org.apache.ambari.server.AmbariService; +import org.apache.ambari.server.configuration.Configuration; import org.apache.ambari.server.controller.AmbariManagementController; import org.apache.ambari.server.controller.AmbariServer; import org.apache.commons.collections.CollectionUtils; @@ -77,6 +78,11 @@ public class LogSearchDataRetrievalService extends AbstractService { @Inject private Injector injector; + @Inject + private Configuration ambariServerConfiguration; + + + /** * A Cache of host+component names to a set of log files associated with * that Host/Component combination. This data is retrieved from the @@ -114,10 +120,17 @@ public class LogSearchDataRetrievalService extends AbstractService { protected void doStart() { LOG.debug("Initializing caches"); + + // obtain the max cache expire time from the ambari configuration + final int maxTimeoutForCacheInHours = + ambariServerConfiguration.getLogSearchMetadataCacheExpireTimeout(); + + LOG.debug("Caches configured with a max expire timeout of " + maxTimeoutForCacheInHours + " hours."); + // initialize the log file name cache - logFileNameCache = CacheBuilder.newBuilder().expireAfterWrite(1, TimeUnit.HOURS).build(); + logFileNameCache = CacheBuilder.newBuilder().expireAfterWrite(maxTimeoutForCacheInHours, TimeUnit.HOURS).build(); // initialize the log file tail URI cache - logFileTailURICache = CacheBuilder.newBuilder().expireAfterWrite(1, TimeUnit.HOURS).build(); + logFileTailURICache = CacheBuilder.newBuilder().expireAfterWrite(maxTimeoutForCacheInHours, TimeUnit.HOURS).build(); // initialize the Executor executor = Executors.newSingleThreadExecutor(); @@ -126,7 +139,7 @@ public class LogSearchDataRetrievalService extends AbstractService { @Override protected void doStop() { LOG.debug("Invalidating LogSearch caches"); - // invalidate the cache + // invalidate the caches logFileNameCache.invalidateAll(); logFileTailURICache.invalidateAll(); @@ -230,6 +243,15 @@ public class LogSearchDataRetrievalService extends AbstractService { } /** + * Package-level setter to facilitate simpler unit testing + * + * @param ambariServerConfiguration + */ + void setConfiguration(Configuration ambariServerConfiguration) { + this.ambariServerConfiguration = ambariServerConfiguration; + } + + /** * This protected method allows for simpler unit tests. * * @return the Set of current Requests that are not yet completed http://git-wip-us.apache.org/repos/asf/ambari/blob/60462002/ambari-server/src/test/java/org/apache/ambari/server/controller/logging/LogSearchDataRetrievalServiceTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/logging/LogSearchDataRetrievalServiceTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/logging/LogSearchDataRetrievalServiceTest.java index 033d698..4296004 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/controller/logging/LogSearchDataRetrievalServiceTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/logging/LogSearchDataRetrievalServiceTest.java @@ -29,6 +29,7 @@ import java.util.Set; import java.util.concurrent.Executor; +import org.apache.ambari.server.configuration.Configuration; import org.apache.ambari.server.controller.AmbariManagementController; import org.easymock.EasyMockSupport; import org.junit.Test; @@ -60,13 +61,18 @@ public class LogSearchDataRetrievalServiceTest { LoggingRequestHelper helperMock = mockSupport.createMock(LoggingRequestHelper.class); + Configuration configurationMock = + mockSupport.createMock(Configuration.class); + expect(helperFactoryMock.getHelper(null, expectedClusterName)).andReturn(helperMock); expect(helperMock.createLogFileTailURI("http://localhost", expectedComponentName, expectedHostName)).andReturn(expectedResultURI); + expect(configurationMock.getLogSearchMetadataCacheExpireTimeout()).andReturn(1).atLeastOnce(); mockSupport.replayAll(); LogSearchDataRetrievalService retrievalService = new LogSearchDataRetrievalService(); retrievalService.setLoggingRequestHelperFactory(helperFactoryMock); + retrievalService.setConfiguration(configurationMock); // call the initialization routine called by the Google framework retrievalService.doStart(); @@ -86,16 +92,23 @@ public class LogSearchDataRetrievalServiceTest { EasyMockSupport mockSupport = new EasyMockSupport(); - LoggingRequestHelperFactory helperFactoryMock = mockSupport.createMock(LoggingRequestHelperFactory.class); + LoggingRequestHelperFactory helperFactoryMock = + mockSupport.createMock(LoggingRequestHelperFactory.class); + + Configuration configurationMock = + mockSupport.createMock(Configuration.class); // return null, to simulate the case where LogSearch Server is // not available for some reason expect(helperFactoryMock.getHelper(null, expectedClusterName)).andReturn(null); + expect(configurationMock.getLogSearchMetadataCacheExpireTimeout()).andReturn(1).atLeastOnce(); + mockSupport.replayAll(); LogSearchDataRetrievalService retrievalService = new LogSearchDataRetrievalService(); retrievalService.setLoggingRequestHelperFactory(helperFactoryMock); + retrievalService.setConfiguration(configurationMock); // call the initialization routine called by the Google framework retrievalService.doStart(); @@ -122,6 +135,9 @@ public class LogSearchDataRetrievalServiceTest { Injector injectorMock = mockSupport.createMock(Injector.class); + Configuration configurationMock = + mockSupport.createMock(Configuration.class); + // expect the executor to be called to execute the LogSearch request executorMock.execute(isA(LogSearchDataRetrievalService.LogSearchFileNameRequestRunnable.class)); // executor should only be called once @@ -129,11 +145,14 @@ public class LogSearchDataRetrievalServiceTest { expect(injectorMock.getInstance(LoggingRequestHelperFactory.class)).andReturn(helperFactoryMock); + expect(configurationMock.getLogSearchMetadataCacheExpireTimeout()).andReturn(1).atLeastOnce(); + mockSupport.replayAll(); LogSearchDataRetrievalService retrievalService = new LogSearchDataRetrievalService(); retrievalService.setLoggingRequestHelperFactory(helperFactoryMock); retrievalService.setInjector(injectorMock); + retrievalService.setConfiguration(configurationMock); // call the initialization routine called by the Google framework retrievalService.doStart(); retrievalService.setExecutor(executorMock); @@ -164,10 +183,16 @@ public class LogSearchDataRetrievalServiceTest { Executor executorMock = mockSupport.createMock(Executor.class); + Configuration configurationMock = + mockSupport.createMock(Configuration.class); + + expect(configurationMock.getLogSearchMetadataCacheExpireTimeout()).andReturn(1).atLeastOnce(); + mockSupport.replayAll(); LogSearchDataRetrievalService retrievalService = new LogSearchDataRetrievalService(); retrievalService.setLoggingRequestHelperFactory(helperFactoryMock); + retrievalService.setConfiguration(configurationMock); // call the initialization routine called by the Google framework retrievalService.doStart(); // there should be no expectations set on this mock