Repository: ambari Updated Branches: refs/heads/branch-2.5 b094cccde -> 6d00ee5d5
AMBARI-19587 Log Feeder should be able to handle metrics with https (mgergely) Change-Id: I0516c5f7c4b841340f413ee6bfbb2d03324287c9 Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/6d00ee5d Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/6d00ee5d Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/6d00ee5d Branch: refs/heads/branch-2.5 Commit: 6d00ee5d5f1feb3ab870efac740141cc40986ec8 Parents: b094ccc Author: Miklos Gergely <mgerg...@hortonworks.com> Authored: Thu Jan 19 00:37:50 2017 +0100 Committer: Miklos Gergely <mgerg...@hortonworks.com> Committed: Thu Jan 19 00:37:50 2017 +0100 ---------------------------------------------------------------------- .../logfeeder/metrics/LogFeederAMSClient.java | 38 ++++++++++++++------ .../apache/ambari/logfeeder/util/SSLUtil.java | 26 ++++++++++++++ .../server/upgrade/UpgradeCatalog250.java | 8 ++--- .../LOGSEARCH/0.5.0/package/scripts/params.py | 27 +++++++++++--- .../0.5.0/properties/logfeeder-env.sh.j2 | 2 +- .../0.5.0/properties/logsearch-env.sh.j2 | 2 +- .../server/upgrade/UpgradeCatalog250Test.java | 6 ++-- .../stacks/2.4/LOGSEARCH/test_logfeeder.py | 3 ++ 8 files changed, 87 insertions(+), 25 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/6d00ee5d/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/metrics/LogFeederAMSClient.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/metrics/LogFeederAMSClient.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/metrics/LogFeederAMSClient.java index 2bdd7c9..2d1bf40 100644 --- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/metrics/LogFeederAMSClient.java +++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/metrics/LogFeederAMSClient.java @@ -20,33 +20,51 @@ package org.apache.ambari.logfeeder.metrics; import org.apache.ambari.logfeeder.util.LogFeederUtil; +import org.apache.ambari.logfeeder.util.SSLUtil; import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.metrics2.sink.timeline.AbstractTimelineMetricsSink; import org.apache.hadoop.metrics2.sink.timeline.TimelineMetrics; import org.apache.log4j.Logger; +import com.google.common.base.Splitter; + import java.util.Collection; +import java.util.List; // TODO: Refactor for failover public class LogFeederAMSClient extends AbstractTimelineMetricsSink { private static final Logger LOG = Logger.getLogger(LogFeederAMSClient.class); - private String collectorHosts = null; + private final List<String> collectorHosts; + private final String collectorProtocol; + private final String collectorPort; + private final String collectorPath; public LogFeederAMSClient() { - collectorHosts = LogFeederUtil.getStringProperty("logfeeder.metrics.collector.hosts"); - if (StringUtils.isBlank(collectorHosts)) { + String collectorHostsString = LogFeederUtil.getStringProperty("logfeeder.metrics.collector.hosts"); + if (!StringUtils.isBlank(collectorHostsString)) { + collectorHostsString = collectorHostsString.trim(); + LOG.info("AMS collector Hosts=" + collectorHostsString); + + collectorHosts = Splitter.on(",").splitToList(collectorHostsString); + collectorProtocol = LogFeederUtil.getStringProperty("logfeeder.metrics.collector.protocol"); + collectorPort = LogFeederUtil.getStringProperty("logfeeder.metrics.collector.port"); + collectorPath = LogFeederUtil.getStringProperty("logfeeder.metrics.collector.path"); + } else { collectorHosts = null; + collectorProtocol = null; + collectorPort = null; + collectorPath = null; } - if (collectorHosts != null) { - collectorHosts = collectorHosts.trim(); + + if (StringUtils.isNotBlank(SSLUtil.getTrustStoreLocation())) { + loadTruststore(SSLUtil.getTrustStoreLocation(), SSLUtil.getTrustStoreType(), SSLUtil.getTrustStorePassword()); } - LOG.info("AMS collector URL=" + collectorHosts); } @Override public String getCollectorUri(String host) { - return collectorHosts; + return String.format("%s://%s:%s%s", collectorProtocol, host, collectorPort, collectorPath); } @Override @@ -62,7 +80,7 @@ public class LogFeederAMSClient extends AbstractTimelineMetricsSink { @Override protected Collection<String> getConfiguredCollectorHosts() { - return null; + return collectorHosts; } @Override @@ -77,12 +95,12 @@ public class LogFeederAMSClient extends AbstractTimelineMetricsSink { @Override protected String getCollectorProtocol() { - return null; + return collectorProtocol; } @Override protected String getCollectorPort() { - return null; + return collectorPort; } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/6d00ee5d/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/util/SSLUtil.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/util/SSLUtil.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/util/SSLUtil.java index 317f5ae..ea9f45d 100644 --- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/util/SSLUtil.java +++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/util/SSLUtil.java @@ -27,6 +27,8 @@ import java.io.File; public class SSLUtil { private static final String KEYSTORE_LOCATION_ARG = "javax.net.ssl.keyStore"; private static final String TRUSTSTORE_LOCATION_ARG = "javax.net.ssl.trustStore"; + private static final String KEYSTORE_TYPE_ARG = "javax.net.ssl.keyStoreType"; + private static final String TRUSTSTORE_TYPE_ARG = "javax.net.ssl.trustStoreType"; private static final String KEYSTORE_PASSWORD_ARG = "javax.net.ssl.keyStorePassword"; private static final String TRUSTSTORE_PASSWORD_ARG = "javax.net.ssl.trustStorePassword"; private static final String KEYSTORE_PASSWORD_FILE = "ks_pass.txt"; @@ -39,6 +41,30 @@ public class SSLUtil { throw new UnsupportedOperationException(); } + public static String getKeyStoreLocation() { + return System.getProperty(KEYSTORE_LOCATION_ARG); + } + + public static String getKeyStoreType() { + return System.getProperty(KEYSTORE_TYPE_ARG); + } + + public static String getKeyStorePassword() { + return System.getProperty(KEYSTORE_PASSWORD_ARG); + } + + public static String getTrustStoreLocation() { + return System.getProperty(TRUSTSTORE_LOCATION_ARG); + } + + public static String getTrustStoreType() { + return System.getProperty(TRUSTSTORE_TYPE_ARG); + } + + public static String getTrustStorePassword() { + return System.getProperty(TRUSTSTORE_PASSWORD_ARG); + } + public static void ensureStorePasswords() { ensureStorePassword(KEYSTORE_LOCATION_ARG, KEYSTORE_PASSWORD_ARG, KEYSTORE_PASSWORD_FILE); ensureStorePassword(TRUSTSTORE_LOCATION_ARG, TRUSTSTORE_PASSWORD_ARG, TRUSTSTORE_PASSWORD_FILE); http://git-wip-us.apache.org/repos/asf/ambari/blob/6d00ee5d/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog250.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog250.java b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog250.java index 3a2ebe6..624e5a2 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog250.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog250.java @@ -689,7 +689,7 @@ public class UpgradeCatalog250 extends AbstractUpgradeCatalog { if (logfeederEnvProperties != null) { String content = logfeederEnvProperties.getProperties().get("content"); if (content.contains("infra_solr_ssl_enabled")) { - content = content.replace("infra_solr_ssl_enabled", "logsearch_solr_ssl_enabled"); + content = content.replace("infra_solr_ssl_enabled", "logfeeder_use_ssl"); updateConfigurationPropertiesForCluster(cluster, "logfeeder-env", Collections.singletonMap("content", content), true, true); } } @@ -699,11 +699,7 @@ public class UpgradeCatalog250 extends AbstractUpgradeCatalog { Map<String, String> newProperties = new HashMap<>(); String content = logsearchEnvProperties.getProperties().get("content"); if (content.contains("infra_solr_ssl_enabled or logsearch_ui_protocol == 'https'")) { - content = content.replace("infra_solr_ssl_enabled or logsearch_ui_protocol == 'https'", - "infra_solr_ssl_enabled or logsearch_ui_protocol == 'https' or ambari_server_use_ssl"); - } - if (content.contains("infra_solr_ssl_enabled")) { - content = content.replace("infra_solr_ssl_enabled", "logsearch_solr_ssl_enabled"); + content = content.replace("infra_solr_ssl_enabled or logsearch_ui_protocol == 'https'", "logsearch_use_ssl"); } if (!content.equals(logsearchEnvProperties.getProperties().get("content"))) { newProperties.put("content", content); http://git-wip-us.apache.org/repos/asf/ambari/blob/6d00ee5d/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/params.py b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/params.py index 25e947d..e9ab98f 100644 --- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/params.py +++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/params.py @@ -69,12 +69,19 @@ logfeeder_metadata = get_logfeeder_metadata(logserch_meta_configs) # for now just pick first collector if 'metrics_collector_hosts' in config['clusterHostInfo']: - metrics_collector_hosts_list = ",".join(config['clusterHostInfo']['metrics_collector_hosts']) - metrics_collector_port = str( - get_port_from_url(config['configurations']['ams-site']['timeline.metrics.service.webapp.address'])) - metrics_collector_hosts = format('http://{metrics_collector_hosts_list}:{metrics_collector_port}/ws/v1/timeline/metrics') + metrics_http_policy = config['configurations']['ams-site']['timeline.metrics.service.http.policy'] + metrics_collector_protocol = 'http' + if metrics_http_policy == 'HTTPS_ONLY': + metrics_collector_protocol = 'https' + + metrics_collector_hosts = ",".join(config['clusterHostInfo']['metrics_collector_hosts']) + metrics_collector_port = str(get_port_from_url(config['configurations']['ams-site']['timeline.metrics.service.webapp.address'])) + metrics_collector_path = '/ws/v1/timeline/metrics' else: + metrics_collector_protocol = '' metrics_collector_hosts = '' + metrics_collector_port = '' + metrics_collector_path = '' ##################################### # Infra Solr configs @@ -250,6 +257,10 @@ logsearch_solr_collection_audit_logs = logsearch_properties['logsearch.solr.coll logsearch_audit_logs_split_interval_mins = logsearch_properties['logsearch.audit.logs.split.interval.mins'] logsearch_collection_audit_logs_numshards = logsearch_properties['logsearch.collection.audit.logs.numshards'] +# check if logsearch uses ssl in any way + +logsearch_use_ssl = logsearch_solr_ssl_enabled or logsearch_ui_protocol == 'https' or ambari_server_use_ssl + ##################################### # Logfeeder configs ##################################### @@ -328,6 +339,10 @@ logfeeder_properties['logfeeder.metrics.collector.hosts'] = format(logfeeder_pro logfeeder_properties['logfeeder.config.files'] = format(logfeeder_properties['logfeeder.config.files']) logfeeder_properties['logfeeder.solr.zk_connect_string'] = logsearch_solr_zk_quorum + logsearch_solr_zk_znode +logfeeder_properties['logfeeder.metrics.collector.protocol'] = metrics_collector_protocol +logfeeder_properties['logfeeder.metrics.collector.port'] = metrics_collector_port +logfeeder_properties['logfeeder.metrics.collector.path'] = '/ws/v1/timeline/metrics' + if logsearch_solr_kerberos_enabled: if 'logfeeder.solr.kerberos.enable' not in logfeeder_properties: logfeeder_properties['logfeeder.solr.kerberos.enable'] = 'true' @@ -336,6 +351,10 @@ if logsearch_solr_kerberos_enabled: logfeeder_checkpoint_folder = logfeeder_properties['logfeeder.checkpoint.folder'] +# check if logfeeder uses ssl in any way + +logfeeder_use_ssl = logsearch_solr_ssl_enabled or metrics_collector_protocol == 'https' + ##################################### # Smoke command ##################################### http://git-wip-us.apache.org/repos/asf/ambari/blob/6d00ee5d/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/properties/logfeeder-env.sh.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/properties/logfeeder-env.sh.j2 b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/properties/logfeeder-env.sh.j2 index 6d1c445..f2dd06b 100644 --- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/properties/logfeeder-env.sh.j2 +++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/properties/logfeeder-env.sh.j2 @@ -32,7 +32,7 @@ if [ "$LOGFEEDER_JAVA_MEM" = "" ]; then export LOGFEEDER_JAVA_MEM=-Xmx{{logfeeder_max_mem}} fi -{% if logsearch_solr_ssl_enabled %} +{% if logfeeder_use_ssl %} export LOGFEEDER_SSL="true" export LOGFEEDER_KEYSTORE_LOCATION={{logfeeder_keystore_location}} export LOGFEEDER_KEYSTORE_TYPE={{logfeeder_keystore_type}} http://git-wip-us.apache.org/repos/asf/ambari/blob/6d00ee5d/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/properties/logsearch-env.sh.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/properties/logsearch-env.sh.j2 b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/properties/logsearch-env.sh.j2 index 338c7f7..f21b7be 100644 --- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/properties/logsearch-env.sh.j2 +++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/properties/logsearch-env.sh.j2 @@ -38,7 +38,7 @@ export LOGSEARCH_DEBUG={{logsearch_debug_enabled}} export LOGSEARCH_DEBUG_PORT={{logsearch_debug_port}} -{% if logsearch_solr_ssl_enabled or logsearch_ui_protocol == 'https' or ambari_server_use_ssl %} +{% if logsearch_use_ssl %} export LOGSEARCH_SSL="true" export LOGSEARCH_KEYSTORE_LOCATION={{logsearch_keystore_location}} export LOGSEARCH_KEYSTORE_TYPE={{logsearch_keystore_type}} http://git-wip-us.apache.org/repos/asf/ambari/blob/6d00ee5d/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog250Test.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog250Test.java b/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog250Test.java index e87b9b5..bbda783 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog250Test.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog250Test.java @@ -1026,7 +1026,7 @@ public class UpgradeCatalog250Test { "content", "infra_solr_ssl_enabled"); Map<String, String> expectedLogFeederEnv = ImmutableMap.of( - "content", "logsearch_solr_ssl_enabled"); + "content", "logfeeder_use_ssl"); Config mockLogFeederEnv = easyMockSupport.createNiceMock(Config.class); expect(cluster.getDesiredConfigByType("logfeeder-env")).andReturn(mockLogFeederEnv).atLeastOnce(); @@ -1042,7 +1042,7 @@ public class UpgradeCatalog250Test { "content", "infra_solr_ssl_enabled or logsearch_ui_protocol == 'https'"); Map<String, String> expectedLogSearchEnv = ImmutableMap.of( - "content", "logsearch_solr_ssl_enabled or logsearch_ui_protocol == 'https' or ambari_server_use_ssl"); + "content", "logsearch_use_ssl"); Config mockLogSearchEnv = easyMockSupport.createNiceMock(Config.class); expect(cluster.getDesiredConfigByType("logsearch-env")).andReturn(mockLogSearchEnv).atLeastOnce(); @@ -1102,7 +1102,7 @@ public class UpgradeCatalog250Test { expect(mockLogFeederLog4j.getProperties()).andReturn(oldLogFeederLog4j).anyTimes(); Capture<Map<String, String>> logFeederLog4jCapture = EasyMock.newCapture(); expect(controller.createConfig(anyObject(Cluster.class), anyString(), capture(logFeederLog4jCapture), anyString(), - anyObject(Map.class))).andReturn(config).once(); + EasyMock.<Map<String, Map<String, String>>>anyObject())).andReturn(config).once(); Map<String, String> oldLogSearchLog4j = ImmutableMap.of( "content", http://git-wip-us.apache.org/repos/asf/ambari/blob/6d00ee5d/ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_logfeeder.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_logfeeder.py b/ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_logfeeder.py index b172f64..1c79c5c 100644 --- a/ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_logfeeder.py +++ b/ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_logfeeder.py @@ -84,6 +84,9 @@ class TestLogFeeder(RMFTestCase): properties={'logfeeder.checkpoint.folder': '/etc/ambari-logsearch-logfeeder/conf/checkpoints', 'logfeeder.config.files': 'output.config.json,input.config-ambari.json,global.config.json,input.config-logsearch.json,input.config-zookeeper.json', 'logfeeder.metrics.collector.hosts': '', + 'logfeeder.metrics.collector.path': '/ws/v1/timeline/metrics', + 'logfeeder.metrics.collector.port': '', + 'logfeeder.metrics.collector.protocol': '', 'logfeeder.solr.core.config.name': 'history', 'logfeeder.solr.zk_connect_string': 'c6401.ambari.apache.org:2181/infra-solr' }