http://git-wip-us.apache.org/repos/asf/ambari/blob/4278c4a4/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrCollectionDao.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrCollectionDao.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrCollectionDao.java index 834ba38..da76924 100644 --- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrCollectionDao.java +++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrCollectionDao.java @@ -20,8 +20,12 @@ package org.apache.ambari.logsearch.dao; import org.apache.ambari.logsearch.conf.SolrPropsConfig; import org.apache.commons.lang.StringUtils; +import org.apache.http.HttpResponse; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.impl.client.CloseableHttpClient; import org.apache.solr.client.solrj.SolrServerException; import org.apache.solr.client.solrj.impl.CloudSolrClient; +import org.apache.solr.client.solrj.impl.HttpClientUtil; import org.apache.solr.client.solrj.request.CollectionAdminRequest; import org.apache.solr.client.solrj.response.CollectionAdminResponse; import org.apache.solr.common.SolrException; @@ -33,6 +37,7 @@ import org.slf4j.LoggerFactory; import static org.apache.ambari.logsearch.solr.SolrConstants.CommonLogConstants.ROUTER_FIELD; import javax.inject.Named; +import javax.ws.rs.core.Response; import java.io.IOException; import java.util.ArrayList; import java.util.Collection; @@ -45,6 +50,8 @@ class SolrCollectionDao { private static final Logger LOG = LoggerFactory.getLogger(SolrCollectionDao.class); private static final int SETUP_RETRY_SECOND = 30; + private static final String MODIFY_COLLECTION_QUERY = "/admin/collections?action=MODIFYCOLLECTION&collection=%s&%s=%d"; + private static final String MAX_SHARDS_PER_NODE = "maxShardsPerNode"; /** * This will try to get the collections from the Solr. Ping doesn't work if @@ -189,7 +196,16 @@ class SolrCollectionDao { } } else { LOG.info("Collection " + solrPropsConfig.getCollection() + " is already there. Will check whether it has the required shards"); - Collection<String> existingShards = getShards(solrClient, solrPropsConfig); + Collection<Slice> slices = getSlices(solrClient, solrPropsConfig); + Collection<String> existingShards = getShards(slices, solrPropsConfig); + if (existingShards.size() < shardsList.size()) { + try { + updateMaximumNumberOfShardsPerCore(slices, solrPropsConfig); + } catch (Throwable t) { + returnValue = false; + LOG.error(String.format("Exception during updating collection (%s)", t)); + } + } for (String shard : shardsList) { if (!existingShards.contains(shard)) { try { @@ -216,10 +232,44 @@ class SolrCollectionDao { return returnValue; } - private Collection<String> getShards(CloudSolrClient solrClient, SolrPropsConfig solrPropsConfig) { - Collection<String> list = new HashSet<>(); + private String getRandomBaseUrl(Collection<Slice> slices) { + String coreUrl = null; + if (slices != null) { + for (Slice slice : slices) { + if (!slice.getReplicas().isEmpty()) { + Replica replica = slice.getReplicas().iterator().next(); + coreUrl = replica.getStr("base_url"); + if (coreUrl != null) { + break; + } + } + } + } + return coreUrl; + } + + private void updateMaximumNumberOfShardsPerCore(Collection<Slice> slices, SolrPropsConfig solrPropsConfig) throws IOException { + String baseUrl = getRandomBaseUrl(slices); + if (baseUrl != null) { + CloseableHttpClient httpClient = HttpClientUtil.createClient(null); + HttpGet request = new HttpGet(baseUrl + String.format(MODIFY_COLLECTION_QUERY, + solrPropsConfig.getCollection(), MAX_SHARDS_PER_NODE, calculateMaxShardsPerNode(solrPropsConfig))); + HttpResponse response = httpClient.execute(request); + if (response.getStatusLine().getStatusCode() != Response.Status.OK.getStatusCode()) { + throw new IllegalStateException(String.format("Cannot update collection (%s) - increase max number of nodes per core", solrPropsConfig.getCollection())); + } + } else { + throw new IllegalStateException(String.format("Cannot get any core url for updating collection (%s)", solrPropsConfig.getCollection())); + } + } + + private Collection<Slice> getSlices(CloudSolrClient solrClient, SolrPropsConfig solrPropsConfig) { ZkStateReader reader = solrClient.getZkStateReader(); - Collection<Slice> slices = reader.getClusterState().getSlices(solrPropsConfig.getCollection()); + return reader.getClusterState().getSlices(solrPropsConfig.getCollection()); + } + + private Collection<String> getShards(Collection<Slice> slices, SolrPropsConfig solrPropsConfig) { + Collection<String> list = new HashSet<>(); for (Slice slice : slices) { for (Replica replica : slice.getReplicas()) { LOG.info("colName=" + solrPropsConfig.getCollection() + ", slice.name=" + slice.getName() + ", slice.state=" + slice.getState() + @@ -245,7 +295,7 @@ class SolrCollectionDao { collectionCreateRequest.setNumShards(solrPropsConfig.getNumberOfShards()); collectionCreateRequest.setReplicationFactor(solrPropsConfig.getReplicationFactor()); collectionCreateRequest.setConfigName(solrPropsConfig.getConfigName()); - collectionCreateRequest.setMaxShardsPerNode(solrPropsConfig.getReplicationFactor() * solrPropsConfig.getNumberOfShards()); + collectionCreateRequest.setMaxShardsPerNode(calculateMaxShardsPerNode(solrPropsConfig)); CollectionAdminResponse createResponse = collectionCreateRequest.process(solrClient); if (createResponse.getStatus() != 0) { LOG.error("Error creating collection. collectionName=" + solrPropsConfig.getCollection() + ", response=" + createResponse); @@ -256,4 +306,8 @@ class SolrCollectionDao { return true; } } + + private Integer calculateMaxShardsPerNode(SolrPropsConfig solrPropsConfig) { + return solrPropsConfig.getReplicationFactor() * solrPropsConfig.getNumberOfShards(); + } }
http://git-wip-us.apache.org/repos/asf/ambari/blob/4278c4a4/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/web/listener/LogSearchSessionListener.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/web/listener/LogSearchSessionListener.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/web/listener/LogSearchSessionListener.java new file mode 100644 index 0000000..9fa5c80 --- /dev/null +++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/web/listener/LogSearchSessionListener.java @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.ambari.logsearch.web.listener; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.servlet.http.HttpSessionEvent; +import javax.servlet.http.HttpSessionListener; + +public class LogSearchSessionListener implements HttpSessionListener { + + private Logger LOG = LoggerFactory.getLogger(LogSearchSessionListener.class); + + private int numberOfSessions = 0; + + @Override + public void sessionCreated(HttpSessionEvent event) { + synchronized (this) { + numberOfSessions++; + } + LOG.debug(String.format("New session is created (Id: %s). Number of sessions: %d", event.getSession().getId(), numberOfSessions)); + } + + @Override + public void sessionDestroyed(HttpSessionEvent event) { + synchronized (this) { + numberOfSessions--; + } + LOG.debug(String.format("Session destroyed (Id: %s). Number of sessions: %d", event.getSession().getId(), numberOfSessions)); + } +} http://git-wip-us.apache.org/repos/asf/ambari/blob/4278c4a4/ambari-logsearch/docker/test-config/logsearch/log4j.xml ---------------------------------------------------------------------- diff --git a/ambari-logsearch/docker/test-config/logsearch/log4j.xml b/ambari-logsearch/docker/test-config/logsearch/log4j.xml index b80824b..d0e26ed 100644 --- a/ambari-logsearch/docker/test-config/logsearch/log4j.xml +++ b/ambari-logsearch/docker/test-config/logsearch/log4j.xml @@ -25,7 +25,7 @@ <param name="maxFileSize" value="10MB" /> <param name="maxBackupIndex" value="10" /> <layout class="org.apache.log4j.PatternLayout"> - <param name="ConversionPattern" value="%d [%t] %-5p %C{6} (%F:%L) - %m%n" /> + <param name="ConversionPattern" value="%d %-5p [%t] %C{6} (%F:%L) - %m%n" /> </layout> </appender> http://git-wip-us.apache.org/repos/asf/ambari/blob/4278c4a4/ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/AbstractTimelineMetricsSink.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/AbstractTimelineMetricsSink.java b/ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/AbstractTimelineMetricsSink.java index 769d0c1..9bc3be5 100644 --- a/ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/AbstractTimelineMetricsSink.java +++ b/ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/AbstractTimelineMetricsSink.java @@ -84,6 +84,10 @@ public abstract class AbstractTimelineMetricsSink { public int ZK_CONNECT_TRY_COUNT = 10; public int ZK_SLEEP_BETWEEN_RETRY_TIME = 2000; public boolean shardExpired = true; + private int zookeeperMinBackoffTimeMins = 2; + private int zookeeperMaxBackoffTimeMins = 5; + private long zookeeperBackoffTimeMillis; + private long lastFailedZkRequestTime = 0l; private SSLSocketFactory sslSocketFactory; @@ -132,6 +136,7 @@ public abstract class AbstractTimelineMetricsSink { metricSinkWriteShardStrategy = new MetricSinkWriteShardHostnameHashingStrategy(getHostname()); collectorHAHelper = new MetricCollectorHAHelper(getZookeeperQuorum(), ZK_CONNECT_TRY_COUNT, ZK_SLEEP_BETWEEN_RETRY_TIME); + zookeeperBackoffTimeMillis = getZookeeperBackoffTimeMillis(); isInitializedForHA = true; } @@ -217,6 +222,11 @@ public abstract class AbstractTimelineMetricsSink { collectorHost = findPreferredCollectHost(); } + if (collectorHost == null) { + LOG.warn("No live collector to send metrics to. Metrics to be sent will be discarded."); + return false; + } + String connectUrl = getCollectorUri(collectorHost); String jsonData = null; LOG.debug("EmitMetrics connectUrl = " + connectUrl); @@ -337,13 +347,27 @@ public abstract class AbstractTimelineMetricsSink { } // Reach out to all configured collectors before Zookeeper - refreshCollectorsFromConfigured(); + Collection<String> collectorHosts = getConfiguredCollectorHosts(); + refreshCollectorsFromConfigured(collectorHosts); // Lookup Zookeeper for live hosts - max 10 seconds wait time - if (allKnownLiveCollectors.size() == 0 && getZookeeperQuorum() != null) { - //TODO : Bring back Zk fallback after proper curation. - LOG.info("No live collectors from configuration. Not requesting zookeeper..."); - //allKnownLiveCollectors.addAll(collectorHAHelper.findLiveCollectorHostsFromZNode()); + long currentTime = System.currentTimeMillis(); + if (allKnownLiveCollectors.size() == 0 && getZookeeperQuorum() != null + && (currentTime - lastFailedZkRequestTime) > zookeeperBackoffTimeMillis) { + + LOG.info("No live collectors from configuration. Requesting zookeeper..."); + allKnownLiveCollectors.addAll(collectorHAHelper.findLiveCollectorHostsFromZNode()); + boolean noNewCollectorFromZk = true; + for (String collectorHostFromZk : allKnownLiveCollectors) { + if (!collectorHosts.contains(collectorHostFromZk)) { + noNewCollectorFromZk = false; + break; + } + } + if (noNewCollectorFromZk) { + LOG.info("No new collector was found from Zookeeper. Will not request zookeeper for " + zookeeperBackoffTimeMillis + " millis"); + lastFailedZkRequestTime = System.currentTimeMillis(); + } } if (allKnownLiveCollectors.size() != 0) { @@ -356,7 +380,7 @@ public abstract class AbstractTimelineMetricsSink { // OR // through Expiry (Refresh needed to pick up dead collectors that might have not become alive). if (shardExpired) { - refreshCollectorsFromConfigured(); + refreshCollectorsFromConfigured(getConfiguredCollectorHosts()); } return metricSinkWriteShardStrategy.findCollectorShard(new ArrayList<>(allKnownLiveCollectors)); } @@ -376,8 +400,7 @@ public abstract class AbstractTimelineMetricsSink { return null; } - private void refreshCollectorsFromConfigured() { - Collection<String> collectorHosts = getConfiguredCollectorHosts(); + private void refreshCollectorsFromConfigured(Collection<String> collectorHosts) { LOG.debug("Trying to find live collector host from : " + collectorHosts); if (collectorHosts != null && !collectorHosts.isEmpty()) { @@ -477,7 +500,7 @@ public abstract class AbstractTimelineMetricsSink { return sb.toString(); } /** - * Parses input Sting of format "['host1', 'host2']" into Collection of hostnames + * Parses input Sting of format "host1,host2" into Collection of hostnames */ public Collection<String> parseHostsStringIntoCollection(String hostsString) { Set<String> hosts = new HashSet<>(); @@ -487,10 +510,8 @@ public abstract class AbstractTimelineMetricsSink { return hosts; } - String[] untrimmedHosts = hostsString.split(","); - for (String host : untrimmedHosts) { - host = StringUtils.substringBetween(host, "'"); + for (String host : hostsString.split(",")) { if (StringUtils.isEmpty(host)) continue; hosts.add(host.trim()); @@ -499,6 +520,12 @@ public abstract class AbstractTimelineMetricsSink { return hosts; } + + private long getZookeeperBackoffTimeMillis() { + return (zookeeperMinBackoffTimeMins + + rand.nextInt(zookeeperMaxBackoffTimeMins - zookeeperMinBackoffTimeMins + 1)) * 60*1000l; + } + /** * Get a pre-formatted URI for the collector */ http://git-wip-us.apache.org/repos/asf/ambari/blob/4278c4a4/ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/availability/MetricCollectorHAHelper.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/availability/MetricCollectorHAHelper.java b/ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/availability/MetricCollectorHAHelper.java index 54c2bdf..c6f6beb 100644 --- a/ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/availability/MetricCollectorHAHelper.java +++ b/ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/availability/MetricCollectorHAHelper.java @@ -23,7 +23,6 @@ import org.apache.curator.CuratorZookeeperClient; import org.apache.curator.RetryLoop; import org.apache.curator.RetryPolicy; import org.apache.curator.retry.BoundedExponentialBackoffRetry; -import org.apache.curator.retry.RetryUntilElapsed; import org.apache.zookeeper.ZooKeeper; import org.apache.zookeeper.data.Stat; http://git-wip-us.apache.org/repos/asf/ambari/blob/4278c4a4/ambari-metrics/ambari-metrics-common/src/test/java/org/apache/hadoop/metrics2/sink/timeline/availability/AbstractTimelineMetricSinkTest.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-common/src/test/java/org/apache/hadoop/metrics2/sink/timeline/availability/AbstractTimelineMetricSinkTest.java b/ambari-metrics/ambari-metrics-common/src/test/java/org/apache/hadoop/metrics2/sink/timeline/availability/AbstractTimelineMetricSinkTest.java index 5e016f8..9b0cdbe 100644 --- a/ambari-metrics/ambari-metrics-common/src/test/java/org/apache/hadoop/metrics2/sink/timeline/availability/AbstractTimelineMetricSinkTest.java +++ b/ambari-metrics/ambari-metrics-common/src/test/java/org/apache/hadoop/metrics2/sink/timeline/availability/AbstractTimelineMetricSinkTest.java @@ -32,21 +32,21 @@ public class AbstractTimelineMetricSinkTest { AbstractTimelineMetricsSink sink = new TestTimelineMetricsSink(); Collection<String> hosts; - hosts = sink.parseHostsStringIntoCollection("[]"); + hosts = sink.parseHostsStringIntoCollection(""); Assert.assertTrue(hosts.isEmpty()); - hosts = sink.parseHostsStringIntoCollection("[u'test1.123.abc.def.local']"); + hosts = sink.parseHostsStringIntoCollection("test1.123.abc.def.local"); Assert.assertTrue(hosts.size() == 1); Assert.assertTrue(hosts.contains("test1.123.abc.def.local")); - hosts = sink.parseHostsStringIntoCollection("['test1.123.abc.def.local']"); + hosts = sink.parseHostsStringIntoCollection("test1.123.abc.def.local "); Assert.assertTrue(hosts.size() == 1); Assert.assertTrue(hosts.contains("test1.123.abc.def.local")); - hosts = sink.parseHostsStringIntoCollection("[u'test1.123.abc.def.local', u'test1.456.abc.def.local']"); + hosts = sink.parseHostsStringIntoCollection("test1.123.abc.def.local,test1.456.abc.def.local"); Assert.assertTrue(hosts.size() == 2); - hosts = sink.parseHostsStringIntoCollection("['test1.123.abc.def.local', 'test1.456.abc.def.local']"); + hosts = sink.parseHostsStringIntoCollection("test1.123.abc.def.local, test1.456.abc.def.local"); Assert.assertTrue(hosts.size() == 2); Assert.assertTrue(hosts.contains("test1.123.abc.def.local")); Assert.assertTrue(hosts.contains("test1.456.abc.def.local")); http://git-wip-us.apache.org/repos/asf/ambari/blob/4278c4a4/ambari-metrics/ambari-metrics-common/src/test/java/org/apache/hadoop/metrics2/sink/timeline/availability/MetricCollectorHATest.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-common/src/test/java/org/apache/hadoop/metrics2/sink/timeline/availability/MetricCollectorHATest.java b/ambari-metrics/ambari-metrics-common/src/test/java/org/apache/hadoop/metrics2/sink/timeline/availability/MetricCollectorHATest.java index a0bc2e8..a393a96 100644 --- a/ambari-metrics/ambari-metrics-common/src/test/java/org/apache/hadoop/metrics2/sink/timeline/availability/MetricCollectorHATest.java +++ b/ambari-metrics/ambari-metrics-common/src/test/java/org/apache/hadoop/metrics2/sink/timeline/availability/MetricCollectorHATest.java @@ -49,7 +49,6 @@ import static org.powermock.api.easymock.PowerMock.verifyAll; @PrepareForTest({AbstractTimelineMetricsSink.class, URL.class, HttpURLConnection.class, MetricCollectorHAHelper.class}) public class MetricCollectorHATest { - @Ignore @Test public void findCollectorUsingZKTest() throws Exception { InputStream is = createNiceMock(InputStream.class); http://git-wip-us.apache.org/repos/asf/ambari/blob/4278c4a4/ambari-metrics/ambari-metrics-common/src/test/java/org/apache/hadoop/metrics2/sink/timeline/cache/HandleConnectExceptionTest.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-common/src/test/java/org/apache/hadoop/metrics2/sink/timeline/cache/HandleConnectExceptionTest.java b/ambari-metrics/ambari-metrics-common/src/test/java/org/apache/hadoop/metrics2/sink/timeline/cache/HandleConnectExceptionTest.java index 36ec074..32fe32e 100644 --- a/ambari-metrics/ambari-metrics-common/src/test/java/org/apache/hadoop/metrics2/sink/timeline/cache/HandleConnectExceptionTest.java +++ b/ambari-metrics/ambari-metrics-common/src/test/java/org/apache/hadoop/metrics2/sink/timeline/cache/HandleConnectExceptionTest.java @@ -130,6 +130,10 @@ public class HandleConnectExceptionTest { return super.emitMetrics(metrics); } + @Override + protected synchronized String findPreferredCollectHost() { + return "localhost"; + } } } http://git-wip-us.apache.org/repos/asf/ambari/blob/4278c4a4/ambari-metrics/ambari-metrics-grafana/conf/unix/ambari-metrics-grafana ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-grafana/conf/unix/ambari-metrics-grafana b/ambari-metrics/ambari-metrics-grafana/conf/unix/ambari-metrics-grafana index 472942e..8a8ac8f 100644 --- a/ambari-metrics/ambari-metrics-grafana/conf/unix/ambari-metrics-grafana +++ b/ambari-metrics/ambari-metrics-grafana/conf/unix/ambari-metrics-grafana @@ -104,7 +104,7 @@ case "$1" in # Prepare environment # mkdir -p "$LOG_DIR" "$DATA_DIR" && chown "$GRAFANA_USER":"$GRAFANA_GROUP" "$LOG_DIR" "$DATA_DIR" - # touch "$PID_FILE" && chown "$GRAFANA_USER":"$GRAFANA_GROUP" "$PID_FILE" + touch "$PID_FILE" && chown "$GRAFANA_USER":"$GRAFANA_GROUP" "$PID_FILE" # if [ -n "$MAX_OPEN_FILES" ]; then # ulimit -n $MAX_OPEN_FILES @@ -116,10 +116,10 @@ case "$1" in return=$? if [ $return -eq 0 ] then - sleep 1 + sleep 5 # check if pid file has been written two if ! [[ -s $PID_FILE ]]; then - echo "Start FAILED" >> $LOG_FILE + echo "Start FAILED because daemon did not write pid in pid_file" >> $LOG_FILE exit 1 fi i=0 http://git-wip-us.apache.org/repos/asf/ambari/blob/4278c4a4/ambari-metrics/ambari-metrics-hadoop-sink/src/main/java/org/apache/hadoop/metrics2/sink/timeline/HadoopTimelineMetricsSink.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-hadoop-sink/src/main/java/org/apache/hadoop/metrics2/sink/timeline/HadoopTimelineMetricsSink.java b/ambari-metrics/ambari-metrics-hadoop-sink/src/main/java/org/apache/hadoop/metrics2/sink/timeline/HadoopTimelineMetricsSink.java index 802e684..b2ca52e 100644 --- a/ambari-metrics/ambari-metrics-hadoop-sink/src/main/java/org/apache/hadoop/metrics2/sink/timeline/HadoopTimelineMetricsSink.java +++ b/ambari-metrics/ambari-metrics-hadoop-sink/src/main/java/org/apache/hadoop/metrics2/sink/timeline/HadoopTimelineMetricsSink.java @@ -194,13 +194,13 @@ public class HadoopTimelineMetricsSink extends AbstractTimelineMetricsSink imple } /** - * Parses input Stings array of format "['host1'", '"host2']" into Collection of hostnames + * Parses input Stings array of format "host1,host2" into Collection of hostnames */ protected Collection<String> parseHostsStringArrayIntoCollection(String[] hostStrings) { Collection<String> result = new HashSet<>(); if (hostStrings == null) return result; for (String s : hostStrings) { - result.addAll(parseHostsStringIntoCollection(s)); + result.add(s.trim()); } return result; } http://git-wip-us.apache.org/repos/asf/ambari/blob/4278c4a4/ambari-metrics/ambari-metrics-hadoop-sink/src/test/java/org/apache/hadoop/metrics2/sink/timeline/HadoopTimelineMetricsSinkTest.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-hadoop-sink/src/test/java/org/apache/hadoop/metrics2/sink/timeline/HadoopTimelineMetricsSinkTest.java b/ambari-metrics/ambari-metrics-hadoop-sink/src/test/java/org/apache/hadoop/metrics2/sink/timeline/HadoopTimelineMetricsSinkTest.java index da00155..0cafd9a 100644 --- a/ambari-metrics/ambari-metrics-hadoop-sink/src/test/java/org/apache/hadoop/metrics2/sink/timeline/HadoopTimelineMetricsSinkTest.java +++ b/ambari-metrics/ambari-metrics-hadoop-sink/src/test/java/org/apache/hadoop/metrics2/sink/timeline/HadoopTimelineMetricsSinkTest.java @@ -100,7 +100,7 @@ public class HadoopTimelineMetricsSinkTest { expect(conf.getString("slave.host.name")).andReturn("localhost").anyTimes(); expect(conf.getParent()).andReturn(null).anyTimes(); expect(conf.getPrefix()).andReturn("service").anyTimes(); - expect(conf.getStringArray(eq(COLLECTOR_HOSTS_PROPERTY))).andReturn(new String[]{"['localhost", "'localhost2']"}).anyTimes(); + expect(conf.getStringArray(eq(COLLECTOR_HOSTS_PROPERTY))).andReturn(new String[]{"localhost"," localhost2"}).anyTimes(); expect(conf.getString(eq("serviceName-prefix"), eq(""))).andReturn("").anyTimes(); expect(conf.getString(eq(COLLECTOR_PROTOCOL), eq("http"))).andReturn("http").anyTimes(); expect(conf.getString(eq(COLLECTOR_PORT), eq("6188"))).andReturn("6188").anyTimes(); @@ -172,7 +172,7 @@ public class HadoopTimelineMetricsSinkTest { expect(conf.getString("slave.host.name")).andReturn("localhost").anyTimes(); expect(conf.getParent()).andReturn(null).anyTimes(); expect(conf.getPrefix()).andReturn("service").anyTimes(); - expect(conf.getStringArray(eq(COLLECTOR_HOSTS_PROPERTY))).andReturn(new String[]{"['localhost", "'localhost2']"}).anyTimes(); + expect(conf.getStringArray(eq(COLLECTOR_HOSTS_PROPERTY))).andReturn(new String[]{"localhost", "localhost2"}).anyTimes(); expect(conf.getString(eq("serviceName-prefix"), eq(""))).andReturn("").anyTimes(); expect(conf.getString(eq(COLLECTOR_PROTOCOL), eq("http"))).andReturn("http").anyTimes(); expect(conf.getString(eq(COLLECTOR_PORT), eq("6188"))).andReturn("6188").anyTimes(); @@ -302,7 +302,7 @@ public class HadoopTimelineMetricsSinkTest { expect(conf.getString("slave.host.name")).andReturn("localhost").anyTimes(); expect(conf.getParent()).andReturn(null).anyTimes(); expect(conf.getPrefix()).andReturn("service").anyTimes(); - expect(conf.getStringArray(eq(COLLECTOR_HOSTS_PROPERTY))).andReturn(new String[]{"['localhost", "'localhost2']"}).anyTimes(); + expect(conf.getStringArray(eq(COLLECTOR_HOSTS_PROPERTY))).andReturn(new String[]{"localhost", "localhost2"}).anyTimes(); expect(conf.getString(eq("serviceName-prefix"), eq(""))).andReturn("").anyTimes(); expect(conf.getString(eq(COLLECTOR_PROTOCOL), eq("http"))).andReturn("http").anyTimes(); expect(conf.getString(eq(COLLECTOR_PORT), eq("6188"))).andReturn("6188").anyTimes(); http://git-wip-us.apache.org/repos/asf/ambari/blob/4278c4a4/ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/config_reader.py ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/config_reader.py b/ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/config_reader.py index 9f95c26..7010187 100644 --- a/ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/config_reader.py +++ b/ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/config_reader.py @@ -102,7 +102,7 @@ config_content = """ [default] debug_level = INFO hostname = localhost -metrics_servers = ['localhost','host1','host2'] +metrics_servers = localhost enable_time_threshold = false enable_value_threshold = false @@ -217,16 +217,13 @@ class Configuration: def get_metrics_collector_hosts(self): hosts = self.get("default", "metrics_servers", "localhost") - if hosts is not "localhost": - return ast.literal_eval(hosts) - else: - return hosts + return hosts.split(",") def get_failover_strategy(self): return self.get("collector", "failover_strategy", ROUND_ROBIN_FAILOVER_STRATEGY) def get_failover_strategy_blacklisted_interval_seconds(self): - return self.get("collector", "failover_strategy_blacklisted_interval_seconds", 600) + return self.get("collector", "failover_strategy_blacklisted_interval_seconds", 300) def get_hostname_script(self): if self.hostname_script: http://git-wip-us.apache.org/repos/asf/ambari/blob/4278c4a4/ambari-metrics/ambari-metrics-host-monitoring/src/test/python/core/TestEmitter.py ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-host-monitoring/src/test/python/core/TestEmitter.py b/ambari-metrics/ambari-metrics-host-monitoring/src/test/python/core/TestEmitter.py index de97292..4056ae3 100644 --- a/ambari-metrics/ambari-metrics-host-monitoring/src/test/python/core/TestEmitter.py +++ b/ambari-metrics/ambari-metrics-host-monitoring/src/test/python/core/TestEmitter.py @@ -83,7 +83,7 @@ class TestEmitter(TestCase): emitter.RETRY_SLEEP_INTERVAL = .001 emitter.submit_metrics() - self.assertEqual(request_mock.call_count, 9) + self.assertEqual(request_mock.call_count, 3) self.assertUrlData(request_mock) def assertUrlData(self, request_mock): http://git-wip-us.apache.org/repos/asf/ambari/blob/4278c4a4/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/HBaseTimelineMetricStore.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/HBaseTimelineMetricStore.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/HBaseTimelineMetricStore.java index 75ce845..844ee41 100644 --- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/HBaseTimelineMetricStore.java +++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/HBaseTimelineMetricStore.java @@ -107,16 +107,14 @@ public class HBaseTimelineMetricStore extends AbstractService implements Timelin // Initialize policies before TTL update hBaseAccessor.initPoliciesAndTTL(); // Start HA service - if (configuration.isDistributedOperationModeEnabled()) { - // Start the controller - haController = new MetricCollectorHAController(configuration); - try { - haController.initializeHAController(); - } catch (Exception e) { - LOG.error(e); - throw new MetricsSystemInitializationException("Unable to " + - "initialize HA controller", e); - } + // Start the controller + haController = new MetricCollectorHAController(configuration); + try { + haController.initializeHAController(); + } catch (Exception e) { + LOG.error(e); + throw new MetricsSystemInitializationException("Unable to " + + "initialize HA controller", e); } String whitelistFile = metricsConf.get(TIMELINE_METRICS_WHITELIST_FILE, ""); http://git-wip-us.apache.org/repos/asf/ambari/blob/4278c4a4/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricConfiguration.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricConfiguration.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricConfiguration.java index 9858e4c..27caf88 100644 --- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricConfiguration.java +++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricConfiguration.java @@ -320,6 +320,20 @@ public class TimelineMetricConfiguration { return hbaseConf.getTrimmed("hbase.zookeeper.quorum"); } + public String getClusterZKClientPort() throws MalformedURLException, URISyntaxException { + if (!isInitialized) { + initialize(); + } + return metricsConf.getTrimmed("cluster.zookeeper.property.clientPort", "2181"); + } + + public String getClusterZKQuorum() throws MalformedURLException, URISyntaxException { + if (!isInitialized) { + initialize(); + } + return metricsConf.getTrimmed("cluster.zookeeper.quorum"); + } + public String getInstanceHostnameFromEnv() throws UnknownHostException { String amsInstanceName = System.getProperty("AMS_INSTANCE_NAME"); if (amsInstanceName == null) { http://git-wip-us.apache.org/repos/asf/ambari/blob/4278c4a4/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/AbstractTimelineAggregator.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/AbstractTimelineAggregator.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/AbstractTimelineAggregator.java index de63d4e..62f8137 100644 --- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/AbstractTimelineAggregator.java +++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/AbstractTimelineAggregator.java @@ -317,9 +317,9 @@ public abstract class AbstractTimelineAggregator implements TimelineMetricAggreg protected void downsample(Connection conn, Long startTime, Long endTime) { - LOG.info("Checking for downsampling requests."); + LOG.debug("Checking for downsampling requests."); if (CollectionUtils.isEmpty(configuredDownSamplers)) { - LOG.info("No downsamplers configured"); + LOG.debug("No downsamplers configured"); return; } @@ -424,7 +424,7 @@ public abstract class AbstractTimelineAggregator implements TimelineMetricAggreg PreparedStatement stmt = null; ResultSet rs = null; - LOG.info("Downsampling query : " + condition.getStatement()); + LOG.debug("Downsampling query : " + condition.getStatement()); try { stmt = PhoenixTransactSQL.prepareGetMetricsSqlStmt(conn, condition); http://git-wip-us.apache.org/repos/asf/ambari/blob/4278c4a4/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/availability/MetricCollectorHAController.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/availability/MetricCollectorHAController.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/availability/MetricCollectorHAController.java index edce367..26702b6 100644 --- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/availability/MetricCollectorHAController.java +++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/availability/MetricCollectorHAController.java @@ -81,8 +81,8 @@ public class MetricCollectorHAController { } try { - String zkClientPort = configuration.getZKClientPort(); - String zkQuorum = configuration.getZKQuorum(); + String zkClientPort = configuration.getClusterZKClientPort(); + String zkQuorum = configuration.getClusterZKQuorum(); if (StringUtils.isEmpty(zkClientPort) || StringUtils.isEmpty(zkQuorum)) { throw new Exception("Unable to parse zookeeper quorum. clientPort = " http://git-wip-us.apache.org/repos/asf/ambari/blob/4278c4a4/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryServer.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryServer.java b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryServer.java index 462213d..3688630 100644 --- a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryServer.java +++ b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryServer.java @@ -25,8 +25,10 @@ import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.service.Service.STATE; import org.apache.hadoop.util.ExitUtil; import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.HBaseTimelineMetricStore; import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.PhoenixHBaseAccessor; import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration; +import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.availability.MetricCollectorHAController; import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.query.DefaultPhoenixDataSource; import org.apache.zookeeper.ClientCnxn; import org.easymock.EasyMock; @@ -71,7 +73,7 @@ import static org.powermock.api.support.membermodification.MemberMatcher.method; import static org.powermock.api.support.membermodification.MemberModifier.suppress; @RunWith(PowerMockRunner.class) -@PrepareForTest({ PhoenixHBaseAccessor.class, UserGroupInformation.class, +@PrepareForTest({ PhoenixHBaseAccessor.class, HBaseTimelineMetricStore.class, UserGroupInformation.class, ClientCnxn.class, DefaultPhoenixDataSource.class, ConnectionFactory.class, TimelineMetricConfiguration.class, ApplicationHistoryServer.class }) @PowerMockIgnore( {"javax.management.*"}) @@ -179,6 +181,8 @@ public class TestApplicationHistoryServer { expect(metricConfiguration.getTimelineMetricsServiceHandlerThreadCount()).andReturn(20).anyTimes(); expect(metricConfiguration.getWebappAddress()).andReturn("localhost:9990").anyTimes(); expect(metricConfiguration.getTimelineServiceRpcAddress()).andReturn("localhost:10299").anyTimes(); + expect(metricConfiguration.getClusterZKQuorum()).andReturn("localhost").anyTimes(); + expect(metricConfiguration.getClusterZKClientPort()).andReturn("2181").anyTimes(); Connection connection = createNiceMock(Connection.class); Statement stmt = createNiceMock(Statement.class); @@ -197,6 +201,14 @@ public class TestApplicationHistoryServer { connection.close(); expectLastCall(); + MetricCollectorHAController haControllerMock = PowerMock.createMock(MetricCollectorHAController.class); + expectNew(MetricCollectorHAController.class, metricConfiguration) + .andReturn(haControllerMock); + + haControllerMock.initializeHAController(); + expectLastCall().once(); + expect(haControllerMock.isInitialized()).andReturn(false).anyTimes(); + org.apache.hadoop.hbase.client.Connection conn = createNiceMock(org.apache.hadoop.hbase.client.Connection.class); mockStatic(ConnectionFactory.class); expect(ConnectionFactory.createConnection((Configuration) anyObject())).andReturn(conn); http://git-wip-us.apache.org/repos/asf/ambari/blob/4278c4a4/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/availability/MetricCollectorHAControllerTest.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/availability/MetricCollectorHAControllerTest.java b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/availability/MetricCollectorHAControllerTest.java index 1e4bac0..875f013 100644 --- a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/availability/MetricCollectorHAControllerTest.java +++ b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/availability/MetricCollectorHAControllerTest.java @@ -51,8 +51,8 @@ public class MetricCollectorHAControllerTest extends AbstractMiniHBaseClusterTes String port = zkUrl.split(":")[3]; String quorum = zkUrl.split(":")[2]; - expect(configuration.getZKClientPort()).andReturn(port); - expect(configuration.getZKQuorum()).andReturn(quorum); + expect(configuration.getClusterZKClientPort()).andReturn(port); + expect(configuration.getClusterZKQuorum()).andReturn(quorum); replay(configuration); } http://git-wip-us.apache.org/repos/asf/ambari/blob/4278c4a4/ambari-project/pom.xml ---------------------------------------------------------------------- diff --git a/ambari-project/pom.xml b/ambari-project/pom.xml index a38c6a2..52c9992 100644 --- a/ambari-project/pom.xml +++ b/ambari-project/pom.xml @@ -30,6 +30,8 @@ <ambari.dir>${project.parent.basedir}</ambari.dir> <powermock.version>1.6.3</powermock.version> <jetty.version>8.1.19.v20160209</jetty.version> + <checkstyle.version>6.19</checkstyle.version> <!-- last version that does not require Java 8 --> + <checkstyle.skip>false</checkstyle.skip> </properties> <profiles> <profile> @@ -477,6 +479,11 @@ <artifactId>jline</artifactId> <version>2.11</version> </dependency> + <dependency> + <groupId>com.puppycrawl.tools</groupId> + <artifactId>checkstyle</artifactId> + <version>${checkstyle.version}</version> + </dependency> </dependencies> </dependencyManagement> <build> @@ -486,6 +493,40 @@ <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-surefire-plugin</artifactId> </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-checkstyle-plugin</artifactId> + <version>2.17</version> + <executions> + <execution> + <id>checkstyle</id> + <phase>test</phase> + <configuration> + <configLocation>${project.basedir}/checkstyle.xml</configLocation> + <encoding>UTF-8</encoding> + <consoleOutput>true</consoleOutput> + <failsOnError>true</failsOnError> + <linkXRef>false</linkXRef> + <skip>${checkstyle.skip}</skip> + </configuration> + <goals> + <goal>check</goal> + </goals> + </execution> + </executions> + <dependencies> + <dependency> + <groupId>com.puppycrawl.tools</groupId> + <artifactId>checkstyle</artifactId> + <version>${checkstyle.version}</version> + </dependency> + <dependency> + <groupId>utility</groupId> + <artifactId>utility</artifactId> + <version>1.0.0.0-SNAPSHOT</version> + </dependency> + </dependencies> + </plugin> </plugins> </pluginManagement> <plugins> http://git-wip-us.apache.org/repos/asf/ambari/blob/4278c4a4/ambari-server/checkstyle.xml ---------------------------------------------------------------------- diff --git a/ambari-server/checkstyle.xml b/ambari-server/checkstyle.xml new file mode 100644 index 0000000..81f6380 --- /dev/null +++ b/ambari-server/checkstyle.xml @@ -0,0 +1,17 @@ +<?xml version="1.0"?> +<!-- Licensed under the Apache License, Version 2.0 (the "License"); you + may not use this file except in compliance with the License. You may obtain + a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless + required by applicable law or agreed to in writing, software distributed + under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES + OR CONDITIONS OF ANY KIND, either express or implied. See the License for + the specific language governing permissions and limitations under the License. + See accompanying LICENSE file. --> +<!DOCTYPE module PUBLIC + "-//Puppy Crawl//DTD Check Configuration 1.3//EN" + "http://www.puppycrawl.com/dtds/configuration_1_3.dtd"> +<module name="Checker"> + <module name="TreeWalker"> + <module name="AvoidTransactionalOnPrivateMethodsCheck"/> + </module> +</module> http://git-wip-us.apache.org/repos/asf/ambari/blob/4278c4a4/ambari-server/pom.xml ---------------------------------------------------------------------- diff --git a/ambari-server/pom.xml b/ambari-server/pom.xml index d1277c3..08757a2 100644 --- a/ambari-server/pom.xml +++ b/ambari-server/pom.xml @@ -686,6 +686,10 @@ <artifactId>jetty-maven-plugin</artifactId> <version>${jetty.version}</version> </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-checkstyle-plugin</artifactId> + </plugin> </plugins> <resources> <resource> http://git-wip-us.apache.org/repos/asf/ambari/blob/4278c4a4/ambari-server/src/main/java/org/apache/ambari/server/agent/AgentRequests.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/agent/AgentRequests.java b/ambari-server/src/main/java/org/apache/ambari/server/agent/AgentRequests.java index 2980f38..01195bf 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/agent/AgentRequests.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/agent/AgentRequests.java @@ -45,9 +45,9 @@ public class AgentRequests { public void setExecutionDetailsRequest(String host, String component, String requestExecutionCmd) { if (StringUtils.isNotBlank(requestExecutionCmd)) { - LOG.debug("Setting need for exec command to " + requestExecutionCmd + " for " + component); Map<String, Boolean> perHostRequiresExecCmdDetails = getPerHostRequiresExecCmdDetails(host); if (Boolean.TRUE.toString().toUpperCase().equals(requestExecutionCmd.toUpperCase())) { + LOG.info("Setting need for exec command to " + requestExecutionCmd + " for " + component); perHostRequiresExecCmdDetails.put(component, Boolean.TRUE); } else { perHostRequiresExecCmdDetails.put(component, Boolean.FALSE); http://git-wip-us.apache.org/repos/asf/ambari/blob/4278c4a4/ambari-server/src/main/java/org/apache/ambari/server/agent/ExecutionCommand.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/agent/ExecutionCommand.java b/ambari-server/src/main/java/org/apache/ambari/server/agent/ExecutionCommand.java index 29737ee..ef1ee4f 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/agent/ExecutionCommand.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/agent/ExecutionCommand.java @@ -343,6 +343,7 @@ public class ExecutionCommand extends AgentCommand { String SCRIPT_TYPE = "script_type"; String SERVICE_PACKAGE_FOLDER = "service_package_folder"; String HOOKS_FOLDER = "hooks_folder"; + String CUSTOM_FOLDER = "custom_folder"; String STACK_NAME = "stack_name"; String SERVICE_TYPE = "service_type"; String STACK_VERSION = "stack_version"; http://git-wip-us.apache.org/repos/asf/ambari/blob/4278c4a4/ambari-server/src/main/java/org/apache/ambari/server/checks/CheckDescription.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/checks/CheckDescription.java b/ambari-server/src/main/java/org/apache/ambari/server/checks/CheckDescription.java index 7f24bf4..45497b5 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/checks/CheckDescription.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/checks/CheckDescription.java @@ -236,10 +236,10 @@ public class CheckDescription { public static CheckDescription HARDCODED_STACK_VERSION_PROPERTIES_CHECK = new CheckDescription("HARDCODED_STACK_VERSION_PROPERTIES_CHECK", PrereqCheckType.CLUSTER, - "Found hardcoded hdp stack version in property value.", + "Found hardcoded stack version in property value.", new ImmutableMap.Builder<String, String>() .put(AbstractCheckDescriptor.DEFAULT, - "Some properties seem to contain hardcoded hdp version string \"%s\"." + + "Some properties seem to contain hardcoded stack version string \"%s\"." + " That is a potential problem when doing stack update.").build()); public static CheckDescription VERSION_MISMATCH = new CheckDescription("VERSION_MISMATCH", http://git-wip-us.apache.org/repos/asf/ambari/blob/4278c4a4/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java index 2a78905..c3cd82e 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java @@ -25,6 +25,7 @@ import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.AMBARI_DB import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.CLIENTS_TO_UPDATE_CONFIGS; import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.COMMAND_RETRY_ENABLED; import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.COMMAND_TIMEOUT; +import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.CUSTOM_FOLDER; import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.DB_DRIVER_FILENAME; import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.GROUP_LIST; import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.HOOKS_FOLDER; @@ -54,7 +55,6 @@ import java.util.EnumMap; import java.util.EnumSet; import java.util.HashMap; import java.util.HashSet; -import java.util.Iterator; import java.util.LinkedHashSet; import java.util.LinkedList; import java.util.List; @@ -79,10 +79,10 @@ import org.apache.ambari.server.ServiceComponentNotFoundException; import org.apache.ambari.server.ServiceNotFoundException; import org.apache.ambari.server.StackAccessException; import org.apache.ambari.server.actionmanager.ActionManager; +import org.apache.ambari.server.actionmanager.CommandExecutionType; import org.apache.ambari.server.actionmanager.HostRoleCommand; import org.apache.ambari.server.actionmanager.RequestFactory; import org.apache.ambari.server.actionmanager.Stage; -import org.apache.ambari.server.actionmanager.CommandExecutionType; import org.apache.ambari.server.actionmanager.StageFactory; import org.apache.ambari.server.agent.ExecutionCommand; import org.apache.ambari.server.agent.ExecutionCommand.KeyNames; @@ -894,17 +894,11 @@ public class AmbariManagementControllerImpl implements AmbariManagementControlle @Override public Config createConfig(Cluster cluster, String type, Map<String, String> properties, String versionTag, Map<String, Map<String, String>> propertiesAttributes) { - Config config = configFactory.createNew(cluster, type, - properties, propertiesAttributes); - if (!StringUtils.isEmpty(versionTag)) { - config.setTag(versionTag); - } - - config.persist(); + Config config = configFactory.createNew(cluster, type, versionTag, properties, + propertiesAttributes); cluster.addConfig(config); - return config; } @@ -1253,7 +1247,12 @@ public class AmbariManagementControllerImpl implements AmbariManagementControlle throw new HostNotFoundException(cluster.getClusterName(), sch.getHostName()); } - r.setMaintenanceState(maintenanceStateHelper.getEffectiveState(sch, host).name()); + MaintenanceState effectiveMaintenanceState = maintenanceStateHelper.getEffectiveState(sch, host); + if(filterByMaintenanceState(request, effectiveMaintenanceState)) { + continue; + } + r.setMaintenanceState(effectiveMaintenanceState.name()); + response.add(r); } catch (ServiceComponentHostNotFoundException e) { if (request.getServiceName() == null || request.getComponentName() == null) { @@ -1304,7 +1303,12 @@ public class AmbariManagementControllerImpl implements AmbariManagementControlle throw new HostNotFoundException(cluster.getClusterName(), sch.getHostName()); } - r.setMaintenanceState(maintenanceStateHelper.getEffectiveState(sch, host).name()); + MaintenanceState effectiveMaintenanceState = maintenanceStateHelper.getEffectiveState(sch, host); + if(filterByMaintenanceState(request, effectiveMaintenanceState)) { + continue; + } + r.setMaintenanceState(effectiveMaintenanceState.name()); + response.add(r); } } @@ -1313,6 +1317,24 @@ public class AmbariManagementControllerImpl implements AmbariManagementControlle return response; } + private boolean filterByMaintenanceState(ServiceComponentHostRequest request, MaintenanceState effectiveMaintenanceState) { + if (request.getMaintenanceState() != null) { + MaintenanceState desiredMaintenanceState = MaintenanceState.valueOf(request.getMaintenanceState()); + if (desiredMaintenanceState.equals(MaintenanceState.ON)) { + /* + * if we want components with ON state it can be one of IMPLIED_FROM_SERVICE, + * IMPLIED_FROM_SERVICE_AND_HOST, IMPLIED_FROM_HOST, ON, ro simply - not OFF + */ + if (effectiveMaintenanceState.equals(MaintenanceState.OFF)) { + return true; + } + } else if (!desiredMaintenanceState.equals(effectiveMaintenanceState)){ + return true; + } + } + return false; + } + @Override public MaintenanceState getEffectiveMaintenanceState(ServiceComponentHost sch) throws AmbariException { @@ -2217,6 +2239,14 @@ public class AmbariManagementControllerImpl implements AmbariManagementControlle serviceInfo.getServicePackageFolder()); commandParams.put(HOOKS_FOLDER, stackInfo.getStackHooksFolder()); + String customCacheDirectory = componentInfo.getCustomFolder(); + if (customCacheDirectory != null) { + File customCache = new File(configs.getResourceDirPath(), customCacheDirectory); + if (customCache.exists() && customCache.isDirectory()) { + commandParams.put(CUSTOM_FOLDER, customCacheDirectory); + } + } + String clusterName = cluster.getClusterName(); if (customCommandExecutionHelper.isTopologyRefreshRequired(roleCommand.name(), clusterName, serviceName)) { commandParams.put(ExecutionCommand.KeyNames.REFRESH_TOPOLOGY, "True"); http://git-wip-us.apache.org/repos/asf/ambari/blob/4278c4a4/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/AlertTargetResourceProvider.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/AlertTargetResourceProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/AlertTargetResourceProvider.java index 8e5674e..4010528 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/AlertTargetResourceProvider.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/AlertTargetResourceProvider.java @@ -374,7 +374,7 @@ public class AlertTargetResourceProvider extends */ @Transactional @SuppressWarnings("unchecked") - private void updateAlertTargets(long alertTargetId, + void updateAlertTargets(long alertTargetId, Map<String, Object> requestMap) throws AmbariException { http://git-wip-us.apache.org/repos/asf/ambari/blob/4278c4a4/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java index ec846f8..b26e056 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java @@ -2526,9 +2526,10 @@ public class BlueprintConfigurationProcessor { } boolean isAtlasInCluster = topology.getBlueprint().getServices().contains("ATLAS"); + boolean isAtlasHiveHookEnabled = Boolean.parseBoolean(properties.get("hive-env").get("hive.atlas.hook")); // Append atlas hook if not already present. - if (isAtlasInCluster) { + if (isAtlasInCluster || isAtlasHiveHookEnabled) { if (!hiveHooksClean.contains(atlasHookClass)) { hiveHooksClean.add(atlasHookClass); } http://git-wip-us.apache.org/repos/asf/ambari/blob/4278c4a4/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/CalculatedStatus.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/CalculatedStatus.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/CalculatedStatus.java index 3a86aef..3c415df 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/CalculatedStatus.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/CalculatedStatus.java @@ -53,6 +53,17 @@ public class CalculatedStatus { */ private final double percent; + /** + * A status which represents a COMPLETED state at 0% + */ + public static final CalculatedStatus COMPLETED = new CalculatedStatus(HostRoleStatus.COMPLETED, + HostRoleStatus.COMPLETED, 100.0); + + /** + * A status which represents a PENDING state at 0% + */ + public static final CalculatedStatus PENDING = new CalculatedStatus(HostRoleStatus.PENDING, + HostRoleStatus.PENDING, 0.0); // ----- Constructors ------------------------------------------------------ @@ -79,12 +90,6 @@ public class CalculatedStatus { this.percent = percent; } - /** - * Static factory method to get Status that represents a Completed state - */ - public static CalculatedStatus getCompletedStatus() { - return new CalculatedStatus(HostRoleStatus.COMPLETED, HostRoleStatus.COMPLETED, 100.0); - } // ----- CalculatedStatus -------------------------------------------------- @@ -291,14 +296,25 @@ public class CalculatedStatus { } /** - * Calculates the overall status of an upgrade. - * @param stageDto the map of stage-to-summary value objects - * @param stageIds the stage ids to consider from the value objects + * Calculates the overall status of an upgrade. If there are no tasks, then a + * status of {@link HostRoleStatus#COMPLETED} is returned. + * + * @param stageDto + * the map of stage-to-summary value objects + * @param stageIds + * the stage ids to consider from the value objects * @return the calculated status */ public static CalculatedStatus statusFromStageSummary(Map<Long, HostRoleCommandStatusSummaryDTO> stageDto, Set<Long> stageIds) { + // if either are empty, then we have no tasks and therefore no status - we + // should return COMPLETED. This can happen if someone removes all tasks but + // leaves the stages and request + if (stageDto.isEmpty() || stageIds.isEmpty()) { + return COMPLETED; + } + Collection<HostRoleStatus> stageStatuses = new HashSet<>(); Collection<HostRoleStatus> stageDisplayStatuses = new HashSet<>(); Collection<HostRoleStatus> taskStatuses = new ArrayList<>(); @@ -378,19 +394,28 @@ public class CalculatedStatus { */ public static HostRoleStatus calculateSummaryStatusOfStage(Map<HostRoleStatus, Integer> counters, int total, boolean skippable) { + + // when there are 0 tasks, return COMPLETED + if (total == 0) { + return HostRoleStatus.COMPLETED; + } + if (counters.get(HostRoleStatus.PENDING) == total) { return HostRoleStatus.PENDING; } + // By definition, any tasks in a future stage must be held in a PENDING status. if (counters.get(HostRoleStatus.HOLDING) > 0 || counters.get(HostRoleStatus.HOLDING_FAILED) > 0 || counters.get(HostRoleStatus.HOLDING_TIMEDOUT) > 0) { return counters.get(HostRoleStatus.HOLDING) > 0 ? HostRoleStatus.HOLDING : counters.get(HostRoleStatus.HOLDING_FAILED) > 0 ? HostRoleStatus.HOLDING_FAILED : HostRoleStatus.HOLDING_TIMEDOUT; } + // Because tasks are not skippable, guaranteed to be FAILED if (counters.get(HostRoleStatus.FAILED) > 0 && !skippable) { return HostRoleStatus.FAILED; } + // Because tasks are not skippable, guaranteed to be TIMEDOUT if (counters.get(HostRoleStatus.TIMEDOUT) > 0 && !skippable) { return HostRoleStatus.TIMEDOUT; @@ -401,9 +426,11 @@ public class CalculatedStatus { if (counters.get(HostRoleStatus.ABORTED) > 0 && numActiveTasks == 0) { return HostRoleStatus.ABORTED; } + if (counters.get(HostRoleStatus.COMPLETED) == total) { return HostRoleStatus.COMPLETED; } + return HostRoleStatus.IN_PROGRESS; } @@ -415,7 +442,8 @@ public class CalculatedStatus { * * @return summary request status based on statuses of tasks in different states. */ - private static HostRoleStatus calculateSummaryStatusOfUpgrade(Map<HostRoleStatus, Integer> counters, int total) { + protected static HostRoleStatus calculateSummaryStatusOfUpgrade( + Map<HostRoleStatus, Integer> counters, int total) { return calculateSummaryStatusOfStage(counters, total, false); } @@ -428,8 +456,8 @@ public class CalculatedStatus { * * @return summary request status based on statuses of tasks in different states. */ - private static HostRoleStatus calculateSummaryDisplayStatus(Map<HostRoleStatus, Integer> counters, - int total, boolean skippable) { + protected static HostRoleStatus calculateSummaryDisplayStatus( + Map<HostRoleStatus, Integer> counters, int total, boolean skippable) { return counters.get(HostRoleStatus.SKIPPED_FAILED) > 0 ? HostRoleStatus.SKIPPED_FAILED : counters.get(HostRoleStatus.FAILED) > 0 ? HostRoleStatus.FAILED: calculateSummaryStatusOfStage(counters, total, skippable); http://git-wip-us.apache.org/repos/asf/ambari/blob/4278c4a4/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClusterStackVersionResourceProvider.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClusterStackVersionResourceProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClusterStackVersionResourceProvider.java index 42aae84..d417ec2 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClusterStackVersionResourceProvider.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClusterStackVersionResourceProvider.java @@ -442,7 +442,7 @@ public class ClusterStackVersionResourceProvider extends AbstractControllerResou } @Transactional - private void createHostVersions(Cluster cluster, List<Host> hosts, StackId stackId, + void createHostVersions(Cluster cluster, List<Host> hosts, StackId stackId, String desiredRepoVersion, RepositoryVersionState repoState) throws AmbariException, SystemException { final String clusterName = cluster.getClusterName(); @@ -482,7 +482,7 @@ public class ClusterStackVersionResourceProvider extends AbstractControllerResou } @Transactional - private RequestStageContainer createOrchestration(Cluster cluster, StackId stackId, + RequestStageContainer createOrchestration(Cluster cluster, StackId stackId, List<Host> hosts, RepositoryVersionEntity repoVersionEnt, Map<String, Object> propertyMap) throws AmbariException, SystemException { final AmbariManagementController managementController = getManagementController(); http://git-wip-us.apache.org/repos/asf/ambari/blob/4278c4a4/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ConfigGroupResourceProvider.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ConfigGroupResourceProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ConfigGroupResourceProvider.java index 96bb8f9..2373068 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ConfigGroupResourceProvider.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ConfigGroupResourceProvider.java @@ -17,7 +17,16 @@ */ package org.apache.ambari.server.controller.internal; -import com.google.inject.Inject; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.EnumSet; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Set; + import org.apache.ambari.server.AmbariException; import org.apache.ambari.server.ClusterNotFoundException; import org.apache.ambari.server.ConfigGroupNotFoundException; @@ -48,7 +57,7 @@ import org.apache.ambari.server.security.authorization.RoleAuthorization; import org.apache.ambari.server.state.Cluster; import org.apache.ambari.server.state.Clusters; import org.apache.ambari.server.state.Config; -import org.apache.ambari.server.state.ConfigImpl; +import org.apache.ambari.server.state.ConfigFactory; import org.apache.ambari.server.state.Host; import org.apache.ambari.server.state.configgroup.ConfigGroup; import org.apache.ambari.server.state.configgroup.ConfigGroupFactory; @@ -56,15 +65,7 @@ import org.apache.commons.lang.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.EnumSet; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.Set; +import com.google.inject.Inject; @StaticallyInject public class ConfigGroupResourceProvider extends @@ -102,6 +103,12 @@ public class ConfigGroupResourceProvider extends private static HostDAO hostDAO; /** + * Used for creating {@link Config} instances to return in the REST response. + */ + @Inject + private static ConfigFactory configFactory; + + /** * Create a new resource provider for the given management controller. * * @param propertyIds the property ids @@ -568,22 +575,19 @@ public class ConfigGroupResourceProvider extends } } + configLogger.info("User {} is creating new configuration group {} for tag {} in cluster {}", + getManagementController().getAuthName(), request.getGroupName(), request.getTag(), + cluster.getClusterName()); + ConfigGroup configGroup = configGroupFactory.createNew(cluster, request.getGroupName(), request.getTag(), request.getDescription(), request.getConfigs(), hosts); - verifyConfigs(configGroup.getConfigurations(), cluster.getClusterName()); configGroup.setServiceName(serviceName); - // Persist before add, since id is auto-generated - configLogger.info("Persisting new Config group" - + ", clusterName = " + cluster.getClusterName() - + ", name = " + configGroup.getName() - + ", tag = " + configGroup.getTag() - + ", user = " + getManagementController().getAuthName()); + verifyConfigs(configGroup.getConfigurations(), cluster.getClusterName()); - configGroup.persist(); cluster.addConfigGroup(configGroup); if (serviceName != null) { cluster.createServiceConfigVersion(serviceName, getManagementController().getAuthName(), @@ -634,6 +638,11 @@ public class ConfigGroupResourceProvider extends + ", clusterName = " + request.getClusterName() + ", groupId = " + request.getId()); } + + configLogger.info("User {} is updating configuration group {} for tag {} in cluster {}", + getManagementController().getAuthName(), request.getGroupName(), request.getTag(), + cluster.getClusterName()); + String serviceName = configGroup.getServiceName(); String requestServiceName = cluster.getServiceForConfigTypes(request.getConfigs().keySet()); if (StringUtils.isEmpty(serviceName) && StringUtils.isEmpty(requestServiceName)) { @@ -682,13 +691,6 @@ public class ConfigGroupResourceProvider extends configGroup.setDescription(request.getDescription()); configGroup.setTag(request.getTag()); - configLogger.info("Persisting updated Config group" - + ", clusterName = " + configGroup.getClusterName() - + ", id = " + configGroup.getId() - + ", tag = " + configGroup.getTag() - + ", user = " + getManagementController().getAuthName()); - - configGroup.persist(); if (serviceName != null) { cluster.createServiceConfigVersion(serviceName, getManagementController().getAuthName(), request.getServiceConfigVersionNote(), configGroup); @@ -781,11 +783,7 @@ public class ConfigGroupResourceProvider extends } } - Config config = new ConfigImpl(type); - config.setTag(tag); - config.setProperties(configProperties); - config.setPropertiesAttributes(configAttributes); - + Config config = configFactory.createReadOnly(type, tag, configProperties, configAttributes); configurations.put(config.getType(), config); } } catch (Exception e) { http://git-wip-us.apache.org/repos/asf/ambari/blob/4278c4a4/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/RequestResourceProvider.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/RequestResourceProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/RequestResourceProvider.java index 8c1bc57..0690ee7 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/RequestResourceProvider.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/RequestResourceProvider.java @@ -69,10 +69,10 @@ import org.apache.ambari.server.state.Cluster; import org.apache.ambari.server.state.Clusters; import org.apache.ambari.server.topology.LogicalRequest; import org.apache.ambari.server.topology.TopologyManager; +import org.apache.commons.lang.StringUtils; import com.google.common.collect.Sets; import com.google.inject.Inject; -import org.apache.commons.lang.StringUtils; /** * Resource provider for request resources. @@ -566,13 +566,18 @@ public class RequestResourceProvider extends AbstractControllerResourceProvider @Override public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } ServiceComponentTuple that = (ServiceComponentTuple) o; - if (serviceName != null ? !serviceName.equals(that.serviceName) : that.serviceName != null) + if (serviceName != null ? !serviceName.equals(that.serviceName) : that.serviceName != null) { return false; + } return !(componentName != null ? !componentName.equals(that.componentName) : that.componentName != null); } @@ -739,15 +744,20 @@ public class RequestResourceProvider extends AbstractControllerResourceProvider // get summaries from TopologyManager for logical requests summary.putAll(topologyManager.getStageSummaries(entity.getRequestId())); + // summary might be empty due to delete host have cleared all + // HostRoleCommands or due to hosts haven't registered yet with the cluster + // when the cluster is provisioned with a Blueprint + final CalculatedStatus status; LogicalRequest logicalRequest = topologyManager.getRequest(entity.getRequestId()); - - CalculatedStatus status = CalculatedStatus.statusFromStageSummary(summary, summary.keySet()); - if (summary.isEmpty() && logicalRequest == null) { - - // summary might be empty due to delete host have cleared all HostRoleCommands - // or due to hosts haven't registered yet with the cluster when the cluster is provisioned - // with a Blueprint - status = CalculatedStatus.getCompletedStatus(); + if (summary.isEmpty() && null != logicalRequest) { + // in this case, it appears that there are no tasks but this is a logical + // topology request, so it's a matter of hosts simply not registering yet + // for tasks to be created + status = CalculatedStatus.PENDING; + } else { + // there are either tasks or this is not a logical request, so do normal + // status calculations + status = CalculatedStatus.statusFromStageSummary(summary, summary.keySet()); } setResourceProperty(resource, REQUEST_STATUS_PROPERTY_ID, status.getStatus().toString(), requestedPropertyIds); http://git-wip-us.apache.org/repos/asf/ambari/blob/4278c4a4/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/StageResourceProvider.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/StageResourceProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/StageResourceProvider.java index 59dd9d9..a778882 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/StageResourceProvider.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/StageResourceProvider.java @@ -339,7 +339,7 @@ public class StageResourceProvider extends AbstractControllerResourceProvider im CalculatedStatus status; if (summary.isEmpty()) { // Delete host might have cleared all HostRoleCommands - status = CalculatedStatus.getCompletedStatus(); + status = CalculatedStatus.COMPLETED; } else { status = CalculatedStatus.statusFromStageSummary(summary, Collections.singleton(entity.getStageId())); } http://git-wip-us.apache.org/repos/asf/ambari/blob/4278c4a4/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeResourceProvider.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeResourceProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeResourceProvider.java index fd38d17..a45ccff 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeResourceProvider.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeResourceProvider.java @@ -956,7 +956,7 @@ public class UpgradeResourceProvider extends AbstractControllerResourceProvider * @throws AmbariException */ @Transactional - private UpgradeEntity createUpgradeInsideTransaction(Cluster cluster, + UpgradeEntity createUpgradeInsideTransaction(Cluster cluster, RequestStageContainer request, UpgradeEntity upgradeEntity) throws AmbariException { http://git-wip-us.apache.org/repos/asf/ambari/blob/4278c4a4/ambari-server/src/main/java/org/apache/ambari/server/controller/logging/LogSearchDataRetrievalService.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/logging/LogSearchDataRetrievalService.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/logging/LogSearchDataRetrievalService.java index 1c135b2..5c0bdb1 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/logging/LogSearchDataRetrievalService.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/logging/LogSearchDataRetrievalService.java @@ -23,9 +23,9 @@ import com.google.common.collect.Sets; import com.google.common.util.concurrent.AbstractService; import com.google.inject.Inject; import org.apache.ambari.server.AmbariService; -import org.apache.ambari.server.configuration.Configuration; import org.apache.ambari.server.controller.AmbariManagementController; import org.apache.ambari.server.controller.AmbariServer; +import org.apache.commons.collections.CollectionUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -64,9 +64,6 @@ public class LogSearchDataRetrievalService extends AbstractService { private static Logger LOG = LoggerFactory.getLogger(LogSearchDataRetrievalService.class); @Inject - private Configuration configuration; - - @Inject private LoggingRequestHelperFactory loggingRequestHelperFactory; /** @@ -288,7 +285,7 @@ public class LogSearchDataRetrievalService extends AbstractService { helper.sendGetLogFileNamesRequest(component, host); // update the cache if result is available - if (logFileNamesResult != null) { + if (CollectionUtils.isNotEmpty(logFileNamesResult)) { LOG.debug("LogSearchFileNameRequestRunnable: request was successful, updating cache"); final String key = generateKey(component, host); // update cache with returned result http://git-wip-us.apache.org/repos/asf/ambari/blob/4278c4a4/ambari-server/src/main/java/org/apache/ambari/server/controller/logging/LoggingCookieStore.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/logging/LoggingCookieStore.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/logging/LoggingCookieStore.java new file mode 100644 index 0000000..a779068 --- /dev/null +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/logging/LoggingCookieStore.java @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.ambari.server.controller.logging; + +import java.util.HashMap; +import java.util.Map; + +/** + * Package protected singleton for storing Cookie key value pairs for Logging Service. + * This has chosen instead of using CookieManager to avoid using system wide Cookie handling + */ +class LoggingCookieStore { + public static final LoggingCookieStore INSTANCE = new LoggingCookieStore(); + + private final Map<String, String> cookiesMap = new HashMap<>(); + + private LoggingCookieStore() { + } + + public Map<String, String> getCookiesMap() { + return cookiesMap; + } + + public void addCookie(String cookieName, String cookieValue) { + cookiesMap.put(cookieName, cookieValue); + } + +} http://git-wip-us.apache.org/repos/asf/ambari/blob/4278c4a4/ambari-server/src/main/java/org/apache/ambari/server/controller/logging/LoggingRequestHelperImpl.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/logging/LoggingRequestHelperImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/logging/LoggingRequestHelperImpl.java index eab0c04..358c1b7 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/logging/LoggingRequestHelperImpl.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/logging/LoggingRequestHelperImpl.java @@ -26,6 +26,7 @@ import org.apache.ambari.server.security.encryption.CredentialStoreService; import org.apache.ambari.server.state.Cluster; import org.apache.ambari.server.state.Config; import org.apache.commons.codec.binary.Base64; +import org.apache.commons.lang.StringUtils; import org.apache.http.client.utils.URIBuilder; import org.apache.log4j.Logger; import org.codehaus.jackson.map.AnnotationIntrospector; @@ -39,11 +40,14 @@ import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; import java.io.StringReader; +import java.net.HttpCookie; import java.net.HttpURLConnection; import java.net.URI; import java.net.URISyntaxException; +import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.Set; import java.util.concurrent.atomic.AtomicInteger; @@ -76,6 +80,10 @@ public class LoggingRequestHelperImpl implements LoggingRequestHelper { private static final String PAGE_SIZE_QUERY_PARAMETER_NAME = "pageSize"; + private static final String COOKIE_HEADER = "Cookie"; + + private static final String SET_COOKIES_HEADER = "Set-Cookie"; + private static final int DEFAULT_LOGSEARCH_CONNECT_TIMEOUT_IN_MILLISECONDS = 5000; private static final int DEFAULT_LOGSEARCH_READ_TIMEOUT_IN_MILLISECONDS = 5000; @@ -109,20 +117,20 @@ public class LoggingRequestHelperImpl implements LoggingRequestHelper { // use the Apache builder to create the correct URI URI logSearchURI = createLogSearchQueryURI("http", queryParameters); LOG.debug("Attempting to connect to LogSearch server at " + logSearchURI); - - HttpURLConnection httpURLConnection = (HttpURLConnection)logSearchURI.toURL().openConnection(); + HttpURLConnection httpURLConnection = (HttpURLConnection) logSearchURI.toURL().openConnection(); httpURLConnection.setRequestMethod("GET"); httpURLConnection.setConnectTimeout(DEFAULT_LOGSEARCH_CONNECT_TIMEOUT_IN_MILLISECONDS); httpURLConnection.setReadTimeout(DEFAULT_LOGSEARCH_READ_TIMEOUT_IN_MILLISECONDS); + addCookiesFromCookieStore(httpURLConnection); setupCredentials(httpURLConnection); StringBuffer buffer = networkConnection.readQueryResponseFromServer(httpURLConnection); + addCookiesToCookieStoreFromResponse(httpURLConnection); // setup a reader for the JSON response - StringReader stringReader = - new StringReader(buffer.toString()); + StringReader stringReader = new StringReader(buffer.toString()); ObjectReader logQueryResponseReader = createObjectReader(LogQueryResponse.class); @@ -137,6 +145,27 @@ public class LoggingRequestHelperImpl implements LoggingRequestHelper { return null; } + private void addCookiesFromCookieStore(HttpURLConnection httpURLConnection) { + if (LoggingCookieStore.INSTANCE.getCookiesMap().size() > 0) { + List<String> cookiesStrList = new ArrayList<>(); + for (Map.Entry<String, String> entry : LoggingCookieStore.INSTANCE.getCookiesMap().entrySet()) { + cookiesStrList.add(String.format("%s=%s", entry.getKey(), entry.getValue())); + } + httpURLConnection.setRequestProperty(COOKIE_HEADER, StringUtils.join(cookiesStrList, "; ")); + } + } + + private void addCookiesToCookieStoreFromResponse(HttpURLConnection httpURLConnection) { + Map<String, List<String>> headerFields = httpURLConnection.getHeaderFields(); + List<String> cookiesHeader = headerFields.get(SET_COOKIES_HEADER); + if (cookiesHeader != null) { + for (String cookie : cookiesHeader) { + HttpCookie cookie1 = HttpCookie.parse(cookie).get(0); + LoggingCookieStore.INSTANCE.addCookie(cookie1.getName(), cookie1.getValue()); + } + } + } + private void setupCredentials(HttpURLConnection httpURLConnection) { final String logSearchAdminUser = @@ -224,10 +253,14 @@ public class LoggingRequestHelperImpl implements LoggingRequestHelper { HttpURLConnection httpURLConnection = (HttpURLConnection) logLevelQueryURI.toURL().openConnection(); httpURLConnection.setRequestMethod("GET"); + addCookiesFromCookieStore(httpURLConnection); + setupCredentials(httpURLConnection); StringBuffer buffer = networkConnection.readQueryResponseFromServer(httpURLConnection); + addCookiesToCookieStoreFromResponse(httpURLConnection); + // setup a reader for the JSON response StringReader stringReader = new StringReader(buffer.toString()); @@ -374,6 +407,7 @@ public class LoggingRequestHelperImpl implements LoggingRequestHelper { BufferedReader reader = new BufferedReader(new InputStreamReader(resultStream)); LOG.debug("Response code from LogSearch Service is = " + httpURLConnection.getResponseCode()); + String line = reader.readLine(); StringBuffer buffer = new StringBuffer(); while (line != null) {