Repository: ambari Updated Branches: refs/heads/trunk 26b093a1a -> b47318902
AMBARI-13035. BE / Hosts Page Performance: alerts_summary takes too long to load (srimanth) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/b4731890 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/b4731890 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/b4731890 Branch: refs/heads/trunk Commit: b4731890272e216ab327656101ec26e051d6b9f8 Parents: 26b093a Author: Srimanth Gunturi <sgunt...@hortonworks.com> Authored: Tue Sep 8 11:21:56 2015 -0700 Committer: Srimanth Gunturi <sgunt...@hortonworks.com> Committed: Wed Sep 9 16:42:33 2015 -0700 ---------------------------------------------------------------------- .../internal/AlertSummaryPropertyProvider.java | 57 ++++++++++++-- .../apache/ambari/server/orm/dao/AlertsDAO.java | 41 ++++++++++ .../server/orm/dao/HostAlertSummaryDTO.java | 63 +++++++++++++++ .../ambari/server/orm/dao/AlertsDAOTest.java | 81 ++++++++++++++++++++ 4 files changed, 237 insertions(+), 5 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/b4731890/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/AlertSummaryPropertyProvider.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/AlertSummaryPropertyProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/AlertSummaryPropertyProvider.java index 2e8cf83..5c6bc85 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/AlertSummaryPropertyProvider.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/AlertSummaryPropertyProvider.java @@ -89,22 +89,60 @@ public class AlertSummaryPropertyProvider extends BaseProvider implements Proper @Override public Set<Resource> populateResources(Set<Resource> resources, Request request, Predicate predicate) throws SystemException { - Set<String> propertyIds = getRequestPropertyIds(request, predicate); try { + // Optimization: + // Some information can be determined more efficiently when requested in bulk + // for an entire cluster at once. + // For Example: + // (1) Cluster level alert-status counts + // (2) Per host alert-status counts + // These can be determined in 1 SQL call per cluster, and results used multiple times. + Map<Long, Map<String, AlertSummaryDTO>> perHostSummaryMap = new HashMap<Long, Map<String, AlertSummaryDTO>>(); + Map<Long, AlertHostSummaryDTO> hostsSummaryMap = new HashMap<Long, AlertHostSummaryDTO>(); + Map<String, Cluster> resourcesClusterMap = new HashMap<String, Cluster>(); + for (Resource res : resources) { + String clusterName = (String) res.getPropertyValue(m_clusterPropertyId); + if (clusterName == null || resourcesClusterMap.containsKey(clusterName)) { + continue; + } + Cluster cluster = s_clusters.get().getCluster(clusterName); + resourcesClusterMap.put(clusterName, cluster); + } + for (Cluster cluster : resourcesClusterMap.values()) { + long clusterId = cluster.getClusterId(); + switch (m_resourceType.getInternalType()) { + case Cluster: + // only make the calculation if asked + if (BaseProvider.isPropertyRequested(ALERTS_SUMMARY_HOSTS, propertyIds)) { + hostsSummaryMap.put(clusterId, s_dao.findCurrentHostCounts(clusterId)); + } + break; + case Host: + if (resources.size() > 1) { + // More efficient to get information for all hosts in 1 call + Map<String, AlertSummaryDTO> perHostCounts = s_dao.findCurrentPerHostCounts(clusterId); + perHostSummaryMap.put(clusterId, perHostCounts); + } + break; + default: + break; + } + } + for (Resource res : resources) { - populateResource(res, propertyIds); + populateResource(res, propertyIds, perHostSummaryMap, hostsSummaryMap); } } catch (AmbariException e) { LOG.error("Could not load built-in alerts - Executor exception ({})", e.getMessage()); } - return resources; } - private void populateResource(Resource resource, Set<String> requestedIds) throws AmbariException { + private void populateResource(Resource resource, Set<String> requestedIds, Map<Long, Map<String, + AlertSummaryDTO>> perHostSummaryMap, Map<Long, AlertHostSummaryDTO> hostsSummaryMap) throws AmbariException { AlertSummaryDTO summary = null; AlertHostSummaryDTO hostSummary = null; @@ -130,7 +168,11 @@ public class AlertSummaryPropertyProvider extends BaseProvider implements Proper // only make the calculation if asked if (BaseProvider.isPropertyRequested(ALERTS_SUMMARY_HOSTS, requestedIds)) { - hostSummary = s_dao.findCurrentHostCounts(clusterId); + if (hostsSummaryMap.containsKey(cluster.getClusterId())) { + hostSummary = hostsSummaryMap.get(cluster.getClusterId()); + } else { + hostSummary = s_dao.findCurrentHostCounts(clusterId); + } } break; @@ -138,7 +180,12 @@ public class AlertSummaryPropertyProvider extends BaseProvider implements Proper summary = s_dao.findCurrentCounts(cluster.getClusterId(), typeId, null); break; case Host: + if (perHostSummaryMap.containsKey(cluster.getClusterId()) && + perHostSummaryMap.get(cluster.getClusterId()).containsKey(typeId)) { + summary = perHostSummaryMap.get(cluster.getClusterId()).get(typeId); + } else { summary = s_dao.findCurrentCounts(cluster.getClusterId(), null, typeId); + } break; default: break; http://git-wip-us.apache.org/repos/asf/ambari/blob/b4731890/ambari-server/src/main/java/org/apache/ambari/server/orm/dao/AlertsDAO.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/orm/dao/AlertsDAO.java b/ambari-server/src/main/java/org/apache/ambari/server/orm/dao/AlertsDAO.java index adcc710..73ca637 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/orm/dao/AlertsDAO.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/orm/dao/AlertsDAO.java @@ -20,6 +20,7 @@ package org.apache.ambari.server.orm.dao; import java.util.Collections; import java.util.Date; import java.util.EnumSet; +import java.util.HashMap; import java.util.List; import java.util.Map; @@ -86,6 +87,15 @@ public class AlertsDAO { + "SUM(CASE WHEN alert.maintenanceState != :maintenanceStateOff THEN 1 ELSE 0 END)) " + "FROM AlertCurrentEntity alert JOIN alert.alertHistory history WHERE history.clusterId = :clusterId"; + private static final String ALERT_COUNT_PER_HOST_SQL_TEMPLATE = "SELECT NEW %s(" + + "history.hostName, " + + "SUM(CASE WHEN history.alertState = :okState AND alert.maintenanceState = :maintenanceStateOff THEN 1 ELSE 0 END), " + + "SUM(CASE WHEN history.alertState = :warningState AND alert.maintenanceState = :maintenanceStateOff THEN 1 ELSE 0 END), " + + "SUM(CASE WHEN history.alertState = :criticalState AND alert.maintenanceState = :maintenanceStateOff THEN 1 ELSE 0 END), " + + "SUM(CASE WHEN history.alertState = :unknownState AND alert.maintenanceState = :maintenanceStateOff THEN 1 ELSE 0 END), " + + "SUM(CASE WHEN alert.maintenanceState != :maintenanceStateOff THEN 1 ELSE 0 END)) " + + "FROM AlertCurrentEntity alert JOIN alert.alertHistory history WHERE history.clusterId = :clusterId GROUP BY history.hostName"; + /** * JPA entity manager */ @@ -445,6 +455,37 @@ public class AlertsDAO { } /** + * Retrieves the summary information for all the hosts in the provided cluster. + * The result is mapping from hostname to summary DTO. + * + * @param clusterId + * the cluster id + * @return map from hostnames to summary DTO + */ + @RequiresSession + public Map<String, AlertSummaryDTO> findCurrentPerHostCounts(long clusterId) { + String sql = String.format(ALERT_COUNT_PER_HOST_SQL_TEMPLATE, HostAlertSummaryDTO.class.getName()); + + StringBuilder sb = new StringBuilder(sql); + + TypedQuery<HostAlertSummaryDTO> query = m_entityManagerProvider.get().createQuery(sb.toString(), HostAlertSummaryDTO.class); + + query.setParameter("clusterId", Long.valueOf(clusterId)); + query.setParameter("okState", AlertState.OK); + query.setParameter("warningState", AlertState.WARNING); + query.setParameter("criticalState", AlertState.CRITICAL); + query.setParameter("unknownState", AlertState.UNKNOWN); + query.setParameter("maintenanceStateOff", MaintenanceState.OFF); + + Map<String, AlertSummaryDTO> map = new HashMap<String, AlertSummaryDTO>(); + List<HostAlertSummaryDTO> resultList = m_daoUtils.selectList(query); + for (HostAlertSummaryDTO result : resultList) { + map.put(result.getHostName(), result); + } + return map; + } + + /** * Retrieve the summary alert information for all hosts. This is different * from {@link #findCurrentCounts(long, String, String)} since this will * return only alerts related to hosts and those values will be the total http://git-wip-us.apache.org/repos/asf/ambari/blob/b4731890/ambari-server/src/main/java/org/apache/ambari/server/orm/dao/HostAlertSummaryDTO.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/orm/dao/HostAlertSummaryDTO.java b/ambari-server/src/main/java/org/apache/ambari/server/orm/dao/HostAlertSummaryDTO.java new file mode 100644 index 0000000..4f9fe66 --- /dev/null +++ b/ambari-server/src/main/java/org/apache/ambari/server/orm/dao/HostAlertSummaryDTO.java @@ -0,0 +1,63 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ambari.server.orm.dao; + +/** + * Used when getting per host alert summary in bulk where a single database call will + * return alert summaries for multiple hosts. In addition to the information + * returned by {@link AlertSummaryDTO}, a host name is provided. + */ +public class HostAlertSummaryDTO extends AlertSummaryDTO { + + private String hostName; + + /** + * Constructor, used by JPA. JPA invokes this constructor, even if there are no + * records in the resultset. In that case, all arguments are {@code null}. + * + * @param hostName + * @param ok + * @param warning + * @param critical + * @param unknown + * @param maintenance + */ + public HostAlertSummaryDTO(String hostName, Number ok, Number warning, Number critical, Number unknown, Number maintenance) { + super(ok, warning, critical, unknown, maintenance); + this.setHostName(hostName); + } + + /** + * Provide host name for this alerts summary + * + * @return + */ + public String getHostName() { + return hostName; + } + + /** + * Set host name for this alerts summary + * + * @param hostName + */ + public void setHostName(String hostName) { + this.hostName = hostName; + } + +} http://git-wip-us.apache.org/repos/asf/ambari/blob/b4731890/ambari-server/src/test/java/org/apache/ambari/server/orm/dao/AlertsDAOTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/orm/dao/AlertsDAOTest.java b/ambari-server/src/test/java/org/apache/ambari/server/orm/dao/AlertsDAOTest.java index f267544..0bbe998 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/orm/dao/AlertsDAOTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/orm/dao/AlertsDAOTest.java @@ -29,6 +29,7 @@ import java.util.Calendar; import java.util.Collections; import java.util.Date; import java.util.List; +import java.util.Map; import java.util.TimeZone; import java.util.UUID; @@ -653,6 +654,86 @@ public class AlertsDAOTest { * */ @Test + public void testFindCurrentPerHostSummary() throws Exception { + // Add extra host and alerts + m_helper.addHost(m_clusters, m_cluster, "h2"); + List<AlertDefinitionEntity> definitions = m_definitionDao.findAll(); + AlertDefinitionEntity definition = definitions.get(0); + AlertHistoryEntity h2CriticalHistory = new AlertHistoryEntity(); + h2CriticalHistory.setServiceName(definition.getServiceName()); + h2CriticalHistory.setClusterId(m_cluster.getClusterId()); + h2CriticalHistory.setAlertDefinition(definition); + h2CriticalHistory.setAlertLabel(definition.getDefinitionName() + " h2"); + h2CriticalHistory.setAlertText(definition.getDefinitionName() + " h2"); + h2CriticalHistory.setAlertTimestamp(calendar.getTimeInMillis()); + h2CriticalHistory.setComponentName(definition.getComponentName()); + h2CriticalHistory.setHostName("h2"); + h2CriticalHistory.setAlertState(AlertState.CRITICAL); + m_dao.create(h2CriticalHistory); + AlertCurrentEntity h2CriticalCurrent = new AlertCurrentEntity(); + h2CriticalCurrent.setAlertHistory(h2CriticalHistory); + h2CriticalCurrent.setLatestTimestamp(new Date().getTime()); + h2CriticalCurrent.setOriginalTimestamp(new Date().getTime() - 10800000); + h2CriticalCurrent.setMaintenanceState(MaintenanceState.OFF); + m_dao.create(h2CriticalCurrent); + + try { + long clusterId = m_cluster.getClusterId(); + AlertSummaryDTO summary = m_dao.findCurrentCounts(clusterId, null, null); + assertEquals(5, summary.getOkCount()); + + AlertHistoryEntity h1 = m_dao.findCurrentByCluster(clusterId).get(2).getAlertHistory(); + AlertHistoryEntity h2 = m_dao.findCurrentByCluster(clusterId).get(3).getAlertHistory(); + AlertHistoryEntity h3 = m_dao.findCurrentByCluster(clusterId).get(4).getAlertHistory(); + + h1.setAlertState(AlertState.WARNING); + m_dao.merge(h1); + h2.setAlertState(AlertState.CRITICAL); + m_dao.merge(h2); + h3.setAlertState(AlertState.UNKNOWN); + m_dao.merge(h3); + + Map<String, AlertSummaryDTO> perHostSummary = m_dao.findCurrentPerHostCounts(clusterId); + + AlertSummaryDTO h1summary = m_dao.findCurrentCounts(clusterId, null, "h1"); + assertEquals(2, h1summary.getOkCount()); + assertEquals(1, h1summary.getWarningCount()); + assertEquals(1, h1summary.getCriticalCount()); + assertEquals(1, h1summary.getUnknownCount()); + assertEquals(0, h1summary.getMaintenanceCount()); + + AlertSummaryDTO h2summary = m_dao.findCurrentCounts(clusterId, null, "h2"); + assertEquals(0, h2summary.getOkCount()); + assertEquals(0, h2summary.getWarningCount()); + assertEquals(1, h2summary.getCriticalCount()); + assertEquals(0, h2summary.getUnknownCount()); + assertEquals(0, h2summary.getMaintenanceCount()); + + AlertSummaryDTO h1PerHostSummary = perHostSummary.get("h1"); + assertEquals(h1PerHostSummary.getOkCount(), h1summary.getOkCount()); + assertEquals(h1PerHostSummary.getWarningCount(), h1summary.getWarningCount()); + assertEquals(h1PerHostSummary.getCriticalCount(), h1summary.getCriticalCount()); + assertEquals(h1PerHostSummary.getUnknownCount(), h1summary.getUnknownCount()); + assertEquals(h1PerHostSummary.getMaintenanceCount(), h1summary.getMaintenanceCount()); + + AlertSummaryDTO h2PerHostSummary = perHostSummary.get("h2"); + assertEquals(h2PerHostSummary.getOkCount(), h2summary.getOkCount()); + assertEquals(h2PerHostSummary.getWarningCount(), h2summary.getWarningCount()); + assertEquals(h2PerHostSummary.getCriticalCount(), h2summary.getCriticalCount()); + assertEquals(h2PerHostSummary.getUnknownCount(), h2summary.getUnknownCount()); + assertEquals(h2PerHostSummary.getMaintenanceCount(), h2summary.getMaintenanceCount()); + } finally { + // Cleanup extra host and alerts to not effect other tests + m_dao.remove(h2CriticalCurrent); + m_dao.remove(h2CriticalHistory); + m_clusters.unmapHostFromCluster("h2", m_cluster.getClusterName()); + } + } + + /** + * + */ + @Test public void testFindCurrentHostSummary() throws Exception { // start out with 1 since all alerts are for a single host and are OK AlertHostSummaryDTO summary = m_dao.findCurrentHostCounts(m_cluster.getClusterId());
