Merge branch 'trunk' into branch-feature-AMBARI-18456
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/276d1244 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/276d1244 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/276d1244 Branch: refs/heads/branch-feature-AMBARI-18456 Commit: 276d1244e1b1d8200cf644b7cccfaa0a757f8146 Parents: 1f804d1 85c9104 Author: Jonathan Hurley <jhur...@hortonworks.com> Authored: Tue Nov 29 16:51:58 2016 -0500 Committer: Jonathan Hurley <jhur...@hortonworks.com> Committed: Tue Nov 29 17:00:59 2016 -0500 ---------------------------------------------------------------------- .../views/stackVersions/stackVersionPage.html | 2 +- .../org/apache/ambari/logsearch/LogSearch.java | 15 +- .../logsearch/common/ExternalServerClient.java | 12 +- .../logsearch/common/LogSearchConstants.java | 1 + .../AbstractOperationHolderConverter.java | 9 + ...actServiceLogRequestFacetQueryConverter.java | 44 ++ .../BaseServiceLogRequestQueryConverter.java | 1 + ...ServiceLogAnyGraphRequestQueryConverter.java | 13 + ...eLogComponentLevelRequestQueryConverter.java | 15 +- ...eLogComponentRequestFacetQueryConverter.java | 15 +- ...rviceLogLevelCountRequestQueryConverter.java | 16 + ...eLogLevelDateRangeRequestQueryConverter.java | 16 +- ...erviceLogTreeRequestFacetQueryConverter.java | 17 +- .../ambari/logsearch/doc/DocConstants.java | 1 + .../logsearch/graph/GraphDataGenerator.java | 26 +- .../logsearch/manager/ServiceLogsManager.java | 2 +- .../request/ServiceLogParamDefinition.java | 6 + .../request/impl/BaseServiceLogRequest.java | 13 + .../LogsearchKRBAuthenticationFilter.java | 4 +- .../src/main/resources/logsearch.properties | 2 +- .../dashboard/BubbleGraphTableLayoutView.js | 4 +- .../views/dashboard/ComponentListView.js | 2 +- .../scripts/views/dashboard/HostListView.js | 14 +- .../main/webapp/scripts/views/tabs/TreeView.js | 10 +- .../webapp/templates/tabs/TreeView_tmpl.html | 115 ++-- ...ComponentLevelRequestQueryConverterTest.java | 7 +- ...ComponentRequestFacetQueryConverterTest.java | 4 +- .../conf/unix/ambari-metrics-grafana | 34 +- .../ambari/server/checks/CheckDescription.java | 13 + .../server/checks/ServicePresenceCheck.java | 177 ++++++ .../internal/ClusterControllerImpl.java | 2 +- .../ServiceConfigVersionResourceProvider.java | 3 +- .../logging/LoggingSearchPropertyProvider.java | 11 +- .../ambari/server/orm/dao/ClusterDAO.java | 24 + .../ambari/server/orm/dao/ServiceConfigDAO.java | 13 +- .../orm/entities/ClusterConfigEntity.java | 7 +- .../entities/ClusterConfigMappingEntity.java | 6 + .../orm/entities/ServiceConfigEntity.java | 5 +- .../server/orm/helpers/dbms/MySqlHelper.java | 18 + .../upgrades/RangerKmsProxyConfig.java | 94 +++ .../server/state/cluster/ClusterImpl.java | 42 +- .../server/upgrade/SchemaUpgradeHelper.java | 35 +- .../main/resources/Ambari-DDL-MySQL-CREATE.sql | 1 + .../FLUME/1.4.0.2.0/package/scripts/flume.py | 1 + .../package/files/hbaseSmokeVerify.sh | 2 +- .../HDFS/2.1.0.2.0/kerberos.json | 2 +- .../HIVE/0.12.0.2.0/configuration/hive-env.xml | 19 + .../HIVE/0.12.0.2.0/package/scripts/webhcat.py | 10 - .../configuration/logfeeder-ambari-config.xml | 37 ++ .../configuration/logfeeder-output-config.xml | 37 ++ .../configuration/logsearch-properties.xml | 40 ++ .../LOGSEARCH/0.5.0/metainfo.xml | 5 + .../LOGSEARCH/0.5.0/package/scripts/params.py | 17 +- .../0.5.0/package/scripts/setup_logfeeder.py | 10 + .../templates/input.config-ambari.json.j2 | 602 ------------------- .../package/templates/output.config.json.j2 | 61 -- .../properties/input.config-ambari.json.j2 | 602 +++++++++++++++++++ .../0.5.0/properties/output.config.json.j2 | 61 ++ .../OOZIE/4.0.0.2.0/configuration/oozie-env.xml | 19 + .../0.6.0.2.5/package/scripts/master.py | 15 +- .../0.6.0.2.5/package/scripts/params.py | 28 +- .../ZEPPELIN/0.6.0.2.5/role_command_order.json | 2 +- .../HDP/2.0.6/configuration/cluster-env.xml | 6 + .../HDP/2.2/services/HIVE/themes/theme.json | 107 +++- .../services/HIVE/configuration/hive-env.xml | 35 ++ .../services/OOZIE/configuration/oozie-env.xml | 33 + .../stacks/HDP/2.3/services/OOZIE/metainfo.xml | 6 + .../HDP/2.3/services/OOZIE/themes/theme.json | 116 ++++ .../HDP/2.3/upgrades/nonrolling-upgrade-2.5.xml | 7 + .../HDP/2.3/upgrades/nonrolling-upgrade-2.6.xml | 7 + .../stacks/HDP/2.3/upgrades/upgrade-2.5.xml | 11 + .../stacks/HDP/2.3/upgrades/upgrade-2.6.xml | 11 + .../HDP/2.4/upgrades/nonrolling-upgrade-2.5.xml | 7 + .../HDP/2.4/upgrades/nonrolling-upgrade-2.6.xml | 7 + .../stacks/HDP/2.4/upgrades/upgrade-2.5.xml | 11 + .../stacks/HDP/2.4/upgrades/upgrade-2.6.xml | 11 + .../stacks/HDP/2.5/services/HDFS/kerberos.json | 2 +- .../stacks/HDP/2.5/services/OOZIE/metainfo.xml | 6 + .../HDP/2.5/services/OOZIE/themes/theme.json | 116 ++++ .../HDP/2.5/upgrades/host-ordered-upgrade.xml | 11 +- .../server/checks/ServicePresenceCheckTest.java | 217 +++++++ .../upgrades/RangerKmsProxyConfigTest.java | 125 ++++ .../stacks/2.0.6/HIVE/test_webhcat_server.py | 4 - .../stacks/2.4/LOGSEARCH/test_logfeeder.py | 14 +- .../test/python/stacks/2.4/configs/default.json | 6 + ambari-web/app/app.js | 4 - .../main/admin/service_auto_start.js | 2 +- .../controllers/main/service/add_controller.js | 11 +- .../data/configs/services/hive_properties.js | 4 - .../configs/services/logsearch_properties.js | 51 +- .../data/configs/services/oozie_properties.js | 4 - ambari-web/app/messages.js | 3 + ambari-web/app/styles/application.less | 10 +- ambari-web/app/styles/bootstrap_overrides.less | 9 + .../app/styles/enhanced_service_dashboard.less | 343 ++++------- ambari-web/app/styles/hosts.less | 3 - .../app/styles/theme/bootstrap-ambari.css | 5 +- .../modal_popups/widget_browser_popup.hbs | 108 ++-- .../templates/main/admin/service_auto_start.hbs | 2 +- .../main/service/widgets/create/expression.hbs | 36 +- .../main/service/widgets/create/step1.hbs | 18 +- .../main/service/widgets/create/step2.hbs | 22 +- .../service/widgets/create/step2_add_metric.hbs | 12 +- .../main/service/widgets/create/step2_graph.hbs | 4 +- .../service/widgets/create/step2_number.hbs | 4 +- .../service/widgets/create/step2_template.hbs | 4 +- .../main/service/widgets/create/step3.hbs | 42 +- .../create/widget_property_threshold.hbs | 18 +- ambari-web/app/views/common/chosen_plugin.js | 21 +- ambari-web/app/views/common/controls_view.js | 2 +- ambari-web/app/views/main/dashboard/widgets.js | 17 +- .../main/service/widgets/create/step2_view.js | 2 + .../main/service/add_controller_test.js | 29 +- .../test/views/main/dashboard/widgets_test.js | 11 +- contrib/utils/perf/deploy-gce-perf-cluster.py | 243 +++++--- docs/pom.xml | 4 +- 116 files changed, 2957 insertions(+), 1408 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/276d1244/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/RangerKmsProxyConfig.java ---------------------------------------------------------------------- diff --cc ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/RangerKmsProxyConfig.java index 0000000,bb88f55..25387cc mode 000000,100644..100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/RangerKmsProxyConfig.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/RangerKmsProxyConfig.java @@@ -1,0 -1,95 +1,94 @@@ + /** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.apache.ambari.server.serveraction.upgrades; + + import java.text.MessageFormat; + import java.util.Map; + import java.util.concurrent.ConcurrentMap; + + import org.apache.ambari.server.AmbariException; + import org.apache.ambari.server.actionmanager.HostRoleStatus; + import org.apache.ambari.server.agent.CommandReport; + import org.apache.ambari.server.serveraction.AbstractServerAction; + import org.apache.ambari.server.state.Cluster; + import org.apache.ambari.server.state.Clusters; + import org.apache.ambari.server.state.Config; + import org.apache.ambari.server.state.SecurityType; -import org.apache.commons.lang.StringUtils; + + import com.google.inject.Inject; + + /** + * Computes Ranger KMS Proxy properties in kms-site + */ + + public class RangerKmsProxyConfig extends AbstractServerAction { + private static final String RANGER_ENV_CONFIG_TYPE = "ranger-env"; + private static final String RANGER_KMS_SITE_CONFIG_TYPE = "kms-site"; + + @Inject + private Clusters m_clusters; + + @Override + public CommandReport execute(ConcurrentMap<String, Object> requestSharedDataContext) + throws AmbariException, InterruptedException { + + String clusterName = getExecutionCommand().getClusterName(); + Cluster cluster = m_clusters.getCluster(clusterName); + String outputMsg = ""; + + Config rangerEnv = cluster.getDesiredConfigByType(RANGER_ENV_CONFIG_TYPE); + + if (null == rangerEnv) { + return createCommandReport(0, HostRoleStatus.COMPLETED, "{}", + MessageFormat.format("Config source type {0} not found, skipping adding properties to {1}.", RANGER_ENV_CONFIG_TYPE, RANGER_KMS_SITE_CONFIG_TYPE), ""); + } + + String rangerUserProp = "ranger_user"; + String rangerUser = rangerEnv.getProperties().get(rangerUserProp); + + if (null == rangerUser) { + return createCommandReport(0, HostRoleStatus.COMPLETED, "{}", + MessageFormat.format("Required user service user value from {0}/{1} not found, skipping adding properties to {2}.", RANGER_ENV_CONFIG_TYPE, rangerUserProp, RANGER_KMS_SITE_CONFIG_TYPE), ""); + } + + Config kmsSite = cluster.getDesiredConfigByType(RANGER_KMS_SITE_CONFIG_TYPE); + + if (null == kmsSite) { + return createCommandReport(0, HostRoleStatus.COMPLETED, "{}", + MessageFormat.format("Config type {0} not found, skipping adding properties to it.", RANGER_KMS_SITE_CONFIG_TYPE), ""); + } + + Map<String, String> targetValues = kmsSite.getProperties(); + if (cluster.getSecurityType() == SecurityType.KERBEROS) { + String userProp = "hadoop.kms.proxyuser." + rangerUser + ".users"; + String groupProp = "hadoop.kms.proxyuser." + rangerUser + ".groups"; + String hostProp = "hadoop.kms.proxyuser." + rangerUser + ".hosts"; + targetValues.put(userProp, "*"); + targetValues.put(groupProp, "*"); + targetValues.put(hostProp, "*"); + kmsSite.setProperties(targetValues); - kmsSite.persist(false); ++ kmsSite.save(); + outputMsg = outputMsg + MessageFormat.format("Successfully added properties to {0}", RANGER_KMS_SITE_CONFIG_TYPE); + } else { + outputMsg = outputMsg + MessageFormat.format("Kerberos not enable, not setting proxy properties to {0}", RANGER_KMS_SITE_CONFIG_TYPE); + } + + return createCommandReport(0, HostRoleStatus.COMPLETED, "{}", outputMsg, ""); + + } + } http://git-wip-us.apache.org/repos/asf/ambari/blob/276d1244/ambari-server/src/main/java/org/apache/ambari/server/state/cluster/ClusterImpl.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/276d1244/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/RangerKmsProxyConfigTest.java ---------------------------------------------------------------------- diff --cc ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/RangerKmsProxyConfigTest.java index 0000000,e000c65..7a0d66f mode 000000,100644..100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/RangerKmsProxyConfigTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/RangerKmsProxyConfigTest.java @@@ -1,0 -1,141 +1,125 @@@ + /** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.apache.ambari.server.serveraction.upgrades; + + import static org.easymock.EasyMock.anyObject; + import static org.easymock.EasyMock.expect; + import static org.easymock.EasyMock.replay; + import static org.junit.Assert.assertEquals; + import static org.junit.Assert.assertNotNull; + import static org.junit.Assert.assertTrue; + + import java.lang.reflect.Field; + import java.util.HashMap; + import java.util.Map; + + import org.apache.ambari.server.actionmanager.ExecutionCommandWrapper; + import org.apache.ambari.server.actionmanager.HostRoleCommand; + import org.apache.ambari.server.agent.CommandReport; + import org.apache.ambari.server.agent.ExecutionCommand; + import org.apache.ambari.server.state.Cluster; + import org.apache.ambari.server.state.Clusters; -import org.apache.ambari.server.state.SecurityType; + import org.apache.ambari.server.state.Config; -import org.apache.ambari.server.state.ConfigImpl; ++import org.apache.ambari.server.state.SecurityType; + import org.easymock.EasyMock; + import org.junit.Before; + import org.junit.Test; + + import com.google.inject.Injector; + + + public class RangerKmsProxyConfigTest { + private Injector m_injector; + private Clusters m_clusters; + private Field m_clusterField; + + @Before + public void setup() throws Exception { + m_injector = EasyMock.createMock(Injector.class); + m_clusters = EasyMock.createMock(Clusters.class); + Cluster cluster = EasyMock.createMock(Cluster.class); + - Config rangerEnv = new ConfigImpl("ranger-env") { - Map<String, String> mockProperties = new HashMap<String, String>() {{ ++ Map<String, String> mockProperties = new HashMap<String, String>() { ++ { + put("ranger_user", "ranger"); - }}; - - @Override - public Map<String, String> getProperties() { - return mockProperties; + } + }; + - Config kmsSite = new ConfigImpl("kms-site") { - Map<String, String> mockProperties = new HashMap<String, String>(); - @Override - public Map<String, String> getProperties() { - return mockProperties; - } - - @Override - public void setProperties(Map<String, String> properties) { - mockProperties.putAll(properties); - } ++ Config rangerEnv = EasyMock.createNiceMock(Config.class); ++ expect(rangerEnv.getType()).andReturn("ranger-env").anyTimes(); ++ expect(rangerEnv.getProperties()).andReturn(mockProperties).anyTimes(); + - @Override - public void persist(boolean newConfig) { - // no-op - } - }; ++ Config kmsSite = EasyMock.createNiceMock(Config.class); ++ expect(kmsSite.getType()).andReturn("kms-site").anyTimes(); ++ expect(kmsSite.getProperties()).andReturn(mockProperties).anyTimes(); + + expect(cluster.getDesiredConfigByType("ranger-env")).andReturn(rangerEnv).atLeastOnce(); + expect(cluster.getDesiredConfigByType("kms-site")).andReturn(kmsSite).atLeastOnce(); + expect(m_clusters.getCluster((String) anyObject())).andReturn(cluster).anyTimes(); + expect(m_injector.getInstance(Clusters.class)).andReturn(m_clusters).atLeastOnce(); + expect(cluster.getSecurityType()).andReturn(SecurityType.KERBEROS).anyTimes(); + - replay(m_injector, m_clusters, cluster); ++ replay(m_injector, m_clusters, cluster, rangerEnv, kmsSite); + + m_clusterField = RangerKmsProxyConfig.class.getDeclaredField("m_clusters"); + m_clusterField.setAccessible(true); + } + + @Test + public void testAction() throws Exception { + + Map<String, String> commandParams = new HashMap<String, String>(); + commandParams.put("clusterName", "c1"); + + ExecutionCommand executionCommand = new ExecutionCommand(); + executionCommand.setCommandParams(commandParams); + executionCommand.setClusterName("c1"); + + HostRoleCommand hrc = EasyMock.createMock(HostRoleCommand.class); + expect(hrc.getRequestId()).andReturn(1L).anyTimes(); + expect(hrc.getStageId()).andReturn(2L).anyTimes(); + expect(hrc.getExecutionCommandWrapper()).andReturn(new ExecutionCommandWrapper(executionCommand)).anyTimes(); + replay(hrc); + + RangerKmsProxyConfig action = new RangerKmsProxyConfig(); + m_clusterField.set(action, m_clusters); + + action.setExecutionCommand(executionCommand); + action.setHostRoleCommand(hrc); + + CommandReport report = action.execute(null); + assertNotNull(report); + + Cluster c = m_clusters.getCluster("c1"); + Config config = c.getDesiredConfigByType("kms-site"); + Map<String, String> map = config.getProperties(); + + assertTrue(map.containsKey("hadoop.kms.proxyuser.ranger.users")); + assertTrue(map.containsKey("hadoop.kms.proxyuser.ranger.groups")); + assertTrue(map.containsKey("hadoop.kms.proxyuser.ranger.hosts")); + + + assertEquals("*", map.get("hadoop.kms.proxyuser.ranger.users")); + assertEquals("*", map.get("hadoop.kms.proxyuser.ranger.groups")); + assertEquals("*", map.get("hadoop.kms.proxyuser.ranger.hosts")); + + report = action.execute(null); + assertNotNull(report); + + } + }