Repository: ambari Updated Branches: refs/heads/branch-2.2 7228c76fb -> b1149fa7e
AMBARI-14251. Blueprints fix stale configs and kerberos-env stack defaults are not being applied during cluster deployment. (Sandor Magyari via rnettleton) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/b1149fa7 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/b1149fa7 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/b1149fa7 Branch: refs/heads/branch-2.2 Commit: b1149fa7e827c94ab929a25b9b7f9bc6165b8605 Parents: 7228c76 Author: Bob Nettleton <rnettle...@hortonworks.com> Authored: Tue Dec 8 17:51:23 2015 -0500 Committer: Bob Nettleton <rnettle...@hortonworks.com> Committed: Tue Dec 8 17:51:23 2015 -0500 ---------------------------------------------------------------------- .../controller/AmbariManagementController.java | 6 + .../AmbariManagementControllerImpl.java | 5 + .../topology/ClusterConfigurationRequest.java | 38 ++++ .../ambari/server/topology/TopologyManager.java | 24 ++- .../ClusterConfigurationRequestTest.java | 172 +++++++++++++++++++ 5 files changed, 236 insertions(+), 9 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/b1149fa7/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementController.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementController.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementController.java index ea7603f..2d11265 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementController.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementController.java @@ -772,5 +772,11 @@ public interface AmbariManagementController { Set<StackConfigurationDependencyResponse> getStackConfigurationDependencies(Set<StackConfigurationDependencyRequest> requests) throws AmbariException; TimelineMetricCacheProvider getTimelineMetricCacheProvider(); + + /** + * Returns KerberosHelper instance + * @return + */ + KerberosHelper getKerberosHelper(); } http://git-wip-us.apache.org/repos/asf/ambari/blob/b1149fa7/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java index d0cbd20..3dae467 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java @@ -4395,6 +4395,11 @@ public class AmbariManagementControllerImpl implements AmbariManagementControlle return injector.getInstance(TimelineMetricCacheProvider.class); } + @Override + public KerberosHelper getKerberosHelper() { + return kerberosHelper; + } + /** * Queries the CredentialStoreService to gather properties about it. * <p/> http://git-wip-us.apache.org/repos/asf/ambari/blob/b1149fa7/ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterConfigurationRequest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterConfigurationRequest.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterConfigurationRequest.java index 1a650ee..6e8b8a3 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterConfigurationRequest.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterConfigurationRequest.java @@ -26,6 +26,8 @@ import org.apache.ambari.server.controller.internal.BlueprintConfigurationProces import org.apache.ambari.server.controller.internal.ClusterResourceProvider; import org.apache.ambari.server.controller.internal.ConfigurationTopologyException; import org.apache.ambari.server.controller.internal.Stack; +import org.apache.ambari.server.serveraction.kerberos.KerberosInvalidConfigurationException; +import org.apache.ambari.server.state.Cluster; import org.apache.ambari.server.state.SecurityType; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -33,6 +35,7 @@ import org.slf4j.LoggerFactory; import java.util.Collection; import java.util.Collections; import java.util.HashMap; +import java.util.HashSet; import java.util.LinkedList; import java.util.List; import java.util.Map; @@ -50,6 +53,7 @@ public class ClusterConfigurationRequest { private BlueprintConfigurationProcessor configurationProcessor; private StackAdvisorBlueprintProcessor stackAdvisorBlueprintProcessor; private Stack stack; + private boolean configureSecurity = false; public ClusterConfigurationRequest(AmbariContext ambariContext, ClusterTopology clusterTopology, boolean setInitial, StackAdvisorBlueprintProcessor stackAdvisorBlueprintProcessor) { @@ -65,6 +69,11 @@ public class ClusterConfigurationRequest { } } + public ClusterConfigurationRequest(AmbariContext ambariContext, ClusterTopology topology, boolean setInitial, StackAdvisorBlueprintProcessor stackAdvisorBlueprintProcessor, boolean configureSecurity) { + this(ambariContext, topology, setInitial, stackAdvisorBlueprintProcessor); + this.configureSecurity = configureSecurity; + } + // get names of required host groups public Collection<String> getRequiredHostGroups() { return configurationProcessor.getRequiredHostGroups(); @@ -79,6 +88,11 @@ public class ClusterConfigurationRequest { if (!ConfigRecommendationStrategy.NEVER_APPLY.equals(this.clusterTopology.getConfigRecommendationStrategy())) { stackAdvisorBlueprintProcessor.adviseConfiguration(this.clusterTopology); } + + if (configureSecurity) { + configureKerberos(); + } + updatedConfigTypes = configurationProcessor.doUpdateForClusterCreate(); } catch (ConfigurationTopologyException e) { @@ -89,6 +103,30 @@ public class ClusterConfigurationRequest { setConfigurationsOnCluster(clusterTopology, TopologyManager.TOPOLOGY_RESOLVED_TAG, updatedConfigTypes); } + private void configureKerberos() throws AmbariException { + String clusterName = ambariContext.getClusterName(clusterTopology.getClusterId()); + Cluster cluster = AmbariContext.getController().getClusters().getCluster(clusterName); + Blueprint blueprint = clusterTopology.getBlueprint(); + Configuration clusterConfiguration = clusterTopology.getConfiguration(); + + try { + Map<String, Map<String, String>> updatedConfigs = AmbariContext.getController().getKerberosHelper() + .getServiceConfigurationUpdates(cluster, clusterConfiguration.getFullProperties(), + new HashSet<String>(blueprint.getServices())); + for (String configType : updatedConfigs.keySet()) { + Map<String, String> propertyMap = updatedConfigs.get(configType); + for (String property : propertyMap.keySet()) { + LOG.debug("Update Kerberos related config property: {} {} {}", configType, property, propertyMap.get + (property)); + clusterConfiguration.setProperty(configType, property, propertyMap.get(property)); + } + } + + } catch (KerberosInvalidConfigurationException e) { + LOG.error("An exception occurred while doing Kerberos related configuration update: " + e, e); + } + } + /** * Set all configurations on the cluster resource. * @param clusterTopology cluster topology http://git-wip-us.apache.org/repos/asf/ambari/blob/b1149fa7/ambari-server/src/main/java/org/apache/ambari/server/topology/TopologyManager.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/TopologyManager.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/TopologyManager.java index 960915f..4b45806 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/topology/TopologyManager.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/TopologyManager.java @@ -18,16 +18,20 @@ package org.apache.ambari.server.topology; +import com.google.inject.Injector; import com.google.inject.Singleton; import org.apache.ambari.server.AmbariException; import org.apache.ambari.server.actionmanager.HostRoleCommand; import org.apache.ambari.server.actionmanager.Request; import org.apache.ambari.server.api.services.stackadvisor.StackAdvisorBlueprintProcessor; +import org.apache.ambari.server.controller.ClusterRequest; +import org.apache.ambari.server.controller.KerberosHelper; import org.apache.ambari.server.controller.RequestStatusResponse; import org.apache.ambari.server.controller.internal.ArtifactResourceProvider; import org.apache.ambari.server.controller.internal.CredentialResourceProvider; import org.apache.ambari.server.controller.internal.ProvisionClusterRequest; import org.apache.ambari.server.controller.internal.RequestImpl; +import org.apache.ambari.server.controller.internal.RequestStageContainer; import org.apache.ambari.server.controller.internal.ScaleClusterRequest; import org.apache.ambari.server.controller.internal.Stack; import org.apache.ambari.server.controller.spi.NoSuchParentResourceException; @@ -40,6 +44,8 @@ import org.apache.ambari.server.controller.spi.UnsupportedPropertyException; import org.apache.ambari.server.orm.dao.HostRoleCommandStatusSummaryDTO; import org.apache.ambari.server.orm.entities.StageEntity; import org.apache.ambari.server.security.encryption.CredentialStoreService; +import org.apache.ambari.server.serveraction.kerberos.KerberosOperationException; +import org.apache.ambari.server.state.Cluster; import org.apache.ambari.server.state.SecurityType; import org.apache.ambari.server.state.host.HostImpl; import org.apache.ambari.server.utils.RetryHelper; @@ -99,10 +105,6 @@ public class TopologyManager { @Inject private SecurityConfigurationFactory securityConfigurationFactory; - @Inject - private CredentialStoreService credentialStoreService; - - /** * A boolean not cached thread-local (volatile) to prevent double-checked * locking on the synchronized keyword. @@ -141,14 +143,20 @@ public class TopologyManager { // get the id prior to creating ambari resources which increments the counter Long provisionId = ambariContext.getNextRequestId(); + final Stack stack = topology.getBlueprint().getStack(); + boolean configureSecurity = false; SecurityConfiguration securityConfiguration = processSecurityConfiguration(request); if (securityConfiguration != null && securityConfiguration.getType() == SecurityType.KERBEROS) { - + configureSecurity = true; addKerberosClient(topology); + // refresh default stack config after adding KERBEROS_CLIENT component to topology + topology.getBlueprint().getConfiguration().setParentConfiguration(stack.getConfiguration(topology.getBlueprint + ().getServices())); + // create Cluster resource with security_type = KERBEROS, this will trigger cluster Kerberization // upon host install task execution - ambariContext.createAmbariResources(topology, clusterName, securityConfiguration.getType()); + ambariContext.createAmbariResources(topology, clusterName, SecurityType.KERBEROS); if (securityConfiguration.getDescriptor() != null) { submitKerberosDescriptorAsArtifact(clusterName, securityConfiguration.getDescriptor()); } @@ -177,10 +185,8 @@ public class TopologyManager { clusterTopologyMap.put(clusterId, topology); - final Stack stack = topology.getBlueprint().getStack(); - addClusterConfigRequest(topology, new ClusterConfigurationRequest( - ambariContext, topology, true, stackAdvisorBlueprintProcessor)); + ambariContext, topology, true, stackAdvisorBlueprintProcessor, configureSecurity)); LogicalRequest logicalRequest = processRequest(persistedRequest, topology, provisionId); //todo: this should be invoked as part of a generic lifecycle event which could possibly http://git-wip-us.apache.org/repos/asf/ambari/blob/b1149fa7/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterConfigurationRequestTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterConfigurationRequestTest.java b/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterConfigurationRequestTest.java new file mode 100644 index 0000000..df32684 --- /dev/null +++ b/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterConfigurationRequestTest.java @@ -0,0 +1,172 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ambari.server.topology; + +import java.lang.reflect.Field; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Future; + +import org.apache.ambari.server.api.services.stackadvisor.StackAdvisorBlueprintProcessor; +import org.apache.ambari.server.controller.AmbariManagementController; +import org.apache.ambari.server.controller.ClusterRequest; +import org.apache.ambari.server.controller.ConfigurationRequest; +import org.apache.ambari.server.controller.KerberosHelper; +import org.apache.ambari.server.controller.RequestStatusResponse; +import org.apache.ambari.server.controller.internal.ProvisionClusterRequest; +import org.apache.ambari.server.controller.internal.Stack; +import org.apache.ambari.server.controller.spi.ClusterController; +import org.apache.ambari.server.controller.spi.Resource; +import org.apache.ambari.server.controller.spi.ResourceProvider; +import org.apache.ambari.server.security.encryption.CredentialStoreService; +import org.apache.ambari.server.state.Cluster; +import org.apache.ambari.server.state.Clusters; +import org.apache.ambari.server.state.SecurityType; +import org.easymock.Capture; +import org.easymock.EasyMock; +import org.easymock.EasyMockRule; +import org.easymock.EasyMockSupport; +import org.easymock.Mock; +import org.easymock.MockType; +import org.easymock.TestSubject; +import org.junit.After; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.powermock.api.easymock.PowerMock; +import org.powermock.core.classloader.annotations.PrepareForTest; +import org.powermock.modules.junit4.PowerMockRunner; + +import static org.easymock.EasyMock.anyObject; +import static org.easymock.EasyMock.anyString; +import static org.easymock.EasyMock.capture; +import static org.easymock.EasyMock.eq; +import static org.easymock.EasyMock.expect; +import static org.easymock.EasyMock.expectLastCall; +import static org.easymock.EasyMock.isA; +import static org.easymock.EasyMock.isNull; +import static org.easymock.EasyMock.newCapture; +import static org.easymock.EasyMock.replay; +import static org.easymock.EasyMock.reset; +import static org.easymock.EasyMock.verify; + +/** + * ClusterConfigurationRequest unit tests + */ + +@RunWith(PowerMockRunner.class) +@PrepareForTest({AmbariContext.class}) +public class ClusterConfigurationRequestTest { + + @Rule + public EasyMockRule mocks = new EasyMockRule(this); + + @Mock(type = MockType.NICE) + private Blueprint blueprint; + + @Mock(type = MockType.NICE) + private AmbariContext ambariContext; + + @Mock(type = MockType.NICE) + private Cluster cluster; + + @Mock(type = MockType.NICE) + private Clusters clusters; + + @Mock(type = MockType.NICE) + private ClusterTopology topology; + + @Mock(type = MockType.NICE) + private StackAdvisorBlueprintProcessor stackAdvisorBlueprintProcessor; + + @Mock(type = MockType.NICE) + private Stack stack; + + @Mock(type = MockType.NICE) + private AmbariManagementController controller; + + @Mock(type = MockType.NICE) + private KerberosHelper kerberosHelper; + + @Test + public void testProcessClusterConfigRequestIncludeKererosConfigs() throws Exception { + + Map<String, Map<String, String>> existingConfig = new HashMap<String, Map<String, String>>(); + Configuration stackConfig = new Configuration(existingConfig, + new HashMap<String, Map<String, Map<String, String>>>()); + + PowerMock.mockStatic(AmbariContext.class); + AmbariContext.getController(); + expectLastCall().andReturn(controller).anyTimes(); + + expect(controller.getClusters()).andReturn(clusters).anyTimes(); + expect(controller.getKerberosHelper()).andReturn(kerberosHelper).once(); + + expect(clusters.getCluster("testCluster")).andReturn(cluster).anyTimes(); + + expect(blueprint.getStack()).andReturn(stack).anyTimes(); + expect(stack.getAllConfigurationTypes(anyString())).andReturn(Collections.<String>singletonList("testConfigType") + ).anyTimes(); + expect(stack.getExcludedConfigurationTypes(anyString())).andReturn(Collections.<String>emptySet()).anyTimes(); + expect(stack.getConfigurationPropertiesWithMetadata(anyString(), anyString())).andReturn(Collections.<String, + Stack.ConfigProperty>emptyMap()).anyTimes(); + + Set<String> services = new HashSet<>(); + services.add("HDFS"); + services.add("KERBEROS"); + services.add("ZOOKEPER"); + expect(blueprint.getServices()).andReturn(services).anyTimes(); + expect(topology.getConfigRecommendationStrategy()).andReturn(ConfigRecommendationStrategy.NEVER_APPLY).anyTimes(); + expect(topology.getBlueprint()).andReturn(blueprint).anyTimes(); + expect(topology.getConfiguration()).andReturn(stackConfig).anyTimes(); + expect(topology.getHostGroupInfo()).andReturn(Collections.<String, HostGroupInfo>emptyMap()); + expect(topology.getClusterId()).andReturn(Long.valueOf(1)).anyTimes(); + expect(ambariContext.getClusterName(Long.valueOf(1))).andReturn("testCluster").anyTimes(); + expect(ambariContext.createConfigurationRequests(anyObject(Map.class))).andReturn(Collections + .<ConfigurationRequest>emptyList()).anyTimes(); + + Map<String, Map<String, String>> kerberosConfig = new HashMap<String, Map<String, String>>(); + Map<String, String> properties = new HashMap<>(); + properties.put("testPorperty", "testValue"); + kerberosConfig.put("testConfigType", properties); + expect(kerberosHelper.getServiceConfigurationUpdates(anyObject(Cluster.class), anyObject(Map.class), anyObject + (Set.class))).andReturn(kerberosConfig).anyTimes(); + + PowerMock.replay(stack, blueprint, topology, controller, clusters, kerberosHelper, ambariContext, + AmbariContext + .class); + + ClusterConfigurationRequest clusterConfigurationRequest = new ClusterConfigurationRequest( + ambariContext, topology, false, stackAdvisorBlueprintProcessor, true); + clusterConfigurationRequest.process(); + + verify(blueprint, topology, ambariContext, controller, kerberosHelper); + + } + +}
