AMBARI-18335. After upgrading cluster from HDP-2.4.x to HDP-2.5.x and added atlas service - missing kafka security properties (rlevas)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/7aff03f5 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/7aff03f5 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/7aff03f5 Branch: refs/heads/branch-2.5 Commit: 7aff03f59d05cec0a491a2a4ab5b79bfaf5dc5f6 Parents: 79afcb9 Author: Robert Levas <rle...@hortonworks.com> Authored: Fri Sep 9 15:41:15 2016 -0400 Committer: Robert Levas <rle...@hortonworks.com> Committed: Fri Sep 9 15:41:15 2016 -0400 ---------------------------------------------------------------------- .../upgrades/UpgradeUserKerberosDescriptor.java | 205 ++ .../kerberos/AbstractKerberosDescriptor.java | 4 +- .../AbstractKerberosDescriptorContainer.java | 86 +- .../KerberosConfigurationDescriptor.java | 23 +- .../state/kerberos/KerberosDescriptor.java | 32 +- .../KerberosDescriptorUpdateHelper.java | 585 +++++ .../kerberos/KerberosKeytabDescriptor.java | 71 +- .../kerberos/KerberosPrincipalDescriptor.java | 4 +- .../kerberos/KerberosServiceDescriptor.java | 4 +- .../HBASE/0.96.0.2.0/kerberos.json | 3 +- .../HDP/2.3/upgrades/nonrolling-upgrade-2.5.xml | 10 + .../stacks/HDP/2.3/upgrades/upgrade-2.5.xml | 10 + .../HDP/2.4/upgrades/nonrolling-upgrade-2.5.xml | 10 + .../stacks/HDP/2.4/upgrades/upgrade-2.5.xml | 10 + .../stacks/HDP/2.5/services/HBASE/kerberos.json | 3 +- .../HDP/2.5/upgrades/nonrolling-upgrade-2.5.xml | 10 + .../stacks/HDP/2.5/upgrades/upgrade-2.5.xml | 10 + ...rKerberosDescriptorResourceProviderTest.java | 270 +-- .../KerberosComponentDescriptorTest.java | 69 +- .../KerberosConfigurationDescriptorTest.java | 56 +- .../state/kerberos/KerberosDescriptorTest.java | 126 +- .../KerberosDescriptorUpdateHelperTest.java | 2272 ++++++++++++++++++ .../KerberosIdentityDescriptorTest.java | 88 +- .../kerberos/KerberosKeytabDescriptorTest.java | 33 +- .../KerberosPrincipalDescriptorTest.java | 32 +- .../kerberos/KerberosServiceDescriptorTest.java | 70 +- 26 files changed, 3650 insertions(+), 446 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/7aff03f5/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/UpgradeUserKerberosDescriptor.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/UpgradeUserKerberosDescriptor.java b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/UpgradeUserKerberosDescriptor.java new file mode 100644 index 0000000..f1eab38 --- /dev/null +++ b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/UpgradeUserKerberosDescriptor.java @@ -0,0 +1,205 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ambari.server.serveraction.upgrades; + +import com.google.inject.Inject; +import org.apache.ambari.server.AmbariException; +import org.apache.ambari.server.actionmanager.HostRoleCommand; +import org.apache.ambari.server.actionmanager.HostRoleStatus; +import org.apache.ambari.server.agent.CommandReport; +import org.apache.ambari.server.api.services.AmbariMetaInfo; +import org.apache.ambari.server.orm.dao.ArtifactDAO; +import org.apache.ambari.server.orm.entities.ArtifactEntity; +import org.apache.ambari.server.serveraction.AbstractServerAction; +import org.apache.ambari.server.state.Cluster; +import org.apache.ambari.server.state.Clusters; +import org.apache.ambari.server.state.StackId; +import org.apache.ambari.server.state.kerberos.KerberosDescriptor; +import org.apache.ambari.server.state.kerberos.KerberosDescriptorFactory; +import org.apache.ambari.server.state.kerberos.KerberosDescriptorUpdateHelper; +import org.apache.ambari.server.state.stack.upgrade.Direction; +import org.apache.commons.lang.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.List; +import java.util.TreeMap; +import java.util.concurrent.ConcurrentMap; + +/** + * Update the user-defined Kerberos Descriptor to work with the current stack. + * + * @see org.apache.ambari.server.state.kerberos.KerberosDescriptorUpdateHelper + */ +public class UpgradeUserKerberosDescriptor extends AbstractServerAction { + private static final Logger LOG = LoggerFactory.getLogger(UpgradeUserKerberosDescriptor.class); + + /** + * The upgrade direction. + * + * @see Direction + */ + private static final String UPGRADE_DIRECTION_KEY = "upgrade_direction"; + + /** + * The original "current" stack of the cluster before the upgrade started. + * This is the same regardless of whether the current direction is + * {@link Direction#UPGRADE} or {@link Direction#DOWNGRADE}. + * + * @see Direction + */ + private static final String ORIGINAL_STACK_KEY = "original_stack"; + + /** + * The target upgrade stack before the upgrade started. This is the same + * regardless of whether the current direction is {@link Direction#UPGRADE} or + * {@link Direction#DOWNGRADE}. + * + * @see Direction + */ + private static final String TARGET_STACK_KEY = "target_stack"; + + @Inject + private ArtifactDAO artifactDAO; + + @Inject + private Clusters clusters; + + @Inject + private AmbariMetaInfo ambariMetaInfo; + + @Inject + private KerberosDescriptorFactory kerberosDescriptorFactory; + + protected UpgradeUserKerberosDescriptor() { + } + + /** + * Update Kerberos Descriptor Storm properties when upgrading to Storm 1.0 + * <p/> + * Finds the relevant artifact entities and iterates through them to process each independently. + */ + @Override + public CommandReport execute(ConcurrentMap<String, Object> requestSharedDataContext) + throws AmbariException, InterruptedException { + HostRoleCommand hostRoleCommand = getHostRoleCommand(); + String clusterName = hostRoleCommand.getExecutionCommandWrapper().getExecutionCommand().getClusterName(); + Cluster cluster = clusters.getCluster(clusterName); + List<String> messages = new ArrayList<String>(); + List<String> errorMessages = new ArrayList<String>(); + + if (cluster != null) { + logMessage(messages, "Obtaining the user-defined Kerberos descriptor"); + + TreeMap<String, String> foreignKeys = new TreeMap<String, String>(); + foreignKeys.put("cluster", String.valueOf(cluster.getClusterId())); + + ArtifactEntity entity = artifactDAO.findByNameAndForeignKeys("kerberos_descriptor", foreignKeys); + KerberosDescriptor userDescriptor = (entity == null) ? null : kerberosDescriptorFactory.createInstance(entity.getArtifactData()); + + if (userDescriptor != null) { + StackId originalStackId = getStackIdFromCommandParams(ORIGINAL_STACK_KEY); + StackId targetStackId = getStackIdFromCommandParams(TARGET_STACK_KEY); + boolean isDowngrade = isDowngrade(); + + StackId newVersion = (isDowngrade) ? originalStackId : targetStackId; + StackId previousVersion = (isDowngrade) ? targetStackId : originalStackId; + KerberosDescriptor newDescriptor = null; + KerberosDescriptor previousDescriptor = null; + + if (newVersion == null) { + logErrorMessage(messages, errorMessages, "The new stack version information was not found."); + } else { + logMessage(messages, String.format("Obtaining new stack Kerberos descriptor for %s.", newVersion.toString())); + newDescriptor = ambariMetaInfo.getKerberosDescriptor(newVersion.getStackName(), newVersion.getStackVersion()); + + if (newDescriptor == null) { + logErrorMessage(messages, errorMessages, String.format("The Kerberos descriptor for the new stack version, %s, was not found.", newVersion.toString())); + } + } + + if (previousVersion == null) { + logErrorMessage(messages, errorMessages, "The previous stack version information was not found."); + } else { + logMessage(messages, String.format("Obtaining previous stack Kerberos descriptor for %s.", previousVersion.toString())); + previousDescriptor = ambariMetaInfo.getKerberosDescriptor(previousVersion.getStackName(), previousVersion.getStackVersion()); + + if (newDescriptor == null) { + logErrorMessage(messages, errorMessages, String.format("The Kerberos descriptor for the previous stack version, %s, was not found.", previousVersion.toString())); + } + } + + if (errorMessages.isEmpty()) { + logMessage(messages, "Updating the user-specified Kerberos descriptor."); + + KerberosDescriptor updatedDescriptor = KerberosDescriptorUpdateHelper.updateUserKerberosDescriptor( + previousDescriptor, + newDescriptor, + userDescriptor); + + logMessage(messages, "Storing updated user-specified Kerberos descriptor."); + + entity.setArtifactData(updatedDescriptor.toMap()); + artifactDAO.merge(entity); + + logMessage(messages, "Successfully updated the user-specified Kerberos descriptor."); + } + } else { + logMessage(messages, "A user-specified Kerberos descriptor was not found. No updates are necessary."); + } + } else { + logErrorMessage(messages, errorMessages, String.format("The cluster named %s was not found.", clusterName)); + } + + if (!errorMessages.isEmpty()) { + logErrorMessage(messages, errorMessages, "No updates have been performed due to previous issues."); + } + + return createCommandReport(0, HostRoleStatus.COMPLETED, "{}", StringUtils.join(messages, "\n"), StringUtils.join(errorMessages, "\n")); + } + + /** + * Determines if upgrade direction is {@link Direction#UPGRADE} or {@link Direction#DOWNGRADE}. + * + * @return {@code true} if {@link Direction#DOWNGRADE}; {@code false} if {@link Direction#UPGRADE} + */ + private boolean isDowngrade() { + return Direction.DOWNGRADE.name().equalsIgnoreCase(getCommandParameterValue(UPGRADE_DIRECTION_KEY)); + } + + private StackId getStackIdFromCommandParams(String commandParamKey) { + String stackId = getCommandParameterValue(commandParamKey); + if (stackId == null) { + return null; + } else { + return new StackId(stackId); + } + } + + private void logMessage(List<String> messages, String message) { + LOG.info(message); + messages.add(message); + } + + private void logErrorMessage(List<String> messages, List<String> errorMessages, String message) { + LOG.error(message); + messages.add(message); + errorMessages.add(message); + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/7aff03f5/ambari-server/src/main/java/org/apache/ambari/server/state/kerberos/AbstractKerberosDescriptor.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/kerberos/AbstractKerberosDescriptor.java b/ambari-server/src/main/java/org/apache/ambari/server/state/kerberos/AbstractKerberosDescriptor.java index 84a9111..2112fcc 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/state/kerberos/AbstractKerberosDescriptor.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/state/kerberos/AbstractKerberosDescriptor.java @@ -18,8 +18,8 @@ package org.apache.ambari.server.state.kerberos; -import java.util.HashMap; import java.util.Map; +import java.util.TreeMap; /** * AbstractKerberosDescriptor is the base class for all Kerberos*Descriptor and associated classes. @@ -65,7 +65,7 @@ public abstract class AbstractKerberosDescriptor { * @return a Map of date representing this AbstractKerberosDescriptor implementation */ public Map<String, Object> toMap() { - HashMap<String, Object> dataMap = new HashMap<String, Object>(); + TreeMap<String, Object> dataMap = new TreeMap<String, Object>(); String name = getName(); if (name != null) { http://git-wip-us.apache.org/repos/asf/ambari/blob/7aff03f5/ambari-server/src/main/java/org/apache/ambari/server/state/kerberos/AbstractKerberosDescriptorContainer.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/kerberos/AbstractKerberosDescriptorContainer.java b/ambari-server/src/main/java/org/apache/ambari/server/state/kerberos/AbstractKerberosDescriptorContainer.java index 39ebdaf..ad2437a 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/state/kerberos/AbstractKerberosDescriptorContainer.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/state/kerberos/AbstractKerberosDescriptorContainer.java @@ -23,12 +23,12 @@ import org.apache.ambari.server.AmbariException; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; +import java.util.TreeMap; import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Set; +import java.util.TreeSet; import java.util.regex.Pattern; /** @@ -165,8 +165,8 @@ public abstract class AbstractKerberosDescriptorContainer extends AbstractKerber /** * Returns a specific named child container - * @param name the name of the child container to retrieve * + * @param name the name of the child container to retrieve * @return an {@link AbstractKerberosDescriptorContainer} */ public abstract AbstractKerberosDescriptorContainer getChildContainer(String name); @@ -193,6 +193,17 @@ public abstract class AbstractKerberosDescriptorContainer extends AbstractKerber } /** + * Set the {@link KerberosIdentityDescriptor} for this {@link AbstractKerberosDescriptorContainer}. + * + * @param identities a {@link List} of {@link KerberosIdentityDescriptor}s + */ + public void setIdentities(List<KerberosIdentityDescriptor> identities) { + this.identities = (identities == null) + ? null + : new ArrayList<KerberosIdentityDescriptor>(identities); + } + + /** * Returns a List of KerberosIdentityDescriptors contained within this * AbstractKerberosDescriptorContainer. * <p/> @@ -215,9 +226,9 @@ public abstract class AbstractKerberosDescriptorContainer extends AbstractKerber * (false) * @return a List of the requested KerberosIdentityDescriptors */ - public List<KerberosIdentityDescriptor> getIdentities(boolean resolveReferences, Map<String,Object> contextForFilter) throws AmbariException { + public List<KerberosIdentityDescriptor> getIdentities(boolean resolveReferences, Map<String, Object> contextForFilter) throws AmbariException { if (identities == null) { - return Collections.emptyList(); + return null; } else { List<KerberosIdentityDescriptor> list = new ArrayList<KerberosIdentityDescriptor>(); @@ -317,6 +328,18 @@ public abstract class AbstractKerberosDescriptorContainer extends AbstractKerber } /** + * Sets the {@link Map} of {@link KerberosConfigurationDescriptor}s for this + * {@link AbstractKerberosDescriptorContainer}. + * + * @param configurations a {@link Map} of {@link KerberosConfigurationDescriptor}s + */ + public void setConfigurations(Map<String, KerberosConfigurationDescriptor> configurations) { + this.configurations = (configurations == null) + ? null + : new TreeMap<String, KerberosConfigurationDescriptor>(configurations); + } + + /** * Returns a Map of raw KerberosConfigurationDescriptors contained within this * AbstractKerberosDescriptorContainer. * <p/> @@ -353,7 +376,7 @@ public abstract class AbstractKerberosDescriptorContainer extends AbstractKerber */ public Map<String, KerberosConfigurationDescriptor> getConfigurations(boolean includeInherited) { if (includeInherited) { - Map<String, KerberosConfigurationDescriptor> mergedConfigurations = new HashMap<String, KerberosConfigurationDescriptor>(); + Map<String, KerberosConfigurationDescriptor> mergedConfigurations = new TreeMap<String, KerberosConfigurationDescriptor>(); List<Map<String, KerberosConfigurationDescriptor>> configurationSets = new ArrayList<Map<String, KerberosConfigurationDescriptor>>(); AbstractKerberosDescriptor currentDescriptor = this; @@ -417,7 +440,7 @@ public abstract class AbstractKerberosDescriptorContainer extends AbstractKerber } if (configurations == null) { - configurations = new HashMap<String, KerberosConfigurationDescriptor>(); + configurations = new TreeMap<String, KerberosConfigurationDescriptor>(); } configurations.put(type, configuration); @@ -448,7 +471,7 @@ public abstract class AbstractKerberosDescriptorContainer extends AbstractKerber public void putAuthToLocalProperty(String authToLocalProperty) { if (authToLocalProperty != null) { if (authToLocalProperties == null) { - authToLocalProperties = new HashSet<String>(); + authToLocalProperties = new TreeSet<String>(); } authToLocalProperties.add(authToLocalProperty); @@ -456,6 +479,17 @@ public abstract class AbstractKerberosDescriptorContainer extends AbstractKerber } /** + * Sets the set of <code>auth_to_local</code> property names. + * + * @param authToLocalProperties a Set of String values; or null if not set + */ + public void setAuthToLocalProperties(Set<String> authToLocalProperties) { + this.authToLocalProperties = (authToLocalProperties == null) + ? null + : new TreeSet<String>(authToLocalProperties); + } + + /** * Gets the set of <code>auth_to_local</code> property names. * * @return a Set of String values; or null if not set @@ -582,13 +616,13 @@ public abstract class AbstractKerberosDescriptorContainer extends AbstractKerber KerberosIdentityDescriptor identityDescriptor = null; if (path != null) { - if(path.startsWith("../")) { + if (path.startsWith("../")) { // Resolve parent path AbstractKerberosDescriptor parent = getParent(); path = path.substring(2); - while(parent != null) { + while (parent != null) { String name = parent.getName(); if (name != null) { @@ -694,24 +728,27 @@ public abstract class AbstractKerberosDescriptorContainer extends AbstractKerber Map<String, Object> map = super.toMap(); if (identities != null) { - List<Map<String, Object>> list = new ArrayList<Map<String, Object>>(); + // Use a TreeMap to force the identities definitions to be ordered by name, alphebetically. + // This helps with readability and comparisons. + Map<String, Map<String, Object>> list = new TreeMap<String, Map<String, Object>>(); for (KerberosIdentityDescriptor identity : identities) { - list.add(identity.toMap()); + list.put(identity.getName(), identity.toMap()); } - map.put(Type.IDENTITY.getDescriptorPluralName(), list); + map.put(Type.IDENTITY.getDescriptorPluralName(), list.values()); } if (configurations != null) { - List<Map<String, Object>> list = new ArrayList<Map<String, Object>>(); + // Use a TreeMap to force the configurations to be ordered by configuration type, alphebetically. + // This helps with readability and comparisons. + Map<String, Map<String, Object>> list = new TreeMap<String, Map<String, Object>>(); for (KerberosConfigurationDescriptor configuration : configurations.values()) { - list.add(configuration.toMap()); + list.put(configuration.getType(), configuration.toMap()); } - map.put(Type.CONFIGURATION.getDescriptorPluralName(), list); + map.put(Type.CONFIGURATION.getDescriptorPluralName(), list.values()); } if (authToLocalProperties != null) { - List<String> list = new ArrayList<String>(authToLocalProperties); - map.put(Type.AUTH_TO_LOCAL_PROPERTY.getDescriptorPluralName(), list); + map.put(Type.AUTH_TO_LOCAL_PROPERTY.getDescriptorPluralName(), authToLocalProperties); } return map; @@ -723,6 +760,9 @@ public abstract class AbstractKerberosDescriptorContainer extends AbstractKerber ((getIdentities() == null) ? 0 : getIdentities().hashCode()) + + ((getAuthToLocalProperties() == null) + ? 0 + : getAuthToLocalProperties().hashCode()) + ((getConfigurations() == null) ? 0 : getConfigurations().hashCode()); @@ -743,6 +783,11 @@ public abstract class AbstractKerberosDescriptorContainer extends AbstractKerber : getIdentities().equals(descriptor.getIdentities()) ) && ( + (getAuthToLocalProperties() == null) + ? (descriptor.getAuthToLocalProperties() == null) + : getAuthToLocalProperties().equals(descriptor.getAuthToLocalProperties()) + ) && + ( (getConfigurations() == null) ? (descriptor.getConfigurations() == null) : getConfigurations().equals(descriptor.getConfigurations()) @@ -768,7 +813,7 @@ public abstract class AbstractKerberosDescriptorContainer extends AbstractKerber private KerberosIdentityDescriptor dereferenceIdentity(KerberosIdentityDescriptor identity) throws AmbariException { KerberosIdentityDescriptor dereferencedIdentity = null; - if(identity != null) { + if (identity != null) { KerberosIdentityDescriptor referencedIdentity; try { if (identity.getReference() != null) { @@ -790,8 +835,7 @@ public abstract class AbstractKerberosDescriptorContainer extends AbstractKerber dereferencedIdentity = new KerberosIdentityDescriptor(referencedIdentity.toMap()); dereferencedIdentity.update(identity); } - } - else { + } else { dereferencedIdentity = new KerberosIdentityDescriptor(identity.toMap()); } } http://git-wip-us.apache.org/repos/asf/ambari/blob/7aff03f5/ambari-server/src/main/java/org/apache/ambari/server/state/kerberos/KerberosConfigurationDescriptor.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/kerberos/KerberosConfigurationDescriptor.java b/ambari-server/src/main/java/org/apache/ambari/server/state/kerberos/KerberosConfigurationDescriptor.java index 3cdd9908..bc5f936 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/state/kerberos/KerberosConfigurationDescriptor.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/state/kerberos/KerberosConfigurationDescriptor.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -18,7 +18,7 @@ package org.apache.ambari.server.state.kerberos; -import java.util.HashMap; +import java.util.TreeMap; import java.util.Map; import java.util.Set; @@ -105,6 +105,19 @@ public class KerberosConfigurationDescriptor extends AbstractKerberosDescriptor } /** + * Set the properties of the configuration data represented by this KerberosConfigurationDescriptor + * + * @param properties a Map of properties + */ + public void setProperties(Map<String, String> properties) { + if (properties == null) { + this.properties = null; + } else { + this.properties = new TreeMap<String, String>(properties); + } + } + + /** * Gets the properties of the configuration data represented by this KerberosConfigurationDescriptor * * @return a Map of properties @@ -137,7 +150,7 @@ public class KerberosConfigurationDescriptor extends AbstractKerberosDescriptor } if (properties == null) { - properties = new HashMap<String, String>(); + properties = new TreeMap<String, String>(); } properties.put(name, value); @@ -173,8 +186,8 @@ public class KerberosConfigurationDescriptor extends AbstractKerberosDescriptor */ @Override public Map<String, Object> toMap() { - Map<String, Object> map = new HashMap<String, Object>(); - map.put(getName(), (properties == null) ? null : new HashMap<String, Object>(properties)); + Map<String, Object> map = new TreeMap<String, Object>(); + map.put(getName(), (properties == null) ? null : new TreeMap<String, Object>(properties)); return map; } http://git-wip-us.apache.org/repos/asf/ambari/blob/7aff03f5/ambari-server/src/main/java/org/apache/ambari/server/state/kerberos/KerberosDescriptor.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/kerberos/KerberosDescriptor.java b/ambari-server/src/main/java/org/apache/ambari/server/state/kerberos/KerberosDescriptor.java index 484f65c..e7be589 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/state/kerberos/KerberosDescriptor.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/state/kerberos/KerberosDescriptor.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -20,7 +20,7 @@ package org.apache.ambari.server.state.kerberos; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; -import java.util.HashMap; +import java.util.TreeMap; import java.util.HashSet; import java.util.List; import java.util.Map; @@ -147,6 +147,17 @@ public class KerberosDescriptor extends AbstractKerberosDescriptorContainer { } /** + * Set the KerberosServiceDescriptors in this KerberosDescriptor + * + * @param services a Map of String to KerberosServiceDescriptor + */ + public void setServices(Map<String, KerberosServiceDescriptor> services) { + this.services = (services == null) + ? null + : new TreeMap<String, KerberosServiceDescriptor>(services); + } + + /** * Returns a Map of the KerberosServiceDescriptors in this KerberosDescriptor * * @return a Map of String to KerberosServiceDescriptor @@ -182,7 +193,7 @@ public class KerberosDescriptor extends AbstractKerberosDescriptorContainer { } if (services == null) { - services = new HashMap<String, KerberosServiceDescriptor>(); + services = new TreeMap<String, KerberosServiceDescriptor>(); } services.put(name, service); @@ -193,6 +204,17 @@ public class KerberosDescriptor extends AbstractKerberosDescriptorContainer { } /** + * Set the Map of properties for this KerberosDescriptor + * + * @param properties a Map of String to String values + */ + public void setProperties(Map<String, String> properties) { + this.properties = (properties == null) + ? null + : new TreeMap<String, String>(properties); + } + + /** * Gets the Map of properties for this KerberosDescriptor * * @return a Map of String to String values @@ -225,7 +247,7 @@ public class KerberosDescriptor extends AbstractKerberosDescriptorContainer { } if (properties == null) { - properties = new HashMap<String, String>(); + properties = new TreeMap<String, String>(); } properties.put(name, value); @@ -304,7 +326,7 @@ public class KerberosDescriptor extends AbstractKerberosDescriptorContainer { } if (properties != null) { - map.put("properties", new HashMap<String, String>(properties)); + map.put("properties", new TreeMap<String, String>(properties)); } return map; http://git-wip-us.apache.org/repos/asf/ambari/blob/7aff03f5/ambari-server/src/main/java/org/apache/ambari/server/state/kerberos/KerberosDescriptorUpdateHelper.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/kerberos/KerberosDescriptorUpdateHelper.java b/ambari-server/src/main/java/org/apache/ambari/server/state/kerberos/KerberosDescriptorUpdateHelper.java new file mode 100644 index 0000000..2eef4b9 --- /dev/null +++ b/ambari-server/src/main/java/org/apache/ambari/server/state/kerberos/KerberosDescriptorUpdateHelper.java @@ -0,0 +1,585 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ambari.server.state.kerberos; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.TreeMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TreeSet; + +/** + * KerberosDescriptorUpdateHelper provides routines for upgrading the user-specified Kerberos descriptor + * when changing stacks. + * <p> + * This implementation should work for stack upgrades and downgrades. + */ +public class KerberosDescriptorUpdateHelper { + private static final Logger LOG = LoggerFactory.getLogger(KerberosDescriptorUpdateHelper.class); + + /** + * The entry point into upgrading a user-specified Kerberos descriptor. + * <p> + * The supplied Kerberos descriptors will remain untouched and new Kerberos descriptor instance will + * created and returned with the update data. + * + * @param beginningStackKerberosDescriptor the Kerberos descriptor for the previous stack version + * @param endingStackKerberosDescriptor the Kerberos descriptor for the new stack version + * @param userKerberosDescriptor the user-specified Kerberos descriptor + * @return a new Kerberos descriptor containing the updated user-specified data + */ + public static KerberosDescriptor updateUserKerberosDescriptor(KerberosDescriptor beginningStackKerberosDescriptor, + KerberosDescriptor endingStackKerberosDescriptor, + KerberosDescriptor userKerberosDescriptor) { + KerberosDescriptor updated = new KerberosDescriptor(userKerberosDescriptor.toMap()); + + updated.setProperties(processProperties( + beginningStackKerberosDescriptor.getProperties(), + endingStackKerberosDescriptor.getProperties(), + updated.getProperties())); + + updated.setConfigurations(processConfigurations( + beginningStackKerberosDescriptor.getConfigurations(), + endingStackKerberosDescriptor.getConfigurations(), + updated.getConfigurations())); + + updated.setIdentities(processIdentities( + beginningStackKerberosDescriptor.getIdentities(), + endingStackKerberosDescriptor.getIdentities(), + updated.getIdentities())); + + updated.setAuthToLocalProperties(processAuthToLocalProperties( + beginningStackKerberosDescriptor.getAuthToLocalProperties(), + endingStackKerberosDescriptor.getAuthToLocalProperties(), + updated.getAuthToLocalProperties())); + + updated.setServices(processServices( + beginningStackKerberosDescriptor.getServices(), + endingStackKerberosDescriptor.getServices(), + updated.getServices())); + + return updated; + } + + /** + * Processes the service-level Kerberos descriptors to add, remove, or update data in the user-specified + * Kerberos descriptor. + * + * @param previousStackServices a map of {@link KerberosServiceDescriptor}s from the previous stack version's Kerberos descriptor + * @param newStackServices a map of {@link KerberosServiceDescriptor}s from the new stack version's Kerberos descriptor + * @param userServices a map of {@link KerberosServiceDescriptor}s from the user-supplied Kerberos descriptor + * @return a map of updated {@link KerberosServiceDescriptor}s + */ + private static Map<String, KerberosServiceDescriptor> processServices(Map<String, KerberosServiceDescriptor> previousStackServices, + Map<String, KerberosServiceDescriptor> newStackServices, + Map<String, KerberosServiceDescriptor> userServices) { + if ((userServices == null) || userServices.isEmpty() || ((previousStackServices == null) && (newStackServices == null))) { + return userServices; + } + + Map<String, KerberosServiceDescriptor> updatedServices = new TreeMap<String, KerberosServiceDescriptor>(); + + if (previousStackServices == null) { + previousStackServices = Collections.emptyMap(); + } + + if (newStackServices == null) { + newStackServices = Collections.emptyMap(); + } + + for (Map.Entry<String, KerberosServiceDescriptor> entry : userServices.entrySet()) { + String name = entry.getKey(); + KerberosServiceDescriptor userValue = entry.getValue(); + + if (userValue != null) { + if (newStackServices.containsKey(name)) { + KerberosServiceDescriptor oldValue = previousStackServices.get(name); + KerberosServiceDescriptor newValue = newStackServices.get(name); + + LOG.debug("Processing service {} for modifications", name); + updatedServices.put(name, processService(oldValue, newValue, userValue)); + } else if (previousStackServices.containsKey(name)) { + LOG.debug("Removing service {} from user-specified Kerberos Descriptor", name); + // Nothing to do here, just don't add it to the updated configurations map... + } else { + LOG.debug("Leaving service {} in user-specified Kerberos Descriptor unchanged since it was user-defined.", name); + updatedServices.put(name, userValue); + } + } + } + + // Note: there is no need to add service definitions that do not exist since they will get + // added dynamically when merged with the stack default value. + + return updatedServices; + } + + /** + * Processes a {@link KerberosServiceDescriptor} to change the user-supplied data based on the changes + * observed between the previous stack version's data and the new stack version's data. + * + * @param previousStackService a {@link KerberosServiceDescriptor} from the previous stack version's Kerberos descriptor + * @param newStackService a {@link KerberosServiceDescriptor} from the new stack version's Kerberos descriptor + * @param userService a {@link KerberosServiceDescriptor} from the user-specified Kerberos descriptor + * @return the updated {@link KerberosServiceDescriptor} + */ + private static KerberosServiceDescriptor processService(KerberosServiceDescriptor previousStackService, + KerberosServiceDescriptor newStackService, + KerberosServiceDescriptor userService) { + + KerberosServiceDescriptor updatedService = new KerberosServiceDescriptor(userService.toMap()); + + updatedService.setAuthToLocalProperties(processAuthToLocalProperties( + (previousStackService == null) ? null : previousStackService.getAuthToLocalProperties(), + (newStackService == null) ? null : newStackService.getAuthToLocalProperties(), + updatedService.getAuthToLocalProperties())); + + updatedService.setConfigurations(processConfigurations( + (previousStackService == null) ? null : previousStackService.getConfigurations(), + (newStackService == null) ? null : newStackService.getConfigurations(), + updatedService.getConfigurations())); + + updatedService.setIdentities(processIdentities( + (previousStackService == null) ? null : previousStackService.getIdentities(), + (newStackService == null) ? null : newStackService.getIdentities(), + updatedService.getIdentities())); + + Map<String, KerberosComponentDescriptor> userServiceComponents = updatedService.getComponents(); + Map<String, KerberosComponentDescriptor> newServiceComponents = (newStackService == null) ? null : newStackService.getComponents(); + Map<String, KerberosComponentDescriptor> oldServiceComponents = (previousStackService == null) ? null : previousStackService.getComponents(); + + if (newServiceComponents == null) { + newServiceComponents = Collections.emptyMap(); + } + + if (oldServiceComponents == null) { + oldServiceComponents = Collections.emptyMap(); + } + + if (userServiceComponents != null) { + Iterator<Map.Entry<String, KerberosComponentDescriptor>> iterator = userServiceComponents.entrySet().iterator(); + while (iterator.hasNext()) { + Map.Entry<String, KerberosComponentDescriptor> entry = iterator.next(); + String name = entry.getKey(); + KerberosComponentDescriptor userValue = entry.getValue(); + + if (userValue == null) { + iterator.remove(); // This is a bad entry... remove it. + } else if (newServiceComponents.containsKey(name)) { + KerberosComponentDescriptor oldValue = oldServiceComponents.get(name); + KerberosComponentDescriptor newValue = newServiceComponents.get(name); + + LOG.debug("Processing component {}/{} for modifications", updatedService.getName(), name); + processComponent(oldValue, newValue, userValue); + } else { + LOG.debug("Removing component {}/{} from user-specified Kerberos Descriptor", updatedService.getName(), name); + iterator.remove(); + } + } + } + + return updatedService; + } + + /** + * Processes a {@link KerberosComponentDescriptor} to change the user-supplied data based on the changes + * observed between the previous stack version's data and the new stack version's data. + * <p> + * The supplied userComponent value is updated in place. + * + * @param previousStackComponent a {@link KerberosComponentDescriptor} from the previous stack version's Kerberos descriptor + * @param newStackComponent a {@link KerberosComponentDescriptor} from the new stack version's Kerberos descriptor + * @param userComponent a {@link KerberosComponentDescriptor} from the user-specified Kerberos descriptor + * @return the updated {@link KerberosComponentDescriptor} + */ + private static KerberosComponentDescriptor processComponent(KerberosComponentDescriptor previousStackComponent, + KerberosComponentDescriptor newStackComponent, + KerberosComponentDescriptor userComponent) { + userComponent.setAuthToLocalProperties(processAuthToLocalProperties( + (previousStackComponent == null) ? null : previousStackComponent.getAuthToLocalProperties(), + (newStackComponent == null) ? null : newStackComponent.getAuthToLocalProperties(), + userComponent.getAuthToLocalProperties())); + + userComponent.setConfigurations(processConfigurations( + (previousStackComponent == null) ? null : previousStackComponent.getConfigurations(), + (newStackComponent == null) ? null : newStackComponent.getConfigurations(), + userComponent.getConfigurations())); + + userComponent.setIdentities(processIdentities( + (previousStackComponent == null) ? null : previousStackComponent.getIdentities(), + (newStackComponent == null) ? null : newStackComponent.getIdentities(), + userComponent.getIdentities())); + + return userComponent; + } + + /** + * Processes a the list of configuration specification (<code><configuration type>/<property name></code>) + * identifying the properties that should be automatically updated with generated auth-to-local rules. + * <p> + * If no user-specified properties are set, <code>null</code> is returned. + * <p> + * Else the configuration specifications from the previous stack are removed from the user-specified + * data and the configuration specifications from the new stack are added to the user-specified, + * leaving the new list of configuration specifications as well as any user-specified changes. + * + * @param previousStackAuthToLocalProperties the auth-to-local properties from the previous stack version's Kerberos descriptor + * @param newStackAuthToLocalProperties the auth-to-local properties from the new stack version's Kerberos descriptor + * @param userAuthToLocalProperties the auth-to-local properties from the user-specified Kerberos descriptor + * @return an updated {@link Set} of configuration specifications + */ + private static Set<String> processAuthToLocalProperties(Set<String> previousStackAuthToLocalProperties, + Set<String> newStackAuthToLocalProperties, + Set<String> userAuthToLocalProperties) { + if (userAuthToLocalProperties == null) { + return null; + } + + TreeSet<String> updatedAuthToLocalProperties = new TreeSet<String>(userAuthToLocalProperties); + + // Remove old configuration specifications, leaving the user-specified ones. + if (previousStackAuthToLocalProperties != null) { + updatedAuthToLocalProperties.removeAll(previousStackAuthToLocalProperties); + } + + // Add the new configuration specifications + if (newStackAuthToLocalProperties != null) { + updatedAuthToLocalProperties.addAll(newStackAuthToLocalProperties); + } + + return updatedAuthToLocalProperties; + } + + /** + * Processes the identity-level Kerberos descriptors to add, remove, or update data in the user-specified + * Kerberos descriptor. + * + * @param previousStackIdentities a map of {@link KerberosIdentityDescriptor}s from the previous stack version's Kerberos descriptor + * @param newStackIdentities a map of {@link KerberosIdentityDescriptor}s from the new stack version's Kerberos descriptor + * @param userIdentities a map of {@link KerberosIdentityDescriptor}s from the user-supplied Kerberos descriptor + * @return a list of updated {@link KerberosIdentityDescriptor}s + */ + private static List<KerberosIdentityDescriptor> processIdentities(List<KerberosIdentityDescriptor> previousStackIdentities, + List<KerberosIdentityDescriptor> newStackIdentities, + List<KerberosIdentityDescriptor> userIdentities) { + + if ((userIdentities == null) || userIdentities.isEmpty() || ((previousStackIdentities == null) && (newStackIdentities == null))) { + return userIdentities; + } + + // Create maps to make processing easier.... + Map<String, KerberosIdentityDescriptor> previousStackIdentityMap = toMap(previousStackIdentities); + Map<String, KerberosIdentityDescriptor> newStackIdentityMap = toMap(newStackIdentities); + Map<String, KerberosIdentityDescriptor> userStackIdentityMap = toMap(userIdentities); + + Map<String, KerberosIdentityDescriptor> updatedIdentities = new TreeMap<String, KerberosIdentityDescriptor>(); + + if (previousStackIdentityMap == null) { + previousStackIdentityMap = Collections.emptyMap(); + } + + if (newStackIdentityMap == null) { + newStackIdentityMap = Collections.emptyMap(); + } + + // Find identities to modify or remove + for (Map.Entry<String, KerberosIdentityDescriptor> entry : userStackIdentityMap.entrySet()) { + String name = entry.getKey(); + KerberosIdentityDescriptor userValue = entry.getValue(); + + if (userValue != null) { + if (newStackIdentityMap.containsKey(name)) { + // Modify the new stack identity value by changing on the principal value and/or keytab + // file value since they are the only fields in this structure that should be changed + // by a user. However, the new stack identity may have been converted to a pure reference + // where the user changes will then be ignored. + KerberosIdentityDescriptor newValue = newStackIdentityMap.get(name); + KerberosIdentityDescriptor previousValue = previousStackIdentityMap.get(name); + + updatedIdentities.put(name, processIdentity(previousValue, newValue, userValue)); + + } else if (previousStackIdentityMap.containsKey(name)) { + LOG.debug("Removing identity named {} from user-specified Kerberos Descriptor", name); + // Nothing to do here, just don't add it to the updated identity map... + } else { + LOG.debug("Leaving identity named {} in user-specified Kerberos Descriptor unchanged since it was user-defined.", name); + updatedIdentities.put(name, userValue); + } + } + } + + // Note: there is no need to add identity definitions that do not exist since they will get + // added dynamically when merged with the stack default value. + + return new ArrayList<KerberosIdentityDescriptor>(updatedIdentities.values()); + } + + + /** + * Processes a {@link KerberosIdentityDescriptor} to change the user-supplied data based on the changes + * observed between the previous stack version's data and the new stack version's data. + * + * @param previousStackIdentity a {@link KerberosIdentityDescriptor} from the previous stack version's Kerberos descriptor + * @param newStackIdentity a {@link KerberosIdentityDescriptor} from the new stack version's Kerberos descriptor + * @param userIdentity a {@link KerberosIdentityDescriptor} from the user-specified Kerberos descriptor + * @return a new, updated, {@link KerberosIdentityDescriptor} + */ + private static KerberosIdentityDescriptor processIdentity(KerberosIdentityDescriptor previousStackIdentity, + KerberosIdentityDescriptor newStackIdentity, + KerberosIdentityDescriptor userIdentity) { + + KerberosIdentityDescriptor updatedValue = new KerberosIdentityDescriptor(newStackIdentity.toMap()); + KerberosPrincipalDescriptor updatedValuePrincipal = updatedValue.getPrincipalDescriptor(); + KerberosKeytabDescriptor updatedValueKeytab = updatedValue.getKeytabDescriptor(); + + // If the new identity definition is a reference and no longer has a principal definition, + // Ignore any user changes to the old principal definition. + if (updatedValuePrincipal != null) { + KerberosPrincipalDescriptor oldValuePrincipal = previousStackIdentity.getPrincipalDescriptor(); + String previousValuePrincipalValue = null; + KerberosPrincipalDescriptor userValuePrincipal = userIdentity.getPrincipalDescriptor(); + String userValuePrincipalValue = null; + + if (oldValuePrincipal != null) { + previousValuePrincipalValue = oldValuePrincipal.getValue(); + } + + if (userValuePrincipal != null) { + userValuePrincipalValue = userValuePrincipal.getValue(); + } + + // If the user changed the stack default, replace the new stack default value with the user's + // changed value + if ((userValuePrincipalValue != null) && !userValuePrincipalValue.equals(previousValuePrincipalValue)) { + updatedValuePrincipal.setValue(userValuePrincipalValue); + } + } + + // If the new identity definition is a reference and no longer has a keytab definition, + // Ignore any user changes to the old keytab definition. + if (updatedValueKeytab != null) { + KerberosKeytabDescriptor oldValueKeytab = previousStackIdentity.getKeytabDescriptor(); + String previousValueKeytabFile = null; + KerberosKeytabDescriptor userValueKeytab = userIdentity.getKeytabDescriptor(); + String userValueKeytabFile = null; + + if (oldValueKeytab != null) { + previousValueKeytabFile = oldValueKeytab.getFile(); + } + + if (userValueKeytab != null) { + userValueKeytabFile = userValueKeytab.getFile(); + } + + // If the user changed the stack default, replace the new stack default value with the user's + // changed value + if ((userValueKeytabFile != null) && !userValueKeytabFile.equals(previousValueKeytabFile)) { + updatedValueKeytab.setFile(userValueKeytabFile); + } + } + + // Remove the when clause + updatedValue.setWhen(null); + + return updatedValue; + } + + /** + * Processes the configuration-level Kerberos descriptors to add, remove, or update data in the user-specified + * Kerberos descriptor. + * + * @param previousStackConfigurations a map of {@link KerberosConfigurationDescriptor}s from the previous stack version's Kerberos descriptor + * @param newStackConfigurations a map of {@link KerberosConfigurationDescriptor}s from the new stack version's Kerberos descriptor + * @param userConfigurations a map of {@link KerberosConfigurationDescriptor}s from the user-supplied Kerberos descriptor + * @return a map of updated {@link KerberosConfigurationDescriptor}s + */ + private static Map<String, KerberosConfigurationDescriptor> processConfigurations(Map<String, KerberosConfigurationDescriptor> previousStackConfigurations, + Map<String, KerberosConfigurationDescriptor> newStackConfigurations, + Map<String, KerberosConfigurationDescriptor> userConfigurations) { + + if ((userConfigurations == null) || ((previousStackConfigurations == null) && (newStackConfigurations == null))) { + return userConfigurations; + } + + Map<String, KerberosConfigurationDescriptor> updatedConfigurations = new TreeMap<String, KerberosConfigurationDescriptor>(); + + if (previousStackConfigurations == null) { + previousStackConfigurations = Collections.emptyMap(); + } + + if (newStackConfigurations == null) { + newStackConfigurations = Collections.emptyMap(); + } + + // Find configurations to modify or remove + for (Map.Entry<String, KerberosConfigurationDescriptor> entry : userConfigurations.entrySet()) { + String name = entry.getKey(); + KerberosConfigurationDescriptor userValue = entry.getValue(); + + if (userValue != null) { + if (newStackConfigurations.containsKey(name)) { + KerberosConfigurationDescriptor oldValue = previousStackConfigurations.get(name); + KerberosConfigurationDescriptor newValue = newStackConfigurations.get(name); + + LOG.debug("Processing configuration type {} for modifications", name); + updatedConfigurations.put(name, processConfiguration(oldValue, newValue, userValue)); + } else if (previousStackConfigurations.containsKey(name)) { + LOG.debug("Removing configuration type {} from user-specified Kerberos Descriptor", name); + // Nothing to do here, just don't add it to the updated configurations map... + } else { + LOG.debug("Leaving configuration type {} in user-specified Kerberos Descriptor unchanged since it was user-defined.", name); + updatedConfigurations.put(name, userValue); + } + } + } + + // Note: there is no need to add configuration definitions that do not exist in the user-specified + // descriptor since they will get added dynamically when merged with the stack default value. + + return updatedConfigurations; + } + + /** + * Processes a {@link KerberosConfigurationDescriptor} to change the user-supplied data based on the changes + * observed between the previous stack version's data and the new stack version's data. + * + * @param previousStackConfiguration a {@link KerberosConfigurationDescriptor} from the previous stack version's Kerberos descriptor + * @param newStackConfiguration a {@link KerberosConfigurationDescriptor} from the new stack version's Kerberos descriptor + * @param userConfiguration a {@link KerberosConfigurationDescriptor} from the user-specified Kerberos descriptor + * @return an updated {@link KerberosConfigurationDescriptor} + */ + private static KerberosConfigurationDescriptor processConfiguration(KerberosConfigurationDescriptor previousStackConfiguration, + KerberosConfigurationDescriptor newStackConfiguration, + KerberosConfigurationDescriptor userConfiguration) { + + KerberosConfigurationDescriptor updatedValue = new KerberosConfigurationDescriptor((userConfiguration == null) ? null : userConfiguration.toMap()); + + Map<String, String> previousValue = (previousStackConfiguration == null) ? null : previousStackConfiguration.getProperties(); + Map<String, String> newValue = (newStackConfiguration == null) ? null : newStackConfiguration.getProperties(); + Map<String, String> userValue = updatedValue.getProperties(); + + updatedValue.setProperties(processProperties(previousValue, newValue, userValue)); + + return updatedValue; + } + + /** + * Processes a map of global or configuration properties to change the user-supplied data based on + * the changes observed between the previous stack version's data and the new stack version's data. + * <p> + * If a property exists in both the previous and new stacks, and the user has not changed it; then + * the value of the property will be updated to the new stack version's value. Else, if the user + * changed the value, the changed value will be left as-is. + * <p> + * If a property exists only in the previous stack, then it will be removed. + * <p> + * If a property exists only in the new stack, then it will be added. + * + * @param previousStackProperties properties from the previous stack version's Kerberos descriptor + * @param newStackProperties properties from the new stack version's Kerberos descriptor + * @param userProperties properties from the user-specified Kerberos descriptor + * @return a new map of updated properties + */ + private static Map<String, String> processProperties(Map<String, String> previousStackProperties, + Map<String, String> newStackProperties, + Map<String, String> userProperties) { + + if ((previousStackProperties == null) && (newStackProperties == null)) { + return userProperties; + } else { + Map<String, String> updatedProperties = new TreeMap<String, String>(); + if (userProperties != null) { + updatedProperties.putAll(userProperties); + } + + if (previousStackProperties == null) { + previousStackProperties = Collections.emptyMap(); + } + + if (newStackProperties == null) { + newStackProperties = Collections.emptyMap(); + } + + // Find properties to modify and remove + for (Map.Entry<String, String> entry : previousStackProperties.entrySet()) { + String name = entry.getKey(); + + if (newStackProperties.containsKey(name)) { + String previousValue = entry.getValue(); + String newValue = newStackProperties.get(name); + String userValue = updatedProperties.get(name); + + // See if the user property should be modified... + // Test if the old property value is different than the new property value and that the user did + // not update the value from the old default. If the user updated the value, then it would be + // risky to change it to the new stack default value. + if (((previousValue == null) ? (newValue != null) : !previousValue.equals(newValue)) && + ((previousValue == null) ? (userValue == null) : previousValue.equals(userValue))) { + LOG.debug("Modifying property named {} from user-specified Kerberos Descriptor", name); + updatedProperties.put(name, newValue); + } + } else { + LOG.debug("Removing property named {} from user-specified Kerberos Descriptor", name); + updatedProperties.remove(name); + } + } + + // Find properties to add + for (Map.Entry<String, String> entry : newStackProperties.entrySet()) { + String name = entry.getKey(); + + if (!previousStackProperties.containsKey(name) && !updatedProperties.containsKey(name)) { + // A new property was found, add it... + LOG.debug("Adding property named {} to user-specified Kerberos Descriptor", name); + updatedProperties.put(name, entry.getValue()); + } + } + + return updatedProperties; + } + } + + /** + * A convenience method used to change a list of {@link KerberosIdentityDescriptor} items into a map + * of identity names or {@link org.apache.ambari.server.api.services.HostKerberosIdentityService} + * items. + * + * @param identities a list of {@link KerberosIdentityDescriptor}s + * @return a map of identity names or {@link org.apache.ambari.server.api.services.HostKerberosIdentityService} + */ + private static Map<String, KerberosIdentityDescriptor> toMap(List<KerberosIdentityDescriptor> identities) { + if (identities == null) { + return null; + } else { + Map<String, KerberosIdentityDescriptor> map = new TreeMap<String, KerberosIdentityDescriptor>(); + + for (KerberosIdentityDescriptor identity : identities) { + map.put(identity.getName(), identity); + } + + return map; + } + } +} http://git-wip-us.apache.org/repos/asf/ambari/blob/7aff03f5/ambari-server/src/main/java/org/apache/ambari/server/state/kerberos/KerberosKeytabDescriptor.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/kerberos/KerberosKeytabDescriptor.java b/ambari-server/src/main/java/org/apache/ambari/server/state/kerberos/KerberosKeytabDescriptor.java index 7ce1c9f..ccb4efe 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/state/kerberos/KerberosKeytabDescriptor.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/state/kerberos/KerberosKeytabDescriptor.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -17,8 +17,8 @@ */ package org.apache.ambari.server.state.kerberos; -import java.util.HashMap; import java.util.Map; +import java.util.TreeMap; /** * KerberosKeytabDescriptor is an implementation of an AbstractKerberosDescriptor that @@ -157,13 +157,13 @@ public class KerberosKeytabDescriptor extends AbstractKerberosDescriptor { /** * Creates a new KerberosKeytabDescriptor * - * @param file the path to the keytab file - * @param ownerName the local username of the file owner - * @param ownerAccess the file access privileges for the file owner ("r", "rw", "") - * @param groupName the local group name with privileges to access the file - * @param groupAccess the file access privileges for the group ("r", "rw", "") + * @param file the path to the keytab file + * @param ownerName the local username of the file owner + * @param ownerAccess the file access privileges for the file owner ("r", "rw", "") + * @param groupName the local group name with privileges to access the file + * @param groupAccess the file access privileges for the group ("r", "rw", "") * @param configuration the configuration used to store the principal name - * @param cachable true if the keytab may be cached by Ambari; otherwise false + * @param cachable true if the keytab may be cached by Ambari; otherwise false */ public KerberosKeytabDescriptor(String file, String ownerName, String ownerAccess, String groupName, String groupAccess, String configuration, boolean cachable) { @@ -175,6 +175,7 @@ public class KerberosKeytabDescriptor extends AbstractKerberosDescriptor { setConfiguration(configuration); setCachable(cachable); } + /** * Creates a new KerberosKeytabDescriptor * <p/> @@ -416,21 +417,53 @@ public class KerberosKeytabDescriptor extends AbstractKerberosDescriptor { */ @Override public Map<String, Object> toMap() { - Map<String, Object> map = new HashMap<String, Object>(); + Map<String, Object> map = new TreeMap<String, Object>(); + + String data; - map.put("file", getFile()); + data = getFile(); + map.put("file", data); - map.put("owner", new HashMap<String, Object>() {{ - put("name", getOwnerName()); - put("access", getOwnerAccess()); - }}); + // Build file owner map + Map<String, String> owner = new TreeMap<String, String>(); + + data = getOwnerName(); + if (data != null) { + owner.put("name", data); + } + + data = getOwnerAccess(); + if (data != null) { + owner.put("access", data); + } + + if (!owner.isEmpty()) { + map.put("owner", owner); + } + // Build file owner map (end) - map.put("group", new HashMap<String, Object>() {{ - put("name", getGroupName()); - put("access", getGroupAccess()); - }}); + // Build file owner map + Map<String, String> group = new TreeMap<String, String>(); - map.put("configuration", getConfiguration()); + data = getGroupName(); + if (data != null) { + group.put("name", data); + } + + data = getGroupAccess(); + if (data != null) { + group.put("access", data); + } + + if (!owner.isEmpty()) { + map.put("group", group); + } + // Build file owner map (end) + + data = getConfiguration(); + if (data != null) { + map.put("configuration", data); + } return map; } http://git-wip-us.apache.org/repos/asf/ambari/blob/7aff03f5/ambari-server/src/main/java/org/apache/ambari/server/state/kerberos/KerberosPrincipalDescriptor.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/kerberos/KerberosPrincipalDescriptor.java b/ambari-server/src/main/java/org/apache/ambari/server/state/kerberos/KerberosPrincipalDescriptor.java index 0156e4a..83dd953 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/state/kerberos/KerberosPrincipalDescriptor.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/state/kerberos/KerberosPrincipalDescriptor.java @@ -17,7 +17,7 @@ */ package org.apache.ambari.server.state.kerberos; -import java.util.HashMap; +import java.util.TreeMap; import java.util.Map; /** @@ -267,7 +267,7 @@ public class KerberosPrincipalDescriptor extends AbstractKerberosDescriptor { */ @Override public Map<String, Object> toMap() { - Map<String, Object> map = new HashMap<String, Object>(); + Map<String, Object> map = new TreeMap<String, Object>(); map.put("value", getValue()); map.put("type", KerberosPrincipalType.translate(getType())); http://git-wip-us.apache.org/repos/asf/ambari/blob/7aff03f5/ambari-server/src/main/java/org/apache/ambari/server/state/kerberos/KerberosServiceDescriptor.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/kerberos/KerberosServiceDescriptor.java b/ambari-server/src/main/java/org/apache/ambari/server/state/kerberos/KerberosServiceDescriptor.java index 72dbcfe..0f14ca6 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/state/kerberos/KerberosServiceDescriptor.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/state/kerberos/KerberosServiceDescriptor.java @@ -20,7 +20,7 @@ package org.apache.ambari.server.state.kerberos; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; -import java.util.HashMap; +import java.util.TreeMap; import java.util.List; import java.util.Map; @@ -196,7 +196,7 @@ public class KerberosServiceDescriptor extends AbstractKerberosDescriptorContain } if (components == null) { - components = new HashMap<String, KerberosComponentDescriptor>(); + components = new TreeMap<String, KerberosComponentDescriptor>(); } components.put(name, component); http://git-wip-us.apache.org/repos/asf/ambari/blob/7aff03f5/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/kerberos.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/kerberos.json b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/kerberos.json index d569447..636d36e 100644 --- a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/kerberos.json +++ b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/kerberos.json @@ -104,7 +104,8 @@ "name": "PHOENIX_QUERY_SERVER", "identities": [ { - "name": "/spnego", + "name": "phoenix_spnego", + "reference": "/spnego", "principal": { "configuration": "hbase-site/phoenix.queryserver.kerberos.principal" }, http://git-wip-us.apache.org/repos/asf/ambari/blob/7aff03f5/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.5.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.5.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.5.xml index f5f06c0..ff4ed18 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.5.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.5.xml @@ -449,6 +449,16 @@ </group> <!-- + After processing this group, the user-specified Kerberos descriptor will be updated to work with + the new stack-level Kerberos descriptor. + --> + <group xsi:type="cluster" name="UPDATE_KERBEROS_DESCRIPTORS" title="Update Kerberos Descriptors"> + <execute-stage title="Update the user-specified Kerberos descriptor" service="" component=""> + <task xsi:type="server_action" class="org.apache.ambari.server.serveraction.upgrades.UpgradeUserKerberosDescriptor"/> + </execute-stage> + </group> + + <!-- Invoke "hdp-select set all" to change any components we may have missed that are installed on the hosts but not known by Ambari. --> http://git-wip-us.apache.org/repos/asf/ambari/blob/7aff03f5/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.5.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.5.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.5.xml index 09bd2ac..46e11e4 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.5.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.5.xml @@ -197,6 +197,16 @@ </execute-stage> </group> + <!-- + After processing this group, the user-specified Kerberos descriptor will be updated to work with + the new stack-level Kerberos descriptor. + --> + <group xsi:type="cluster" name="UPDATE_KERBEROS_DESCRIPTORS" title="Update Kerberos Descriptors"> + <execute-stage title="Update the user-specified Kerberos descriptor" service="" component=""> + <task xsi:type="server_action" class="org.apache.ambari.server.serveraction.upgrades.UpgradeUserKerberosDescriptor"/> + </execute-stage> + </group> + <group name="CORE_MASTER" title="Core Masters"> <service-check>false</service-check> <service name="HDFS"> http://git-wip-us.apache.org/repos/asf/ambari/blob/7aff03f5/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/nonrolling-upgrade-2.5.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/nonrolling-upgrade-2.5.xml b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/nonrolling-upgrade-2.5.xml index 1cc9529..3478603 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/nonrolling-upgrade-2.5.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/nonrolling-upgrade-2.5.xml @@ -405,6 +405,16 @@ </group> <!-- + After processing this group, the user-specified Kerberos descriptor will be updated to work with + the new stack-level Kerberos descriptor. + --> + <group xsi:type="cluster" name="UPDATE_KERBEROS_DESCRIPTORS" title="Update Kerberos Descriptors"> + <execute-stage title="Update the user-specified Kerberos descriptor" service="" component=""> + <task xsi:type="server_action" class="org.apache.ambari.server.serveraction.upgrades.UpgradeUserKerberosDescriptor"/> + </execute-stage> + </group> + + <!-- Invoke "hdp-select set all" to change any components we may have missed that are installed on the hosts but not known by Ambari. --> http://git-wip-us.apache.org/repos/asf/ambari/blob/7aff03f5/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.5.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.5.xml b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.5.xml index f20ad06..8a7f0fa 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.5.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.5.xml @@ -183,6 +183,16 @@ </execute-stage> </group> + <!-- + After processing this group, the user-specified Kerberos descriptor will be updated to work with + the new stack-level Kerberos descriptor. + --> + <group xsi:type="cluster" name="UPDATE_KERBEROS_DESCRIPTORS" title="Update Kerberos Descriptors"> + <execute-stage title="Update the user-specified Kerberos descriptor" service="" component=""> + <task xsi:type="server_action" class="org.apache.ambari.server.serveraction.upgrades.UpgradeUserKerberosDescriptor"/> + </execute-stage> + </group> + <group name="CORE_MASTER" title="Core Masters"> <service-check>false</service-check> <service name="HDFS"> http://git-wip-us.apache.org/repos/asf/ambari/blob/7aff03f5/ambari-server/src/main/resources/stacks/HDP/2.5/services/HBASE/kerberos.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/HBASE/kerberos.json b/ambari-server/src/main/resources/stacks/HDP/2.5/services/HBASE/kerberos.json index 501bcd3..9ed40ef 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/HBASE/kerberos.json +++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/HBASE/kerberos.json @@ -142,7 +142,8 @@ "name": "PHOENIX_QUERY_SERVER", "identities": [ { - "name": "/spnego", + "name": "phoenix_spnego", + "reference": "/spnego", "principal": { "configuration": "hbase-site/phoenix.queryserver.kerberos.principal" }, http://git-wip-us.apache.org/repos/asf/ambari/blob/7aff03f5/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/nonrolling-upgrade-2.5.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/nonrolling-upgrade-2.5.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/nonrolling-upgrade-2.5.xml index cbab386..a54c830 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/nonrolling-upgrade-2.5.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/nonrolling-upgrade-2.5.xml @@ -295,6 +295,16 @@ </group> <!-- + After processing this group, the user-specified Kerberos descriptor will be updated to work with + the new stack-level Kerberos descriptor. + --> + <group xsi:type="cluster" name="UPDATE_KERBEROS_DESCRIPTORS" title="Update Kerberos Descriptors"> + <execute-stage title="Update the user-specified Kerberos descriptor" service="" component=""> + <task xsi:type="server_action" class="org.apache.ambari.server.serveraction.upgrades.UpgradeUserKerberosDescriptor"/> + </execute-stage> + </group> + + <!-- Invoke "hdp-select set all" to change any components we may have missed that are installed on the hosts but not known by Ambari. --> http://git-wip-us.apache.org/repos/asf/ambari/blob/7aff03f5/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/upgrade-2.5.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/upgrade-2.5.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/upgrade-2.5.xml index 60e81c0..e3e632b 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/upgrade-2.5.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/upgrade-2.5.xml @@ -148,6 +148,16 @@ </service> </group> + <!-- + After processing this group, the user-specified Kerberos descriptor will be updated to work with + the new stack-level Kerberos descriptor. + --> + <group xsi:type="cluster" name="UPDATE_KERBEROS_DESCRIPTORS" title="Update Kerberos Descriptors"> + <execute-stage title="Update the user-specified Kerberos descriptor" service="" component=""> + <task xsi:type="server_action" class="org.apache.ambari.server.serveraction.upgrades.UpgradeUserKerberosDescriptor"/> + </execute-stage> + </group> + <group name="CORE_MASTER" title="Core Masters"> <service-check>false</service-check> <service name="HDFS">
