AMBARI-21602. Pre-configure services when Kerberos is enabled to reduce number of core service restarts when services are added (rlevas)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/4bce5782 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/4bce5782 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/4bce5782 Branch: refs/heads/feature-branch-AMBARI-21307 Commit: 4bce57823da7d972df9e422e3f920ebbc60362f3 Parents: f5f9a75 Author: Robert Levas <rle...@hortonworks.com> Authored: Thu Aug 24 17:01:12 2017 -0400 Committer: Robert Levas <rle...@hortonworks.com> Committed: Thu Aug 24 17:01:12 2017 -0400 ---------------------------------------------------------------------- .../server/api/services/AmbariMetaInfo.java | 115 +-- .../stackadvisor/StackAdvisorRequest.java | 7 +- .../controller/DeleteIdentityHandler.java | 2 +- .../server/controller/KerberosHelper.java | 114 ++- .../server/controller/KerberosHelperImpl.java | 818 +++++++++++++------ .../BlueprintConfigurationProcessor.java | 2 +- ...usterKerberosDescriptorResourceProvider.java | 3 +- .../internal/StackVersionResourceProvider.java | 66 -- .../utilities/RemovableIdentities.java | 4 +- .../controller/utilities/UsedIdentities.java | 2 +- .../customactions/ActionDefinitionManager.java | 4 +- .../AbstractPrepareKerberosServerAction.java | 2 +- .../kerberos/KerberosServerAction.java | 6 + .../kerberos/PreconfigureServiceType.java | 46 ++ .../PrepareDisableKerberosServerAction.java | 2 +- .../PrepareEnableKerberosServerAction.java | 38 +- .../PrepareKerberosIdentitiesServerAction.java | 48 +- .../upgrades/UpgradeUserKerberosDescriptor.java | 4 +- .../server/stack/ConfigurationDirectory.java | 5 +- .../ambari/server/stack/ExtensionDirectory.java | 3 +- .../ambari/server/stack/ServiceDirectory.java | 14 +- .../ambari/server/stack/ServiceModule.java | 6 +- .../ambari/server/stack/StackDirectory.java | 92 ++- .../ambari/server/stack/StackManager.java | 13 +- .../apache/ambari/server/stack/StackModule.java | 9 +- .../ambari/server/state/ConfigHelper.java | 3 +- .../apache/ambari/server/state/ServiceInfo.java | 11 +- .../apache/ambari/server/state/StackInfo.java | 20 + .../kerberos/AbstractKerberosDescriptor.java | 36 +- .../state/kerberos/KerberosDescriptor.java | 42 +- .../kerberos/KerberosServiceDescriptor.java | 86 +- .../server/upgrade/AbstractUpgradeCatalog.java | 2 +- .../1.10.3-10/configuration/kerberos-env.xml | 27 + .../stacks/HDP/2.6/kerberos_preconfigure.json | 24 + .../render/ClusterBlueprintRendererTest.java | 2 +- .../server/api/services/AmbariMetaInfoTest.java | 55 +- .../server/controller/KerberosHelperTest.java | 707 ++++++++++++---- .../BlueprintConfigurationProcessorTest.java | 2 +- ...rKerberosDescriptorResourceProviderTest.java | 14 +- .../utilities/KerberosIdentityCleanerTest.java | 2 +- .../UpgradeUserKerberosDescriptorTest.java | 6 +- .../ambari/server/stack/ServiceModuleTest.java | 3 +- .../KerberosDescriptorUpdateHelperTest.java | 4 +- .../kerberos/KerberosServiceDescriptorTest.java | 2 + .../stacks/HDP/2.0.8/kerberos_preconfigure.json | 23 + 45 files changed, 1780 insertions(+), 716 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/4bce5782/ambari-server/src/main/java/org/apache/ambari/server/api/services/AmbariMetaInfo.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/api/services/AmbariMetaInfo.java b/ambari-server/src/main/java/org/apache/ambari/server/api/services/AmbariMetaInfo.java index 7eb82e4..aad35a9 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/api/services/AmbariMetaInfo.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/api/services/AmbariMetaInfo.java @@ -24,7 +24,6 @@ import static org.apache.ambari.server.controller.utilities.PropertyHelper.AGGRE import java.io.File; import java.io.FileReader; -import java.io.FilenameFilter; import java.io.IOException; import java.lang.reflect.Type; import java.util.ArrayList; @@ -55,7 +54,6 @@ import org.apache.ambari.server.metadata.AmbariServiceAlertDefinitions; import org.apache.ambari.server.orm.dao.AlertDefinitionDAO; import org.apache.ambari.server.orm.dao.MetainfoDAO; import org.apache.ambari.server.orm.entities.AlertDefinitionEntity; -import org.apache.ambari.server.stack.StackDirectory; import org.apache.ambari.server.stack.StackManager; import org.apache.ambari.server.stack.StackManagerFactory; import org.apache.ambari.server.state.Cluster; @@ -97,30 +95,6 @@ import com.google.inject.Singleton; */ @Singleton public class AmbariMetaInfo { - public static final String SERVICE_CONFIG_FOLDER_NAME = "configuration"; - public static final String SERVICE_PROPERTIES_FOLDER_NAME = "properties"; - public static final String SERVICE_THEMES_FOLDER_NAME = "themes"; - public static final String SERVICE_QUICKLINKS_CONFIGURATIONS_FOLDER_NAME = "quicklinks"; - public static final String SERVICE_CONFIG_FILE_NAME_POSTFIX = ".xml"; - public static final String RCO_FILE_NAME = "role_command_order.json"; - public static final String SERVICE_METRIC_FILE_NAME = "metrics.json"; - public static final String SERVICE_ALERT_FILE_NAME = "alerts.json"; - public static final String SERVICE_ADVISOR_FILE_NAME = "service_advisor.py"; - - /** - * The filename for a Kerberos descriptor file at either the stack or service level - */ - public static final String KERBEROS_DESCRIPTOR_FILE_NAME = "kerberos.json"; - - /** - * The filename for a Widgets descriptor file at either the stack or service level - */ - public static final String WIDGETS_DESCRIPTOR_FILE_NAME = "widgets.json"; - - /** - * Filename for theme file at service layer - */ - public static final String SERVICE_THEME_FILE_NAME = "theme.json"; /** * This string is used in placeholder in places that are common for @@ -131,13 +105,6 @@ public class AmbariMetaInfo { * Version of XML files with support of custom services and custom commands */ public static final String SCHEMA_VERSION_2 = "2.0"; - public static final FilenameFilter FILENAME_FILTER = new FilenameFilter() { - @Override - public boolean accept(File dir, String s) { - return !(s.equals(".svn") || s.equals(".git") || - s.equals(StackDirectory.HOOKS_FOLDER_NAME)); - } - }; private final static Logger LOG = LoggerFactory.getLogger(AmbariMetaInfo.class); @@ -1276,40 +1243,52 @@ public class AmbariMetaInfo { * All of the kerberos.json files from the specified stack (and version) are read, parsed and * complied into a complete Kerberos descriptor hierarchy. * - * @param stackName a String declaring the stack name - * @param stackVersion a String declaring the stack version + * @param stackName a String declaring the stack name + * @param stackVersion a String declaring the stack version + * @param includePreconfigureData a Boolean value indicating whether to include the pre-configuration + * data (<code>true</code>), or not (<code>false</code>) * @return a new complete KerberosDescriptor, or null if no Kerberos descriptor information is available * @throws AmbariException if an error occurs reading or parsing the stack's kerberos.json files */ - public KerberosDescriptor getKerberosDescriptor(String stackName, String stackVersion) throws AmbariException { + public KerberosDescriptor getKerberosDescriptor(String stackName, String stackVersion, boolean includePreconfigureData) throws AmbariException { StackInfo stackInfo = getStack(stackName, stackVersion); - String kerberosDescriptorFileLocation = stackInfo.getKerberosDescriptorFileLocation(); - KerberosDescriptor kerberosDescriptor = null; - // Read in the stack-level Kerberos descriptor - if (kerberosDescriptorFileLocation != null) { - File file = new File(kerberosDescriptorFileLocation); + // Read in the stack-level Kerberos descriptor pre-configuration data + if (includePreconfigureData) { + kerberosDescriptor = readKerberosDescriptorFromFile(stackInfo.getKerberosDescriptorPreConfigurationFileLocation()); - if (file.canRead()) { - try { - kerberosDescriptor = kerberosDescriptorFactory.createInstance(file); - } catch (IOException e) { - throw new AmbariException(String.format("Failed to parse kerberos descriptor file %s", - file.getAbsolutePath()), e); + if (kerberosDescriptor != null) { + // Ensure the all services to be pre-configured are flagged appropriately. + Map<String, KerberosServiceDescriptor> serviceDescriptors = kerberosDescriptor.getServices(); + if (serviceDescriptors != null) { + for (KerberosServiceDescriptor serviceDescriptor : serviceDescriptors.values()) { + serviceDescriptor.setPreconfigure(true); + } } - } else { - throw new AmbariException(String.format("Unable to read kerberos descriptor file %s", - file.getAbsolutePath())); } } - if (kerberosDescriptor == null) { - kerberosDescriptor = new KerberosDescriptor(); + // Read in the base stack-level Kerberos descriptor. + KerberosDescriptor stackKerberosDescriptor = readKerberosDescriptorFromFile(stackInfo.getKerberosDescriptorFileLocation()); + if (stackKerberosDescriptor == null) { + // If kerberosDescriptor is null and stackKerberosDescriptor is null, then ensure + // kerberosDescriptor is an empty KerberosDescriptor. + if (kerberosDescriptor == null) { + kerberosDescriptor = new KerberosDescriptor(); + } + } else { + if (kerberosDescriptor == null) { + // If kerberosDescriptor is null; then set it to stackKerberosDescriptor. + kerberosDescriptor = stackKerberosDescriptor; + } else { + // If kerberosDescriptor is not null; then update it using stackKerberosDescriptor. + kerberosDescriptor.update(stackKerberosDescriptor); + } } - // Read in the service-level Kerberos descriptors + // Read in the individual service-level Kerberos descriptors Map<String, ServiceInfo> services = getServices(stackName, stackVersion); if (services != null) { @@ -1318,6 +1297,9 @@ public class AmbariMetaInfo { if (serviceDescriptors != null) { for (KerberosServiceDescriptor serviceDescriptor : serviceDescriptors) { + // Add the service-level Kerberos descriptor to kerberosDescriptor. If the service was + // previously added (possible from the pre-configure data), update the existing service-level + // Kerberos descriptor with this one. kerberosDescriptor.putService(serviceDescriptor); } } @@ -1425,5 +1407,32 @@ public class AmbariMetaInfo { return versionDefinitions; } + /** + * Reads a Kerberos descriptor from the specified file path. + * + * @param fileLocation the path to the file + * @return a KerberosDescriptor or <code>null</code>, if no path is specified + * @throws AmbariException if an error occurs reading or parsing the Kerberos descriptor file + */ + KerberosDescriptor readKerberosDescriptorFromFile(String fileLocation) throws AmbariException { + if (!StringUtils.isEmpty(fileLocation)) { + File file = new File(fileLocation); + + if (file.canRead()) { + try { + return kerberosDescriptorFactory.createInstance(file); + } catch (IOException e) { + throw new AmbariException(String.format("Failed to parse Kerberos descriptor file %s", + file.getAbsolutePath()), e); + } + } else { + throw new AmbariException(String.format("Unable to read Kerberos descriptor file %s", + file.getAbsolutePath())); + } + } else { + LOG.debug("Missing path to Kerberos descriptor, returning null"); + } + return null; + } } http://git-wip-us.apache.org/repos/asf/ambari/blob/4bce5782/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorRequest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorRequest.java b/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorRequest.java index 7ba1b18..3a2b488 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorRequest.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorRequest.java @@ -20,6 +20,7 @@ package org.apache.ambari.server.api.services.stackadvisor; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collection; import java.util.HashMap; import java.util.LinkedList; import java.util.List; @@ -39,7 +40,7 @@ public class StackAdvisorRequest { private String stackVersion; private StackAdvisorRequestType requestType; private List<String> hosts = new ArrayList<>(); - private List<String> services = new ArrayList<>(); + private Collection<String> services = new ArrayList<>(); private Map<String, Set<String>> componentHostsMap = new HashMap<>(); private Map<String, Set<String>> hostComponents = new HashMap<>(); private Map<String, Set<String>> hostGroupBindings = new HashMap<>(); @@ -64,7 +65,7 @@ public class StackAdvisorRequest { return hosts; } - public List<String> getServices() { + public Collection<String> getServices() { return services; } @@ -142,7 +143,7 @@ public class StackAdvisorRequest { return this; } - public StackAdvisorRequestBuilder forServices(List<String> services) { + public StackAdvisorRequestBuilder forServices(Collection<String> services) { this.instance.services = services; return this; } http://git-wip-us.apache.org/repos/asf/ambari/blob/4bce5782/ambari-server/src/main/java/org/apache/ambari/server/controller/DeleteIdentityHandler.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/DeleteIdentityHandler.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/DeleteIdentityHandler.java index 3329e76..978b329 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/DeleteIdentityHandler.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/DeleteIdentityHandler.java @@ -279,7 +279,7 @@ class DeleteIdentityHandler { } private KerberosDescriptor getKerberosDescriptor() throws AmbariException { - return getKerberosHelper().getKerberosDescriptor(getCluster()); + return getKerberosHelper().getKerberosDescriptor(getCluster(), false); } } http://git-wip-us.apache.org/repos/asf/ambari/blob/4bce5782/ambari-server/src/main/java/org/apache/ambari/server/controller/KerberosHelper.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/KerberosHelper.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/KerberosHelper.java index a334542..9bdb377 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/KerberosHelper.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/KerberosHelper.java @@ -80,6 +80,10 @@ public interface KerberosHelper { */ String AMBARI_SERVER_HOST_NAME = "ambari_server"; /** + * The name of the Kerberos environment configuration type + */ + String KERBEROS_ENV = "kerberos-env"; + /** * The name of the Ambari server's Kerberos identities as defined in the Kerberos descriptor */ String AMBARI_SERVER_KERBEROS_IDENTITY_NAME = "ambari-server"; @@ -93,6 +97,32 @@ public interface KerberosHelper { * identities or not */ String MANAGE_IDENTITIES = "manage_identities"; + /** + * The kerberos-env property name declaring the default realm + */ + String DEFAULT_REALM = "realm"; + /** + * The kerberos-env property name declaring the kdc_type + * + * @see org.apache.ambari.server.serveraction.kerberos.KDCType + */ + String KDC_TYPE = "kdc_type"; + /** + * The kerberos-env property name declaring whether to manage auth-to-local rules or not + */ + String MANAGE_AUTH_TO_LOCAL_RULES = "manage_auth_to_local"; + /** + * The kerberos-env property name declaring whether auth-to-local rules should be case-insensitive or not + */ + String CASE_INSENSITIVE_USERNAME_RULES = "case_insensitive_username_rules"; + /** + * The kerberos-env property name declaring how to preprocess services. + * <p> + * Expected values are <code>"ALL"</code>, <code>"DEFAULT"</code>, <code>"NONE"</code> + * + * @see org.apache.ambari.server.serveraction.kerberos.PreconfigureServiceType + */ + String PRECONFIGURE_SERVICES = "preconfigure_services"; /** * Toggles Kerberos security to enable it or remove it depending on the state of the cluster. @@ -379,18 +409,22 @@ public interface KerberosHelper { * Sets the relevant auth-to-local rule configuration properties using the services installed on * the cluster and their relevant Kerberos descriptors to determine the rules to be created. * - * @param kerberosDescriptor the current Kerberos descriptor - * @param realm the default realm - * @param installedServices the map of services and relevant components to process - * @param existingConfigurations a map of the current configurations - * @param kerberosConfigurations a map of the configurations to update, this where the generated - * auth-to-local values will be stored + * @param cluster cluster instance + * @param kerberosDescriptor the current Kerberos descriptor + * @param realm the default realm + * @param installedServices the map of services and relevant components to process + * @param existingConfigurations a map of the current configurations + * @param kerberosConfigurations a map of the configurations to update, this where the generated + * auth-to-local values will be stored + * @param includePreconfigureData <code>true</code> to include the preconfigure data; <code>false</code> otherwise * @throws AmbariException */ - void setAuthToLocalRules(KerberosDescriptor kerberosDescriptor, String realm, + void setAuthToLocalRules(Cluster cluster, KerberosDescriptor kerberosDescriptor, + String realm, Map<String, Set<String>> installedServices, Map<String, Map<String, String>> existingConfigurations, - Map<String, Map<String, String>> kerberosConfigurations) + Map<String, Map<String, String>> kerberosConfigurations, + boolean includePreconfigureData) throws AmbariException; /** @@ -437,12 +471,13 @@ public interface KerberosHelper { * descriptor and the composite is returned. If not, the default cluster descriptor is returned * as-is. * - * @param cluster cluster instance + * @param cluster cluster instance + * @param includePreconfigureData <code>true</code> to include the preconfigure data; <code>false</code> otherwise * @return the kerberos descriptor associated with the specified cluster * @throws AmbariException if unable to obtain the descriptor - * @see #getKerberosDescriptor(KerberosDescriptorType, Cluster, boolean, Collection) + * @see #getKerberosDescriptor(KerberosDescriptorType, Cluster, boolean, Collection, boolean) */ - KerberosDescriptor getKerberosDescriptor(Cluster cluster) throws AmbariException; + KerberosDescriptor getKerberosDescriptor(Cluster cluster, boolean includePreconfigureData) throws AmbariException; /** * Gets the requested Kerberos descriptor. @@ -456,39 +491,62 @@ public interface KerberosHelper { * <dd>A Kerberos descriptor built using user-specified data stored as an artifact of the cluster, only</dd> * <dt>{@link KerberosDescriptorType#COMPOSITE}</dt> * <dd>A Kerberos descriptor built using data from the current stack definition with user-specified data stored as an artifact of the cluster applied - * - see {@link #getKerberosDescriptor(Cluster)}</dd> + * - see {@link #getKerberosDescriptor(Cluster, boolean)}</dd> * </dl> * - * @param kerberosDescriptorType the type of Kerberos descriptor to retrieve - see {@link KerberosDescriptorType} - * @param cluster the relevant Cluster - * @param evaluateWhenClauses true to evaluate Kerberos identity <code>when</code> clauses and - * prune if necessary; false otherwise. - * @param additionalServices an optional collection of service names declaring additional - * services to add to the set of currently installed services to use - * while evaluating <code>when</code> clauses + * @param kerberosDescriptorType the type of Kerberos descriptor to retrieve - see {@link KerberosDescriptorType} + * @param cluster the relevant Cluster + * @param evaluateWhenClauses true to evaluate Kerberos identity <code>when</code> clauses and + * prune if necessary; false otherwise. + * @param additionalServices an optional collection of service names declaring additional + * services to add to the set of currently installed services to use + * while evaluating <code>when</code> clauses + * @param includePreconfigureData <code>true</code> to include the preconfigure data; <code>false</code> otherwise * @return a Kerberos descriptor * @throws AmbariException */ KerberosDescriptor getKerberosDescriptor(KerberosDescriptorType kerberosDescriptorType, Cluster cluster, - boolean evaluateWhenClauses, Collection<String> additionalServices) + boolean evaluateWhenClauses, Collection<String> additionalServices, boolean includePreconfigureData) throws AmbariException; /** - * Merges configuration from a Map of configuration updates into a main configurations Map. Each - * property in the updates Map is processed to replace variables using the replacement Map. + * Merges configurations from a Map of configuration updates into a main configurations Map. + * <p> + * Each property in the updates Map is processed to replace variables using the replacement Map, + * unless it has been filtered out using an optionally supplied configuration type filter + * (configurationTypeFilter). * <p/> * See {@link org.apache.ambari.server.state.kerberos.VariableReplacementHelper#replaceVariables(String, java.util.Map)} * for information on variable replacement. * - * @param configurations a Map of configurations - * @param updates a Map of configuration updates - * @param replacements a Map of (grouped) replacement values - * @return the merged Map - * @throws AmbariException + * @param configurations a Map of existing configurations (updated in-place) + * @param updates a Map of configuration updates + * @param replacements a Map of (grouped) replacement values + * @param configurationTypeFilter a Set of config types to filter from the map of updates; + * <code>null</code> indicate no filter + * @return the updated configurations map + * @throws AmbariException if an issue occurs */ Map<String, Map<String, String>> mergeConfigurations(Map<String, Map<String, String>> configurations, Map<String, KerberosConfigurationDescriptor> updates, - Map<String, Map<String, String>> replacements) + Map<String, Map<String, String>> replacements, + Set<String> configurationTypeFilter) + throws AmbariException; + + /** + * Determines which services, not currently installed, should be preconfigured to aid in reducing + * the number of service restarts when new services are added to a cluster where Kerberos is enabled. + * + * @param configurations a Map of existing configurations (updated in-place) + * @param replacements a Map of (grouped) replacement values + * @param cluster the cluster + * @param kerberosDescriptor the Kerberos Descriptor + * @return the updated configurations map + * @throws AmbariException if an issue occurs + */ + Map<String, Map<String, String>> processPreconfiguredServiceConfigurations(Map<String, Map<String, String>> configurations, + Map<String, Map<String, String>> replacements, Cluster cluster, + KerberosDescriptor kerberosDescriptor) throws AmbariException; /**
