Repository: ambari Updated Branches: refs/heads/trunk bf499456c -> b62846d25
AMBARI-8247. Expose stack and service kerberos descriptors via REST API * Note, this patch doesn't expose new resources for these descriptors but returns them as new properties in the existing stack and service resources. This is temporary and these properties will be removed shortly, before the 2.0 release, and replaced with proper sub-resources Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/b62846d2 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/b62846d2 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/b62846d2 Branch: refs/heads/trunk Commit: b62846d25f4a3d65ee3e01a9d1b03697548a96bc Parents: bf49945 Author: Robert Levas <rle...@hortonworks.com> Authored: Fri Dec 12 22:25:05 2014 -0500 Committer: John Speidel <jspei...@hortonworks.com> Committed: Fri Dec 12 22:25:05 2014 -0500 ---------------------------------------------------------------------- .../server/api/services/AmbariMetaInfo.java | 5 + .../server/controller/StackServiceResponse.java | 29 ++++ .../server/controller/StackVersionResponse.java | 64 +++++++- .../internal/StackServiceResourceProvider.java | 34 +++++ .../internal/StackVersionResourceProvider.java | 63 ++++++++ .../ambari/server/stack/ServiceDirectory.java | 19 +++ .../ambari/server/stack/ServiceModule.java | 4 + .../ambari/server/stack/StackDirectory.java | 20 +++ .../apache/ambari/server/stack/StackModule.java | 1 + .../apache/ambari/server/state/ServiceInfo.java | 19 ++- .../apache/ambari/server/state/StackInfo.java | 44 +++++- .../src/main/resources/properties.json | 2 + .../server/api/services/AmbariMetaInfoTest.java | 58 ++++++++ .../ambari/server/stack/ServiceModuleTest.java | 38 +++++ .../HDP/2.0.8/services/HDFS/kerberos.json | 147 +++++++++++++++++++ .../stacks/HDP/2.1.1/services/HDFS/metainfo.xml | 146 ++++++++++++++++++ 16 files changed, 689 insertions(+), 4 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/b62846d2/ambari-server/src/main/java/org/apache/ambari/server/api/services/AmbariMetaInfo.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/api/services/AmbariMetaInfo.java b/ambari-server/src/main/java/org/apache/ambari/server/api/services/AmbariMetaInfo.java index 60d4f9b..2dd3351 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/api/services/AmbariMetaInfo.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/api/services/AmbariMetaInfo.java @@ -91,6 +91,11 @@ public class AmbariMetaInfo { public static final String SERVICE_ALERT_FILE_NAME = "alerts.json"; /** + * The filename name for a Kerberos descriptor file at either the stack or service level + */ + public static final String KERBEROS_DESCRIPTOR_FILE_NAME = "kerberos.json"; + + /** * This string is used in placeholder in places that are common for * all operating systems or in situations where os type is not important. */ http://git-wip-us.apache.org/repos/asf/ambari/blob/b62846d2/ambari-server/src/main/java/org/apache/ambari/server/controller/StackServiceResponse.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/StackServiceResponse.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/StackServiceResponse.java index 2794c18..0eecb3f 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/StackServiceResponse.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/StackServiceResponse.java @@ -21,6 +21,7 @@ package org.apache.ambari.server.controller; import org.apache.ambari.server.state.CustomCommandDefinition; import org.apache.ambari.server.state.ServiceInfo; +import java.io.File; import java.util.ArrayList; import java.util.Collections; import java.util.List; @@ -42,6 +43,13 @@ public class StackServiceResponse { private List<String> requiredServices; /** + * A File pointing to the service-level Kerberos descriptor file + * + * This may be null if a relevant file is not available. + */ + private File kerberosDescriptorFile; + + /** * Constructor. * * @param service @@ -67,6 +75,8 @@ public class StackServiceResponse { customCommands.add(command.getName()); } } + + kerberosDescriptorFile = service.getKerberosDescriptorFile(); } public String getStackName() { @@ -138,6 +148,25 @@ public class StackServiceResponse { } /** + * Gets a File pointing to the service-level Kerberos descriptor + * + * @return a File pointing to the service-level Kerberos descriptor, or null if no relevant file is + * available + */ + public File getKerberosDescriptorFile() { + return kerberosDescriptorFile; + } + + /** + * Sets the service-level Kerberos descriptor File + * + * @param kerberosDescriptorFile a File pointing to the service-level Kerberos descriptor + */ + public void setKerberosDescriptorFile(File kerberosDescriptorFile) { + this.kerberosDescriptorFile = kerberosDescriptorFile; + } + + /** * Gets whether the service represented by this response supports running * "Service Checks". A service check is possible where there is a custom * command defined in the {@code metainfo.xml} of the service definition. This http://git-wip-us.apache.org/repos/asf/ambari/blob/b62846d2/ambari-server/src/main/java/org/apache/ambari/server/controller/StackVersionResponse.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/StackVersionResponse.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/StackVersionResponse.java index a338a12..1263553 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/StackVersionResponse.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/StackVersionResponse.java @@ -18,6 +18,8 @@ package org.apache.ambari.server.controller; +import java.io.File; +import java.util.Collection; import java.util.Map; @@ -30,14 +32,32 @@ public class StackVersionResponse { private String parentVersion; private Map<String, Map<String, Map<String, String>>> configTypes; + /** + * A File pointing to the stack-level Kerberos descriptor file + * + * This may be null if a relevant file is not available. + */ + private File stackKerberosDescriptorFile; + + /** + * A Collection of Files pointing to the service-level Kerberos descriptor files + * + * This may be null or empty if no relevant files are available. + */ + private Collection<File> serviceKerberosDescriptorFiles; + public StackVersionResponse(String stackVersion, String minUpgradeVersion, - boolean active, String parentVersion, - Map<String, Map<String, Map<String, String>>> configTypes) { + boolean active, String parentVersion, + Map<String, Map<String, Map<String, String>>> configTypes, + File stackKerberosDescriptorFile, + Collection<File> serviceKerberosDescriptorFiles) { setStackVersion(stackVersion); setMinUpgradeVersion(minUpgradeVersion); setActive(active); setParentVersion(parentVersion); setConfigTypes(configTypes); + setKerberosDescriptorFile(stackKerberosDescriptorFile); + setServiceKerberosDescriptorFiles(serviceKerberosDescriptorFiles); } public String getStackName() { @@ -86,4 +106,44 @@ public class StackVersionResponse { Map<String, Map<String, Map<String, String>>> configTypes) { this.configTypes = configTypes; } + + /** + * Gets a File pointing to the stack-level Kerberos descriptor + * + * @return a File pointing to the stack-level Kerberos descriptor, or null if no relevant file is + * available + */ + public File getStackKerberosDescriptorFile() { + return stackKerberosDescriptorFile; + } + + /** + * Sets the stack-level Kerberos descriptor File + * + * @param stackKerberosDescriptorFile a File pointing to the stack-level Kerberos descriptor + */ + public void setKerberosDescriptorFile(File stackKerberosDescriptorFile) { + this.stackKerberosDescriptorFile = stackKerberosDescriptorFile; + } + + /** + * Gets the Collection of Files pointing to the stack-specific service-level Kerberos descriptor + * files + * + * @return a Collection of Files pointing to the stack-specific service-level Kerberos descriptor + * files, or null if no relevant files are available + */ + public Collection<File> getServiceKerberosDescriptorFiles() { + return serviceKerberosDescriptorFiles; + } + + /** + * Sets the Collection of stack-specific service-level Kerberos descriptor Files + * + * @param serviceKerberosDescriptorFiles a Collection of stack-specific service-level Kerberos + * descriptor Files + */ + public void setServiceKerberosDescriptorFiles(Collection<File> serviceKerberosDescriptorFiles) { + this.serviceKerberosDescriptorFiles = serviceKerberosDescriptorFiles; + } } http://git-wip-us.apache.org/repos/asf/ambari/blob/b62846d2/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/StackServiceResourceProvider.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/StackServiceResourceProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/StackServiceResourceProvider.java index 1296ba1..89ac9d7 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/StackServiceResourceProvider.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/StackServiceResourceProvider.java @@ -19,6 +19,7 @@ package org.apache.ambari.server.controller.internal; +import com.google.gson.Gson; import org.apache.ambari.server.AmbariException; import org.apache.ambari.server.controller.AmbariManagementController; import org.apache.ambari.server.controller.StackServiceRequest; @@ -26,7 +27,12 @@ import org.apache.ambari.server.controller.StackServiceResponse; import org.apache.ambari.server.controller.spi.*; import org.apache.ambari.server.controller.spi.Resource.Type; import org.apache.ambari.server.controller.utilities.PropertyHelper; +import org.apache.ambari.server.state.kerberos.KerberosServiceDescriptor; +import java.io.File; +import java.io.FileNotFoundException; +import java.io.FileReader; +import java.io.IOException; import java.util.*; public class StackServiceResourceProvider extends ReadOnlyResourceProvider { @@ -64,6 +70,9 @@ public class StackServiceResourceProvider extends ReadOnlyResourceProvider { private static final String CUSTOM_COMMANDS_PROPERTY_ID = PropertyHelper.getPropertyId( "StackServices", "custom_commands"); + private static final String KERBEROS_DESCRIPTOR_PROPERTY_ID = PropertyHelper.getPropertyId( + "StackServices", "kerberos_descriptor"); + private static Set<String> pkPropertyIds = new HashSet<String>( Arrays.asList(new String[] { STACK_NAME_PROPERTY_ID, STACK_VERSION_PROPERTY_ID, SERVICE_NAME_PROPERTY_ID })); @@ -136,6 +145,31 @@ public class StackServiceResourceProvider extends ReadOnlyResourceProvider { setResourceProperty(resource, CUSTOM_COMMANDS_PROPERTY_ID, response.getCustomCommands(), requestedIds); + // TODO (rlevas): Convert this to an official resource + File kerberosDescriptorFile = response.getKerberosDescriptorFile(); + if (kerberosDescriptorFile != null) { + KerberosServiceDescriptor[] descriptors; + try { + descriptors = KerberosServiceDescriptor.fromFile(kerberosDescriptorFile); + } catch (IOException e) { + throw new SystemException("Failed to parse the service's Kerberos descriptor", e); + } + + if (descriptors != null) { + String serviceName = response.getServiceName(); + + // Iterate over the KerberosServiceDescriptors to find the one for this service since + // Kerberos descriptor files can contain details about more than one service + for(KerberosServiceDescriptor descriptor:descriptors) { + if(serviceName.equals(descriptor.getName())) { + setResourceProperty(resource, KERBEROS_DESCRIPTOR_PROPERTY_ID, + descriptor.toMap(), requestedIds); + break; // Stop looping, this was the service we are looking for. + } + } + } + } + resources.add(resource); } http://git-wip-us.apache.org/repos/asf/ambari/blob/b62846d2/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/StackVersionResourceProvider.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/StackVersionResourceProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/StackVersionResourceProvider.java index 8c9a139..fa17f69 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/StackVersionResourceProvider.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/StackVersionResourceProvider.java @@ -19,7 +19,10 @@ package org.apache.ambari.server.controller.internal; +import java.io.File; +import java.io.IOException; import java.util.Arrays; +import java.util.Collection; import java.util.Collections; import java.util.HashSet; import java.util.Map; @@ -38,6 +41,8 @@ import org.apache.ambari.server.controller.spi.Resource.Type; import org.apache.ambari.server.controller.spi.SystemException; import org.apache.ambari.server.controller.spi.UnsupportedPropertyException; import org.apache.ambari.server.controller.utilities.PropertyHelper; +import org.apache.ambari.server.state.kerberos.KerberosDescriptor; +import org.apache.ambari.server.state.kerberos.KerberosServiceDescriptor; public class StackVersionResourceProvider extends ReadOnlyResourceProvider { @@ -47,6 +52,7 @@ public class StackVersionResourceProvider extends ReadOnlyResourceProvider { public static final String STACK_ACTIVE_PROPERTY_ID = PropertyHelper.getPropertyId("Versions", "active"); public static final String STACK_CONFIG_TYPES = PropertyHelper.getPropertyId("Versions", "config_types"); public static final String STACK_PARENT_PROPERTY_ID = PropertyHelper.getPropertyId("Versions", "parent_stack_version"); + public static final String KERBEROS_DESCRIPTOR_PROPERTY_ID = PropertyHelper.getPropertyId("Versions", "kerberos_descriptor"); private static Set<String> pkPropertyIds = new HashSet<String>( Arrays.asList(new String[] { STACK_NAME_PROPERTY_ID, STACK_VERSION_PROPERTY_ID })); @@ -104,12 +110,69 @@ public class StackVersionResourceProvider extends ReadOnlyResourceProvider { setResourceProperty(resource, STACK_CONFIG_TYPES, response.getConfigTypes(), requestedIds); + // TODO (rlevas): Convert this to an official resource + KerberosDescriptor kerberosDescriptor; + try { + kerberosDescriptor = buildKerberosDescriptor(response); + } catch (IOException e) { + throw new SystemException("Failed to build composite Kerberos descriptor data", e); + } + if (kerberosDescriptor != null) { + setResourceProperty(resource, KERBEROS_DESCRIPTOR_PROPERTY_ID, + kerberosDescriptor.toMap(), requestedIds); + } + resources.add(resource); } return resources; } + /** + * Given data from a StackVersionResponse build a complete Kerberos descriptor hierarchy. + * + * @param stackVersionResponse the StackVersionResponse instance containing the details of the + * stack and the relevant Kerberos descriptor files + * @return a KerberosDescriptor containing the complete hierarchy for the stack + * @throws IOException if the specified File is not found or not a readable + * @throws AmbariException if the specified File does not contain valid JSON-encoded Kerberos + * descriptor + */ + private KerberosDescriptor buildKerberosDescriptor(StackVersionResponse stackVersionResponse) + throws IOException { + KerberosDescriptor kerberosDescriptor = null; + + // Process the stack-level Kerberos descriptor file + File stackKerberosDescriptorFile = stackVersionResponse.getStackKerberosDescriptorFile(); + if (stackKerberosDescriptorFile != null) { + kerberosDescriptor = KerberosDescriptor.fromFile(stackKerberosDescriptorFile); + } + + // Process the service-level Kerberos descriptor files + Collection<File> serviceDescriptorFiles = stackVersionResponse.getServiceKerberosDescriptorFiles(); + if ((serviceDescriptorFiles != null) && !serviceDescriptorFiles.isEmpty()) { + // Make sure kerberosDescriptor is not null. This will be the case if there is no stack-level + // Kerberos descriptor file. + if (kerberosDescriptor == null) { + kerberosDescriptor = new KerberosDescriptor(); + } + + // For each service-level Kerberos descriptor file, parse into an array of KerberosServiceDescriptors + // and then append each to the KerberosDescriptor hierarchy. + for (File file : serviceDescriptorFiles) { + KerberosServiceDescriptor[] serviceDescriptors = KerberosServiceDescriptor.fromFile(file); + + if (serviceDescriptors != null) { + for (KerberosServiceDescriptor serviceDescriptor : serviceDescriptors) { + kerberosDescriptor.putService(serviceDescriptor); + } + } + } + } + + return kerberosDescriptor; + } + private StackVersionRequest getRequest(Map<String, Object> properties) { return new StackVersionRequest( (String) properties.get(STACK_NAME_PROPERTY_ID), http://git-wip-us.apache.org/repos/asf/ambari/blob/b62846d2/ambari-server/src/main/java/org/apache/ambari/server/stack/ServiceDirectory.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/stack/ServiceDirectory.java b/ambari-server/src/main/java/org/apache/ambari/server/stack/ServiceDirectory.java index 843df0b..6008b1a 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/stack/ServiceDirectory.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/stack/ServiceDirectory.java @@ -31,6 +31,7 @@ import java.io.File; * Encapsulates IO operations on a stack definition service directory. */ public class ServiceDirectory extends StackDefinitionDirectory { + /** * metrics file */ @@ -42,6 +43,11 @@ public class ServiceDirectory extends StackDefinitionDirectory { private File alertsFile; /** + * kerberos descriptor file + */ + private File kerberosDescriptorFile; + + /** * package directory path */ private String packageDir; @@ -94,6 +100,10 @@ public class ServiceDirectory extends StackDefinitionDirectory { File af = new File(directory.getAbsolutePath() + File.separator + AmbariMetaInfo.SERVICE_ALERT_FILE_NAME); alertsFile = af.exists() ? af : null; + + File kdf = new File(directory.getAbsolutePath() + + File.separator + AmbariMetaInfo.KERBEROS_DESCRIPTOR_FILE_NAME); + kerberosDescriptorFile = kdf.exists() ? kdf : null; } /** @@ -124,6 +134,15 @@ public class ServiceDirectory extends StackDefinitionDirectory { } /** + * Obtain the Kerberos Descriptor file. + * + * @return Kerberos Descriptor file + */ + public File getKerberosDescriptorFile() { + return kerberosDescriptorFile; + } + + /** * Obtain the service metainfo file object representation. * * @return http://git-wip-us.apache.org/repos/asf/ambari/blob/b62846d2/ambari-server/src/main/java/org/apache/ambari/server/stack/ServiceModule.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/stack/ServiceModule.java b/ambari-server/src/main/java/org/apache/ambari/server/stack/ServiceModule.java index ae7dd28..400dd27 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/stack/ServiceModule.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/stack/ServiceModule.java @@ -96,6 +96,7 @@ public class ServiceModule extends BaseModule<ServiceModule, ServiceInfo> { serviceInfo.setMetricsFile(serviceDirectory.getMetricsFile()); serviceInfo.setAlertsFile(serviceDirectory.getAlertsFile()); + serviceInfo.setKerberosDescriptorFile(serviceDirectory.getKerberosDescriptorFile()); serviceInfo.setSchemaVersion(AmbariMetaInfo.SCHEMA_VERSION_2); serviceInfo.setServicePackageFolder(serviceDirectory.getPackageDir()); @@ -152,6 +153,9 @@ public class ServiceModule extends BaseModule<ServiceModule, ServiceInfo> { if (serviceInfo.getAlertsFile() == null) { serviceInfo.setAlertsFile(parent.getAlertsFile()); } + if (serviceInfo.getKerberosDescriptorFile() == null) { + serviceInfo.setKerberosDescriptorFile(parent.getKerberosDescriptorFile()); + } mergeCustomCommands(parent.getCustomCommands(), serviceInfo.getCustomCommands()); mergeConfigDependencies(parent); http://git-wip-us.apache.org/repos/asf/ambari/blob/b62846d2/ambari-server/src/main/java/org/apache/ambari/server/stack/StackDirectory.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/stack/StackDirectory.java b/ambari-server/src/main/java/org/apache/ambari/server/stack/StackDirectory.java index 1e1e3a0..dc84feb 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/stack/StackDirectory.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/stack/StackDirectory.java @@ -63,6 +63,11 @@ public class StackDirectory extends StackDefinitionDirectory { private String rcoFilePath; /** + * kerberos descriptor file path + */ + private String kerberosDescriptorFilePath; + + /** * repository file */ private RepositoryXml repoFile; @@ -182,6 +187,15 @@ public class StackDirectory extends StackDefinitionDirectory { } /** + * Obtain the path to the (stack-level) Kerberos descriptor file + * + * @return the path to the (stack-level) Kerberos descriptor file + */ + public String getKerberosDescriptorFilePath() { + return kerberosDescriptorFilePath; + } + + /** * Obtain the repository directory path. * * @return repository directory path @@ -257,6 +271,12 @@ public class StackDirectory extends StackDefinitionDirectory { rcoFilePath = getAbsolutePath() + File.separator + AmbariMetaInfo.RCO_FILE_NAME; } + + if (subDirs.contains(AmbariMetaInfo.KERBEROS_DESCRIPTOR_FILE_NAME)) { + // kerberosDescriptorFilePath is expected to be absolute + kerberosDescriptorFilePath = getAbsolutePath() + File.separator + AmbariMetaInfo.KERBEROS_DESCRIPTOR_FILE_NAME; + } + parseUpgradePacks(subDirs); parseServiceDirectories(subDirs); parseRepoFile(subDirs); http://git-wip-us.apache.org/repos/asf/ambari/blob/b62846d2/ambari-server/src/main/java/org/apache/ambari/server/stack/StackModule.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/stack/StackModule.java b/ambari-server/src/main/java/org/apache/ambari/server/stack/StackModule.java index 6f6577a..933f187 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/stack/StackModule.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/stack/StackModule.java @@ -378,6 +378,7 @@ public class StackModule extends BaseModule<StackModule, StackInfo> { stackInfo.setParentStackVersion(smx.getExtends()); stackInfo.setStackHooksFolder(stackDirectory.getHooksDir()); stackInfo.setRcoFileLocation(stackDirectory.getRcoFilePath()); + stackInfo.setKerberosDescriptorFileLocation(stackDirectory.getKerberosDescriptorFilePath()); stackInfo.setUpgradesFolder(stackDirectory.getUpgradesDir()); stackInfo.setUpgradePacks(stackDirectory.getUpgradePacks()); stackInfo.setRoleCommandOrder(stackDirectory.getRoleCommandOrder()); http://git-wip-us.apache.org/repos/asf/ambari/blob/b62846d2/ambari-server/src/main/java/org/apache/ambari/server/state/ServiceInfo.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/ServiceInfo.java b/ambari-server/src/main/java/org/apache/ambari/server/state/ServiceInfo.java index 5224aaa..d469c17 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/state/ServiceInfo.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/state/ServiceInfo.java @@ -108,6 +108,9 @@ public class ServiceInfo { @XmlTransient private File alertsFile = null; + @XmlTransient + private File kerberosDescriptorFile = null; + /** * Internal list of os-specific details (loaded from xml). Added at schema ver 2 */ @@ -533,7 +536,7 @@ public class ServiceInfo { public void setAlertsFile(File file) { alertsFile = file; } - + /** * @return the alerts file, or <code>null</code> if none exists */ @@ -542,6 +545,20 @@ public class ServiceInfo { } /** + * @param file the file containing the alert definitions + */ + public void setKerberosDescriptorFile(File file) { + kerberosDescriptorFile = file; + } + + /** + * @return the kerberos descriptor file, or <code>null</code> if none exists + */ + public File getKerberosDescriptorFile() { + return kerberosDescriptorFile; + } + + /** * @return config types this service contains configuration for, but which are primarily related to another service */ public Set<String> getExcludedConfigTypes() { http://git-wip-us.apache.org/repos/asf/ambari/blob/b62846d2/ambari-server/src/main/java/org/apache/ambari/server/state/StackInfo.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/StackInfo.java b/ambari-server/src/main/java/org/apache/ambari/server/state/StackInfo.java index b99dcd6..7c5d411 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/state/StackInfo.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/state/StackInfo.java @@ -18,10 +18,12 @@ package org.apache.ambari.server.state; +import java.io.File; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; @@ -35,6 +37,7 @@ public class StackInfo implements Comparable<StackInfo>{ private String minUpgradeVersion; private boolean active; private String rcoFileLocation; + private String kerberosDescriptorFileLocation; private List<RepositoryInfo> repositories; private Collection<ServiceInfo> services; private String parentStackVersion; @@ -188,8 +191,28 @@ public class StackInfo implements Comparable<StackInfo>{ public StackVersionResponse convertToResponse() { + // Get the stack-level Kerberos descriptor file path + String stackDescriptorFileFilePath = getKerberosDescriptorFileLocation(); + + // Collect the services' Kerberos descriptor files + Collection<ServiceInfo> serviceInfos = getServices(); + // The collection of service descriptor files. A Set is being used because some Kerberos descriptor + // files contain multiple services, therefore the same File may be encountered more than once. + // For example the YARN directory may contain YARN and MAPREDUCE2 services. + Collection<File> serviceDescriptorFiles = new HashSet<File>(); + if (serviceInfos != null) { + for (ServiceInfo serviceInfo : serviceInfos) { + File file = serviceInfo.getKerberosDescriptorFile(); + if (file != null) { + serviceDescriptorFiles.add(file); + } + } + } + return new StackVersionResponse(getVersion(), getMinUpgradeVersion(), - isActive(), getParentStackVersion(), getConfigTypeAttributes()); + isActive(), getParentStackVersion(), getConfigTypeAttributes(), + (stackDescriptorFileFilePath == null) ? null : new File(stackDescriptorFileFilePath), + serviceDescriptorFiles); } public String getMinUpgradeVersion() { @@ -232,6 +255,25 @@ public class StackInfo implements Comparable<StackInfo>{ this.rcoFileLocation = rcoFileLocation; } + /** + * Gets the path to the stack-level Kerberos descriptor file + * + * @return a String containing the path to the stack-level Kerberos descriptor file + */ + public String getKerberosDescriptorFileLocation() { + return kerberosDescriptorFileLocation; + } + + /** + * Sets the path to the stack-level Kerberos descriptor file + * + * @param kerberosDescriptorFileLocation a String containing the path to the stack-level Kerberos + * descriptor file + */ + public void setKerberosDescriptorFileLocation(String kerberosDescriptorFileLocation) { + this.kerberosDescriptorFileLocation = kerberosDescriptorFileLocation; + } + public String getStackHooksFolder() { return stackHooksFolder; } http://git-wip-us.apache.org/repos/asf/ambari/blob/b62846d2/ambari-server/src/main/resources/properties.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/properties.json b/ambari-server/src/main/resources/properties.json index fe67de9..c54e85e 100644 --- a/ambari-server/src/main/resources/properties.json +++ b/ambari-server/src/main/resources/properties.json @@ -203,6 +203,7 @@ "Versions/active", "Versions/parent_stack_version", "Versions/config_types", + "Versions/kerberos_descriptor", "_" ], "StackService":[ @@ -217,6 +218,7 @@ "StackServices/service_check_supported", "StackServices/custom_commands", "StackServices/required_services", + "StackServices/kerberos_descriptor", "_" ], "StackConfiguration":[ http://git-wip-us.apache.org/repos/asf/ambari/blob/b62846d2/ambari-server/src/test/java/org/apache/ambari/server/api/services/AmbariMetaInfoTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/api/services/AmbariMetaInfoTest.java b/ambari-server/src/test/java/org/apache/ambari/server/api/services/AmbariMetaInfoTest.java index 734f4b9..cb43321 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/api/services/AmbariMetaInfoTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/api/services/AmbariMetaInfoTest.java @@ -30,6 +30,7 @@ import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; import java.io.File; +import java.io.FileReader; import java.lang.reflect.Field; import java.util.Collection; import java.util.HashSet; @@ -41,6 +42,7 @@ import java.util.Set; import javax.persistence.EntityManager; import javax.xml.bind.JAXBException; +import com.google.gson.Gson; import junit.framework.Assert; import org.apache.ambari.server.AmbariException; @@ -819,6 +821,33 @@ public class AmbariMetaInfoTest { } @Test + public void testKerberosJson() throws Exception { + ServiceInfo svc; + + svc = metaInfo.getService(STACK_NAME_HDP, "2.0.8", "HDFS"); + Assert.assertNotNull(svc); + + File kerberosDescriptorFile1 = svc.getKerberosDescriptorFile(); + Assert.assertNotNull(kerberosDescriptorFile1); + Assert.assertTrue(kerberosDescriptorFile1.exists()); + + svc = metaInfo.getService(STACK_NAME_HDP, "2.1.1", "HDFS"); + Assert.assertNotNull(svc); + + File kerberosDescriptorFile2 = svc.getKerberosDescriptorFile(); + Assert.assertNotNull(kerberosDescriptorFile1); + Assert.assertTrue(kerberosDescriptorFile1.exists()); + + Assert.assertEquals(kerberosDescriptorFile1, kerberosDescriptorFile2); + + svc = metaInfo.getService(STACK_NAME_HDP, "2.0.7", "HDFS"); + Assert.assertNotNull(svc); + + File kerberosDescriptorFile3 = svc.getKerberosDescriptorFile(); + Assert.assertNull(kerberosDescriptorFile3); + } + + @Test public void testGanglia134Dependencies() throws Exception { ServiceInfo service = metaInfo.getService(STACK_NAME_HDP, "1.3.4", "GANGLIA"); List<ComponentInfo> componentList = service.getComponents(); @@ -1679,6 +1708,35 @@ public class AmbariMetaInfoTest { } } + @Test + public void testKerberosDescriptor() throws Exception { + ServiceInfo service; + + // Test that kerberos descriptor file is not available when not supplied in service definition + service = metaInfo.getService(STACK_NAME_HDP, "2.1.1", "PIG"); + Assert.assertNotNull(service); + Assert.assertNull(service.getKerberosDescriptorFile()); + + // Test that kerberos descriptor file is available when supplied in service definition + service = metaInfo.getService(STACK_NAME_HDP, "2.0.8", "HDFS"); + Assert.assertNotNull(service); + Assert.assertNotNull(service.getKerberosDescriptorFile()); + + // Test that kerberos descriptor file is available from inherited stack version + service = metaInfo.getService(STACK_NAME_HDP, "2.1.1", "HDFS"); + Assert.assertNotNull(service); + Assert.assertNotNull(service.getKerberosDescriptorFile()); + + + // Test that kerberos.json file can be parsed into mapped data + Map<?,?> kerberosDescriptorData = new Gson() + .fromJson(new FileReader(service.getKerberosDescriptorFile()), Map.class); + + Assert.assertNotNull(kerberosDescriptorData); + Assert.assertEquals(2, kerberosDescriptorData.size()); + } + + private TestAmbariMetaInfo setupTempAmbariMetaInfo(String buildDir, boolean replayMocks) throws Exception { File stackRootTmp = new File(buildDir + "/ambari-metaInfo"); File stackRoot = new File("src/test/resources/stacks"); http://git-wip-us.apache.org/repos/asf/ambari/blob/b62846d2/ambari-server/src/test/java/org/apache/ambari/server/stack/ServiceModuleTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/stack/ServiceModuleTest.java b/ambari-server/src/test/java/org/apache/ambari/server/stack/ServiceModuleTest.java index b6b7190..fc12d75 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/stack/ServiceModuleTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/stack/ServiceModuleTest.java @@ -391,6 +391,43 @@ public class ServiceModuleTest { } @Test + public void testResolve_KerberosDescriptorFile() throws Exception { + File kerberosDescriptorFile = new File("testKerberosDescriptorFile"); + + // specified in child only + ServiceInfo info = new ServiceInfo(); + ServiceInfo parentInfo = new ServiceInfo(); + + ServiceModule child = createServiceModule(info); + ServiceModule parent = createServiceModule(parentInfo); + + // set in the module constructor from a value obtained from service directory which is mocked + assertEquals(kerberosDescriptorFile, child.getModuleInfo().getKerberosDescriptorFile()); + parent.getModuleInfo().setKerberosDescriptorFile(null); + + resolveService(child, parent); + assertEquals(kerberosDescriptorFile, child.getModuleInfo().getKerberosDescriptorFile()); + + // specified in parent only + child = createServiceModule(info); + parent = createServiceModule(parentInfo); + parent.getModuleInfo().setKerberosDescriptorFile(kerberosDescriptorFile); + child.getModuleInfo().setKerberosDescriptorFile(null); + + resolveService(child, parent); + assertEquals(kerberosDescriptorFile, child.getModuleInfo().getKerberosDescriptorFile()); + + // specified in both + child = createServiceModule(info); + parent = createServiceModule(parentInfo); + parent.getModuleInfo().setKerberosDescriptorFile(new File("someOtherDir")); + child.getModuleInfo().setKerberosDescriptorFile(kerberosDescriptorFile); + + resolveService(child, parent); + assertEquals(kerberosDescriptorFile, child.getModuleInfo().getKerberosDescriptorFile()); + } + + @Test public void testResolve_CustomCommands() throws Exception { List<CustomCommandDefinition> customCommands = new ArrayList<CustomCommandDefinition>(); CustomCommandDefinition cmd1 = new CustomCommandDefinition(); @@ -918,6 +955,7 @@ public class ServiceModuleTest { expect(serviceDirectory.getConfigurationDirectory(dir)).andReturn(configDir).anyTimes(); expect(serviceDirectory.getMetricsFile()).andReturn(new File("testMetricsFile")).anyTimes(); expect(serviceDirectory.getAlertsFile()).andReturn(new File("testAlertsFile")).anyTimes(); + expect(serviceDirectory.getKerberosDescriptorFile()).andReturn(new File("testKerberosDescriptorFile")).anyTimes(); expect(serviceDirectory.getPackageDir()).andReturn("packageDir").anyTimes(); replay(serviceDirectory); http://git-wip-us.apache.org/repos/asf/ambari/blob/b62846d2/ambari-server/src/test/resources/stacks/HDP/2.0.8/services/HDFS/kerberos.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/resources/stacks/HDP/2.0.8/services/HDFS/kerberos.json b/ambari-server/src/test/resources/stacks/HDP/2.0.8/services/HDFS/kerberos.json new file mode 100644 index 0000000..b61c8b5 --- /dev/null +++ b/ambari-server/src/test/resources/stacks/HDP/2.0.8/services/HDFS/kerberos.json @@ -0,0 +1,147 @@ +{ + "name": "HDFS", + "components": [ + { + "name": "NAMENODE", + "identities": [ + { + "name": "namenode_nn", + "principal": { + "value": "nn/_HOST@${realm}", + "configuration": "hdfs-site/dfs.namenode.kerberos.principal" + }, + "keytab": { + "file": "${keytab_dir}/nn.service.keytab", + "owner": { + "name": "${hadoop-env/hdfs_user}", + "access": "r" + }, + "group": { + "name": "${cluster-env/user_group}", + "access": "" + }, + "configuration": "hdfs-site/dfs.namenode.keytab.file" + } + }, + { + "name": "namenode_host", + "principal": { + "value": "host/_HOST@${realm}" + }, + "keytab": { + "file": "${keytab_dir}/nn.service.keytab", + "owner": { + "name": "${hadoop-env/hdfs_user}", + "access": "r" + }, + "group": { + "name": "${cluster-env/user_group}", + "access": "" + } + } + }, + { + "name": "/spnego", + "principal": { + "configuration": "hdfs-site/dfs.namenode.kerberos.internal.spnego.principal" + } + } + ], + "configurations": [ + { + "hdfs-site": { + "dfs.block.access.token.enable": "true" + } + } + ] + }, + { + "name": "DATANODE", + "identities": [ + { + "name": "datanode_dn", + "principal": { + "value": "dn/_HOST@${realm}", + "configuration": "hdfs-site/dfs.datanode.kerberos.principal" + }, + "keytab": { + "file": "${keytab_dir}/dn.service.keytab", + "owner": { + "name": "${hadoop-env/hdfs_user}", + "access": "r" + }, + "group": { + "name": "${cluster-env/user_group}", + "access": "" + }, + "configuration": "hdfs-site/dfs.datanode.keytab.file" + } + }, + { + "name": "datanode_host", + "principal": { + "value": "host/_HOST@${realm}" + }, + "keytab": { + "file": "${keytab_dir}/dn.service.keytab", + "owner": { + "name": "${hadoop-env/hdfs_user}", + "access": "r" + }, + "group": { + "name": "${cluster-env/user_group}", + "access": "" + } + } + } + ] + }, + { + "name": "SECONDARY_NAMENODE", + "identities": [ + { + "name": "secondary_namenode_nn", + "principal": { + "value": "nn/_HOST@${realm}", + "configuration": "hdfs-site/dfs.secondary.namenode.kerberos.principal" + }, + "keytab": { + "file": "${keytab_dir}/snn.service.keytab", + "owner": { + "name": "${hadoop-env/hdfs_user}", + "access": "r" + }, + "group": { + "name": "${cluster-env/user_group}", + "access": "" + }, + "configuration": "hdfs-site/dfs.secondary.namenode.kerberos.principal" + } + }, + { + "name": "secondary_namenode_host", + "principal": { + "value": "host/_HOST@${realm}" + }, + "keytab": { + "file": "${keytab_dir}/snn.service.keytab", + "owner": { + "name": "${hadoop-env/hdfs_user}", + "access": "r" + }, + "group": { + "name": "${cluster-env/user_group}", + "access": "" + } + } + }, + { + "name": "/spnego", + "principal": { + "configuration": "hdfs-site/dfs.secondary.namenode.kerberos.internal.spnego.principal" + } + } + ] + } + ] +} http://git-wip-us.apache.org/repos/asf/ambari/blob/b62846d2/ambari-server/src/test/resources/stacks/HDP/2.1.1/services/HDFS/metainfo.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/resources/stacks/HDP/2.1.1/services/HDFS/metainfo.xml b/ambari-server/src/test/resources/stacks/HDP/2.1.1/services/HDFS/metainfo.xml new file mode 100644 index 0000000..d3bef74 --- /dev/null +++ b/ambari-server/src/test/resources/stacks/HDP/2.1.1/services/HDFS/metainfo.xml @@ -0,0 +1,146 @@ +<?xml version="1.0"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<metainfo> + <schemaVersion>2.0</schemaVersion> + <services> + <service> + <name>HDFS</name> + <comment>Apache Hadoop Distributed File System</comment> + <version>2.1.0.2.0.6.0</version> + + <components> + <component> + <name>NAMENODE</name> + <category>MASTER</category> + <commandScript> + <script>scripts/namenode.py</script> + <scriptType>PYTHON</scriptType> + <timeout>600</timeout> + </commandScript> + <customCommands> + <customCommand> + <name>DECOMMISSION</name> + <commandScript> + <script>scripts/namenode_dec_overr.py</script> + <scriptType>PYTHON</scriptType> + <timeout>600</timeout> + </commandScript> + </customCommand> + <customCommand> + <name>YET_ANOTHER_CHILD_COMMAND</name> + <commandScript> + <script>scripts/yet_another_child_command.py</script> + <scriptType>PYTHON</scriptType> + <timeout>600</timeout> + </commandScript> + </customCommand> + </customCommands> + </component> + + <component> + <name>DATANODE</name> + <category>SLAVE</category> + <commandScript> + <script>scripts/datanode.py</script> + <scriptType>PYTHON</scriptType> + <timeout>600</timeout> + </commandScript> + </component> + + <component> + <name>SECONDARY_NAMENODE</name> + <category>MASTER</category> + <commandScript> + <script>scripts/snamenode.py</script> + <scriptType>PYTHON</scriptType> + <timeout>600</timeout> + </commandScript> + </component> + + <component> + <name>HDFS_CLIENT</name> + <category>CLIENT</category> + <commandScript> + <script>scripts/hdfs_client_overridden.py</script> + <scriptType>PYTHON</scriptType> + <timeout>600</timeout> + </commandScript> + </component> + + <component> + <name>JOURNALNODE</name> + <category>MASTER</category> + <commandScript> + <script>scripts/journalnode.py</script> + <scriptType>PYTHON</scriptType> + <timeout>600</timeout> + </commandScript> + </component> + + <component> + <name>ZKFC</name> + <category>SLAVE</category> + <commandScript> + <script>scripts/zkfc_slave.py</script> + <scriptType>PYTHON</scriptType> + <timeout>600</timeout> + </commandScript> + </component> + </components> + + <osSpecifics> + <osSpecific> + <osFamily>any</osFamily> + <packages> + <package> + <name>child-package-def</name> + </package> + </packages> + </osSpecific> + </osSpecifics> + + <commandScript> + <script>scripts/service_check_2.py</script> + <scriptType>PYTHON</scriptType> + <timeout>300</timeout> + </commandScript> + + <customCommands> + <customCommand> + <name>RESTART</name> + <commandScript> + <script>scripts/restart_child.py</script> + <scriptType>PYTHON</scriptType> + <timeout>600</timeout> + </commandScript> + </customCommand> + <customCommand> + <name>YET_ANOTHER_CHILD_SRV_COMMAND</name> + <commandScript> + <script>scripts/yet_another_child_srv_command.py</script> + <scriptType>PYTHON</scriptType> + <timeout>600</timeout> + </commandScript> + </customCommand> + </customCommands> + + <configuration-dependencies> + </configuration-dependencies> + </service> + </services> +</metainfo>
