AMBARI-7113. Property: javax.jdo.option.ConnectionPassword is absent in hive-site.xml. (aonishuk)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/0dd2063f Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/0dd2063f Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/0dd2063f Branch: refs/heads/branch-alerts-dev Commit: 0dd2063f1762af03a4fbe70e0df7db6a45b4087a Parents: 4adf44e Author: Andrew Onishuk <aonis...@hortonworks.com> Authored: Fri Sep 5 16:35:51 2014 +0300 Committer: Andrew Onishuk <aonis...@hortonworks.com> Committed: Fri Sep 5 16:35:51 2014 +0300 ---------------------------------------------------------------------- .../server/api/util/StackExtensionHelper.java | 8 +- .../internal/ClientConfigResourceProvider.java | 5 + .../api/util/StackExtensionHelperTest.java | 27 +++ .../ClientConfigResourceProviderTest.java | 175 ++++++++++++++++++- .../HDP/2.0.5/services/ZOOKEEPER/metainfo.xml | 12 ++ .../HDP/2.0.7/services/ZOOKEEPER/metainfo.xml | 12 ++ 6 files changed, 227 insertions(+), 12 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/0dd2063f/ambari-server/src/main/java/org/apache/ambari/server/api/util/StackExtensionHelper.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/api/util/StackExtensionHelper.java b/ambari-server/src/main/java/org/apache/ambari/server/api/util/StackExtensionHelper.java index c4dd022..1f6fda0 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/api/util/StackExtensionHelper.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/api/util/StackExtensionHelper.java @@ -334,13 +334,7 @@ public class StackExtensionHelper { child.getConfigDependencies() : parent.getConfigDependencies()); -// HashSet downloadSource = child.getDownloadSource(); -// if (downloadSource != null) { -// result.setDownloadSource(child.getDownloadSource()); -// } else { -// result.setDownloadSource(parent.getDownloadSource()); -// } -// + //Merge client config file definitions List<ClientConfigFileDefinition> clientConfigFiles = child.getClientConfigFiles(); if (clientConfigFiles != null) { result.setClientConfigFiles(child.getClientConfigFiles()); http://git-wip-us.apache.org/repos/asf/ambari/blob/0dd2063f/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClientConfigResourceProvider.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClientConfigResourceProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClientConfigResourceProvider.java index 7bec61a..cc97eba 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClientConfigResourceProvider.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClientConfigResourceProvider.java @@ -201,6 +201,11 @@ public class ClientConfigResourceProvider extends AbstractControllerResourceProv } } + // Hack - Remove passwords from configs + if (configurations.get(Configuration.HIVE_CONFIG_TAG)!=null) { + configurations.get(Configuration.HIVE_CONFIG_TAG).remove(Configuration.HIVE_METASTORE_PASSWORD_PROPERTY); + } + Map<String, Set<String>> clusterHostInfo = null; ServiceInfo serviceInfo = null; String osFamily = null; http://git-wip-us.apache.org/repos/asf/ambari/blob/0dd2063f/ambari-server/src/test/java/org/apache/ambari/server/api/util/StackExtensionHelperTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/api/util/StackExtensionHelperTest.java b/ambari-server/src/test/java/org/apache/ambari/server/api/util/StackExtensionHelperTest.java index 9a7946a..dfb1b81 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/api/util/StackExtensionHelperTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/api/util/StackExtensionHelperTest.java @@ -269,6 +269,33 @@ public class StackExtensionHelperTest { } @Test + public void testClientConfigFilesInheritance() throws Exception{ + File stackRoot = new File(stackRootStr); + StackInfo stackInfo = new StackInfo(); + stackInfo.setName("HDP"); + stackInfo.setVersion("2.0.6"); + StackExtensionHelper helper = new StackExtensionHelper(injector, stackRoot); + helper.populateServicesForStack(stackInfo); + helper.fillInfo(); + List<ServiceInfo> allServices = helper.getAllApplicableServices(stackInfo); + for (ServiceInfo serviceInfo : allServices) { + if (serviceInfo.getName().equals("ZOOKEEPER")) { + List<ComponentInfo> components = serviceInfo.getComponents(); + assertTrue(components.size() == 2); + ComponentInfo componentInfo = components.get(1); + List<ClientConfigFileDefinition> clientConfigs = componentInfo.getClientConfigFiles(); + assertEquals(2,clientConfigs.size()); + assertEquals("zookeeper-env",clientConfigs.get(0).getDictionaryName()); + assertEquals("zookeeper-env.sh",clientConfigs.get(0).getFileName()); + assertEquals("env",clientConfigs.get(0).getType()); + assertEquals("zookeeper-log4j",clientConfigs.get(1).getDictionaryName()); + assertEquals("log4j.properties",clientConfigs.get(1).getFileName()); + assertEquals("env",clientConfigs.get(1).getType()); + } + } + } + + @Test public void testMonitoringServicePropertyInheritance() throws Exception{ File stackRoot = new File(stackRootStr); StackInfo stackInfo = new StackInfo(); http://git-wip-us.apache.org/repos/asf/ambari/blob/0dd2063f/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/ClientConfigResourceProviderTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/ClientConfigResourceProviderTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/ClientConfigResourceProviderTest.java index 2f54273..e7ecb88 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/ClientConfigResourceProviderTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/ClientConfigResourceProviderTest.java @@ -18,24 +18,32 @@ package org.apache.ambari.server.controller.internal; -import org.apache.ambari.server.controller.AmbariManagementController; -import org.apache.ambari.server.controller.RequestStatusResponse; -import org.apache.ambari.server.controller.TaskStatusResponse; +import org.apache.ambari.server.api.services.AmbariMetaInfo; +import org.apache.ambari.server.configuration.Configuration; +import org.apache.ambari.server.controller.*; import org.apache.ambari.server.controller.spi.*; import org.apache.ambari.server.controller.utilities.PredicateBuilder; import org.apache.ambari.server.controller.utilities.PropertyHelper; +import org.apache.ambari.server.state.*; +import org.apache.ambari.server.state.PropertyInfo; import org.junit.Assert; import org.junit.Test; +import org.junit.runner.RunWith; +import org.powermock.api.easymock.PowerMock; +import org.powermock.core.classloader.annotations.PrepareForTest; +import org.powermock.modules.junit4.PowerMockRunner; +import java.io.File; +import java.io.PrintWriter; import java.util.*; import static org.easymock.EasyMock.*; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNull; /** * TaskResourceProvider tests. */ +@RunWith(PowerMockRunner.class) +@PrepareForTest( {ClientConfigResourceProvider.class} ) public class ClientConfigResourceProviderTest { @Test public void testCreateResources() throws Exception { @@ -116,6 +124,163 @@ public class ClientConfigResourceProviderTest { } @Test + public void testGetResources() throws Exception { + Resource.Type type = Resource.Type.ClientConfig; + + AmbariManagementController managementController = createNiceMock(AmbariManagementController.class); + Clusters clusters = createNiceMock(Clusters.class); + + Cluster cluster = createNiceMock(Cluster.class); + AmbariMetaInfo ambariMetaInfo = createNiceMock(AmbariMetaInfo.class); + StackId stackId = createNiceMock(StackId.class); + ComponentInfo componentInfo = createNiceMock(ComponentInfo.class); + ServiceInfo serviceInfo = createNiceMock(ServiceInfo.class); + CommandScriptDefinition commandScriptDefinition = createNiceMock(CommandScriptDefinition.class); + Config clusterConfig = createNiceMock(Config.class); + Host host = createNiceMock(Host.class); + Service service = createNiceMock(Service.class); + ServiceComponent serviceComponent = createNiceMock(ServiceComponent.class); + ServiceComponentHost serviceComponentHost = createNiceMock(ServiceComponentHost.class); + ServiceOsSpecific serviceOsSpecific = createNiceMock(ServiceOsSpecific.class); + ConfigHelper configHelper = createNiceMock(ConfigHelper.class); + + File mockFile = PowerMock.createNiceMock(File.class); + Runtime runtime = createMock(Runtime.class); + Process process = createNiceMock(Process.class); + + Collection<Config> clusterConfigs = new HashSet<Config>(); + //Config clusterConfig = new ConfigImpl("config"); + clusterConfigs.add(clusterConfig); + Map<String, Map<String, String>> allConfigTags = new HashMap<String, Map<String, String>>(); + Map<String, Map<String, String>> properties = new HashMap<String, Map<String, String>>(); + Map<String, Map<String, String>> configTags = new HashMap<String, + Map<String, String>>(); + Map<String, Map<String, Map<String, String>>> attributes = new HashMap<String, + Map<String, Map<String, String>>>(); + + ClientConfigFileDefinition clientConfigFileDefinition = new ClientConfigFileDefinition(); + clientConfigFileDefinition.setDictionaryName("pig-env"); + clientConfigFileDefinition.setFileName("pig-env.sh"); + clientConfigFileDefinition.setType("env"); + List <ClientConfigFileDefinition> clientConfigFileDefinitionList = new LinkedList<ClientConfigFileDefinition>(); + clientConfigFileDefinitionList.add(clientConfigFileDefinition); + + ResourceProvider provider = AbstractControllerResourceProvider.getResourceProvider( + type, + PropertyHelper.getPropertyIds(type), + PropertyHelper.getKeyPropertyIds(type), + managementController); + + // create the request + Request request = PropertyHelper.getReadRequest(ClientConfigResourceProvider.COMPONENT_CLUSTER_NAME_PROPERTY_ID, "c1", + ClientConfigResourceProvider.COMPONENT_COMPONENT_NAME_PROPERTY_ID, + ClientConfigResourceProvider.COMPONENT_SERVICE_NAME_PROPERTY_ID); + + Predicate predicate = new PredicateBuilder().property(ClientConfigResourceProvider.COMPONENT_CLUSTER_NAME_PROPERTY_ID).equals("c1"). + toPredicate(); + + String clusterName = "C1"; + String serviceName = "PIG"; + String componentName = "PIG"; + String hostName = "Host100"; + String desiredState = "INSTALLED"; + + String stackName = "S1"; + String stackVersion = "V1"; + + String stackRoot="/tmp/stacks/S1/V1"; + + String packageFolder="PIG/package"; + + HashMap<String, Host> hosts = new HashMap<String, Host>(); + hosts.put(hostName,host); + HashMap<String, Service> services = new HashMap<String, Service>(); + services.put(serviceName,service); + HashMap<String, ServiceComponent> serviceComponentMap = new HashMap<String, ServiceComponent>(); + serviceComponentMap.put(componentName,serviceComponent); + HashMap<String, ServiceComponentHost> serviceComponentHosts = new HashMap<String, ServiceComponentHost>(); + serviceComponentHosts.put(componentName, serviceComponentHost); + HashMap<String, ServiceOsSpecific> serviceOsSpecificHashMap = new HashMap<String, ServiceOsSpecific>(); + serviceOsSpecificHashMap.put("key",serviceOsSpecific); + + ServiceComponentHostResponse shr1 = new ServiceComponentHostResponse(clusterName, serviceName, componentName, hostName, desiredState, "", null, null, null); + + Set<ServiceComponentHostResponse> responses = new LinkedHashSet<ServiceComponentHostResponse>(); + responses.add(shr1); + + // set expectations + expect(managementController.getConfigHelper()).andReturn(configHelper); + expect(managementController.getAmbariMetaInfo()).andReturn(ambariMetaInfo).anyTimes(); + expect(managementController.getClusters()).andReturn(clusters).anyTimes(); + expect(clusters.getCluster(clusterName)).andReturn(cluster).anyTimes(); + expect(configHelper.getEffectiveConfigProperties(cluster, configTags)).andReturn(properties); + expect(clusterConfig.getType()).andReturn(Configuration.HIVE_CONFIG_TAG).anyTimes(); + expect(configHelper.getEffectiveConfigAttributes(cluster, configTags)).andReturn(attributes); + //!!!! + Map<String,String> props = new HashMap<String, String>(); + props.put(Configuration.HIVE_METASTORE_PASSWORD_PROPERTY, "pass"); + props.put("key","value"); + expect(clusterConfig.getProperties()).andReturn(props); + expect(configHelper.getEffectiveDesiredTags(cluster, hostName)).andReturn(allConfigTags); + //!!!! + expect(cluster.getClusterName()).andReturn(clusterName); + expect(managementController.getHostComponents((Set<ServiceComponentHostRequest>) anyObject())).andReturn(responses).anyTimes(); + expect(cluster.getCurrentStackVersion()).andReturn(stackId); + + expect(stackId.getStackName()).andReturn(stackName).anyTimes(); + expect(stackId.getStackVersion()).andReturn(stackVersion).anyTimes(); + + expect(ambariMetaInfo.getComponent(stackName, stackVersion, serviceName, componentName)).andReturn(componentInfo); + expect(ambariMetaInfo.getServiceInfo(stackName, stackVersion, serviceName)).andReturn(serviceInfo); + expect(serviceInfo.getServicePackageFolder()).andReturn(packageFolder); + expect(ambariMetaInfo.getComponentCategory((String) anyObject(), (String) anyObject(), + (String) anyObject(), (String) anyObject())).andReturn(componentInfo).anyTimes(); + expect(componentInfo.getCommandScript()).andReturn(commandScriptDefinition); + expect(componentInfo.getClientConfigFiles()).andReturn(clientConfigFileDefinitionList); + expect(ambariMetaInfo.getStackRoot()).andReturn(new File(stackRoot)); + expect(cluster.getAllConfigs()).andReturn(clusterConfigs); + expect(clusters.getHostsForCluster(clusterName)).andReturn(hosts); + expect(cluster.getServices()).andReturn(services); + expect(service.getServiceComponents()).andReturn(serviceComponentMap); + expect(serviceComponent.getName()).andReturn(componentName); + expect(serviceComponent.getServiceComponentHosts()).andReturn(serviceComponentHosts); + expect(clusters.getHost(hostName)).andReturn(host); + + HashMap<String, String> rcaParams = new HashMap<String, String>(); + rcaParams.put("key","value"); + expect(managementController.getRcaParameters()).andReturn(rcaParams).anyTimes(); + expect(ambariMetaInfo.getServiceInfo(stackName, stackVersion, serviceName)).andReturn(serviceInfo); + expect(serviceInfo.getOsSpecifics()).andReturn(new HashMap<String, ServiceOsSpecific>()).anyTimes(); + Set<String> userSet = new HashSet<String>(); + userSet.add("hdfs"); + expect(configHelper.getPropertyValuesWithPropertyType(stackId, PropertyInfo.PropertyType.USER, cluster)).andReturn(userSet); + PowerMock.expectNew(File.class, new Class<?>[]{String.class}, anyObject(String.class)).andReturn(mockFile).anyTimes(); + PowerMock.createNiceMockAndExpectNew(PrintWriter.class, anyObject()); + expect(mockFile.getParent()).andReturn(""); + PowerMock.mockStatic(Runtime.class); + expect(Runtime.getRuntime()).andReturn(runtime); + expect(mockFile.exists()).andReturn(true); + expect(runtime.exec("ambari-python-wrap /tmp/stacks/S1/V1/PIG/package/null generate_configs null " + + "/tmp/stacks/S1/V1/PIG/package /tmp/ambari-server/structured-out.json INFO /tmp/ambari-server")) + .andReturn(process).once(); + + // replay + replay(managementController, clusters, cluster, ambariMetaInfo, stackId, componentInfo, commandScriptDefinition, + clusterConfig, host, service, serviceComponent, serviceComponentHost, serviceInfo, configHelper, + runtime, process); + PowerMock.replayAll(); + + provider.getResources(request, predicate); + + + + // verify + verify(managementController, clusters, cluster, ambariMetaInfo, stackId, componentInfo,commandScriptDefinition, + clusterConfig, host, service, serviceComponent, serviceComponentHost, serviceInfo, configHelper, + runtime, process); + } + + @Test public void testDeleteResources() throws Exception { Resource.Type type = Resource.Type.ClientConfig; http://git-wip-us.apache.org/repos/asf/ambari/blob/0dd2063f/ambari-server/src/test/resources/stacks/HDP/2.0.5/services/ZOOKEEPER/metainfo.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/resources/stacks/HDP/2.0.5/services/ZOOKEEPER/metainfo.xml b/ambari-server/src/test/resources/stacks/HDP/2.0.5/services/ZOOKEEPER/metainfo.xml index 2875b77..e17bbd3 100644 --- a/ambari-server/src/test/resources/stacks/HDP/2.0.5/services/ZOOKEEPER/metainfo.xml +++ b/ambari-server/src/test/resources/stacks/HDP/2.0.5/services/ZOOKEEPER/metainfo.xml @@ -44,6 +44,18 @@ <script>scripts/zookeeper_client.py</script> <scriptType>PYTHON</scriptType> </commandScript> + <configFiles> + <configFile> + <type>env</type> + <fileName>zookeeper-env.sh</fileName> + <dictionaryName>zookeeper-env</dictionaryName> + </configFile> + <configFile> + <type>env</type> + <fileName>log4j.properties</fileName> + <dictionaryName>zookeeper-log4j</dictionaryName> + </configFile> + </configFiles> </component> </components> http://git-wip-us.apache.org/repos/asf/ambari/blob/0dd2063f/ambari-server/src/test/resources/stacks/HDP/2.0.7/services/ZOOKEEPER/metainfo.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/resources/stacks/HDP/2.0.7/services/ZOOKEEPER/metainfo.xml b/ambari-server/src/test/resources/stacks/HDP/2.0.7/services/ZOOKEEPER/metainfo.xml index d397d19..c4cdb22 100644 --- a/ambari-server/src/test/resources/stacks/HDP/2.0.7/services/ZOOKEEPER/metainfo.xml +++ b/ambari-server/src/test/resources/stacks/HDP/2.0.7/services/ZOOKEEPER/metainfo.xml @@ -46,6 +46,18 @@ <script>scripts/zookeeper_client.py</script> <scriptType>PYTHON</scriptType> </commandScript> + <configFiles> + <configFile> + <type>env</type> + <fileName>zookeeper-env.sh</fileName> + <dictionaryName>zookeeper-env</dictionaryName> + </configFile> + <configFile> + <type>env</type> + <fileName>log4j.properties</fileName> + <dictionaryName>zookeeper-log4j</dictionaryName> + </configFile> + </configFiles> </component> </components>