Updated Branches: refs/heads/trunk e8d045af2 -> c4a778da0
AMBARI-4465 - Update component name mapping in ClusterDefinition Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/c4a778da Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/c4a778da Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/c4a778da Branch: refs/heads/trunk Commit: c4a778da041ee56d01ef2dc88fd7fcfebe2a0680 Parents: e8d045a Author: Artem Baranchuk <[email protected]> Authored: Wed Jan 29 21:39:00 2014 +0200 Committer: Artem Baranchuk <[email protected]> Committed: Thu Jan 30 22:19:05 2014 +0200 ---------------------------------------------------------------------- .../apache/ambari/msi/ClusterDefinition.java | 86 +++++++++++++++----- .../apache/ambari/scom/SQLProviderModule.java | 3 + .../ambari/msi/ClusterDefinitionTest.java | 24 ++++++ .../scom/TestClusterDefinitionProvider.java | 6 ++ .../test/resources/clusterproperties_HDP2.txt | 49 +++++++++++ 5 files changed, 149 insertions(+), 19 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/c4a778da/contrib/ambari-scom/ambari-scom-server/src/main/java/org/apache/ambari/msi/ClusterDefinition.java ---------------------------------------------------------------------- diff --git a/contrib/ambari-scom/ambari-scom-server/src/main/java/org/apache/ambari/msi/ClusterDefinition.java b/contrib/ambari-scom/ambari-scom-server/src/main/java/org/apache/ambari/msi/ClusterDefinition.java index 3e95ebd..a2da723 100644 --- a/contrib/ambari-scom/ambari-scom-server/src/main/java/org/apache/ambari/msi/ClusterDefinition.java +++ b/contrib/ambari-scom/ambari-scom-server/src/main/java/org/apache/ambari/msi/ClusterDefinition.java @@ -22,6 +22,7 @@ import org.apache.ambari.scom.ClusterDefinitionProvider; import org.apache.ambari.scom.HostInfoProvider; import org.apache.ambari.server.controller.internal.ResourceImpl; import org.apache.ambari.server.controller.spi.Resource; +import org.apache.commons.lang.StringUtils; import java.io.BufferedReader; import java.io.IOException; @@ -39,7 +40,7 @@ import java.util.Set; public class ClusterDefinition { private static final String HEADER_TAG = "#"; - private static final String HOSTS_HEADER = "Hosts"; + private static final String HOSTS_HEADER = "hosts"; private final Set<String> services = new HashSet<String>(); private final Set<String> hosts = new HashSet<String>(); @@ -64,9 +65,9 @@ public class ClusterDefinition { * Component name mapping to account for differences in what is provided by the MSI * and what is expected by the Ambari providers. */ - private static final Map<String, Set<String>> componentNameMap = new HashMap<String, Set<String>>(); + private final Map<String, Set<String>> componentNameMap = new HashMap<String, Set<String>>(); - static { + private void initComponentNameMap() { componentNameMap.put("NAMENODE_HOST", Collections.singleton("NAMENODE")); componentNameMap.put("SECONDARY_NAMENODE_HOST", Collections.singleton("SECONDARY_NAMENODE")); componentNameMap.put("OOZIE_SERVER_HOST", Collections.singleton("OOZIE_SERVER")); @@ -76,15 +77,8 @@ public class ClusterDefinition { componentNameMap.put("HBASE_REGIONSERVERS", Collections.singleton("HBASE_REGIONSERVER")); componentNameMap.put("ZOOKEEPER_HOSTS", Collections.singleton("ZOOKEEPER_SERVER")); - Set<String> mapReduceComponents = new HashSet<String>(); - mapReduceComponents.add("JOBTRACKER"); - mapReduceComponents.add("HISTORY_SERVER"); - componentNameMap.put("JOBTRACKER_HOST", mapReduceComponents); - Set<String> slaveComponents = new HashSet<String>(); slaveComponents.add("DATANODE"); - slaveComponents.add("TASKTRACKER"); - componentNameMap.put("SLAVE_HOSTS", slaveComponents); Set<String> hiveComponents = new HashSet<String>(); @@ -92,22 +86,44 @@ public class ClusterDefinition { hiveComponents.add("HIVE_SERVER2"); hiveComponents.add("HIVE_METASTORE"); hiveComponents.add("HIVE_CLIENT"); - componentNameMap.put("HIVE_SERVER_HOST", hiveComponents); + + Integer majorStackVersion = getMajorStackVersion(); + if(majorStackVersion != null) { + if(majorStackVersion == 1) { + Set<String> mapReduceComponents = new HashSet<String>(); + mapReduceComponents.add("JOBTRACKER"); + mapReduceComponents.add("HISTORY_SERVER"); + componentNameMap.put("JOBTRACKER_HOST", mapReduceComponents); + + slaveComponents.add("TASKTRACKER"); + } + if(majorStackVersion == 2) { + componentNameMap.put("JOURNALNODE_HOST", Collections.singleton("JOURNALNODE")); + + Set<String> mapReduce2Components = new HashSet<String>(); + mapReduce2Components.add("HISTORY_SERVER"); + mapReduce2Components.add("RESOURCEMANAGER"); + componentNameMap.put("RESOURCEMANAGER_HOST", mapReduce2Components); + + slaveComponents.add("NODEMANAGER"); + //hiveComponents.add("MYSQL_SERVER"); + + Set<String> clientHosts = new HashSet<String>(); + componentNameMap.put("CLIENT_HOSTS", clientHosts); + } + } } /** * Component service mapping . */ - private static final Map<String, String> componentServiceMap = new HashMap<String, String>(); + private final Map<String, String> componentServiceMap = new HashMap<String, String>(); - static { + private void initComponentServiceMap() { componentServiceMap.put("NAMENODE", "HDFS"); componentServiceMap.put("DATANODE", "HDFS"); componentServiceMap.put("SECONDARY_NAMENODE", "HDFS"); - componentServiceMap.put("JOBTRACKER", "MAPREDUCE"); - componentServiceMap.put("HISTORY_SERVER", "MAPREDUCE"); - componentServiceMap.put("TASKTRACKER", "MAPREDUCE"); componentServiceMap.put("HIVE_SERVER", "HIVE"); componentServiceMap.put("HIVE_SERVER2", "HIVE"); componentServiceMap.put("HIVE_METASTORE", "HIVE"); @@ -118,8 +134,23 @@ public class ClusterDefinition { componentServiceMap.put("HBASE_MASTER", "HBASE"); componentServiceMap.put("HBASE_REGIONSERVER", "HBASE"); componentServiceMap.put("ZOOKEEPER_SERVER", "ZOOKEEPER"); - } + Integer majorStackVersion = getMajorStackVersion(); + if(majorStackVersion != null) { + if(majorStackVersion == 1) { + componentServiceMap.put("JOBTRACKER", "MAPREDUCE"); + componentServiceMap.put("HISTORY_SERVER", "MAPREDUCE"); + componentServiceMap.put("TASKTRACKER", "MAPREDUCE"); + } + if(majorStackVersion == 2) { + componentServiceMap.put("HISTORY_SERVER", "MAPREDUCE2"); + componentServiceMap.put("JOURNALNODE", "HDFS"); + componentServiceMap.put("NODEMANAGER", "YARN"); + componentServiceMap.put("RESOURCEMANAGER", "YARN"); + //componentServiceMap.put("MYSQL_SERVER", "HIVE"); + } + } + } // ----- Constructors ------------------------------------------------------ @@ -137,6 +168,8 @@ public class ClusterDefinition { this.clusterName = definitionProvider.getClusterName(); this.versionId = definitionProvider.getVersionId(); + init(); + try { readClusterDefinition(); } catch (IOException e) { @@ -145,6 +178,21 @@ public class ClusterDefinition { } } + private void init() { + initComponentNameMap(); + initComponentServiceMap(); + } + + private Integer getMajorStackVersion() { + if(StringUtils.isNotEmpty(versionId)) { + String majorVersion = StringUtils.substring(versionId, 4, 5); + if(StringUtils.isNotEmpty(majorVersion)) { + return Integer.parseInt(majorVersion); + } + } + return null; + } + // ----- ClusterDefinition ------------------------------------------------- /** @@ -481,8 +529,8 @@ public class ClusterDefinition { if (line.startsWith(HEADER_TAG)) { - String header = line.substring(HEADER_TAG.length()); - hostsSection = header.equals(HOSTS_HEADER); + String header = line.substring(HEADER_TAG.length()).toLowerCase(); + hostsSection = header.equalsIgnoreCase(HOSTS_HEADER); if (!hostsSection && (header.startsWith(HOSTS_HEADER) ) ){ char c = header.charAt(HOSTS_HEADER.length()); http://git-wip-us.apache.org/repos/asf/ambari/blob/c4a778da/contrib/ambari-scom/ambari-scom-server/src/main/java/org/apache/ambari/scom/SQLProviderModule.java ---------------------------------------------------------------------- diff --git a/contrib/ambari-scom/ambari-scom-server/src/main/java/org/apache/ambari/scom/SQLProviderModule.java b/contrib/ambari-scom/ambari-scom-server/src/main/java/org/apache/ambari/scom/SQLProviderModule.java index 439ac00..cbf2bfe 100644 --- a/contrib/ambari-scom/ambari-scom-server/src/main/java/org/apache/ambari/scom/SQLProviderModule.java +++ b/contrib/ambari-scom/ambari-scom-server/src/main/java/org/apache/ambari/scom/SQLProviderModule.java @@ -76,6 +76,9 @@ public class SQLProviderModule extends DefaultProviderModule implements HostInfo serviceNames.put("DATANODE", "datanode"); serviceNames.put("TASKTRACKER", "tasktracker"); serviceNames.put("WEBHCAT_SERVER", "templeton"); + serviceNames.put("NODEMANAGER", "nodemanager"); + serviceNames.put("RESOURCEMANAGER", "resourcemanager"); + serviceNames.put("JOURNALNODE", "journalnode"); } private static final String STATE_PREFIX = "STATE : "; http://git-wip-us.apache.org/repos/asf/ambari/blob/c4a778da/contrib/ambari-scom/ambari-scom-server/src/test/java/org/apache/ambari/msi/ClusterDefinitionTest.java ---------------------------------------------------------------------- diff --git a/contrib/ambari-scom/ambari-scom-server/src/test/java/org/apache/ambari/msi/ClusterDefinitionTest.java b/contrib/ambari-scom/ambari-scom-server/src/test/java/org/apache/ambari/msi/ClusterDefinitionTest.java index c048df2..0652ea6 100644 --- a/contrib/ambari-scom/ambari-scom-server/src/test/java/org/apache/ambari/msi/ClusterDefinitionTest.java +++ b/contrib/ambari-scom/ambari-scom-server/src/test/java/org/apache/ambari/msi/ClusterDefinitionTest.java @@ -283,4 +283,28 @@ public class ClusterDefinitionTest { verify(mockClusterDefinitionProvider, mockHostInfoProvider, mockStateProvider); } + + @Test + public void testHDP2ServicesAndComponents() { + StateProvider mockStateProvider = createStrictMock(StateProvider.class); + ClusterDefinitionProvider mockClusterDefinitionProvider = createStrictMock(ClusterDefinitionProvider.class); + HostInfoProvider mockHostInfoProvider = createStrictMock(HostInfoProvider.class); + + TestClusterDefinitionProvider testClusterDefinitionProvider = new TestClusterDefinitionProvider("clusterproperties_HDP2.txt", "myCluster", "HDP-2.0.6"); + + expect(mockClusterDefinitionProvider.getClusterName()).andDelegateTo(testClusterDefinitionProvider); + expect(mockClusterDefinitionProvider.getVersionId()).andDelegateTo(testClusterDefinitionProvider); + expect(mockClusterDefinitionProvider.getInputStream()).andDelegateTo(testClusterDefinitionProvider); + + replay(mockClusterDefinitionProvider, mockHostInfoProvider, mockStateProvider); + + ClusterDefinition clusterDefinition = new ClusterDefinition(mockStateProvider, mockClusterDefinitionProvider, mockHostInfoProvider); + Assert.assertFalse(clusterDefinition.getServices().contains("MAPREDUCE")); + Assert.assertTrue(clusterDefinition.getServices().contains("YARN")); + Assert.assertTrue(clusterDefinition.getServices().contains("MAPREDUCE2")); + Assert.assertTrue(clusterDefinition.getComponents("YARN").contains("NODEMANAGER")); + Assert.assertTrue(clusterDefinition.getComponents("YARN").contains("RESOURCEMANAGER")); + + verify(mockClusterDefinitionProvider, mockHostInfoProvider, mockStateProvider); + } } http://git-wip-us.apache.org/repos/asf/ambari/blob/c4a778da/contrib/ambari-scom/ambari-scom-server/src/test/java/org/apache/ambari/scom/TestClusterDefinitionProvider.java ---------------------------------------------------------------------- diff --git a/contrib/ambari-scom/ambari-scom-server/src/test/java/org/apache/ambari/scom/TestClusterDefinitionProvider.java b/contrib/ambari-scom/ambari-scom-server/src/test/java/org/apache/ambari/scom/TestClusterDefinitionProvider.java index 98c6869..58e057b 100644 --- a/contrib/ambari-scom/ambari-scom-server/src/test/java/org/apache/ambari/scom/TestClusterDefinitionProvider.java +++ b/contrib/ambari-scom/ambari-scom-server/src/test/java/org/apache/ambari/scom/TestClusterDefinitionProvider.java @@ -27,4 +27,10 @@ public class TestClusterDefinitionProvider extends ClusterDefinitionProvider { setClusterName("myCluster"); setVersionId("HDP-1.2.9"); } + + public TestClusterDefinitionProvider (String fileName, String clusterName, String versionId) { + setFileName(fileName); + setClusterName(clusterName); + setVersionId(versionId); + } } http://git-wip-us.apache.org/repos/asf/ambari/blob/c4a778da/contrib/ambari-scom/ambari-scom-server/src/test/resources/clusterproperties_HDP2.txt ---------------------------------------------------------------------- diff --git a/contrib/ambari-scom/ambari-scom-server/src/test/resources/clusterproperties_HDP2.txt b/contrib/ambari-scom/ambari-scom-server/src/test/resources/clusterproperties_HDP2.txt new file mode 100644 index 0000000..e359aa9 --- /dev/null +++ b/contrib/ambari-scom/ambari-scom-server/src/test/resources/clusterproperties_HDP2.txt @@ -0,0 +1,49 @@ +### Licensed to the Apache Software Foundation (ASF) under one or more +### contributor license agreements. See the NOTICE file distributed with +### this work for additional information regarding copyright ownership. +### The ASF licenses this file to You under the Apache License, Version 2.0 +### (the "License"); you may not use this file except in compliance with +### the License. You may obtain a copy of the License at +### +### http://www.apache.org/licenses/LICENSE-2.0 +### +### Unless required by applicable law or agreed to in writing, software +### distributed under the License is distributed on an "AS IS" BASIS, +### WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +### See the License for the specific language governing permissions and +### limitations under the License. + +#Log directory +HDP_LOG_DIR=c:\hadoop\logs + +#Data directory +HDP_DATA_DIR=c:\hadoop\data + +#hosts +NAMENODE_HOST=tbeerb-ascom21 +SECONDARY_NAMENODE_HOST=tbeerb-ascom21 +RESOURCEMANAGER_HOST=tbeerb-ascom21 +HIVE_SERVER_HOST=tbeerb-ascom21 +OOZIE_SERVER_HOST=tbeerb-ascom21 +WEBHCAT_HOST=tbeerb-ascom21 +SLAVE_HOSTS=tbeerb-ascom21 +CLIENT_HOSTS=tbeerb-ascom21 +HBASE_MASTER=tbeerb-ascom21 +HBASE_REGIONSERVERS=tbeerb-ascom21 +ZOOKEEPER_HOSTS=tbeerb-ascom21 +FLUME_HOSTS=tbeerb-ascom21 + +#Database host +DB_FLAVOR=derby +DB_HOSTNAME=tbeerb-ascom21 +DB_PORT=1527 + +#Hive properties +HIVE_DB_NAME=hive +HIVE_DB_USERNAME=hive +HIVE_DB_PASSWORD=hive + +#Oozie properties +OOZIE_DB_NAME=oozie +OOZIE_DB_USERNAME=oozie +OOZIE_DB_PASSWORD=oozie
