AMBARI-22325 Cluster template object improvements (benyoka)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/67d1a008 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/67d1a008 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/67d1a008 Branch: refs/heads/branch-feature-AMBARI-14714-blueprintv2 Commit: 67d1a008ff331c6ef128d89731d7d8eb4979a937 Parents: 104e1fb Author: Balazs Bence Sari <beny...@apache.org> Authored: Tue Nov 21 10:32:23 2017 +0100 Committer: Doroszlai, Attila <adorosz...@hortonworks.com> Committed: Fri Dec 8 20:24:25 2017 +0100 ---------------------------------------------------------------------- .../ambari/server/topology/ComponentV2.java | 6 +- .../topology/ProvisionClusterTemplate.java | 72 ++++++++++++++++---- .../ProvisionClusterTemplateFactory.java | 9 +-- .../apache/ambari/server/topology/Service.java | 6 +- .../ambari/server/topology/ServiceId.java | 27 +++----- .../server/topology/BlueprintV2FactoryTest.java | 16 ++--- .../topology/ProvisionClusterTemplateTest.java | 49 ++++++++++++- .../blueprintv2/cluster_template_v2.json | 15 +++- .../cluster_template_v2_invalid_hostgroup.json | 70 +++++++++++++++++++ 9 files changed, 215 insertions(+), 55 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/67d1a008/ambari-server/src/main/java/org/apache/ambari/server/topology/ComponentV2.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/ComponentV2.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/ComponentV2.java index 0d26ef6..42158f4 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/topology/ComponentV2.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/ComponentV2.java @@ -30,7 +30,7 @@ public class ComponentV2 implements Configurable { private String name; - private ServiceId serviceId = new ServiceId(); + private ServiceId serviceId = new ServiceId(null, null); private ProvisionAction provisionAction = ProvisionAction.INSTALL_AND_START; @@ -110,12 +110,12 @@ public class ComponentV2 implements Configurable { @JsonProperty("service_group") public void setServiceGroup(String serviceGroup) { - serviceId.setServiceGroup(serviceGroup); + this.serviceId = new ServiceId(this.serviceId.getName(), serviceGroup); } @JsonProperty("service_name") public void setServiceName(String serviceName) { - serviceId.setName(serviceName); + this.serviceId = new ServiceId(serviceName, this.serviceId.getServiceGroup()); } public String getServiceName() { http://git-wip-us.apache.org/repos/asf/ambari/blob/67d1a008/ambari-server/src/main/java/org/apache/ambari/server/topology/ProvisionClusterTemplate.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/ProvisionClusterTemplate.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/ProvisionClusterTemplate.java index 5ef6517..90da776 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/topology/ProvisionClusterTemplate.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/ProvisionClusterTemplate.java @@ -18,26 +18,39 @@ */ package org.apache.ambari.server.topology; +import static java.util.stream.Collectors.toMap; + import java.util.Collection; +import java.util.Collections; +import java.util.Map; + +import javax.annotation.Nullable; import org.apache.ambari.server.controller.internal.ProvisionAction; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.base.Preconditions; public class ProvisionClusterTemplate { - private String blueprint; + private String blueprint = null; + @JsonProperty("default_password") - private String defaultPassword; + private String defaultPassword = null; + @JsonProperty("config_recommendation_strategy") private ConfigRecommendationStrategy configRecommendationStrategy; + @JsonProperty("provision_action") private ProvisionAction provisionAction; - private Collection<ProvisionClusterTemplate.Service> services; - @JsonProperty("host_groups") - private Collection<ProvisionClusterTemplate.HostGroup> hostGroups; + + private Map<ServiceId, ProvisionClusterTemplate.Service> servicesById = Collections.emptyMap(); + + private Map<String, ProvisionClusterTemplate.HostGroup> hostGroups = Collections.emptyMap(); + private Collection<Credential> credentials; + @JsonProperty("security") private SecurityConfiguration securityConfiguration; @@ -57,12 +70,21 @@ public class ProvisionClusterTemplate { this.defaultPassword = defaultPassword; } + public @Nullable Service getServiceById(ServiceId serviceId) { + return servicesById.get(serviceId); + } + + @JsonProperty("services") public Collection<Service> getServices() { - return services; + return servicesById.values(); } + @JsonProperty("services") public void setServices(Collection<Service> services) { - this.services = services; + this.servicesById = services.stream().collect(toMap( + s -> s.getId(), + s -> s + )); } public Collection<Credential> getCredentials() { @@ -97,21 +119,34 @@ public class ProvisionClusterTemplate { this.provisionAction = provisionAction; } + @JsonProperty("host_groups") public Collection<HostGroup> getHostGroups() { - return hostGroups; + return hostGroups.values(); + } + + public HostGroup getHostGroupByName(String name) { + return hostGroups.get(name); } + @JsonProperty("host_groups") public void setHostGroups(Collection<HostGroup> hostGroups) { - this.hostGroups = hostGroups; + this.hostGroups = hostGroups.stream().collect(toMap( + hg -> hg.getName(), + hg -> hg + )); + } + + public void validate() throws IllegalStateException { + getHostGroups().forEach(HostGroup::validate); } public static class HostGroup implements Configurable { private String name; @JsonIgnore private Configuration configuration; - private Collection<Host> hosts; + private Collection<Host> hosts = Collections.emptyList(); @JsonProperty("host_count") - private Integer hostCount; + private int hostCount = 0; @JsonProperty("host_predicate") private String hostPredicate; @@ -141,11 +176,11 @@ public class ProvisionClusterTemplate { this.hosts = hosts; } - public Integer getHostCount() { + public int getHostCount() { return hostCount; } - public void setHostCount(Integer hostCount) { + public void setHostCount(int hostCount) { this.hostCount = hostCount; } @@ -156,6 +191,12 @@ public class ProvisionClusterTemplate { public void setHostPredicate(String hostPredicate) { this.hostPredicate = hostPredicate; } + + void validate() throws IllegalStateException { + Preconditions.checkState((hostCount == 0 && null == hostPredicate) || getHosts().isEmpty(), + "Invalid custer topology template. Host group %s must have either declatere its hosts or " + + "hostcount (and optionally host predicate)", name); + } } public static class Service implements Configurable { @@ -175,6 +216,11 @@ public class ProvisionClusterTemplate { this.configuration = configuration; } + @JsonIgnore + public ServiceId getId() { + return new ServiceId(name, serviceGroup); + } + public String getName() { return name; } http://git-wip-us.apache.org/repos/asf/ambari/blob/67d1a008/ambari-server/src/main/java/org/apache/ambari/server/topology/ProvisionClusterTemplateFactory.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/ProvisionClusterTemplateFactory.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/ProvisionClusterTemplateFactory.java index c0c280f..cd99fa1 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/topology/ProvisionClusterTemplateFactory.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/ProvisionClusterTemplateFactory.java @@ -51,16 +51,13 @@ public class ProvisionClusterTemplateFactory { private void createObjectMapper() { objectMapper = new ObjectMapper(); -// SimpleModule module = new SimpleModule("CustomModel", Version.unknownVersion()); -// SimpleAbstractTypeResolver resolver = new SimpleAbstractTypeResolver(); -// resolver.addMapping(HostGroupV2.class, HostGroupV2Impl.class); -// module.setAbstractTypes(resolver); -// objectMapper.registerModule(module); objectMapper.setSerializationInclusion(JsonInclude.Include.NON_NULL); } public ProvisionClusterTemplate convertFromJson(String clusterTemplateJson) throws IOException { - return objectMapper.readValue(clusterTemplateJson, ProvisionClusterTemplate.class); + ProvisionClusterTemplate template = objectMapper.readValue(clusterTemplateJson, ProvisionClusterTemplate.class); + template.validate(); + return template; } } http://git-wip-us.apache.org/repos/asf/ambari/blob/67d1a008/ambari-server/src/main/java/org/apache/ambari/server/topology/Service.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/Service.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/Service.java index 4a93ecd..a45386f 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/topology/Service.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/Service.java @@ -36,7 +36,7 @@ public class Service implements Configurable { private String type; - private ServiceId id = new ServiceId(); + private ServiceId id = new ServiceId(null, null); private String stackId; @@ -115,12 +115,12 @@ public class Service implements Configurable { } public void setName(String name) { - this.id.setName(name); + this.id = new ServiceId(name, this.id.getServiceGroup()); } public void setServiceGroup(ServiceGroup serviceGroup) { this.serviceGroup = serviceGroup; - this.id.setServiceGroup(serviceGroup.getName()); + this.id = new ServiceId(this.id.getName(), serviceGroup.getName()) ; } @JsonProperty("stack_id") http://git-wip-us.apache.org/repos/asf/ambari/blob/67d1a008/ambari-server/src/main/java/org/apache/ambari/server/topology/ServiceId.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/ServiceId.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/ServiceId.java index 2d81a07..9de0b3d 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/topology/ServiceId.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/ServiceId.java @@ -17,38 +17,33 @@ */ package org.apache.ambari.server.topology; +import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; public class ServiceId { - private String serviceGroup; - private String name; + private final String serviceGroup; + private final String name; - public ServiceId() { } + @JsonCreator + public ServiceId(@JsonProperty("service_name") String name, @JsonProperty("service_group") String serviceGroup) { + this.name = name; + this.serviceGroup = serviceGroup; + } public static ServiceId of(String name, String serviceGroup) { - ServiceId id = new ServiceId(); - id.name = name; - id.serviceGroup = serviceGroup; - return id; + return new ServiceId(name, serviceGroup); } + @JsonProperty("service_group") public String getServiceGroup() { return serviceGroup; } - @JsonProperty("service_group") - public void setServiceGroup(String serviceGroup) { - this.serviceGroup = serviceGroup; - } - + @JsonProperty("service_name") public String getName() { return name; } - @JsonProperty("service_name") - public void setName(String name) { - this.name = name; - } @Override public boolean equals(Object o) { http://git-wip-us.apache.org/repos/asf/ambari/blob/67d1a008/ambari-server/src/test/java/org/apache/ambari/server/topology/BlueprintV2FactoryTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/topology/BlueprintV2FactoryTest.java b/ambari-server/src/test/java/org/apache/ambari/server/topology/BlueprintV2FactoryTest.java index 78aa98c..ab9adea 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/topology/BlueprintV2FactoryTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/topology/BlueprintV2FactoryTest.java @@ -37,6 +37,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.base.Charsets; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; import com.google.common.io.Resources; public class BlueprintV2FactoryTest { @@ -79,14 +80,11 @@ public class BlueprintV2FactoryTest { assertEquals(2, bp.getServiceGroups().size()); Service zk1 = bp.getServiceGroup("CoreSG").getServiceByName("ZK1"); - Map<String, Map<String, String>> expectedProperties = new HashMap<>(); - Map<String, String> zooCfg = new HashMap<>(); - zooCfg.put("dataDir", "/zookeeper1"); - expectedProperties.put("zoo.cfg", zooCfg); - Map<String, String> zookeeperEnv = new HashMap<>(); - zookeeperEnv.put("zk_user", "zkuser1"); - zookeeperEnv.put("zk_server_heapsize", "256MB"); - expectedProperties.put("zookeeper-env", zookeeperEnv); + Map<String, Map<String, String>> expectedProperties = ImmutableMap.of( + "zoo.cfg", ImmutableMap.of("dataDir", "/zookeeper1"), + "zookeeper-env", ImmutableMap.of( + "zk_user", "zkuser1", + "zk_server_heapsize", "256MB")); assertEquals(expectedProperties, zk1.getConfiguration().getProperties()); } @@ -120,7 +118,7 @@ public class BlueprintV2FactoryTest { verifyBlueprintStructure2(blueprintFactory.convertFromJson(BLUEPRINTV2_2_JSON)); } - private void verifyBlueprintStructure2(BlueprintV2 bp) { + private void verifyBlueprintStructure2(BlueprintV2 bp) { // for "blueprintv2_2.json" assertEquals(new StackId("HDP", "3.0.0"), bp.getServiceGroups().iterator().next().getServices().iterator().next().getStack().getStackId()); assertEquals(1, bp.getStackIds().size()); http://git-wip-us.apache.org/repos/asf/ambari/blob/67d1a008/ambari-server/src/test/java/org/apache/ambari/server/topology/ProvisionClusterTemplateTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/topology/ProvisionClusterTemplateTest.java b/ambari-server/src/test/java/org/apache/ambari/server/topology/ProvisionClusterTemplateTest.java index b02a00a..3bb6ea4 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/topology/ProvisionClusterTemplateTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/topology/ProvisionClusterTemplateTest.java @@ -18,23 +18,68 @@ */ package org.apache.ambari.server.topology; +import static java.util.stream.Collectors.toSet; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; + import java.io.IOException; +import java.util.Map; import org.junit.Test; import com.google.common.base.Charsets; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; import com.google.common.io.Resources; public class ProvisionClusterTemplateTest { - public static final String CLUSTER_TEMPLATE = getResource("blueprintv2/cluster_template_v2.json"); + public static final String CLUSTER_TEMPLATE = + getResource("blueprintv2/cluster_template_v2.json"); + public static final String CLUSTER_TEMPLATE_INVALID = + getResource("blueprintv2/cluster_template_v2_invalid_hostgroup.json"); @Test public void testProvisionClusterTemplate() throws Exception { ProvisionClusterTemplateFactory factory = new ProvisionClusterTemplateFactory(); ProvisionClusterTemplate template = factory.convertFromJson(CLUSTER_TEMPLATE); - System.out.println(template); + verifyClusterTemplate(template); + } + + @Test(expected = IllegalStateException.class) + public void testProvisionClusterTemplateInvalidTemplate() throws Exception { + ProvisionClusterTemplateFactory factory = new ProvisionClusterTemplateFactory(); + ProvisionClusterTemplate template = factory.convertFromJson(CLUSTER_TEMPLATE_INVALID); + } + + + private void verifyClusterTemplate(ProvisionClusterTemplate template) { + ProvisionClusterTemplate.Service zk1 = template.getServiceById(ServiceId.of("ZK1", "CORE_SG")); + assertNotNull(zk1); + Map<String, Map<String, String>> expectedZkProperties = ImmutableMap.of( + "zoo.cfg", ImmutableMap.of("dataDir", "/zookeeper1")); + assertEquals(expectedZkProperties, zk1.getConfiguration().getProperties()); + + ProvisionClusterTemplate.Service hdfs = template.getServiceById(ServiceId.of("HDFS", "CORE_SG")); + Map<String, Map<String, String>> expectedHdfsProperties = ImmutableMap.of( + "hdfs-site", ImmutableMap.of("property-name", "property-value")); + assertNotNull(hdfs); + assertEquals(expectedHdfsProperties, hdfs.getConfiguration().getProperties()); + + ProvisionClusterTemplate.HostGroup hostGroup1 = template.getHostGroupByName("host-group-1"); + assertNotNull(hostGroup1); + assertEquals(2, hostGroup1.getHosts().size()); + assertEquals(0, hostGroup1.getHostCount()); + assertEquals(ImmutableSet.of("host.domain.com", "host2.domain.com"), + hostGroup1.getHosts().stream().map(host -> host.getFqdn()).collect(toSet())); + hostGroup1.getHosts().forEach(host -> assertEquals("/dc1/rack1", host.getRackInfo())); + + ProvisionClusterTemplate.HostGroup hostGroup2 = template.getHostGroupByName("host-group-2"); + assertNotNull(hostGroup2); + assertEquals(0, hostGroup2.getHosts().size()); + assertEquals(2, hostGroup2.getHostCount()); + assertEquals("Hosts/os_type=centos6&Hosts/cpu_count=2", hostGroup2.getHostPredicate()); } http://git-wip-us.apache.org/repos/asf/ambari/blob/67d1a008/ambari-server/src/test/resources/blueprintv2/cluster_template_v2.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/resources/blueprintv2/cluster_template_v2.json b/ambari-server/src/test/resources/blueprintv2/cluster_template_v2.json index 1fb0b04..d80f2f5 100644 --- a/ambari-server/src/test/resources/blueprintv2/cluster_template_v2.json +++ b/ambari-server/src/test/resources/blueprintv2/cluster_template_v2.json @@ -9,7 +9,7 @@ { "zoo.cfg": { "properties": { - "dataDir": "/zookeeper2" + "dataDir": "/zookeeper1" } } } @@ -44,13 +44,22 @@ ], "hosts": [ { - "fqdn": "host.domain.com" + "fqdn": "host.domain.com", + "rack_info": "/dc1/rack1" }, { - "fqdn": "host2.domain.com" + "fqdn": "host2.domain.com", + "rack_info": "/dc1/rack1" } ] + }, + { + "name": "host-group-2", + "configurations": [], + "host_count": "2", + "host_predicate": "Hosts/os_type=centos6&Hosts/cpu_count=2" } + ], "credentials": [ { http://git-wip-us.apache.org/repos/asf/ambari/blob/67d1a008/ambari-server/src/test/resources/blueprintv2/cluster_template_v2_invalid_hostgroup.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/resources/blueprintv2/cluster_template_v2_invalid_hostgroup.json b/ambari-server/src/test/resources/blueprintv2/cluster_template_v2_invalid_hostgroup.json new file mode 100644 index 0000000..abe0aaf --- /dev/null +++ b/ambari-server/src/test/resources/blueprintv2/cluster_template_v2_invalid_hostgroup.json @@ -0,0 +1,70 @@ +{ + "blueprint": "blueprint-name", + "default_password": "super-secret-password", + "services": [ + { + "service_group": "CORE_SG", + "name": "ZK1", + "configurations": [ + { + "zoo.cfg": { + "properties": { + "dataDir": "/zookeeper1" + } + } + } + ] + }, + { + "service_group": "CORE_SG", + "name": "HDFS", + "configurations": [ + { + "hdfs-site": { + "properties": { + "property-name": "property-value" + } + } + } + ] + } + + ], + "host_groups": [ + { + "name": "host-group-1", + "configurations": [ + { + "yarn-site": { + "properties": { + "property-name": "property-value" + } + } + } + ], + "hosts": [ + { + "fqdn": "host.domain.com", + "rack_info": "/dc1/rack1" + }, + { + "fqdn": "host2.domain.com", + "rack_info": "/dc1/rack1" + } + ], + "host_count": "2", + "host_predicate": "Hosts/os_type=centos6&Hosts/cpu_count=2" + } + ], + "credentials": [ + { + "alias": "kdc.admin.credential", + "principal": "principal", + "key": "key", + "type": "TEMPORARY" + } + ], + "security": { + "type": "NONE" + } +}