AMBARI-22480. Validate blueprint does not allow lzo enable without setup with license agreement. (mpapirkovskyy)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/9bbc0ef7 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/9bbc0ef7 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/9bbc0ef7 Branch: refs/heads/branch-3.0-perf Commit: 9bbc0ef720bc91ff39701955984bb9635d811c59 Parents: 41853a1 Author: Myroslav Papirkovskyi <mpapyrkovs...@hortonworks.com> Authored: Wed Nov 29 16:55:34 2017 +0200 Committer: Myroslav Papirkovskyi <mpapyrkovs...@hortonworks.com> Committed: Wed Nov 29 18:30:08 2017 +0200 ---------------------------------------------------------------------- .../internal/BlueprintResourceProvider.java | 3 +- .../ambari/server/topology/Blueprint.java | 2 +- .../ambari/server/topology/BlueprintImpl.java | 2 +- .../server/topology/BlueprintValidator.java | 2 +- .../server/topology/BlueprintValidatorImpl.java | 21 +++++- .../server/topology/BlueprintImplTest.java | 69 ++++++++++++++++++-- .../topology/BlueprintValidatorImplTest.java | 10 ++- 7 files changed, 95 insertions(+), 14 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/9bbc0ef7/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintResourceProvider.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintResourceProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintResourceProvider.java index 67f5448..8f4d62e 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintResourceProvider.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintResourceProvider.java @@ -56,6 +56,7 @@ import org.apache.ambari.server.state.SecurityType; import org.apache.ambari.server.state.StackInfo; import org.apache.ambari.server.topology.Blueprint; import org.apache.ambari.server.topology.BlueprintFactory; +import org.apache.ambari.server.topology.GPLLicenseNotAcceptedException; import org.apache.ambari.server.topology.InvalidTopologyException; import org.apache.ambari.server.topology.SecurityConfiguration; import org.apache.ambari.server.topology.SecurityConfigurationFactory; @@ -519,7 +520,7 @@ public class BlueprintResourceProvider extends AbstractControllerResourceProvide try { blueprint.validateRequiredProperties(); - } catch (InvalidTopologyException e) { + } catch (InvalidTopologyException | GPLLicenseNotAcceptedException e) { throw new IllegalArgumentException("Blueprint configuration validation failed: " + e.getMessage(), e); } http://git-wip-us.apache.org/repos/asf/ambari/blob/9bbc0ef7/ambari-server/src/main/java/org/apache/ambari/server/topology/Blueprint.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/Blueprint.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/Blueprint.java index 779a02d..6ed38f8 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/topology/Blueprint.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/Blueprint.java @@ -152,7 +152,7 @@ public interface Blueprint { * * @throws InvalidTopologyException if the blueprint doesn't contain all required properties */ - void validateRequiredProperties() throws InvalidTopologyException; + void validateRequiredProperties() throws InvalidTopologyException, GPLLicenseNotAcceptedException; /** * http://git-wip-us.apache.org/repos/asf/ambari/blob/9bbc0ef7/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintImpl.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintImpl.java index 8c83ed6..6801e33 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintImpl.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintImpl.java @@ -353,7 +353,7 @@ public class BlueprintImpl implements Blueprint { * @throws InvalidTopologyException if the blueprint configuration is invalid */ @Override - public void validateRequiredProperties() throws InvalidTopologyException { + public void validateRequiredProperties() throws InvalidTopologyException, GPLLicenseNotAcceptedException { validator.validateRequiredProperties(); } http://git-wip-us.apache.org/repos/asf/ambari/blob/9bbc0ef7/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintValidator.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintValidator.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintValidator.java index be194df..156fe8c 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintValidator.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintValidator.java @@ -37,5 +37,5 @@ public interface BlueprintValidator { * * @throws InvalidTopologyException if required properties are not set in blueprint */ - void validateRequiredProperties() throws InvalidTopologyException; + void validateRequiredProperties() throws InvalidTopologyException, GPLLicenseNotAcceptedException; } http://git-wip-us.apache.org/repos/asf/ambari/blob/9bbc0ef7/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintValidatorImpl.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintValidatorImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintValidatorImpl.java index 1a43b85..87b5936 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintValidatorImpl.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintValidatorImpl.java @@ -26,6 +26,7 @@ import java.util.Iterator; import java.util.Map; import java.util.Set; +import org.apache.ambari.server.StaticallyInject; import org.apache.ambari.server.controller.internal.Stack; import org.apache.ambari.server.state.AutoDeployInfo; import org.apache.ambari.server.state.DependencyConditionInfo; @@ -35,15 +36,24 @@ import org.apache.ambari.server.utils.VersionUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.google.inject.Inject; + /** * Default blueprint validator. */ +@StaticallyInject public class BlueprintValidatorImpl implements BlueprintValidator { private static final Logger LOGGER = LoggerFactory.getLogger(BlueprintValidatorImpl.class); private final Blueprint blueprint; private final Stack stack; + public static final String LZO_CODEC_CLASS_PROPERTY_NAME = "io.compression.codec.lzo.class"; + public static final String LZO_CODEC_CLASS = "com.hadoop.compression.lzo.LzoCodec"; + + @Inject + private static org.apache.ambari.server.configuration.Configuration configuration; + public BlueprintValidatorImpl(Blueprint blueprint) { this.blueprint = blueprint; this.stack = blueprint.getStack(); @@ -84,13 +94,17 @@ public class BlueprintValidatorImpl implements BlueprintValidator { } @Override - public void validateRequiredProperties() throws InvalidTopologyException { + public void validateRequiredProperties() throws InvalidTopologyException, GPLLicenseNotAcceptedException { // we don't want to include default stack properties so we can't just use hostGroup full properties Map<String, Map<String, String>> clusterConfigurations = blueprint.getConfiguration().getProperties(); // we need to have real passwords, not references if (clusterConfigurations != null) { + + // need to reject blueprints that have LZO enabled if the Ambari Server hasn't been configured for it + boolean gplEnabled = configuration.getGplLicenseAccepted(); + StringBuilder errorMessage = new StringBuilder(); boolean containsSecretReferences = false; for (Map.Entry<String, Map<String, String>> configEntry : clusterConfigurations.entrySet()) { @@ -100,6 +114,11 @@ public class BlueprintValidatorImpl implements BlueprintValidator { String propertyName = propertyEntry.getKey(); String propertyValue = propertyEntry.getValue(); if (propertyValue != null) { + if (!gplEnabled && configType.equals("core-site") && propertyName.equals(LZO_CODEC_CLASS_PROPERTY_NAME) + && propertyValue.contains(LZO_CODEC_CLASS)) { + throw new GPLLicenseNotAcceptedException("Your Ambari server has not been configured to download LZO GPL software. " + + "Please refer to documentation to configure Ambari before proceeding."); + } if (SecretReference.isSecret(propertyValue)) { errorMessage.append(" Config:" + configType + " Property:" + propertyName + "\n"); containsSecretReferences = true; http://git-wip-us.apache.org/repos/asf/ambari/blob/9bbc0ef7/ambari-server/src/test/java/org/apache/ambari/server/topology/BlueprintImplTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/topology/BlueprintImplTest.java b/ambari-server/src/test/java/org/apache/ambari/server/topology/BlueprintImplTest.java index 6ac74a3..6d3179e 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/topology/BlueprintImplTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/topology/BlueprintImplTest.java @@ -21,11 +21,13 @@ package org.apache.ambari.server.topology; import static org.easymock.EasyMock.createMock; import static org.easymock.EasyMock.createNiceMock; import static org.easymock.EasyMock.expect; +import static org.easymock.EasyMock.mock; import static org.easymock.EasyMock.replay; import static org.easymock.EasyMock.verify; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; +import java.lang.reflect.Field; import java.util.Arrays; import java.util.Collection; import java.util.Collections; @@ -106,13 +108,14 @@ public class BlueprintImplTest { @Test public void testValidateConfigurations__basic_positive() throws Exception { + org.apache.ambari.server.configuration.Configuration serverConfig = setupConfigurationWithGPLLicense(true); expect(group1.getCardinality()).andReturn("1").atLeastOnce(); expect(group1.getComponents()).andReturn(Arrays.asList(new Component("c1"), new Component("c2"))).atLeastOnce(); expect(group2.getCardinality()).andReturn("1").atLeastOnce(); expect(group2.getComponents()).andReturn(Arrays.asList(new Component("c1"), new Component("c3"))).atLeastOnce(); expect(group2.getConfiguration()).andReturn(EMPTY_CONFIGURATION).atLeastOnce(); - replay(stack, group1, group2); + replay(stack, group1, group2, serverConfig); Map<String, String> category2Props = new HashMap<>(); properties.put("category2", category2Props); @@ -123,7 +126,7 @@ public class BlueprintImplTest { blueprint.validateRequiredProperties(); BlueprintEntity entity = blueprint.toEntity(); - verify(stack, group1, group2); + verify(stack, group1, group2, serverConfig); assertTrue(entity.getSecurityType() == SecurityType.KERBEROS); assertTrue(entity.getSecurityDescriptorReference().equals("testRef")); } @@ -134,6 +137,8 @@ public class BlueprintImplTest { Map<String, String> group2Category2Props = new HashMap<>(); group2Props.put("category2", group2Category2Props); group2Category2Props.put("prop2", "val"); + + org.apache.ambari.server.configuration.Configuration serverConfig = setupConfigurationWithGPLLicense(true); // set config for group2 which contains a required property Configuration group2Configuration = new Configuration(group2Props, EMPTY_ATTRIBUTES, configuration); expect(group2.getConfiguration()).andReturn(group2Configuration).atLeastOnce(); @@ -155,11 +160,11 @@ public class BlueprintImplTest { properties.put("hadoop-env", hadoopProps); hadoopProps.put("dfs_ha_initial_namenode_active", "%HOSTGROUP:group1%"); hadoopProps.put("dfs_ha_initial_namenode_standby", "%HOSTGROUP:group2%"); - replay(stack, group1, group2); + replay(stack, group1, group2, serverConfig); Blueprint blueprint = new BlueprintImpl("test", hostGroups, stack, configuration, null); blueprint.validateRequiredProperties(); BlueprintEntity entity = blueprint.toEntity(); - verify(stack, group1, group2); + verify(stack, group1, group2, serverConfig); assertTrue(entity.getSecurityType() == SecurityType.NONE); assertTrue(entity.getSecurityDescriptorReference() == null); } @@ -277,17 +282,55 @@ public class BlueprintImplTest { verify(stack, group1, group2); } @Test(expected = InvalidTopologyException.class) - public void testValidateConfigurations__secretReference() throws InvalidTopologyException { + public void testValidateConfigurations__secretReference() throws InvalidTopologyException, + GPLLicenseNotAcceptedException, NoSuchFieldException, IllegalAccessException { Map<String, Map<String, String>> group2Props = new HashMap<>(); Map<String, String> group2Category2Props = new HashMap<>(); + + org.apache.ambari.server.configuration.Configuration serverConfig = setupConfigurationWithGPLLicense(true); group2Props.put("category2", group2Category2Props); group2Category2Props.put("prop2", "val"); hdfsProps.put("secret", "SECRET:hdfs-site:1:test"); - replay(stack, group1, group2); + replay(stack, group1, group2, serverConfig); Blueprint blueprint = new BlueprintImpl("test", hostGroups, stack, configuration, null); blueprint.validateRequiredProperties(); - verify(stack, group1, group2); + verify(stack, group1, group2, serverConfig); + } + + @Test(expected = GPLLicenseNotAcceptedException.class) + public void testValidateConfigurations__gplIsNotAllowed() throws InvalidTopologyException, + GPLLicenseNotAcceptedException, NoSuchFieldException, IllegalAccessException { + Map<String, Map<String, String>> lzoProperties = new HashMap<>(); + lzoProperties.put("core-site", new HashMap<String, String>(){{ + put(BlueprintValidatorImpl.LZO_CODEC_CLASS_PROPERTY_NAME, BlueprintValidatorImpl.LZO_CODEC_CLASS); + }}); + Configuration lzoUsageConfiguration = new Configuration(lzoProperties, EMPTY_ATTRIBUTES, EMPTY_CONFIGURATION); + + org.apache.ambari.server.configuration.Configuration serverConfig = setupConfigurationWithGPLLicense(false); + replay(stack, group1, group2, serverConfig); + + Blueprint blueprint = new BlueprintImpl("test", hostGroups, stack, lzoUsageConfiguration, null); + blueprint.validateRequiredProperties(); + verify(stack, group1, group2, serverConfig); + } + + @Test + public void testValidateConfigurations__gplISAllowed() throws InvalidTopologyException, + GPLLicenseNotAcceptedException, NoSuchFieldException, IllegalAccessException { + Map<String, Map<String, String>> lzoProperties = new HashMap<>(); + lzoProperties.put("core-site", new HashMap<String, String>(){{ + put(BlueprintValidatorImpl.LZO_CODEC_CLASS_PROPERTY_NAME, BlueprintValidatorImpl.LZO_CODEC_CLASS); + }}); + Configuration lzoUsageConfiguration = new Configuration(lzoProperties, EMPTY_ATTRIBUTES, EMPTY_CONFIGURATION); + + org.apache.ambari.server.configuration.Configuration serverConfig = setupConfigurationWithGPLLicense(true); + expect(group2.getConfiguration()).andReturn(EMPTY_CONFIGURATION).atLeastOnce(); + replay(stack, group1, group2, serverConfig); + + Blueprint blueprint = new BlueprintImpl("test", hostGroups, stack, lzoUsageConfiguration, null); + blueprint.validateRequiredProperties(); + verify(stack, group1, group2, serverConfig); } @Test @@ -314,6 +357,18 @@ public class BlueprintImplTest { verify(stack, setting); } + public static org.apache.ambari.server.configuration.Configuration setupConfigurationWithGPLLicense(boolean isGPLAllowed) + throws NoSuchFieldException, IllegalAccessException { + org.apache.ambari.server.configuration.Configuration serverConfig = + mock(org.apache.ambari.server.configuration.Configuration.class); + expect(serverConfig.getGplLicenseAccepted()).andReturn(isGPLAllowed).atLeastOnce(); + + Field field = BlueprintValidatorImpl.class.getDeclaredField("configuration"); + field.setAccessible(true); + field.set(null, serverConfig); + return serverConfig; + } + //todo: ensure coverage for these existing tests // private void validateEntity(BlueprintEntity entity, boolean containsConfig) { http://git-wip-us.apache.org/repos/asf/ambari/blob/9bbc0ef7/ambari-server/src/test/java/org/apache/ambari/server/topology/BlueprintValidatorImplTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/topology/BlueprintValidatorImplTest.java b/ambari-server/src/test/java/org/apache/ambari/server/topology/BlueprintValidatorImplTest.java index a706428..75a9d6b 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/topology/BlueprintValidatorImplTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/topology/BlueprintValidatorImplTest.java @@ -228,6 +228,9 @@ public class BlueprintValidatorImplTest { services.addAll(Arrays.asList("HIVE")); + org.apache.ambari.server.configuration.Configuration serverConfig = + BlueprintImplTest.setupConfigurationWithGPLLicense(true); + Configuration config = new Configuration(new HashMap<>(), new HashMap<>()); expect(group1.getConfiguration()).andReturn(config).anyTimes(); @@ -237,7 +240,7 @@ public class BlueprintValidatorImplTest { expect(blueprint.getHostGroupsForComponent("HIVE_METASTORE")).andReturn(Collections.singleton(group1)).anyTimes(); - replay(blueprint, stack, group1, group2, dependency1); + replay(blueprint, stack, group1, group2, dependency1, serverConfig); BlueprintValidator validator = new BlueprintValidatorImpl(blueprint); validator.validateRequiredProperties(); } @@ -252,6 +255,9 @@ public class BlueprintValidatorImplTest { services.addAll(Arrays.asList("OOZIE")); + org.apache.ambari.server.configuration.Configuration serverConfig = + BlueprintImplTest.setupConfigurationWithGPLLicense(true); + Configuration config = new Configuration(new HashMap<>(), new HashMap<>()); expect(group1.getConfiguration()).andReturn(config).anyTimes(); @@ -261,7 +267,7 @@ public class BlueprintValidatorImplTest { expect(blueprint.getHostGroupsForComponent("OOZIE_SERVER")).andReturn(Collections.singleton(group1)).anyTimes(); - replay(blueprint, stack, group1, group2, dependency1); + replay(blueprint, stack, group1, group2, dependency1, serverConfig); BlueprintValidator validator = new BlueprintValidatorImpl(blueprint); validator.validateRequiredProperties(); }
