This is an automated email from the ASF dual-hosted git repository. aldrin pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/nifi-minifi.git
The following commit(s) were added to refs/heads/master by this push: new c6f2340 MINIFI-424 Restructured ConfigTransformer to accept bootstrap properties and created BootstrapTransformer to consolidate future bootstrap use cases c6f2340 is described below commit c6f2340a839efde6fadb68e704c3ee962457bc0a Author: GCHQ-NiFi <gchqn...@gmail.com> AuthorDate: Mon Aug 19 19:23:25 2019 +0100 MINIFI-424 Restructured ConfigTransformer to accept bootstrap properties and created BootstrapTransformer to consolidate future bootstrap use cases This closes #167 Signed-off-by: Aldrin Piri <ald...@apache.org> --- .../apache/nifi/minifi/bootstrap/RunMiNiFi.java | 156 +-------------------- .../bootstrap/status/reporters/StatusLogger.java | 2 +- .../bootstrap/util/BootstrapTransformer.java | 91 ++++++++++++ .../minifi/bootstrap/util/ConfigTransformer.java | 35 +---- .../nifi/minifi/bootstrap/RunMiNiFiTest.java | 11 +- .../bootstrap/util/ConfigTransformerTest.java | 44 ++++-- .../schema/common/BootstrapPropertyKeys.java | 128 +++++++++++++++++ 7 files changed, 262 insertions(+), 205 deletions(-) diff --git a/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/RunMiNiFi.java b/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/RunMiNiFi.java index 93d5f05..5d0951a 100644 --- a/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/RunMiNiFi.java +++ b/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/RunMiNiFi.java @@ -23,10 +23,6 @@ import org.apache.nifi.minifi.bootstrap.configuration.ConfigurationChangeExcepti import org.apache.nifi.minifi.bootstrap.configuration.ConfigurationChangeListener; import org.apache.nifi.minifi.bootstrap.status.PeriodicStatusReporter; import org.apache.nifi.minifi.bootstrap.util.ConfigTransformer; -import org.apache.nifi.minifi.commons.schema.ProvenanceReportingSchema; -import org.apache.nifi.minifi.commons.schema.SecurityPropertiesSchema; -import org.apache.nifi.minifi.commons.schema.SensitivePropsSchema; -import org.apache.nifi.minifi.commons.schema.common.CommonPropertyKeys; import org.apache.nifi.minifi.commons.status.FlowStatusReport; import org.apache.nifi.util.Tuple; import org.apache.nifi.util.file.FileUtils; @@ -69,7 +65,6 @@ import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.Map.Entry; -import java.util.Optional; import java.util.Properties; import java.util.Set; import java.util.concurrent.ExecutorService; @@ -84,11 +79,7 @@ import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; import static java.nio.file.StandardCopyOption.REPLACE_EXISTING; -import static org.apache.nifi.minifi.commons.schema.RemoteProcessGroupSchema.TIMEOUT_KEY; -import static org.apache.nifi.minifi.commons.schema.common.CommonPropertyKeys.COMMENT_KEY; -import static org.apache.nifi.minifi.commons.schema.common.CommonPropertyKeys.SCHEDULING_PERIOD_KEY; -import static org.apache.nifi.minifi.commons.schema.common.CommonPropertyKeys.SCHEDULING_STRATEGY_KEY; -import static org.apache.nifi.minifi.commons.schema.common.CommonPropertyKeys.USE_COMPRESSION_KEY; +import static org.apache.nifi.minifi.commons.schema.common.BootstrapPropertyKeys.STATUS_REPORTER_COMPONENTS_KEY; /** * <p> @@ -137,96 +128,6 @@ public class RunMiNiFi implements QueryableStatusAggregator, ConfigurationFileHo public static final String DUMP_CMD = "DUMP"; public static final String FLOW_STATUS_REPORT_CMD = "FLOW_STATUS_REPORT"; - public static final String NOTIFIER_PROPERTY_PREFIX = "nifi.minifi.notifier"; - public static final String NOTIFIER_COMPONENTS_KEY = NOTIFIER_PROPERTY_PREFIX + ".components"; - - public static final String STATUS_REPORTER_PROPERTY_PREFIX = "nifi.minifi.status.reporter"; - public static final String STATUS_REPORTER_COMPONENTS_KEY = STATUS_REPORTER_PROPERTY_PREFIX + ".components"; - - - public static final String SECURITY_KEYSTORE_KEY = "nifi.minifi.security.keystore"; - public static final String SECURITY_KEYSTORE_TYPE_KEY = "nifi.minifi.security.keystoreType"; - public static final String SECURITY_KEYSTORE_PASSWORD_KEY = "nifi.minifi.security.keystorePasswd"; - public static final String SECURITY_KEY_PASSWORD_KEY = "nifi.minifi.security.keyPasswd"; - public static final String SECURITY_TRUSTSTORE_KEY = "nifi.minifi.security.truststore"; - public static final String SECURITY_TRUSTSTORE_TYPE_KEY = "nifi.minifi.security.truststoreType"; - public static final String SECURITY_TRUSTSTORE_PASSWORD_KEY = "nifi.minifi.security.truststorePasswd"; - public static final String SECURITY_SSL_PROTOCOL_KEY = "nifi.minifi.security.ssl.protocol"; - - public static final String SENSITIVE_PROPS_KEY_KEY = "nifi.minifi.sensitive.props.key"; - public static final String SENSITIVE_PROPS_ALGORITHM_KEY = "nifi.minifi.sensitive.props.algorithm"; - public static final String SENSITIVE_PROPS_PROVIDER_KEY = "nifi.minifi.sensitive.props.provider"; - - public static final Set<String> BOOTSTRAP_SECURITY_PROPERTY_KEYS = new HashSet<>( - Arrays.asList(SECURITY_KEYSTORE_KEY, - SECURITY_KEYSTORE_TYPE_KEY, - SECURITY_KEYSTORE_PASSWORD_KEY, - SECURITY_KEY_PASSWORD_KEY, - SECURITY_TRUSTSTORE_KEY, - SECURITY_TRUSTSTORE_TYPE_KEY, - SECURITY_TRUSTSTORE_PASSWORD_KEY, - SECURITY_SSL_PROTOCOL_KEY)); - - public static final Set<String> BOOTSTRAP_SENSITIVE_PROPERTY_KEYS = new HashSet<>( - Arrays.asList( - SENSITIVE_PROPS_KEY_KEY, - SENSITIVE_PROPS_ALGORITHM_KEY, - SENSITIVE_PROPS_PROVIDER_KEY)); - - public static final String PROVENANCE_REPORTING_COMMENT_KEY = "nifi.minifi.provenance.reporting.comment"; - public static final String PROVENANCE_REPORTING_SCHEDULING_STRATEGY_KEY = "nifi.minifi.provenance.reporting.scheduling.strategy"; - public static final String PROVENANCE_REPORTING_SCHEDULING_PERIOD_KEY = "nifi.minifi.provenance.reporting.scheduling.period"; - public static final String PROVENANCE_REPORTING_DESTINATION_URL_KEY = "nifi.minifi.provenance.reporting.destination.url"; - public static final String PROVENANCE_REPORTING_INPUT_PORT_NAME_KEY = "nifi.minifi.provenance.reporting.input.port.name"; - public static final String PROVENANCE_REPORTING_INSTANCE_URL_KEY = "nifi.minifi.provenance.reporting.instance.url"; - public static final String PROVENANCE_REPORTING_COMPRESS_EVENTS_KEY = "nifi.minifi.provenance.reporting.compress.events"; - public static final String PROVENANCE_REPORTING_BATCH_SIZE_KEY = "nifi.minifi.provenance.reporting.batch.size"; - public static final String PROVENANCE_REPORTING_COMMUNICATIONS_TIMEOUT_KEY = "nifi.minifi.provenance.reporting.communications.timeout"; - - public static final Set<String> BOOTSTRAP_PROVENANCE_REPORTING_KEYS = new HashSet<>( - Arrays.asList(PROVENANCE_REPORTING_COMMENT_KEY, - PROVENANCE_REPORTING_SCHEDULING_STRATEGY_KEY, - PROVENANCE_REPORTING_SCHEDULING_PERIOD_KEY, - PROVENANCE_REPORTING_DESTINATION_URL_KEY, - PROVENANCE_REPORTING_INPUT_PORT_NAME_KEY, - PROVENANCE_REPORTING_INSTANCE_URL_KEY, - PROVENANCE_REPORTING_COMPRESS_EVENTS_KEY, - PROVENANCE_REPORTING_BATCH_SIZE_KEY, - PROVENANCE_REPORTING_COMMUNICATIONS_TIMEOUT_KEY - )); - - public static final Map<String, String> BOOTSTRAP_KEYS_TO_YML_KEYS; - - static { - final Map<String, String> mutableMap = new HashMap<>(); - mutableMap.put(SECURITY_KEYSTORE_KEY, SecurityPropertiesSchema.KEYSTORE_KEY); - mutableMap.put(SECURITY_KEYSTORE_TYPE_KEY, SecurityPropertiesSchema.KEYSTORE_TYPE_KEY); - mutableMap.put(SECURITY_KEYSTORE_PASSWORD_KEY, SecurityPropertiesSchema.KEYSTORE_PASSWORD_KEY); - mutableMap.put(SECURITY_KEY_PASSWORD_KEY, SecurityPropertiesSchema.KEY_PASSWORD_KEY); - - mutableMap.put(SECURITY_TRUSTSTORE_KEY, SecurityPropertiesSchema.TRUSTSTORE_KEY); - mutableMap.put(SECURITY_TRUSTSTORE_TYPE_KEY, SecurityPropertiesSchema.TRUSTSTORE_TYPE_KEY); - mutableMap.put(SECURITY_TRUSTSTORE_PASSWORD_KEY, SecurityPropertiesSchema.TRUSTSTORE_PASSWORD_KEY); - - mutableMap.put(SECURITY_SSL_PROTOCOL_KEY, SecurityPropertiesSchema.SSL_PROTOCOL_KEY); - - mutableMap.put(SENSITIVE_PROPS_KEY_KEY, SensitivePropsSchema.SENSITIVE_PROPS_KEY_KEY); - mutableMap.put(SENSITIVE_PROPS_ALGORITHM_KEY, SensitivePropsSchema.SENSITIVE_PROPS_ALGORITHM_KEY); - mutableMap.put(SENSITIVE_PROPS_PROVIDER_KEY, SensitivePropsSchema.SENSITIVE_PROPS_PROVIDER_KEY); - - mutableMap.put(PROVENANCE_REPORTING_COMMENT_KEY, COMMENT_KEY); - mutableMap.put(PROVENANCE_REPORTING_SCHEDULING_STRATEGY_KEY, SCHEDULING_STRATEGY_KEY); - mutableMap.put(PROVENANCE_REPORTING_SCHEDULING_PERIOD_KEY, SCHEDULING_PERIOD_KEY); - mutableMap.put(PROVENANCE_REPORTING_DESTINATION_URL_KEY, ProvenanceReportingSchema.DESTINATION_URL_KEY); - mutableMap.put(PROVENANCE_REPORTING_INPUT_PORT_NAME_KEY, ProvenanceReportingSchema.PORT_NAME_KEY); - mutableMap.put(PROVENANCE_REPORTING_INSTANCE_URL_KEY, ProvenanceReportingSchema.ORIGINATING_URL_KEY); - mutableMap.put(PROVENANCE_REPORTING_COMPRESS_EVENTS_KEY, USE_COMPRESSION_KEY); - mutableMap.put(PROVENANCE_REPORTING_BATCH_SIZE_KEY, ProvenanceReportingSchema.BATCH_SIZE_KEY); - mutableMap.put(PROVENANCE_REPORTING_COMMUNICATIONS_TIMEOUT_KEY, TIMEOUT_KEY); - - BOOTSTRAP_KEYS_TO_YML_KEYS = Collections.unmodifiableMap(mutableMap); - } - private static final int UNINITIALIZED_CC_PORT = -1; private volatile boolean autoRestartNiFi = true; @@ -1811,8 +1712,7 @@ public class RunMiNiFi implements QueryableStatusAggregator, ConfigurationFileHo ConfigTransformer.transformConfigFile( teeInputStream, configDestinationPath, - buildSecurityPropertiesFromBootstrap(getBootstrapProperties()).orElse(null), - buildProvenanceReportingPropertiesFromBootstrap(getBootstrapProperties()).orElse(null) + getBootstrapProperties() ); return ByteBuffer.wrap(byteArrayOutputStream.toByteArray()); @@ -1823,58 +1723,6 @@ public class RunMiNiFi implements QueryableStatusAggregator, ConfigurationFileHo } } - // TODO extract this and buildProvenanceReportingPropertiesFromBootstrap to separate class BootstrapTransformer, and make private - public Optional<SecurityPropertiesSchema> buildSecurityPropertiesFromBootstrap(final Properties bootstrapProperties) { - - Optional<SecurityPropertiesSchema> securityPropsOptional = Optional.empty(); - - final Map<String, Object> securityProperties = new HashMap<>(); - - BOOTSTRAP_SECURITY_PROPERTY_KEYS.stream() - .filter(key -> StringUtils.isNotBlank(bootstrapProperties.getProperty(key))) - .forEach(key -> - securityProperties.put(BOOTSTRAP_KEYS_TO_YML_KEYS.get(key), bootstrapProperties.getProperty(key)) - ); - - if (!securityProperties.isEmpty()) { - // Determine if sensitive properties were provided - final Map<String, String> sensitiveProperties = new HashMap<>(); - BOOTSTRAP_SENSITIVE_PROPERTY_KEYS.stream() - .filter(key -> StringUtils.isNotBlank(bootstrapProperties.getProperty(key))) - .forEach(key -> - sensitiveProperties.put(BOOTSTRAP_KEYS_TO_YML_KEYS.get(key), bootstrapProperties.getProperty(key)) - ); - if (!sensitiveProperties.isEmpty()) { - securityProperties.put(CommonPropertyKeys.SENSITIVE_PROPS_KEY, sensitiveProperties); - } - - final SecurityPropertiesSchema securityPropertiesSchema = new SecurityPropertiesSchema(securityProperties); - securityPropsOptional = Optional.of(securityPropertiesSchema); - } - - return securityPropsOptional; - } - - public Optional<ProvenanceReportingSchema> buildProvenanceReportingPropertiesFromBootstrap(final Properties bootstrapProperties) { - - Optional<ProvenanceReportingSchema> provenanceReportingPropsOptional = Optional.empty(); - - final Map<String, Object> provenanceReportingProperties = new HashMap<>(); - - BOOTSTRAP_PROVENANCE_REPORTING_KEYS.stream() - .filter(key -> StringUtils.isNotBlank(bootstrapProperties.getProperty(key))) - .forEach(key -> - provenanceReportingProperties.put(BOOTSTRAP_KEYS_TO_YML_KEYS.get(key), bootstrapProperties.getProperty(key)) - ); - - if (!provenanceReportingProperties.isEmpty()) { - final ProvenanceReportingSchema provenanceReportingSchema = new ProvenanceReportingSchema(provenanceReportingProperties); - provenanceReportingPropsOptional = Optional.of(provenanceReportingSchema); - } - - return provenanceReportingPropsOptional; - } - private static class Status { private final Integer port; diff --git a/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/status/reporters/StatusLogger.java b/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/status/reporters/StatusLogger.java index ec7e7fc..da5b33e 100644 --- a/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/status/reporters/StatusLogger.java +++ b/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/status/reporters/StatusLogger.java @@ -27,7 +27,7 @@ import org.slf4j.LoggerFactory; import java.io.IOException; import java.util.Properties; -import static org.apache.nifi.minifi.bootstrap.RunMiNiFi.STATUS_REPORTER_PROPERTY_PREFIX; +import static org.apache.nifi.minifi.commons.schema.common.BootstrapPropertyKeys.STATUS_REPORTER_PROPERTY_PREFIX; public class StatusLogger extends PeriodicStatusReporter { diff --git a/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/util/BootstrapTransformer.java b/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/util/BootstrapTransformer.java new file mode 100644 index 0000000..dc57990 --- /dev/null +++ b/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/util/BootstrapTransformer.java @@ -0,0 +1,91 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.minifi.bootstrap.util; + +import org.apache.commons.lang3.StringUtils; +import org.apache.nifi.minifi.commons.schema.ProvenanceReportingSchema; +import org.apache.nifi.minifi.commons.schema.SecurityPropertiesSchema; +import org.apache.nifi.minifi.commons.schema.common.CommonPropertyKeys; + +import java.util.HashMap; +import java.util.Map; +import java.util.Optional; +import java.util.Properties; + +import static org.apache.nifi.minifi.commons.schema.common.BootstrapPropertyKeys.BOOTSTRAP_KEYS_TO_YML_KEYS; +import static org.apache.nifi.minifi.commons.schema.common.BootstrapPropertyKeys.BOOTSTRAP_PROVENANCE_REPORTING_KEYS; +import static org.apache.nifi.minifi.commons.schema.common.BootstrapPropertyKeys.BOOTSTRAP_SECURITY_PROPERTY_KEYS; +import static org.apache.nifi.minifi.commons.schema.common.BootstrapPropertyKeys.BOOTSTRAP_SENSITIVE_PROPERTY_KEYS; + +public class BootstrapTransformer { + + public static Optional<SecurityPropertiesSchema> buildSecurityPropertiesFromBootstrap(final Properties bootstrapProperties) { + + Optional<SecurityPropertiesSchema> securityPropsOptional = Optional.empty(); + + final Map<String, Object> securityProperties = new HashMap<>(); + + if (bootstrapProperties != null) { + BOOTSTRAP_SECURITY_PROPERTY_KEYS.stream() + .filter(key -> StringUtils.isNotBlank(bootstrapProperties.getProperty(key))) + .forEach(key -> + securityProperties.put(BOOTSTRAP_KEYS_TO_YML_KEYS.get(key), bootstrapProperties.getProperty(key)) + ); + + if (!securityProperties.isEmpty()) { + // Determine if sensitive properties were provided + final Map<String, String> sensitiveProperties = new HashMap<>(); + BOOTSTRAP_SENSITIVE_PROPERTY_KEYS.stream() + .filter(key -> StringUtils.isNotBlank(bootstrapProperties.getProperty(key))) + .forEach(key -> + sensitiveProperties.put(BOOTSTRAP_KEYS_TO_YML_KEYS.get(key), bootstrapProperties.getProperty(key)) + ); + if (!sensitiveProperties.isEmpty()) { + securityProperties.put(CommonPropertyKeys.SENSITIVE_PROPS_KEY, sensitiveProperties); + } + + final SecurityPropertiesSchema securityPropertiesSchema = new SecurityPropertiesSchema(securityProperties); + securityPropsOptional = Optional.of(securityPropertiesSchema); + } + } + + return securityPropsOptional; + } + + public static Optional<ProvenanceReportingSchema> buildProvenanceReportingPropertiesFromBootstrap(final Properties bootstrapProperties) { + + Optional<ProvenanceReportingSchema> provenanceReportingPropsOptional = Optional.empty(); + + final Map<String, Object> provenanceReportingProperties = new HashMap<>(); + if (bootstrapProperties != null) { + BOOTSTRAP_PROVENANCE_REPORTING_KEYS.stream() + .filter(key -> StringUtils.isNotBlank(bootstrapProperties.getProperty(key))) + .forEach(key -> + provenanceReportingProperties.put(BOOTSTRAP_KEYS_TO_YML_KEYS.get(key), bootstrapProperties.getProperty(key)) + ); + + if (!provenanceReportingProperties.isEmpty()) { + final ProvenanceReportingSchema provenanceReportingSchema = new ProvenanceReportingSchema(provenanceReportingProperties); + provenanceReportingPropsOptional = Optional.of(provenanceReportingSchema); + } + } + + return provenanceReportingPropsOptional; + } + +} diff --git a/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/util/ConfigTransformer.java b/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/util/ConfigTransformer.java index 7ba6fe3..0e92ece 100644 --- a/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/util/ConfigTransformer.java +++ b/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/util/ConfigTransformer.java @@ -59,8 +59,6 @@ import javax.xml.transform.TransformerFactory; import javax.xml.transform.TransformerFactoryConfigurationError; import javax.xml.transform.dom.DOMSource; import javax.xml.transform.stream.StreamResult; -import java.io.File; -import java.io.FileInputStream; import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; @@ -73,6 +71,7 @@ import java.util.Collection; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Properties; import java.util.stream.Collectors; import java.util.zip.GZIPOutputStream; @@ -87,37 +86,13 @@ public final class ConfigTransformer { private ConfigTransformer() { } - public static void transformConfigFile(String sourceFile, String destPath) throws Exception { - transformConfigFile(sourceFile, destPath, null); - } - - public static void transformConfigFile(String sourceFile, String destPath, SecurityPropertiesSchema securityProperties) throws Exception { - final File ymlConfigFile = new File(sourceFile); - final InputStream ios = new FileInputStream(ymlConfigFile); - - transformConfigFile(ios, destPath, securityProperties, null); - } - - public static void transformConfigFile(String sourceFile, String destPath, SecurityPropertiesSchema securityProperties, ProvenanceReportingSchema provenanceReportingProperties) throws Exception { - final File ymlConfigFile = new File(sourceFile); - final InputStream ios = new FileInputStream(ymlConfigFile); - - transformConfigFile(ios, destPath, securityProperties, provenanceReportingProperties); - } - - public static void transformConfigFile(InputStream sourceStream, String destPath) throws Exception { - transformConfigFile(sourceStream, destPath, null, null); - } - - - public static void transformConfigFile( - InputStream sourceStream, - String destPath, - SecurityPropertiesSchema securityProperties, - ProvenanceReportingSchema provenanceReportingProperties) throws Exception { + public static void transformConfigFile(InputStream sourceStream, String destPath, Properties bootstrapProperties) throws Exception { ConvertableSchema<ConfigSchema> convertableSchema = throwIfInvalid(SchemaLoader.loadConvertableSchemaFromYaml(sourceStream)); ConfigSchema configSchema = throwIfInvalid(convertableSchema.convert()); + SecurityPropertiesSchema securityProperties = BootstrapTransformer.buildSecurityPropertiesFromBootstrap(bootstrapProperties).orElse(null); + ProvenanceReportingSchema provenanceReportingProperties = BootstrapTransformer.buildProvenanceReportingPropertiesFromBootstrap(bootstrapProperties).orElse(null); + // See if we are providing defined properties from the filesystem configurations and use those as the definitive values if (securityProperties != null) { configSchema.setSecurityProperties(securityProperties); diff --git a/minifi-bootstrap/src/test/java/org/apache/nifi/minifi/bootstrap/RunMiNiFiTest.java b/minifi-bootstrap/src/test/java/org/apache/nifi/minifi/bootstrap/RunMiNiFiTest.java index 8bd651f..079c874 100644 --- a/minifi-bootstrap/src/test/java/org/apache/nifi/minifi/bootstrap/RunMiNiFiTest.java +++ b/minifi-bootstrap/src/test/java/org/apache/nifi/minifi/bootstrap/RunMiNiFiTest.java @@ -16,6 +16,7 @@ */ package org.apache.nifi.minifi.bootstrap; +import org.apache.nifi.minifi.bootstrap.util.BootstrapTransformer; import org.apache.nifi.minifi.commons.schema.ProvenanceReportingSchema; import org.apache.nifi.minifi.commons.schema.SecurityPropertiesSchema; import org.apache.nifi.minifi.commons.schema.SensitivePropsSchema; @@ -33,7 +34,7 @@ public class RunMiNiFiTest { public void buildSecurityPropertiesNotDefined() throws Exception { final RunMiNiFi testMiNiFi = new RunMiNiFi(null); final Properties bootstrapProperties = getTestBootstrapProperties("bootstrap-ssl-ctx/bootstrap.conf.default"); - final Optional<SecurityPropertiesSchema> securityPropsOptional = testMiNiFi.buildSecurityPropertiesFromBootstrap(bootstrapProperties); + final Optional<SecurityPropertiesSchema> securityPropsOptional = BootstrapTransformer.buildSecurityPropertiesFromBootstrap(bootstrapProperties); Assert.assertTrue(!securityPropsOptional.isPresent()); } @@ -41,7 +42,7 @@ public class RunMiNiFiTest { public void buildSecurityPropertiesDefined() throws Exception { final RunMiNiFi testMiNiFi = new RunMiNiFi(null); final Properties bootstrapProperties = getTestBootstrapProperties("bootstrap-ssl-ctx/bootstrap.conf.configured"); - final Optional<SecurityPropertiesSchema> securityPropsOptional = testMiNiFi.buildSecurityPropertiesFromBootstrap(bootstrapProperties); + final Optional<SecurityPropertiesSchema> securityPropsOptional = BootstrapTransformer.buildSecurityPropertiesFromBootstrap(bootstrapProperties); Assert.assertTrue(securityPropsOptional.isPresent()); final SecurityPropertiesSchema securityPropertiesSchema = securityPropsOptional.get(); @@ -71,7 +72,7 @@ public class RunMiNiFiTest { public void buildSecurityPropertiesDefinedButInvalid() throws Exception { final RunMiNiFi testMiNiFi = new RunMiNiFi(null); final Properties bootstrapProperties = getTestBootstrapProperties("bootstrap-ssl-ctx/bootstrap.conf.configured.invalid"); - final Optional<SecurityPropertiesSchema> securityPropsOptional = testMiNiFi.buildSecurityPropertiesFromBootstrap(bootstrapProperties); + final Optional<SecurityPropertiesSchema> securityPropsOptional = BootstrapTransformer.buildSecurityPropertiesFromBootstrap(bootstrapProperties); Assert.assertTrue(securityPropsOptional.isPresent()); final SecurityPropertiesSchema securityPropertiesSchema = securityPropsOptional.get(); @@ -99,7 +100,7 @@ public class RunMiNiFiTest { public void buildProvenanceReportingNotDefined() throws Exception { final RunMiNiFi testMiNiFi = new RunMiNiFi(null); final Properties bootstrapProperties = getTestBootstrapProperties("bootstrap-provenance-reporting/bootstrap.conf.default"); - final Optional<ProvenanceReportingSchema> provenanceReportingPropsOptional = testMiNiFi.buildProvenanceReportingPropertiesFromBootstrap(bootstrapProperties); + final Optional<ProvenanceReportingSchema> provenanceReportingPropsOptional = BootstrapTransformer.buildProvenanceReportingPropertiesFromBootstrap(bootstrapProperties); Assert.assertTrue(!provenanceReportingPropsOptional.isPresent()); } @@ -107,7 +108,7 @@ public class RunMiNiFiTest { public void buildProvenanceReportingDefined() throws Exception { final RunMiNiFi testMiNiFi = new RunMiNiFi(null); final Properties bootstrapProperties = getTestBootstrapProperties("bootstrap-provenance-reporting/bootstrap.conf.configured"); - final Optional<ProvenanceReportingSchema> provenanceReportingPropsOptional = testMiNiFi.buildProvenanceReportingPropertiesFromBootstrap(bootstrapProperties); + final Optional<ProvenanceReportingSchema> provenanceReportingPropsOptional = BootstrapTransformer.buildProvenanceReportingPropertiesFromBootstrap(bootstrapProperties); Assert.assertTrue(provenanceReportingPropsOptional.isPresent()); final ProvenanceReportingSchema provenanceReportingSchema = provenanceReportingPropsOptional.get(); diff --git a/minifi-bootstrap/src/test/java/org/apache/nifi/minifi/bootstrap/util/ConfigTransformerTest.java b/minifi-bootstrap/src/test/java/org/apache/nifi/minifi/bootstrap/util/ConfigTransformerTest.java index b8e5218..5c36cee 100644 --- a/minifi-bootstrap/src/test/java/org/apache/nifi/minifi/bootstrap/util/ConfigTransformerTest.java +++ b/minifi-bootstrap/src/test/java/org/apache/nifi/minifi/bootstrap/util/ConfigTransformerTest.java @@ -222,7 +222,8 @@ public class ConfigTransformerTest { @Test public void doesTransformFile() throws Exception { - ConfigTransformer.transformConfigFile("./src/test/resources/config.yml", "./target/"); + File inputFile = new File("./src/test/resources/config.yml"); + ConfigTransformer.transformConfigFile(new FileInputStream(inputFile), "./target/", null); File nifiPropertiesFile = new File("./target/nifi.properties"); assertTrue(nifiPropertiesFile.exists()); @@ -239,7 +240,8 @@ public class ConfigTransformerTest { @Test public void doesTransformV1File() throws Exception { - ConfigTransformer.transformConfigFile("./src/test/resources/config-v1.yml", "./target/"); + File inputFile = new File("./src/test/resources/config-v1.yml"); + ConfigTransformer.transformConfigFile(new FileInputStream(inputFile), "./target/", null); File nifiPropertiesFile = new File("./target/nifi.properties"); assertTrue(nifiPropertiesFile.exists()); @@ -257,7 +259,7 @@ public class ConfigTransformerTest { @Test public void doesTransformInputStream() throws Exception { File inputFile = new File("./src/test/resources/config.yml"); - ConfigTransformer.transformConfigFile(new FileInputStream(inputFile), "./target/"); + ConfigTransformer.transformConfigFile(new FileInputStream(inputFile), "./target/", null); File nifiPropertiesFile = new File("./target/nifi.properties"); assertTrue(nifiPropertiesFile.exists()); @@ -274,7 +276,8 @@ public class ConfigTransformerTest { @Test public void doesTransformOnDefaultFile() throws Exception { - ConfigTransformer.transformConfigFile("./src/test/resources/default.yml", "./target/"); + File inputFile = new File("./src/test/resources/default.yml"); + ConfigTransformer.transformConfigFile(new FileInputStream(inputFile), "./target/", null); File nifiPropertiesFile = new File("./target/nifi.properties"); assertTrue(nifiPropertiesFile.exists()); @@ -291,7 +294,8 @@ public class ConfigTransformerTest { @Test public void doesTransformOnMultipleProcessors() throws Exception { - ConfigTransformer.transformConfigFile("./src/test/resources/config-multiple-processors.yml", "./target/"); + File inputFile = new File("./src/test/resources/config-multiple-processors.yml"); + ConfigTransformer.transformConfigFile(new FileInputStream(inputFile), "./target/", null); File nifiPropertiesFile = new File("./target/nifi.properties"); assertTrue(nifiPropertiesFile.exists()); @@ -308,7 +312,8 @@ public class ConfigTransformerTest { @Test public void doesTransformOnMultipleRemoteProcessGroups() throws Exception { - ConfigTransformer.transformConfigFile("./src/test/resources/config-multiple-RPGs.yml", "./target/"); + File inputFile = new File("./src/test/resources/config-multiple-RPGs.yml"); + ConfigTransformer.transformConfigFile(new FileInputStream(inputFile), "./target/", null); File nifiPropertiesFile = new File("./target/nifi.properties"); assertTrue(nifiPropertiesFile.exists()); @@ -325,7 +330,8 @@ public class ConfigTransformerTest { @Test public void doesTransformOnMultipleInputPorts() throws Exception { - ConfigTransformer.transformConfigFile("./src/test/resources/config-multiple-input-ports.yml", "./target/"); + File inputFile = new File("./src/test/resources/config-multiple-input-ports.yml"); + ConfigTransformer.transformConfigFile(new FileInputStream(inputFile), "./target/", null); File nifiPropertiesFile = new File("./target/nifi.properties"); assertTrue(nifiPropertiesFile.exists()); @@ -342,7 +348,8 @@ public class ConfigTransformerTest { @Test public void doesTransformOnMinimal() throws Exception { - ConfigTransformer.transformConfigFile("./src/test/resources/config-minimal.yml", "./target/"); + File inputFile = new File("./src/test/resources/config-minimal.yml"); + ConfigTransformer.transformConfigFile(new FileInputStream(inputFile), "./target/", null); File nifiPropertiesFile = new File("./target/nifi.properties"); assertTrue(nifiPropertiesFile.exists()); @@ -359,7 +366,8 @@ public class ConfigTransformerTest { @Test public void doesTransformOnProvenanceRepository() throws Exception { - ConfigTransformer.transformConfigFile("./src/test/resources/config-provenance-repository.yml", "./target/"); + File inputFile = new File("./src/test/resources/config-provenance-repository.yml"); + ConfigTransformer.transformConfigFile(new FileInputStream(inputFile), "./target/", null); File nifiPropertiesFile = new File("./target/nifi.properties"); assertTrue(nifiPropertiesFile.exists()); @@ -379,7 +387,8 @@ public class ConfigTransformerTest { @Test public void doesTransformOnCustomProvenanceRepository() throws Exception { - ConfigTransformer.transformConfigFile("./src/test/resources/config-provenance-custom-repository.yml", "./target/"); + File inputFile = new File("./src/test/resources/config-provenance-custom-repository.yml"); + ConfigTransformer.transformConfigFile(new FileInputStream(inputFile), "./target/", null); File nifiPropertiesFile = new File("./target/nifi.properties"); assertTrue(nifiPropertiesFile.exists()); @@ -400,7 +409,8 @@ public class ConfigTransformerTest { @Test public void handleTransformInvalidFile() throws Exception { try { - ConfigTransformer.transformConfigFile("./src/test/resources/config-invalid.yml", "./target/"); + File inputFile = new File("./src/test/resources/config-invalid.yml"); + ConfigTransformer.transformConfigFile(new FileInputStream(inputFile), "./target/", null); fail("Invalid configuration file was not detected."); } catch (SchemaLoaderException e){ assertEquals("Provided YAML configuration is not a Map", e.getMessage()); @@ -410,7 +420,8 @@ public class ConfigTransformerTest { @Test public void handleTransformMalformedField() throws Exception { try { - ConfigTransformer.transformConfigFile("./src/test/resources/config-malformed-field.yml", "./target/"); + File inputFile = new File("./src/test/resources/config-malformed-field.yml"); + ConfigTransformer.transformConfigFile(new FileInputStream(inputFile), "./target/", null); fail("Invalid configuration file was not detected."); } catch (InvalidConfigurationException e){ assertEquals("Failed to transform config file due to:['threshold' in section 'Swap' because it is found but could not be parsed as a Number]", e.getMessage()); @@ -420,7 +431,8 @@ public class ConfigTransformerTest { @Test public void handleTransformEmptyFile() throws Exception { try { - ConfigTransformer.transformConfigFile("./src/test/resources/config-empty.yml", "./target/"); + File inputFile = new File("./src/test/resources/config-empty.yml"); + ConfigTransformer.transformConfigFile(new FileInputStream(inputFile), "./target/", null); fail("Invalid configuration file was not detected."); } catch (SchemaLoaderException e){ assertEquals("Provided YAML configuration is not a Map", e.getMessage()); @@ -430,7 +442,8 @@ public class ConfigTransformerTest { @Test public void handleTransformFileMissingRequiredField() throws Exception { try { - ConfigTransformer.transformConfigFile("./src/test/resources/config-missing-required-field.yml", "./target/"); + File inputFile = new File("./src/test/resources/config-missing-required-field.yml"); + ConfigTransformer.transformConfigFile(new FileInputStream(inputFile), "./target/", null); fail("Invalid configuration file was not detected."); } catch (InvalidConfigurationException e){ assertEquals("Failed to transform config file due to:['class' in section 'Processors' because it was not found and it is required]", e.getMessage()); @@ -440,7 +453,8 @@ public class ConfigTransformerTest { @Test public void handleTransformFileMultipleProblems() throws Exception { try { - ConfigTransformer.transformConfigFile("./src/test/resources/config-multiple-problems.yml", "./target/"); + File inputFile = new File("./src/test/resources/config-multiple-problems.yml"); + ConfigTransformer.transformConfigFile(new FileInputStream(inputFile), "./target/", null); fail("Invalid configuration file was not detected."); } catch (InvalidConfigurationException e){ assertEquals("Failed to transform config file due to:['class' in section 'Processors' because it was not found and it is required], " + diff --git a/minifi-commons/minifi-commons-schema/src/main/java/org/apache/nifi/minifi/commons/schema/common/BootstrapPropertyKeys.java b/minifi-commons/minifi-commons-schema/src/main/java/org/apache/nifi/minifi/commons/schema/common/BootstrapPropertyKeys.java new file mode 100644 index 0000000..5d7724e --- /dev/null +++ b/minifi-commons/minifi-commons-schema/src/main/java/org/apache/nifi/minifi/commons/schema/common/BootstrapPropertyKeys.java @@ -0,0 +1,128 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.minifi.commons.schema.common; + +import org.apache.nifi.minifi.commons.schema.ProvenanceReportingSchema; +import org.apache.nifi.minifi.commons.schema.SecurityPropertiesSchema; +import org.apache.nifi.minifi.commons.schema.SensitivePropsSchema; + +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; + +import static org.apache.nifi.minifi.commons.schema.RemoteProcessGroupSchema.TIMEOUT_KEY; +import static org.apache.nifi.minifi.commons.schema.common.CommonPropertyKeys.COMMENT_KEY; +import static org.apache.nifi.minifi.commons.schema.common.CommonPropertyKeys.SCHEDULING_PERIOD_KEY; +import static org.apache.nifi.minifi.commons.schema.common.CommonPropertyKeys.SCHEDULING_STRATEGY_KEY; +import static org.apache.nifi.minifi.commons.schema.common.CommonPropertyKeys.USE_COMPRESSION_KEY; + +public class BootstrapPropertyKeys { + + public static final String NOTIFIER_PROPERTY_PREFIX = "nifi.minifi.notifier"; + public static final String NOTIFIER_COMPONENTS_KEY = NOTIFIER_PROPERTY_PREFIX + ".components"; + + public static final String STATUS_REPORTER_PROPERTY_PREFIX = "nifi.minifi.status.reporter"; + public static final String STATUS_REPORTER_COMPONENTS_KEY = STATUS_REPORTER_PROPERTY_PREFIX + ".components"; + + public static final String SECURITY_KEYSTORE_KEY = "nifi.minifi.security.keystore"; + public static final String SECURITY_KEYSTORE_TYPE_KEY = "nifi.minifi.security.keystoreType"; + public static final String SECURITY_KEYSTORE_PASSWORD_KEY = "nifi.minifi.security.keystorePasswd"; + public static final String SECURITY_KEY_PASSWORD_KEY = "nifi.minifi.security.keyPasswd"; + public static final String SECURITY_TRUSTSTORE_KEY = "nifi.minifi.security.truststore"; + public static final String SECURITY_TRUSTSTORE_TYPE_KEY = "nifi.minifi.security.truststoreType"; + public static final String SECURITY_TRUSTSTORE_PASSWORD_KEY = "nifi.minifi.security.truststorePasswd"; + public static final String SECURITY_SSL_PROTOCOL_KEY = "nifi.minifi.security.ssl.protocol"; + + public static final String SENSITIVE_PROPS_KEY_KEY = "nifi.minifi.sensitive.props.key"; + public static final String SENSITIVE_PROPS_ALGORITHM_KEY = "nifi.minifi.sensitive.props.algorithm"; + public static final String SENSITIVE_PROPS_PROVIDER_KEY = "nifi.minifi.sensitive.props.provider"; + + public static final Set<String> BOOTSTRAP_SECURITY_PROPERTY_KEYS = new HashSet<>( + Arrays.asList(SECURITY_KEYSTORE_KEY, + SECURITY_KEYSTORE_TYPE_KEY, + SECURITY_KEYSTORE_PASSWORD_KEY, + SECURITY_KEY_PASSWORD_KEY, + SECURITY_TRUSTSTORE_KEY, + SECURITY_TRUSTSTORE_TYPE_KEY, + SECURITY_TRUSTSTORE_PASSWORD_KEY, + SECURITY_SSL_PROTOCOL_KEY)); + + public static final Set<String> BOOTSTRAP_SENSITIVE_PROPERTY_KEYS = new HashSet<>( + Arrays.asList( + SENSITIVE_PROPS_KEY_KEY, + SENSITIVE_PROPS_ALGORITHM_KEY, + SENSITIVE_PROPS_PROVIDER_KEY)); + + public static final String PROVENANCE_REPORTING_COMMENT_KEY = "nifi.minifi.provenance.reporting.comment"; + public static final String PROVENANCE_REPORTING_SCHEDULING_STRATEGY_KEY = "nifi.minifi.provenance.reporting.scheduling.strategy"; + public static final String PROVENANCE_REPORTING_SCHEDULING_PERIOD_KEY = "nifi.minifi.provenance.reporting.scheduling.period"; + public static final String PROVENANCE_REPORTING_DESTINATION_URL_KEY = "nifi.minifi.provenance.reporting.destination.url"; + public static final String PROVENANCE_REPORTING_INPUT_PORT_NAME_KEY = "nifi.minifi.provenance.reporting.input.port.name"; + public static final String PROVENANCE_REPORTING_INSTANCE_URL_KEY = "nifi.minifi.provenance.reporting.instance.url"; + public static final String PROVENANCE_REPORTING_COMPRESS_EVENTS_KEY = "nifi.minifi.provenance.reporting.compress.events"; + public static final String PROVENANCE_REPORTING_BATCH_SIZE_KEY = "nifi.minifi.provenance.reporting.batch.size"; + public static final String PROVENANCE_REPORTING_COMMUNICATIONS_TIMEOUT_KEY = "nifi.minifi.provenance.reporting.communications.timeout"; + + public static final Set<String> BOOTSTRAP_PROVENANCE_REPORTING_KEYS = new HashSet<>( + Arrays.asList(PROVENANCE_REPORTING_COMMENT_KEY, + PROVENANCE_REPORTING_SCHEDULING_STRATEGY_KEY, + PROVENANCE_REPORTING_SCHEDULING_PERIOD_KEY, + PROVENANCE_REPORTING_DESTINATION_URL_KEY, + PROVENANCE_REPORTING_INPUT_PORT_NAME_KEY, + PROVENANCE_REPORTING_INSTANCE_URL_KEY, + PROVENANCE_REPORTING_COMPRESS_EVENTS_KEY, + PROVENANCE_REPORTING_BATCH_SIZE_KEY, + PROVENANCE_REPORTING_COMMUNICATIONS_TIMEOUT_KEY + )); + + public static final Map<String, String> BOOTSTRAP_KEYS_TO_YML_KEYS; + + static { + final Map<String, String> mutableMap = new HashMap<>(); + mutableMap.put(SECURITY_KEYSTORE_KEY, SecurityPropertiesSchema.KEYSTORE_KEY); + mutableMap.put(SECURITY_KEYSTORE_TYPE_KEY, SecurityPropertiesSchema.KEYSTORE_TYPE_KEY); + mutableMap.put(SECURITY_KEYSTORE_PASSWORD_KEY, SecurityPropertiesSchema.KEYSTORE_PASSWORD_KEY); + mutableMap.put(SECURITY_KEY_PASSWORD_KEY, SecurityPropertiesSchema.KEY_PASSWORD_KEY); + + mutableMap.put(SECURITY_TRUSTSTORE_KEY, SecurityPropertiesSchema.TRUSTSTORE_KEY); + mutableMap.put(SECURITY_TRUSTSTORE_TYPE_KEY, SecurityPropertiesSchema.TRUSTSTORE_TYPE_KEY); + mutableMap.put(SECURITY_TRUSTSTORE_PASSWORD_KEY, SecurityPropertiesSchema.TRUSTSTORE_PASSWORD_KEY); + + mutableMap.put(SECURITY_SSL_PROTOCOL_KEY, SecurityPropertiesSchema.SSL_PROTOCOL_KEY); + + mutableMap.put(SENSITIVE_PROPS_KEY_KEY, SensitivePropsSchema.SENSITIVE_PROPS_KEY_KEY); + mutableMap.put(SENSITIVE_PROPS_ALGORITHM_KEY, SensitivePropsSchema.SENSITIVE_PROPS_ALGORITHM_KEY); + mutableMap.put(SENSITIVE_PROPS_PROVIDER_KEY, SensitivePropsSchema.SENSITIVE_PROPS_PROVIDER_KEY); + + mutableMap.put(PROVENANCE_REPORTING_COMMENT_KEY, COMMENT_KEY); + mutableMap.put(PROVENANCE_REPORTING_SCHEDULING_STRATEGY_KEY, SCHEDULING_STRATEGY_KEY); + mutableMap.put(PROVENANCE_REPORTING_SCHEDULING_PERIOD_KEY, SCHEDULING_PERIOD_KEY); + mutableMap.put(PROVENANCE_REPORTING_DESTINATION_URL_KEY, ProvenanceReportingSchema.DESTINATION_URL_KEY); + mutableMap.put(PROVENANCE_REPORTING_INPUT_PORT_NAME_KEY, ProvenanceReportingSchema.PORT_NAME_KEY); + mutableMap.put(PROVENANCE_REPORTING_INSTANCE_URL_KEY, ProvenanceReportingSchema.ORIGINATING_URL_KEY); + mutableMap.put(PROVENANCE_REPORTING_COMPRESS_EVENTS_KEY, USE_COMPRESSION_KEY); + mutableMap.put(PROVENANCE_REPORTING_BATCH_SIZE_KEY, ProvenanceReportingSchema.BATCH_SIZE_KEY); + mutableMap.put(PROVENANCE_REPORTING_COMMUNICATIONS_TIMEOUT_KEY, TIMEOUT_KEY); + + BOOTSTRAP_KEYS_TO_YML_KEYS = Collections.unmodifiableMap(mutableMap); + } + +}