Repository: nifi-minifi Updated Branches: refs/heads/master b059afef2 -> 5c6add80a
MINIFI-277: Write variable registry property to nifi.properties. This closes #80. Signed-off-by: Bryan Rosander <brosan...@apache.org> Project: http://git-wip-us.apache.org/repos/asf/nifi-minifi/repo Commit: http://git-wip-us.apache.org/repos/asf/nifi-minifi/commit/5c6add80 Tree: http://git-wip-us.apache.org/repos/asf/nifi-minifi/tree/5c6add80 Diff: http://git-wip-us.apache.org/repos/asf/nifi-minifi/diff/5c6add80 Branch: refs/heads/master Commit: 5c6add80a7a264d30392484d3567712f932c1078 Parents: b059afe Author: jzonthemtn <jzemer...@apache.org> Authored: Mon May 1 09:52:38 2017 -0400 Committer: Bryan Rosander <brosan...@apache.org> Committed: Mon May 1 10:30:40 2017 -0400 ---------------------------------------------------------------------- .../bootstrap/util/ConfigTransformer.java | 1 + .../bootstrap/util/ConfigTransformerTest.java | 21 ++++ .../src/test/resources/MINIFI-277/config.yml | 107 ++++++++++++++++ .../test/resources/MINIFI-277/nifi.properties | 123 +++++++++++++++++++ .../commons/schema/CorePropertiesSchema.java | 11 ++ .../commons/schema/v2/ConfigSchemaV2.java | 39 +++--- .../schema/v2/CorePropertiesSchemaV2.java | 72 +++++++++++ .../src/test/resources/CsvToJson.yml | 1 + .../resources/DecompressionCircularFlow.yml | 1 + .../resources/InvokeHttpMiNiFiTemplateTest.yml | 1 + .../test/resources/MultipleRelationships.yml | 1 + .../test/resources/NestedControllerServices.yml | 1 + .../ProcessGroupsAndRemoteProcessGroups.yml | 1 + ...aceTextExpressionLanguageCSVReformatting.yml | 1 + .../test/resources/SimpleRPGToLogAttributes.yml | 1 + .../src/test/resources/SimpleTailFileToRPG.yml | 1 + .../src/test/resources/StressTestFramework.yml | 1 + .../resources/StressTestFrameworkFunnel.yml | 1 + 18 files changed, 365 insertions(+), 20 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/nifi-minifi/blob/5c6add80/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/util/ConfigTransformer.java ---------------------------------------------------------------------- diff --git a/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/util/ConfigTransformer.java b/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/util/ConfigTransformer.java index 64422c4..edd4a36 100644 --- a/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/util/ConfigTransformer.java +++ b/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/util/ConfigTransformer.java @@ -168,6 +168,7 @@ public final class ConfigTransformer { orderedProperties.setProperty("nifi.flowcontroller.graceful.shutdown.period", coreProperties.getFlowControllerGracefulShutdownPeriod()); orderedProperties.setProperty("nifi.flowservice.writedelay.interval", coreProperties.getFlowServiceWriteDelayInterval()); orderedProperties.setProperty("nifi.administrative.yield.duration", coreProperties.getAdministrativeYieldDuration()); + orderedProperties.setProperty("nifi.variable.registry.properties", coreProperties.getVariableRegistryProperties()); orderedProperties.setProperty("nifi.bored.yield.duration", coreProperties.getBoredYieldDuration(), "# If a component has no work to do (is \"bored\"), how long should we wait before checking again for work?"); http://git-wip-us.apache.org/repos/asf/nifi-minifi/blob/5c6add80/minifi-bootstrap/src/test/java/org/apache/nifi/minifi/bootstrap/util/ConfigTransformerTest.java ---------------------------------------------------------------------- diff --git a/minifi-bootstrap/src/test/java/org/apache/nifi/minifi/bootstrap/util/ConfigTransformerTest.java b/minifi-bootstrap/src/test/java/org/apache/nifi/minifi/bootstrap/util/ConfigTransformerTest.java index 67c5916..661da6b 100644 --- a/minifi-bootstrap/src/test/java/org/apache/nifi/minifi/bootstrap/util/ConfigTransformerTest.java +++ b/minifi-bootstrap/src/test/java/org/apache/nifi/minifi/bootstrap/util/ConfigTransformerTest.java @@ -194,6 +194,27 @@ public class ConfigTransformerTest { } @Test + public void testNifiPropertiesVariableRegistry() throws IOException, ConfigurationChangeException, SchemaLoaderException { + Properties initialProperties = new Properties(); + try (InputStream pre216PropertiesStream = ConfigTransformerTest.class.getClassLoader().getResourceAsStream("MINIFI-277/nifi.properties")) { + initialProperties.load(pre216PropertiesStream); + } + initialProperties.setProperty(ConfigTransformer.NIFI_VERSION_KEY, ConfigTransformer.NIFI_VERSION); + + ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); + try (InputStream configStream = ConfigTransformerTest.class.getClassLoader().getResourceAsStream("MINIFI-277/config.yml")) { + ConfigSchema configSchema = SchemaLoader.loadConfigSchemaFromYaml(configStream); + ConfigTransformer.writeNiFiProperties(configSchema, outputStream); + } + Properties properties = new Properties(); + properties.load(new ByteArrayInputStream(outputStream.toByteArray())); + + for (String name : initialProperties.stringPropertyNames()) { + assertEquals("Property key " + name + " doesn't match.", initialProperties.getProperty(name), properties.getProperty(name)); + } + } + + @Test public void doesTransformFile() throws Exception { ConfigTransformer.transformConfigFile("./src/test/resources/config.yml", "./target/"); File nifiPropertiesFile = new File("./target/nifi.properties"); http://git-wip-us.apache.org/repos/asf/nifi-minifi/blob/5c6add80/minifi-bootstrap/src/test/resources/MINIFI-277/config.yml ---------------------------------------------------------------------- diff --git a/minifi-bootstrap/src/test/resources/MINIFI-277/config.yml b/minifi-bootstrap/src/test/resources/MINIFI-277/config.yml new file mode 100644 index 0000000..def2666 --- /dev/null +++ b/minifi-bootstrap/src/test/resources/MINIFI-277/config.yml @@ -0,0 +1,107 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the \"License\"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an \"AS IS\" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +MiNiFi Config Version: 3 +Flow Controller: + name: MiNiFi Flow + comment: '' +Core Properties: + flow controller graceful shutdown period: 10 sec + flow service write delay interval: 500 ms + administrative yield duration: 30 sec + bored yield duration: 10 millis + max concurrent threads: 1 + variable registry properties: additional.properties +FlowFile Repository: + partitions: 256 + checkpoint interval: 2 mins + always sync: false + Swap: + threshold: 20000 + in period: 5 sec + in threads: 1 + out period: 5 sec + out threads: 4 +Content Repository: + content claim max appendable size: 10 MB + content claim max flow files: 100 + always sync: false +Provenance Repository: + provenance rollover time: 1 min +Component Status Repository: + buffer size: 1440 + snapshot frequency: 1 min +Security Properties: + keystore: /tmp/ssl/localhost-ks.jks + keystore type: JKS + keystore password: localtest + key password: localtest + truststore: /tmp/ssl/localhost-ts.jks + truststore type: JKS + truststore password: localtest + ssl protocol: TLS + Sensitive Props: + key: '' + algorithm: PBEWITHMD5AND256BITAES-CBC-OPENSSL + provider: BC +Processors: +- id: 94b8e610-b4ed-3ec9-b26f-c839931bf3e2 + name: TailFile + class: org.apache.nifi.processors.standard.TailFile + max concurrent tasks: 1 + scheduling strategy: TIMER_DRIVEN + scheduling period: 1 sec + penalization period: 30 sec + yield period: 1 sec + run duration nanos: 0 + auto-terminated relationships list: [] + Properties: + File to Tail: logs/minifi-app.log + Initial Start Position: Beginning of File + Rolling Filename Pattern: minifi-app* +Connections: +- id: f6cef9b0-8982-391c-8d70-76f33917ac12 + name: TailToS2S + source id: 94b8e610-b4ed-3ec9-b26f-c839931bf3e2 + source relationship names: + - success + destination id: 8644cbcc-a45c-40e0-964d-5e536e2ada61 + max work queue size: 0 + max work queue data size: 1 MB + flowfile expiration: 60 sec + queue prioritizer class: org.apache.nifi.prioritizer.NewestFlowFileFirstPrioritizer +Remote Process Groups: +- id: 94b8e610-b4ed-3ec9-b26f-c839931bf3e3 + name: NiFi Flow + url: https://localhost:8090/nifi + comment: '' + timeout: 30 secs + yield period: 10 sec + Input Ports: + - id: 8644cbcc-a45c-40e0-964d-5e536e2ada61 + name: tailed log + comment: '' + max concurrent tasks: 1 + use compression: false +Provenance Reporting: + comment: '' + scheduling strategy: TIMER_DRIVEN + scheduling period: 30 sec + destination url: https://localhost:8090/ + port name: provenance + originating url: http://${hostname(true)}:8081/nifi + use compression: true + timeout: 30 secs + batch size: 1000 http://git-wip-us.apache.org/repos/asf/nifi-minifi/blob/5c6add80/minifi-bootstrap/src/test/resources/MINIFI-277/nifi.properties ---------------------------------------------------------------------- diff --git a/minifi-bootstrap/src/test/resources/MINIFI-277/nifi.properties b/minifi-bootstrap/src/test/resources/MINIFI-277/nifi.properties new file mode 100644 index 0000000..53c4ad8 --- /dev/null +++ b/minifi-bootstrap/src/test/resources/MINIFI-277/nifi.properties @@ -0,0 +1,123 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Core Properties # + +nifi.version=1.1.0 + +nifi.flow.configuration.file=./conf/flow.xml.gz +nifi.flow.configuration.archive.enabled=false +nifi.flow.configuration.archive.dir=./conf/archive/ +nifi.flowcontroller.autoResumeState=true +nifi.flowcontroller.graceful.shutdown.period=10 sec +nifi.flowservice.writedelay.interval=500 ms +nifi.administrative.yield.duration=30 sec +# If a component has no work to do (is "bored"), how long should we wait before checking again for work? +nifi.bored.yield.duration=10 millis + +nifi.authority.provider.configuration.file=./conf/authority-providers.xml +nifi.login.identity.provider.configuration.file=./conf/login-identity-providers.xml +nifi.templates.directory=./conf/templates +nifi.ui.banner.text= +nifi.ui.autorefresh.interval=30 sec +nifi.nar.library.directory=./lib +nifi.nar.working.directory=./work/nar/ +nifi.documentation.working.directory=./work/docs/components +nifi.variable.registry.properties=additional.properties + +#################### +# State Management # +#################### +nifi.state.management.configuration.file=./conf/state-management.xml +# The ID of the local state provider +nifi.state.management.provider.local=local-provider + +# H2 Settings +nifi.database.directory=./database_repository +nifi.h2.url.append=;LOCK_TIMEOUT=25000;WRITE_DELAY=0;AUTO_SERVER=FALSE + +# FlowFile Repository +nifi.flowfile.repository.implementation=org.apache.nifi.controller.repository.WriteAheadFlowFileRepository +nifi.flowfile.repository.directory=./flowfile_repository +nifi.flowfile.repository.partitions=256 +nifi.flowfile.repository.checkpoint.interval=2 mins +nifi.flowfile.repository.always.sync=false + +nifi.swap.manager.implementation=org.apache.nifi.controller.FileSystemSwapManager +nifi.queue.swap.threshold=20000 +nifi.swap.in.period=5 sec +nifi.swap.in.threads=1 +nifi.swap.out.period=5 sec +nifi.swap.out.threads=4 + +# Content Repository +nifi.content.repository.implementation=org.apache.nifi.controller.repository.FileSystemRepository +nifi.content.claim.max.appendable.size=10 MB +nifi.content.claim.max.flow.files=100 +nifi.content.repository.archive.max.retention.period= +nifi.content.repository.archive.max.usage.percentage= +nifi.content.repository.archive.enabled=false +nifi.content.repository.directory.default=./content_repository +nifi.content.repository.always.sync=false + +# Provenance Repository Properties +nifi.provenance.repository.implementation=org.apache.nifi.provenance.MiNiFiPersistentProvenanceRepository +nifi.provenance.repository.rollover.time=1 min + +# Volatile Provenance Respository Properties +nifi.provenance.repository.buffer.size=10000 + +# Component Status Repository +nifi.components.status.repository.implementation=org.apache.nifi.controller.status.history.VolatileComponentStatusRepository +nifi.components.status.repository.buffer.size=1440 +nifi.components.status.snapshot.frequency=1 min + +# web properties # +nifi.web.war.directory=./lib +nifi.web.http.host= +nifi.web.http.port=8081 +nifi.web.https.host= +nifi.web.https.port= +nifi.web.jetty.working.directory=./work/jetty +nifi.web.jetty.threads=200 + +# security properties # +nifi.sensitive.props.key= +nifi.sensitive.props.algorithm=PBEWITHMD5AND256BITAES-CBC-OPENSSL +nifi.sensitive.props.provider=BC + +nifi.security.keystore=/tmp/ssl/localhost-ks.jks +nifi.security.keystoreType=JKS +nifi.security.keystorePasswd=localtest +nifi.security.keyPasswd=localtest +nifi.security.truststore=/tmp/ssl/localhost-ts.jks +nifi.security.truststoreType=JKS +nifi.security.truststorePasswd=localtest +nifi.security.needClientAuth= +nifi.security.user.credential.cache.duration=24 hours +nifi.security.user.authority.provider=file-provider +nifi.security.user.login.identity.provider= +nifi.security.support.new.account.requests= +# Valid Authorities include: ROLE_MONITOR,ROLE_DFM,ROLE_ADMIN,ROLE_PROVENANCE,ROLE_NIFI +nifi.security.anonymous.authorities= +nifi.security.ocsp.responder.url= +nifi.security.ocsp.responder.certificate= + + +# cluster node properties (only configure for cluster nodes) # +nifi.cluster.is.node=false + +# cluster manager properties (only configure for cluster manager) # +nifi.cluster.is.manager=false http://git-wip-us.apache.org/repos/asf/nifi-minifi/blob/5c6add80/minifi-commons/minifi-commons-schema/src/main/java/org/apache/nifi/minifi/commons/schema/CorePropertiesSchema.java ---------------------------------------------------------------------- diff --git a/minifi-commons/minifi-commons-schema/src/main/java/org/apache/nifi/minifi/commons/schema/CorePropertiesSchema.java b/minifi-commons/minifi-commons-schema/src/main/java/org/apache/nifi/minifi/commons/schema/CorePropertiesSchema.java index f5f83d8..665c95d 100644 --- a/minifi-commons/minifi-commons-schema/src/main/java/org/apache/nifi/minifi/commons/schema/CorePropertiesSchema.java +++ b/minifi-commons/minifi-commons-schema/src/main/java/org/apache/nifi/minifi/commons/schema/CorePropertiesSchema.java @@ -34,18 +34,21 @@ public class CorePropertiesSchema extends BaseSchema implements WritableSchema { public static final String FLOW_SERVICE_WRITE_DELAY_INTERVAL_KEY = "flow service write delay interval"; public static final String ADMINISTRATIVE_YIELD_DURATION_KEY = "administrative yield duration"; public static final String BORED_YIELD_DURATION_KEY = "bored yield duration"; + public static final String VARIABLE_REGISTRY_PROPERTIES_KEY = "variable registry properties"; public static final String DEFAULT_FLOW_CONTROLLER_GRACEFUL_SHUTDOWN_PERIOD = "10 sec"; public static final String DEFAULT_FLOW_SERVICE_WRITE_DELAY_INTERVAL = "500 ms"; public static final String DEFAULT_ADMINISTRATIVE_YIELD_DURATION = "30 sec"; public static final String DEFAULT_BORED_YIELD_DURATION = "10 millis"; public static final int DEFAULT_MAX_CONCURRENT_THREADS = 1; + public static final String DEFAULT_VARIABLE_REGISTRY_PROPERTIES = ""; private String flowControllerGracefulShutdownPeriod = DEFAULT_FLOW_CONTROLLER_GRACEFUL_SHUTDOWN_PERIOD; private String flowServiceWriteDelayInterval = DEFAULT_FLOW_SERVICE_WRITE_DELAY_INTERVAL; private String administrativeYieldDuration = DEFAULT_ADMINISTRATIVE_YIELD_DURATION; private String boredYieldDuration = DEFAULT_BORED_YIELD_DURATION; private Number maxConcurrentThreads = DEFAULT_MAX_CONCURRENT_THREADS; + private String variableRegistryProperties = DEFAULT_VARIABLE_REGISTRY_PROPERTIES; public CorePropertiesSchema() { } @@ -60,6 +63,8 @@ public class CorePropertiesSchema extends BaseSchema implements WritableSchema { boredYieldDuration = getOptionalKeyAsType(map, BORED_YIELD_DURATION_KEY, String.class, CORE_PROPS_KEY, DEFAULT_BORED_YIELD_DURATION); maxConcurrentThreads = getOptionalKeyAsType(map, MAX_CONCURRENT_THREADS_KEY, Number.class, CORE_PROPS_KEY, DEFAULT_MAX_CONCURRENT_THREADS); + variableRegistryProperties = getOptionalKeyAsType(map, VARIABLE_REGISTRY_PROPERTIES_KEY, String.class, + CORE_PROPS_KEY, DEFAULT_VARIABLE_REGISTRY_PROPERTIES); } @Override @@ -70,6 +75,7 @@ public class CorePropertiesSchema extends BaseSchema implements WritableSchema { result.put(ADMINISTRATIVE_YIELD_DURATION_KEY, administrativeYieldDuration); result.put(BORED_YIELD_DURATION_KEY, boredYieldDuration); result.put(MAX_CONCURRENT_THREADS_KEY, maxConcurrentThreads); + result.put(VARIABLE_REGISTRY_PROPERTIES_KEY, variableRegistryProperties); return result; } @@ -92,4 +98,9 @@ public class CorePropertiesSchema extends BaseSchema implements WritableSchema { public Number getMaxConcurrentThreads() { return maxConcurrentThreads; } + + public String getVariableRegistryProperties() { + return variableRegistryProperties; + } + } http://git-wip-us.apache.org/repos/asf/nifi-minifi/blob/5c6add80/minifi-commons/minifi-commons-schema/src/main/java/org/apache/nifi/minifi/commons/schema/v2/ConfigSchemaV2.java ---------------------------------------------------------------------- diff --git a/minifi-commons/minifi-commons-schema/src/main/java/org/apache/nifi/minifi/commons/schema/v2/ConfigSchemaV2.java b/minifi-commons/minifi-commons-schema/src/main/java/org/apache/nifi/minifi/commons/schema/v2/ConfigSchemaV2.java index ef95988..9daec38 100644 --- a/minifi-commons/minifi-commons-schema/src/main/java/org/apache/nifi/minifi/commons/schema/v2/ConfigSchemaV2.java +++ b/minifi-commons/minifi-commons-schema/src/main/java/org/apache/nifi/minifi/commons/schema/v2/ConfigSchemaV2.java @@ -17,11 +17,26 @@ package org.apache.nifi.minifi.commons.schema.v2; +import static org.apache.nifi.minifi.commons.schema.common.CommonPropertyKeys.COMPONENT_STATUS_REPO_KEY; +import static org.apache.nifi.minifi.commons.schema.common.CommonPropertyKeys.CONTENT_REPO_KEY; +import static org.apache.nifi.minifi.commons.schema.common.CommonPropertyKeys.CORE_PROPS_KEY; +import static org.apache.nifi.minifi.commons.schema.common.CommonPropertyKeys.FLOWFILE_REPO_KEY; +import static org.apache.nifi.minifi.commons.schema.common.CommonPropertyKeys.FLOW_CONTROLLER_PROPS_KEY; +import static org.apache.nifi.minifi.commons.schema.common.CommonPropertyKeys.PROVENANCE_REPORTING_KEY; +import static org.apache.nifi.minifi.commons.schema.common.CommonPropertyKeys.PROVENANCE_REPO_KEY; +import static org.apache.nifi.minifi.commons.schema.common.CommonPropertyKeys.SECURITY_PROPS_KEY; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + import org.apache.nifi.minifi.commons.schema.ComponentStatusRepositorySchema; import org.apache.nifi.minifi.commons.schema.ConfigSchema; import org.apache.nifi.minifi.commons.schema.ConnectionSchema; import org.apache.nifi.minifi.commons.schema.ContentRepositorySchema; -import org.apache.nifi.minifi.commons.schema.CorePropertiesSchema; import org.apache.nifi.minifi.commons.schema.FlowControllerSchema; import org.apache.nifi.minifi.commons.schema.FlowFileRepositorySchema; import org.apache.nifi.minifi.commons.schema.FunnelSchema; @@ -37,22 +52,6 @@ import org.apache.nifi.minifi.commons.schema.common.CollectionOverlap; import org.apache.nifi.minifi.commons.schema.common.ConvertableSchema; import org.apache.nifi.minifi.commons.schema.common.StringUtil; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; - -import static org.apache.nifi.minifi.commons.schema.common.CommonPropertyKeys.COMPONENT_STATUS_REPO_KEY; -import static org.apache.nifi.minifi.commons.schema.common.CommonPropertyKeys.CONTENT_REPO_KEY; -import static org.apache.nifi.minifi.commons.schema.common.CommonPropertyKeys.CORE_PROPS_KEY; -import static org.apache.nifi.minifi.commons.schema.common.CommonPropertyKeys.FLOWFILE_REPO_KEY; -import static org.apache.nifi.minifi.commons.schema.common.CommonPropertyKeys.FLOW_CONTROLLER_PROPS_KEY; -import static org.apache.nifi.minifi.commons.schema.common.CommonPropertyKeys.PROVENANCE_REPORTING_KEY; -import static org.apache.nifi.minifi.commons.schema.common.CommonPropertyKeys.PROVENANCE_REPO_KEY; -import static org.apache.nifi.minifi.commons.schema.common.CommonPropertyKeys.SECURITY_PROPS_KEY; - public class ConfigSchemaV2 extends BaseSchema implements ConvertableSchema<ConfigSchema> { public static final int CONFIG_VERSION = 2; public static final String VERSION = "MiNiFi Config Version"; @@ -70,7 +69,7 @@ public class ConfigSchemaV2 extends BaseSchema implements ConvertableSchema<Conf public static String TOP_LEVEL_NAME = "top level"; private FlowControllerSchema flowControllerProperties; - private CorePropertiesSchema coreProperties; + private CorePropertiesSchemaV2 coreProperties; private FlowFileRepositorySchema flowfileRepositoryProperties; private ContentRepositorySchema contentRepositoryProperties; private ComponentStatusRepositorySchema componentStatusRepositoryProperties; @@ -87,7 +86,7 @@ public class ConfigSchemaV2 extends BaseSchema implements ConvertableSchema<Conf validationIssues.stream().forEach(this::addValidationIssue); flowControllerProperties = getMapAsType(map, FLOW_CONTROLLER_PROPS_KEY, FlowControllerSchema.class, TOP_LEVEL_NAME, true); - coreProperties = getMapAsType(map, CORE_PROPS_KEY, CorePropertiesSchema.class, TOP_LEVEL_NAME, false); + coreProperties = getMapAsType(map, CORE_PROPS_KEY, CorePropertiesSchemaV2.class, TOP_LEVEL_NAME, false); flowfileRepositoryProperties = getMapAsType(map, FLOWFILE_REPO_KEY, FlowFileRepositorySchema.class, TOP_LEVEL_NAME, false); contentRepositoryProperties = getMapAsType(map, CONTENT_REPO_KEY, ContentRepositorySchema.class, TOP_LEVEL_NAME, false); provenanceRepositorySchema = getMapAsType(map, PROVENANCE_REPO_KEY, ProvenanceRepositorySchema.class, TOP_LEVEL_NAME, false); @@ -169,7 +168,7 @@ public class ConfigSchemaV2 extends BaseSchema implements ConvertableSchema<Conf Map<String, Object> result = mapSupplier.get(); result.put(VERSION, getVersion()); putIfNotNull(result, FLOW_CONTROLLER_PROPS_KEY, flowControllerProperties); - putIfNotNull(result, CORE_PROPS_KEY, coreProperties); + putIfNotNull(result, CORE_PROPS_KEY, coreProperties.convert()); putIfNotNull(result, FLOWFILE_REPO_KEY, flowfileRepositoryProperties); putIfNotNull(result, CONTENT_REPO_KEY, contentRepositoryProperties); putIfNotNull(result, PROVENANCE_REPO_KEY, provenanceRepositorySchema); http://git-wip-us.apache.org/repos/asf/nifi-minifi/blob/5c6add80/minifi-commons/minifi-commons-schema/src/main/java/org/apache/nifi/minifi/commons/schema/v2/CorePropertiesSchemaV2.java ---------------------------------------------------------------------- diff --git a/minifi-commons/minifi-commons-schema/src/main/java/org/apache/nifi/minifi/commons/schema/v2/CorePropertiesSchemaV2.java b/minifi-commons/minifi-commons-schema/src/main/java/org/apache/nifi/minifi/commons/schema/v2/CorePropertiesSchemaV2.java new file mode 100644 index 0000000..b3080eb --- /dev/null +++ b/minifi-commons/minifi-commons-schema/src/main/java/org/apache/nifi/minifi/commons/schema/v2/CorePropertiesSchemaV2.java @@ -0,0 +1,72 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.minifi.commons.schema.v2; + +import static org.apache.nifi.minifi.commons.schema.common.CommonPropertyKeys.CORE_PROPS_KEY; +import static org.apache.nifi.minifi.commons.schema.common.CommonPropertyKeys.MAX_CONCURRENT_THREADS_KEY; +import java.util.Map; + +import org.apache.nifi.minifi.commons.schema.CorePropertiesSchema; +import org.apache.nifi.minifi.commons.schema.common.BaseSchema; +import org.apache.nifi.minifi.commons.schema.common.ConvertableSchema; + +public class CorePropertiesSchemaV2 extends BaseSchema implements ConvertableSchema<CorePropertiesSchema> { + + private static final int CONFIG_VERSION = 2; + + private String flowControllerGracefulShutdownPeriod = CorePropertiesSchema.DEFAULT_FLOW_CONTROLLER_GRACEFUL_SHUTDOWN_PERIOD; + private String flowServiceWriteDelayInterval = CorePropertiesSchema.DEFAULT_FLOW_SERVICE_WRITE_DELAY_INTERVAL; + private String administrativeYieldDuration = CorePropertiesSchema.DEFAULT_ADMINISTRATIVE_YIELD_DURATION; + private String boredYieldDuration = CorePropertiesSchema.DEFAULT_BORED_YIELD_DURATION; + private Number maxConcurrentThreads = CorePropertiesSchema.DEFAULT_MAX_CONCURRENT_THREADS; + + public CorePropertiesSchemaV2() { + + } + + public CorePropertiesSchemaV2(Map map) { + flowControllerGracefulShutdownPeriod = getOptionalKeyAsType(map, CorePropertiesSchema.FLOW_CONTROLLER_SHUTDOWN_PERIOD_KEY, String.class, + CORE_PROPS_KEY, CorePropertiesSchema.DEFAULT_FLOW_CONTROLLER_GRACEFUL_SHUTDOWN_PERIOD); + flowServiceWriteDelayInterval = getOptionalKeyAsType(map, CorePropertiesSchema.FLOW_SERVICE_WRITE_DELAY_INTERVAL_KEY, String.class, + CORE_PROPS_KEY, CorePropertiesSchema.DEFAULT_FLOW_SERVICE_WRITE_DELAY_INTERVAL); + administrativeYieldDuration = getOptionalKeyAsType(map, CorePropertiesSchema.ADMINISTRATIVE_YIELD_DURATION_KEY, String.class, + CORE_PROPS_KEY, CorePropertiesSchema.DEFAULT_ADMINISTRATIVE_YIELD_DURATION); + boredYieldDuration = getOptionalKeyAsType(map, CorePropertiesSchema.BORED_YIELD_DURATION_KEY, String.class, CORE_PROPS_KEY, CorePropertiesSchema.DEFAULT_BORED_YIELD_DURATION); + maxConcurrentThreads = getOptionalKeyAsType(map, MAX_CONCURRENT_THREADS_KEY, Number.class, + CORE_PROPS_KEY, CorePropertiesSchema.DEFAULT_MAX_CONCURRENT_THREADS); + } + + @Override + public int getVersion() { + return CONFIG_VERSION; + } + + @Override + public CorePropertiesSchema convert() { + + Map<String, Object> result = mapSupplier.get(); + result.put(CorePropertiesSchema.FLOW_CONTROLLER_SHUTDOWN_PERIOD_KEY, flowControllerGracefulShutdownPeriod); + result.put(CorePropertiesSchema.FLOW_SERVICE_WRITE_DELAY_INTERVAL_KEY, flowServiceWriteDelayInterval); + result.put(CorePropertiesSchema.ADMINISTRATIVE_YIELD_DURATION_KEY, administrativeYieldDuration); + result.put(CorePropertiesSchema.BORED_YIELD_DURATION_KEY, boredYieldDuration); + result.put(MAX_CONCURRENT_THREADS_KEY, maxConcurrentThreads); + return new CorePropertiesSchema(result); + + } + +} http://git-wip-us.apache.org/repos/asf/nifi-minifi/blob/5c6add80/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/CsvToJson.yml ---------------------------------------------------------------------- diff --git a/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/CsvToJson.yml b/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/CsvToJson.yml index 1d6a8a2..7dd6534 100644 --- a/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/CsvToJson.yml +++ b/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/CsvToJson.yml @@ -23,6 +23,7 @@ Core Properties: administrative yield duration: 30 sec bored yield duration: 10 millis max concurrent threads: 1 + variable registry properties: '' FlowFile Repository: partitions: 256 checkpoint interval: 2 mins http://git-wip-us.apache.org/repos/asf/nifi-minifi/blob/5c6add80/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/DecompressionCircularFlow.yml ---------------------------------------------------------------------- diff --git a/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/DecompressionCircularFlow.yml b/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/DecompressionCircularFlow.yml index 8e0f8dc..8c02ae7 100644 --- a/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/DecompressionCircularFlow.yml +++ b/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/DecompressionCircularFlow.yml @@ -23,6 +23,7 @@ Core Properties: administrative yield duration: 30 sec bored yield duration: 10 millis max concurrent threads: 1 + variable registry properties: '' FlowFile Repository: partitions: 256 checkpoint interval: 2 mins http://git-wip-us.apache.org/repos/asf/nifi-minifi/blob/5c6add80/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/InvokeHttpMiNiFiTemplateTest.yml ---------------------------------------------------------------------- diff --git a/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/InvokeHttpMiNiFiTemplateTest.yml b/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/InvokeHttpMiNiFiTemplateTest.yml index 4287987..a043428 100644 --- a/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/InvokeHttpMiNiFiTemplateTest.yml +++ b/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/InvokeHttpMiNiFiTemplateTest.yml @@ -23,6 +23,7 @@ Core Properties: administrative yield duration: 30 sec bored yield duration: 10 millis max concurrent threads: 1 + variable registry properties: '' FlowFile Repository: partitions: 256 checkpoint interval: 2 mins http://git-wip-us.apache.org/repos/asf/nifi-minifi/blob/5c6add80/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/MultipleRelationships.yml ---------------------------------------------------------------------- diff --git a/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/MultipleRelationships.yml b/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/MultipleRelationships.yml index 8c6455a..a91fe9b 100644 --- a/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/MultipleRelationships.yml +++ b/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/MultipleRelationships.yml @@ -23,6 +23,7 @@ Core Properties: administrative yield duration: 30 sec bored yield duration: 10 millis max concurrent threads: 1 + variable registry properties: '' FlowFile Repository: partitions: 256 checkpoint interval: 2 mins http://git-wip-us.apache.org/repos/asf/nifi-minifi/blob/5c6add80/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/NestedControllerServices.yml ---------------------------------------------------------------------- diff --git a/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/NestedControllerServices.yml b/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/NestedControllerServices.yml index 526e55d..957ddd2 100644 --- a/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/NestedControllerServices.yml +++ b/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/NestedControllerServices.yml @@ -23,6 +23,7 @@ Core Properties: administrative yield duration: 30 sec bored yield duration: 10 millis max concurrent threads: 1 + variable registry properties: '' FlowFile Repository: partitions: 256 checkpoint interval: 2 mins http://git-wip-us.apache.org/repos/asf/nifi-minifi/blob/5c6add80/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/ProcessGroupsAndRemoteProcessGroups.yml ---------------------------------------------------------------------- diff --git a/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/ProcessGroupsAndRemoteProcessGroups.yml b/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/ProcessGroupsAndRemoteProcessGroups.yml index 823e56f..2b020c8 100644 --- a/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/ProcessGroupsAndRemoteProcessGroups.yml +++ b/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/ProcessGroupsAndRemoteProcessGroups.yml @@ -23,6 +23,7 @@ Core Properties: administrative yield duration: 30 sec bored yield duration: 10 millis max concurrent threads: 1 + variable registry properties: '' FlowFile Repository: partitions: 256 checkpoint interval: 2 mins http://git-wip-us.apache.org/repos/asf/nifi-minifi/blob/5c6add80/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/ReplaceTextExpressionLanguageCSVReformatting.yml ---------------------------------------------------------------------- diff --git a/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/ReplaceTextExpressionLanguageCSVReformatting.yml b/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/ReplaceTextExpressionLanguageCSVReformatting.yml index 9985966..8e997f9 100644 --- a/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/ReplaceTextExpressionLanguageCSVReformatting.yml +++ b/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/ReplaceTextExpressionLanguageCSVReformatting.yml @@ -23,6 +23,7 @@ Core Properties: administrative yield duration: 30 sec bored yield duration: 10 millis max concurrent threads: 1 + variable registry properties: '' FlowFile Repository: partitions: 256 checkpoint interval: 2 mins http://git-wip-us.apache.org/repos/asf/nifi-minifi/blob/5c6add80/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/SimpleRPGToLogAttributes.yml ---------------------------------------------------------------------- diff --git a/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/SimpleRPGToLogAttributes.yml b/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/SimpleRPGToLogAttributes.yml index e267792..a0458cc 100644 --- a/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/SimpleRPGToLogAttributes.yml +++ b/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/SimpleRPGToLogAttributes.yml @@ -23,6 +23,7 @@ Core Properties: administrative yield duration: 30 sec bored yield duration: 10 millis max concurrent threads: 1 + variable registry properties: '' FlowFile Repository: partitions: 256 checkpoint interval: 2 mins http://git-wip-us.apache.org/repos/asf/nifi-minifi/blob/5c6add80/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/SimpleTailFileToRPG.yml ---------------------------------------------------------------------- diff --git a/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/SimpleTailFileToRPG.yml b/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/SimpleTailFileToRPG.yml index a9108b9..3ca73ca 100644 --- a/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/SimpleTailFileToRPG.yml +++ b/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/SimpleTailFileToRPG.yml @@ -23,6 +23,7 @@ Core Properties: administrative yield duration: 30 sec bored yield duration: 10 millis max concurrent threads: 1 + variable registry properties: '' FlowFile Repository: partitions: 256 checkpoint interval: 2 mins http://git-wip-us.apache.org/repos/asf/nifi-minifi/blob/5c6add80/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/StressTestFramework.yml ---------------------------------------------------------------------- diff --git a/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/StressTestFramework.yml b/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/StressTestFramework.yml index 87fa189..3c41942 100644 --- a/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/StressTestFramework.yml +++ b/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/StressTestFramework.yml @@ -23,6 +23,7 @@ Core Properties: administrative yield duration: 30 sec bored yield duration: 10 millis max concurrent threads: 1 + variable registry properties: '' FlowFile Repository: partitions: 256 checkpoint interval: 2 mins http://git-wip-us.apache.org/repos/asf/nifi-minifi/blob/5c6add80/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/StressTestFrameworkFunnel.yml ---------------------------------------------------------------------- diff --git a/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/StressTestFrameworkFunnel.yml b/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/StressTestFrameworkFunnel.yml index 2502cdb..6897b3f 100644 --- a/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/StressTestFrameworkFunnel.yml +++ b/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/StressTestFrameworkFunnel.yml @@ -23,6 +23,7 @@ Core Properties: administrative yield duration: 30 sec bored yield duration: 10 millis max concurrent threads: 1 + variable registry properties: '' FlowFile Repository: partitions: 256 checkpoint interval: 2 mins
