taz1988 commented on code in PR #6838: URL: https://github.com/apache/nifi/pull/6838#discussion_r1072831472
########## nifi-toolkit/nifi-toolkit-kafka-migrator/src/main/java/org/apache/nifi/toolkit/kafkamigrator/KafkaMigratorMain.java: ########## @@ -0,0 +1,113 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.toolkit.kafkamigrator; + +import org.apache.nifi.toolkit.kafkamigrator.service.KafkaMigrationService; +import org.apache.nifi.xml.processing.parsers.DocumentProvider; +import org.apache.nifi.xml.processing.parsers.StandardDocumentProvider; +import org.apache.nifi.xml.processing.transform.StandardTransformProvider; +import org.w3c.dom.Document; + +import javax.xml.transform.dom.DOMSource; +import javax.xml.transform.stream.StreamResult; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.HashMap; +import java.util.Map; +import java.util.zip.GZIPInputStream; +import java.util.zip.GZIPOutputStream; + +public class KafkaMigratorMain { + + private static void printUsage() { + System.out.println("This application replaces Kafka processors from version 0.8, 0.9, 0.10 and 0.11 to version 2.0 processors" + + " in a flow.xml.gz file."); + System.out.println("\n"); + System.out.println("Usage: kafka-migrator.sh <path to input flow.xml.gz> <path to output flow.xml.gz>" + + " <use transaction true or false>\n<optional: coma separated kafka brokers in <host>:<port> format. " + + "Required for version 0.8 processors only>"); + } + + public static void main(final String[] args) throws Exception { + if (helpRequested(args)) { Review Comment: A plus ```(helpRequested(args) or args.length < 3)```check would be nice. As I see other toolkit main classes check argument length to in a defensive style. Otherwise NPE could be possible ########## nifi-toolkit/nifi-toolkit-kafka-migrator/pom.xml: ########## @@ -0,0 +1,53 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd";> + <modelVersion>4.0.0</modelVersion> + <parent> + <groupId>org.apache.nifi</groupId> + <artifactId>nifi-toolkit</artifactId> + <version>1.20.0-SNAPSHOT</version> + </parent> + + <artifactId>nifi-toolkit-kafka-migrator</artifactId> + <dependencies> + <dependency> + <groupId>org.apache.nifi</groupId> + <artifactId>nifi-xml-processing</artifactId> + <version>1.20.0-SNAPSHOT</version> + </dependency> + <dependency> + <groupId>org.apache.commons</groupId> + <artifactId>commons-lang3</artifactId> + <version>3.12.0</version> Review Comment: I think you don't need to specify the version, as specified in the main pom.xml already and other modules not specified as well ########## nifi-toolkit/nifi-toolkit-kafka-migrator/src/main/java/org/apache/nifi/toolkit/kafkamigrator/service/KafkaMigrationService.java: ########## @@ -0,0 +1,110 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.toolkit.kafkamigrator.service; + +import org.apache.commons.lang3.StringUtils; +import org.apache.nifi.toolkit.kafkamigrator.migrator.Migrator; +import org.w3c.dom.Document; +import org.w3c.dom.Element; +import org.w3c.dom.Node; +import org.w3c.dom.NodeList; + +import javax.xml.xpath.XPath; +import javax.xml.xpath.XPathConstants; +import javax.xml.xpath.XPathExpressionException; +import javax.xml.xpath.XPathFactory; +import java.util.Map; + +public class KafkaMigrationService { + + private final static String REGEX_FOR_REPLACEABLE_PROCESSOR_NAMES = "(Get|Put|Consume|Publish)Kafka(Record)?(_0_1\\d)?"; + private final static String NEW_KAFKA_PROCESSOR_VERSION = "_2_0"; + private final static String ARTIFACT = "nifi-kafka-2-0-nar"; + private final static String PATH_FOR_ARTIFACT = "bundle/artifact"; + protected static final boolean IS_VERSION_EIGHT_PROCESSOR = Boolean.TRUE; + protected static final boolean IS_NOT_VERSION_EIGHT_PROCESSOR = Boolean.FALSE; + + protected String path; + protected String pathForClass; + + public void replaceKafkaProcessors(final Document document, final Map<String, String> arguments) throws XPathExpressionException { + final KafkaFlowMigrationService flowMigrationService = new KafkaFlowMigrationService(); + final KafkaTemplateMigrationService templateMigrationService = new KafkaTemplateMigrationService(); + + flowMigrationService.replaceKafkaProcessors(document, arguments); + templateMigrationService.replaceKafkaProcessors(document, arguments); + } + + protected void replaceProcessors(Document document, Map<String, String> arguments) throws XPathExpressionException { + Migrator migrator; + final XPath xPath = XPathFactory.newInstance().newXPath(); + + final NodeList processors = (NodeList) xPath.evaluate(path, document, XPathConstants.NODESET); + for (int i = 0; i < processors.getLength(); i++) { + final Node processor = processors.item(i); + final String className = xPath.evaluate(pathForClass, processor); + final String processorName = StringUtils.substringAfterLast(className, "."); + + if (replaceableKafkaProcessor(processorName)) { + if (processorName.contains("Publish")) { + migrator = createPublishMigrator(arguments); + } else if (processorName.contains("Put")) { + migrator = createVersionEightPublishMigrator(arguments); + } else if (processorName.contains("Consume")) { + migrator = createConsumeMigrator(arguments); + } else { + migrator = createVersionEightConsumeMigrator(arguments); + } + + migrator.configureProperties(processor); + migrator.configureComponentSpecificSteps(processor, arguments); + migrator.configureDescriptors(processor); + + final String newClassName = replaceClassNameWithNewProcessorName(className, processorName); Review Comment: This way the KafkaMigrationService search the unsupported processors, call migrator service for them and in the end replace processor class here. Since all processor modification is inside migrator, maybe it would be better fit to that part? ########## nifi-toolkit/nifi-toolkit-kafka-migrator/src/main/java/org/apache/nifi/toolkit/kafkamigrator/service/KafkaMigrationService.java: ########## @@ -0,0 +1,110 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.toolkit.kafkamigrator.service; + +import org.apache.commons.lang3.StringUtils; +import org.apache.nifi.toolkit.kafkamigrator.migrator.Migrator; +import org.w3c.dom.Document; +import org.w3c.dom.Element; +import org.w3c.dom.Node; +import org.w3c.dom.NodeList; + +import javax.xml.xpath.XPath; +import javax.xml.xpath.XPathConstants; +import javax.xml.xpath.XPathExpressionException; +import javax.xml.xpath.XPathFactory; +import java.util.Map; + +public class KafkaMigrationService { + + private final static String REGEX_FOR_REPLACEABLE_PROCESSOR_NAMES = "(Get|Put|Consume|Publish)Kafka(Record)?(_0_1\\d)?"; + private final static String NEW_KAFKA_PROCESSOR_VERSION = "_2_0"; + private final static String ARTIFACT = "nifi-kafka-2-0-nar"; + private final static String PATH_FOR_ARTIFACT = "bundle/artifact"; + protected static final boolean IS_VERSION_EIGHT_PROCESSOR = Boolean.TRUE; + protected static final boolean IS_NOT_VERSION_EIGHT_PROCESSOR = Boolean.FALSE; + + protected String path; + protected String pathForClass; + + public void replaceKafkaProcessors(final Document document, final Map<String, String> arguments) throws XPathExpressionException { Review Comment: I agree with @dam4rus, this method could be go to the main class. ########## nifi-toolkit/nifi-toolkit-kafka-migrator/src/main/java/org/apache/nifi/toolkit/kafkamigrator/service/KafkaMigrationService.java: ########## @@ -0,0 +1,110 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.toolkit.kafkamigrator.service; + +import org.apache.commons.lang3.StringUtils; +import org.apache.nifi.toolkit.kafkamigrator.migrator.Migrator; +import org.w3c.dom.Document; +import org.w3c.dom.Element; +import org.w3c.dom.Node; +import org.w3c.dom.NodeList; + +import javax.xml.xpath.XPath; +import javax.xml.xpath.XPathConstants; +import javax.xml.xpath.XPathExpressionException; +import javax.xml.xpath.XPathFactory; +import java.util.Map; + +public class KafkaMigrationService { + + private final static String REGEX_FOR_REPLACEABLE_PROCESSOR_NAMES = "(Get|Put|Consume|Publish)Kafka(Record)?(_0_1\\d)?"; + private final static String NEW_KAFKA_PROCESSOR_VERSION = "_2_0"; + private final static String ARTIFACT = "nifi-kafka-2-0-nar"; + private final static String PATH_FOR_ARTIFACT = "bundle/artifact"; + protected static final boolean IS_VERSION_EIGHT_PROCESSOR = Boolean.TRUE; + protected static final boolean IS_NOT_VERSION_EIGHT_PROCESSOR = Boolean.FALSE; + + protected String path; + protected String pathForClass; + + public void replaceKafkaProcessors(final Document document, final Map<String, String> arguments) throws XPathExpressionException { + final KafkaFlowMigrationService flowMigrationService = new KafkaFlowMigrationService(); + final KafkaTemplateMigrationService templateMigrationService = new KafkaTemplateMigrationService(); + + flowMigrationService.replaceKafkaProcessors(document, arguments); + templateMigrationService.replaceKafkaProcessors(document, arguments); + } + + protected void replaceProcessors(Document document, Map<String, String> arguments) throws XPathExpressionException { + Migrator migrator; + final XPath xPath = XPathFactory.newInstance().newXPath(); + + final NodeList processors = (NodeList) xPath.evaluate(path, document, XPathConstants.NODESET); + for (int i = 0; i < processors.getLength(); i++) { + final Node processor = processors.item(i); + final String className = xPath.evaluate(pathForClass, processor); + final String processorName = StringUtils.substringAfterLast(className, "."); + + if (replaceableKafkaProcessor(processorName)) { + if (processorName.contains("Publish")) { + migrator = createPublishMigrator(arguments); + } else if (processorName.contains("Put")) { + migrator = createVersionEightPublishMigrator(arguments); + } else if (processorName.contains("Consume")) { + migrator = createConsumeMigrator(arguments); + } else { + migrator = createVersionEightConsumeMigrator(arguments); + } + + migrator.configureProperties(processor); Review Comment: I would consider introduce only one method (migrateProcessor(processor, arguments) ?) for the migrator interface and call only that here. In the child this way maybe we could avoid empty methods and call exactly what is needed? -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: issues-unsubscr...@nifi.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org
