This is an automated email from the ASF dual-hosted git repository.
bamaer pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/hop.git
The following commit(s) were added to refs/heads/main by this push:
new 3ffb62ea9f Issue #4500 : Report skipped files in the Kettle import
summary (#8692)
3ffb62ea9f is described below
commit 3ffb62ea9f20ff84a68680c33eaf4a0b197450d1
Author: Matt Casters <[email protected]>
AuthorDate: Thu Oct 1 09:14:48 2026 +0200
Issue #4500 : Report skipped files in the Kettle import summary (#8692)
---
.../hop-vs-kettle/import-kettle-projects.adoc | 9 +-
.../apache/hop/imports/kettle/KettleImport.java | 99 +++++++++---
.../kettle/messages/messages_en_US.properties | 3 +
.../imports/kettle/KettleImportSkipReportTest.java | 178 +++++++++++++++++++++
4 files changed, 264 insertions(+), 25 deletions(-)
diff --git
a/docs/hop-user-manual/modules/ROOT/pages/hop-vs-kettle/import-kettle-projects.adoc
b/docs/hop-user-manual/modules/ROOT/pages/hop-vs-kettle/import-kettle-projects.adoc
index 236fe347e1..0047b6e588 100644
---
a/docs/hop-user-manual/modules/ROOT/pages/hop-vs-kettle/import-kettle-projects.adoc
+++
b/docs/hop-user-manual/modules/ROOT/pages/hop-vs-kettle/import-kettle-projects.adoc
@@ -105,14 +105,15 @@ image:hop-import/import-report.png[Import Report]
The migration summary shows:
-* number of jobs
-* number of transformations
-* number of other files
+* number of jobs imported, and how many were skipped because the target file
already existed
+* number of transformations imported, and how many were skipped because the
target file already existed
+* number of other files imported, and how many were skipped because the target
file already existed
* number of variables
* number of database connections
-NOTE: Only migrated items will be shown.
+NOTE: A line is shown when that kind of file was imported or skipped.
Items that were not available in the specified folders or files for this
import will not be shown.
+Skipped files are left unchanged.
When multiple database connections with the same name but different
configurations were found (see 'Known limitations'), a `connnections.csv` file
will be created in the project folder.
This file contains a list of all jobs and transformations, with the
connections they use.
diff --git
a/plugins/misc/import/src/main/java/org/apache/hop/imports/kettle/KettleImport.java
b/plugins/misc/import/src/main/java/org/apache/hop/imports/kettle/KettleImport.java
index 78a5c6e8aa..ce3df66965 100644
---
a/plugins/misc/import/src/main/java/org/apache/hop/imports/kettle/KettleImport.java
+++
b/plugins/misc/import/src/main/java/org/apache/hop/imports/kettle/KettleImport.java
@@ -125,6 +125,12 @@ public class KettleImport extends HopImportBase implements
IHopImport {
private int kjbCounter;
private int ktrCounter;
private int otherCounter;
+
+ /** Files left unchanged because the target already existed, per source
type. */
+ private int kjbSkippedCounter;
+
+ private int ktrSkippedCounter;
+ private int otherSkippedCounter;
private String variablesTargetConfigFile;
private String connectionsReportFileName;
@@ -152,6 +158,9 @@ public class KettleImport extends HopImportBase implements
IHopImport {
this.kjbCounter = 0;
this.ktrCounter = 0;
this.otherCounter = 0;
+ this.kjbSkippedCounter = 0;
+ this.ktrSkippedCounter = 0;
+ this.otherSkippedCounter = 0;
// Find all files...
//
@@ -286,6 +295,7 @@ public class KettleImport extends HopImportBase implements
IHopImport {
FileObject targetFile = HopVfs.getFileObject(targetFilename);
if (isSkippingExistingTargetFiles() && targetFile.exists()) {
+ recordSkippedExistingTarget(sourceFile, domSource);
continue;
}
@@ -1305,32 +1315,50 @@ public class KettleImport extends HopImportBase
implements IHopImport {
}
}
+ /**
+ * The find phase counts every source file as imported. A later skip leaves
that file unchanged,
+ * so move it from the imported count to the skipped count before the
summary is shown.
+ */
+ private void recordSkippedExistingTarget(FileObject sourceFile, DOMSource
domSource) {
+ String extension = sourceFile.getName().getExtension();
+ if (domSource != null && "kjb".equalsIgnoreCase(extension)) {
+ kjbCounter = Math.max(0, kjbCounter - 1);
+ kjbSkippedCounter++;
+ } else if (domSource != null && "ktr".equalsIgnoreCase(extension)) {
+ ktrCounter = Math.max(0, ktrCounter - 1);
+ ktrSkippedCounter++;
+ } else {
+ otherCounter = Math.max(0, otherCounter - 1);
+ otherSkippedCounter++;
+ }
+ }
+
@Override
public String getImportReport() {
String eol = System.getProperty("line.separator");
String messageString =
BaseMessages.getString(PKG,
"KettleImportDialog.ImportSummary.Imported.Label") + eol;
- if (getKjbCounter() > 0) {
- messageString +=
- getKjbCounter()
- + " "
- + BaseMessages.getString(PKG,
"KettleImportDialog.ImportSummary.ImportedJobs.Label")
- + eol;
- }
- if (getKtrCounter() > 0) {
- messageString +=
- getKtrCounter()
- + " "
- + BaseMessages.getString(PKG,
"KettleImportDialog.ImportSummary.ImportedTransf.Label")
- + eol;
- }
- if (getOtherCounter() > 0) {
- messageString +=
- getOtherCounter()
- + " "
- + BaseMessages.getString(PKG,
"KettleImportDialog.ImportSummary.ImportedOther.Label")
- + eol;
- }
+ messageString +=
+ importedCountLine(
+ getKjbCounter(),
+ kjbSkippedCounter,
+ "KettleImportDialog.ImportSummary.ImportedJobs.Label",
+ "KettleImportDialog.ImportSummary.ImportedJobsSkipped.Label",
+ eol);
+ messageString +=
+ importedCountLine(
+ getKtrCounter(),
+ ktrSkippedCounter,
+ "KettleImportDialog.ImportSummary.ImportedTransf.Label",
+ "KettleImportDialog.ImportSummary.ImportedTransfSkipped.Label",
+ eol);
+ messageString +=
+ importedCountLine(
+ getOtherCounter(),
+ otherSkippedCounter,
+ "KettleImportDialog.ImportSummary.ImportedOther.Label",
+ "KettleImportDialog.ImportSummary.ImportedOtherSkipped.Label",
+ eol);
if (getVariableCounter() > 0) {
messageString +=
getVariableCounter()
@@ -1382,6 +1410,23 @@ public class KettleImport extends HopImportBase
implements IHopImport {
return messageString;
}
+ /**
+ * One summary line. With nothing skipped this stays "{count} {label}".
Otherwise it names both
+ * the files written and the files left in place, including a zero written
count.
+ */
+ private static String importedCountLine(
+ int imported, int skipped, String labelKey, String skippedKey, String
eol) {
+ if (imported <= 0 && skipped <= 0) {
+ return "";
+ }
+ if (skipped > 0) {
+ return BaseMessages.getString(
+ PKG, skippedKey, Integer.toString(imported),
Integer.toString(skipped))
+ + eol;
+ }
+ return imported + " " + BaseMessages.getString(PKG, labelKey) + eol;
+ }
+
/**
* Gets kjbCounter
*
@@ -1430,6 +1475,18 @@ public class KettleImport extends HopImportBase
implements IHopImport {
this.otherCounter = otherCounter;
}
+ public int getKjbSkippedCounter() {
+ return kjbSkippedCounter;
+ }
+
+ public int getKtrSkippedCounter() {
+ return ktrSkippedCounter;
+ }
+
+ public int getOtherSkippedCounter() {
+ return otherSkippedCounter;
+ }
+
/**
* Gets variablesTargetConfigFile
*
diff --git
a/plugins/misc/import/src/main/resources/org/apache/hop/imports/kettle/messages/messages_en_US.properties
b/plugins/misc/import/src/main/resources/org/apache/hop/imports/kettle/messages/messages_en_US.properties
index 9c0870d820..d4a3197a21 100644
---
a/plugins/misc/import/src/main/resources/org/apache/hop/imports/kettle/messages/messages_en_US.properties
+++
b/plugins/misc/import/src/main/resources/org/apache/hop/imports/kettle/messages/messages_en_US.properties
@@ -21,8 +21,11 @@ KettleImportDialog.Tab.Target=Target
KettleImportDialog.Tab.Metadata=Metadata
KettleImportDialog.ImportSummary.Imported.Label=Imported :
KettleImportDialog.ImportSummary.ImportedJobs.Label=jobs
+KettleImportDialog.ImportSummary.ImportedJobsSkipped.Label={0} jobs, {1}
skipped
KettleImportDialog.ImportSummary.ImportedOther.Label=other files
+KettleImportDialog.ImportSummary.ImportedOtherSkipped.Label={0} other files,
{1} skipped
KettleImportDialog.ImportSummary.ImportedTransf.Label=transformations
+KettleImportDialog.ImportSummary.ImportedTransfSkipped.Label={0}
transformations, {1} skipped
KettleImportDialog.ImportSummary.Title=Import summary
KettleImportDialog.Label.ImportFrom=Import from
KettleImportDialog.Label.ImportInExistingProject=Import in existing project
diff --git
a/plugins/misc/import/src/test/java/org/apache/hop/imports/kettle/KettleImportSkipReportTest.java
b/plugins/misc/import/src/test/java/org/apache/hop/imports/kettle/KettleImportSkipReportTest.java
new file mode 100644
index 0000000000..ab1be97551
--- /dev/null
+++
b/plugins/misc/import/src/test/java/org/apache/hop/imports/kettle/KettleImportSkipReportTest.java
@@ -0,0 +1,178 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.imports.kettle;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNotEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.nio.charset.StandardCharsets;
+import java.util.UUID;
+import org.apache.commons.vfs2.FileObject;
+import org.apache.hop.core.HopClientEnvironment;
+import org.apache.hop.core.vfs.HopVfs;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
+
+class KettleImportSkipReportTest {
+
+ private static final String ORIGINAL_WORKFLOW = "original-workflow";
+ private static final String ORIGINAL_NOTES = "original-notes";
+ private static final String SOURCE_NOTES = "source-notes";
+
+ static {
+ System.setProperty(
+ "HOP_CONFIG_FOLDER",
+ System.getProperty("java.io.tmpdir") + "/hop-issue-4500-" +
UUID.randomUUID());
+ }
+
+ @BeforeAll
+ static void setUpBeforeClass() throws Exception {
+ HopClientEnvironment.init();
+ }
+
+ @AfterEach
+ void tearDown() {
+ HopVfs.reset();
+ }
+
+ /**
+ * Re-importing with "Skip existing target files" counted every source file
as imported. The
+ * summary has to say how many were written and how many were left unchanged
(#4500).
+ */
+ @Test
+ void summaryCountsFilesLeftInPlaceWhenSkippingExistingTargets() throws
Exception {
+ String id = UUID.randomUUID().toString();
+ String source = "ram:///" + id + "/src";
+ String target = "ram:///" + id + "/tgt";
+ FileObject sourceFolder = HopVfs.getFileObject(source);
+ sourceFolder.createFolder();
+ writeText(source + "/keep.kjb", "<job></job>");
+ writeText(source + "/skip.kjb", "<job></job>");
+ writeText(source + "/keep.ktr", "<transformation></transformation>");
+ writeText(source + "/skip.txt", SOURCE_NOTES);
+ sourceFolder.refresh();
+
+ FileObject targetFolder = HopVfs.getFileObject(target);
+ targetFolder.createFolder();
+ writeText(target + "/skip.hwf", ORIGINAL_WORKFLOW);
+ writeText(target + "/skip.txt", ORIGINAL_NOTES);
+ targetFolder.refresh();
+
+ KettleImport kettleImport = importFrom(source, target, true);
+
+ assertEquals(1, kettleImport.getKjbCounter());
+ assertEquals(1, kettleImport.getKjbSkippedCounter());
+ assertEquals(1, kettleImport.getKtrCounter());
+ assertEquals(0, kettleImport.getKtrSkippedCounter());
+ assertEquals(0, kettleImport.getOtherCounter());
+ assertEquals(1, kettleImport.getOtherSkippedCounter());
+
+ String eol = System.getProperty("line.separator");
+ assertEquals(
+ "Imported :"
+ + eol
+ + "1 jobs, 1 skipped"
+ + eol
+ + "1 transformations"
+ + eol
+ + "0 other files, 1 skipped"
+ + eol,
+ kettleImport.getImportReport());
+
+ assertEquals(ORIGINAL_WORKFLOW, readText(target + "/skip.hwf"));
+ assertEquals(ORIGINAL_NOTES, readText(target + "/skip.txt"));
+ assertTrue(HopVfs.getFileObject(target + "/keep.hwf").exists(),
kettleImport.getImportReport());
+ assertTrue(HopVfs.getFileObject(target + "/keep.hpl").exists(),
kettleImport.getImportReport());
+ assertFalse(readText(target + "/keep.hwf").contains(ORIGINAL_WORKFLOW));
+ }
+
+ @Test
+ void summaryStaysUnchangedWhenExistingTargetsAreOverwritten() throws
Exception {
+ String id = UUID.randomUUID().toString();
+ String source = "ram:///" + id + "/src";
+ String target = "ram:///" + id + "/tgt";
+ FileObject sourceFolder = HopVfs.getFileObject(source);
+ sourceFolder.createFolder();
+ writeText(source + "/keep.kjb", "<job></job>");
+ writeText(source + "/skip.kjb", "<job></job>");
+ writeText(source + "/keep.ktr", "<transformation></transformation>");
+ writeText(source + "/skip.txt", SOURCE_NOTES);
+ sourceFolder.refresh();
+
+ FileObject targetFolder = HopVfs.getFileObject(target);
+ targetFolder.createFolder();
+ writeText(target + "/keep.hwf", ORIGINAL_WORKFLOW);
+ writeText(target + "/skip.hwf", ORIGINAL_WORKFLOW);
+ writeText(target + "/keep.hpl", "original-pipeline");
+ writeText(target + "/skip.txt", ORIGINAL_NOTES);
+ targetFolder.refresh();
+
+ KettleImport kettleImport = importFrom(source, target, false);
+
+ assertEquals(2, kettleImport.getKjbCounter());
+ assertEquals(1, kettleImport.getKtrCounter());
+ assertEquals(1, kettleImport.getOtherCounter());
+ assertEquals(0, kettleImport.getKjbSkippedCounter());
+ assertEquals(0, kettleImport.getKtrSkippedCounter());
+ assertEquals(0, kettleImport.getOtherSkippedCounter());
+
+ String eol = System.getProperty("line.separator");
+ assertEquals(
+ "Imported :" + eol + "2 jobs" + eol + "1 transformations" + eol + "1
other files" + eol,
+ kettleImport.getImportReport());
+ assertFalse(kettleImport.getImportReport().contains("skipped"));
+ assertEquals(SOURCE_NOTES, readText(target + "/skip.txt"));
+ assertNotEquals(ORIGINAL_WORKFLOW, readText(target + "/skip.hwf"));
+ }
+
+ @Test
+ void summaryOmitsKindsThatWereNeitherImportedNorSkipped() {
+ KettleImport kettleImport = new KettleImport();
+ kettleImport.setKjbCounter(2);
+ String eol = System.getProperty("line.separator");
+ assertEquals("Imported :" + eol + "2 jobs" + eol,
kettleImport.getImportReport());
+ }
+
+ private static KettleImport importFrom(String source, String target, boolean
skipExisting)
+ throws Exception {
+ KettleImport kettleImport = new KettleImport();
+ kettleImport.setValidateInputFolder(source);
+ kettleImport.setValidateOutputFolder(target);
+ kettleImport.setSkippingExistingTargetFiles(skipExisting);
+ kettleImport.findFilesToImport();
+ kettleImport.importFiles();
+ return kettleImport;
+ }
+
+ private static void writeText(String uri, String text) throws Exception {
+ try (OutputStream out = HopVfs.getOutputStream(uri, false)) {
+ out.write(text.getBytes(StandardCharsets.UTF_8));
+ }
+ }
+
+ private static String readText(String uri) throws Exception {
+ try (InputStream in = HopVfs.getInputStream(uri)) {
+ return new String(in.readAllBytes(), StandardCharsets.UTF_8);
+ }
+ }
+}
