This is an automated email from the ASF dual-hosted git repository.
mcasters pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/hop.git
The following commit(s) were added to refs/heads/main by this push:
new ec2fbe52a4 minor fixes to byte tracking, and some cleanup, fixes #6931
(#6933)
ec2fbe52a4 is described below
commit ec2fbe52a483af8f297595c86388ef08f677efd3
Author: Hans Van Akelyen <[email protected]>
AuthorDate: Thu Apr 2 15:19:19 2026 +0200
minor fixes to byte tracking, and some cleanup, fixes #6931 (#6933)
---
.../hop/workflow/actions/unzip/ActionUnZip.java | 26 +-
.../actions/unzip/WorkflowActionUnZipTest.java | 42 ++-
.../transforms/processfiles/ProcessFiles.java | 352 ++++++++++++++----
.../messages/messages_en_US.properties | 39 +-
.../messages/messages_es_AR.properties | 5 +-
.../messages/messages_fr_FR.properties | 6 +-
.../messages/messages_it_IT.properties | 6 +-
.../messages/messages_ja_JP.properties | 5 +-
.../messages/messages_ko_KR.properties | 4 +-
.../messages/messages_pt_BR.properties | 8 +-
.../messages/messages_zh_CN.properties | 6 +-
.../transforms/processfiles/ProcessFilesTest.java | 393 +++++++++++++++++++++
12 files changed, 762 insertions(+), 130 deletions(-)
diff --git
a/plugins/actions/unzip/src/main/java/org/apache/hop/workflow/actions/unzip/ActionUnZip.java
b/plugins/actions/unzip/src/main/java/org/apache/hop/workflow/actions/unzip/ActionUnZip.java
index 6cf735450f..14b362b128 100644
---
a/plugins/actions/unzip/src/main/java/org/apache/hop/workflow/actions/unzip/ActionUnZip.java
+++
b/plugins/actions/unzip/src/main/java/org/apache/hop/workflow/actions/unzip/ActionUnZip.java
@@ -480,7 +480,9 @@ public class ActionUnZip extends ActionBase implements
Cloneable, IAction {
if (!children[i].getType().equals(FileType.FOLDER)) {
boolean unzip = true;
- String filename = children[i].getName().getPath();
+ // Match RegExp against the file name only (like other file
actions). getPath() is the
+ // full path from the FS root, so patterns such as
"daily_.*\\.zip" never matched.
+ String sourceMatchName = children[i].getName().getBaseName();
Pattern patternSource = null;
@@ -490,7 +492,7 @@ public class ActionUnZip extends ActionBase implements
Cloneable, IAction {
// First see if the file matches the regular expression!
if (patternSource != null) {
- Matcher matcher = patternSource.matcher(filename);
+ Matcher matcher = patternSource.matcher(sourceMatchName);
unzip = matcher.matches();
}
if (unzip) {
@@ -781,7 +783,7 @@ public class ActionUnZip extends ActionBase implements
Cloneable, IAction {
// Unzip done...
if (afterUnzip > 0) {
- doUnzipPostProcessing(sourceFileObject, movetodir,
realMovetodirectory);
+ doUnzipPostProcessing(sourceFileObject, movetodir,
realMovetodirectory, result);
}
retval = true;
} catch (Exception e) {
@@ -797,7 +799,7 @@ public class ActionUnZip extends ActionBase implements
Cloneable, IAction {
/** Moving or deleting source file. */
private void doUnzipPostProcessing(
- FileObject sourceFileObject, FileObject movetodir, String
realMovetodirectory)
+ FileObject sourceFileObject, FileObject movetodir, String
realMovetodirectory, Result result)
throws FileSystemException {
if (afterUnzip == 1) {
// delete zip file
@@ -822,8 +824,24 @@ public class ActionUnZip extends ActionBase implements
Cloneable, IAction {
movetodir + Const.FILE_SEPARATOR +
sourceFileObject.getName().getBaseName();
destFile = HopVfs.getFileObject(destinationFilename, getVariables());
+ long archiveSize = 0;
+ try {
+ if (sourceFileObject.getType().hasContent()) {
+ long sz = sourceFileObject.getContent().getSize();
+ if (sz > 0) {
+ archiveSize = sz;
+ }
+ }
+ } catch (Exception ignored) {
+ // best-effort for metrics only
+ }
+
sourceFileObject.moveTo(destFile);
+ if (archiveSize > 0) {
+ result.setBytesWrittenThisAction(result.getBytesWrittenThisAction()
+ archiveSize);
+ }
+
// File moved
if (isDetailed()) {
logDetailed(
diff --git
a/plugins/actions/unzip/src/test/java/org/apache/hop/workflow/actions/unzip/WorkflowActionUnZipTest.java
b/plugins/actions/unzip/src/test/java/org/apache/hop/workflow/actions/unzip/WorkflowActionUnZipTest.java
index d44d5c9a42..1bb603d6c1 100644
---
a/plugins/actions/unzip/src/test/java/org/apache/hop/workflow/actions/unzip/WorkflowActionUnZipTest.java
+++
b/plugins/actions/unzip/src/test/java/org/apache/hop/workflow/actions/unzip/WorkflowActionUnZipTest.java
@@ -16,11 +16,16 @@
*/
package org.apache.hop.workflow.actions.unzip;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
import java.lang.reflect.Method;
import java.util.ArrayList;
import java.util.List;
+import org.apache.commons.vfs2.FileContent;
import org.apache.commons.vfs2.FileName;
import org.apache.commons.vfs2.FileObject;
+import org.apache.commons.vfs2.FileType;
+import org.apache.hop.core.Result;
import org.apache.hop.junit.rules.RestoreHopEngineEnvironmentExtension;
import
org.apache.hop.workflow.action.loadsave.WorkflowActionLoadSaveTestSupport;
import org.junit.jupiter.api.Test;
@@ -50,21 +55,38 @@ class WorkflowActionUnZipTest extends
WorkflowActionLoadSaveTestSupport<ActionUn
jobEntryUnZip
.getClass()
.getDeclaredMethod(
- "doUnzipPostProcessing", FileObject.class, FileObject.class,
String.class);
+ "doUnzipPostProcessing",
+ FileObject.class,
+ FileObject.class,
+ String.class,
+ Result.class);
unzipPostprocessingMethod.setAccessible(true);
- FileObject sourceFileObject = Mockito.mock(FileObject.class);
-
Mockito.doReturn(Mockito.mock(FileName.class)).when(sourceFileObject).getName();
+ FileObject movetodir = Mockito.mock(FileObject.class);
+ Mockito.when(movetodir.toString()).thenReturn("file:///dest");
// delete
+ FileObject sourceForDelete = Mockito.mock(FileObject.class);
+
Mockito.doReturn(Mockito.mock(FileName.class)).when(sourceForDelete).getName();
jobEntryUnZip.afterUnzip = 1;
- unzipPostprocessingMethod.invoke(
- jobEntryUnZip, sourceFileObject, Mockito.mock(FileObject.class), "");
- Mockito.verify(sourceFileObject, Mockito.times(1)).delete();
+ unzipPostprocessingMethod.invoke(jobEntryUnZip, sourceForDelete,
movetodir, "", new Result());
+ Mockito.verify(sourceForDelete, Mockito.times(1)).delete();
+
+ // move (bytes written for moved archive size)
+ FileObject sourceForMove = Mockito.mock(FileObject.class);
+ FileName moveName = Mockito.mock(FileName.class);
+ Mockito.when(moveName.getBaseName()).thenReturn("archive.zip");
+ Mockito.when(sourceForMove.getName()).thenReturn(moveName);
+ FileType moveType = Mockito.mock(FileType.class);
+ Mockito.when(sourceForMove.getType()).thenReturn(moveType);
+ Mockito.when(moveType.hasContent()).thenReturn(true);
+ FileContent moveContent = Mockito.mock(FileContent.class);
+ Mockito.when(sourceForMove.getContent()).thenReturn(moveContent);
+ Mockito.when(moveContent.getSize()).thenReturn(77L);
- // move
jobEntryUnZip.afterUnzip = 2;
- unzipPostprocessingMethod.invoke(
- jobEntryUnZip, sourceFileObject, Mockito.mock(FileObject.class), "");
- Mockito.verify(sourceFileObject, Mockito.times(1)).moveTo(Mockito.any());
+ Result moveResult = new Result();
+ unzipPostprocessingMethod.invoke(jobEntryUnZip, sourceForMove, movetodir,
"", moveResult);
+ Mockito.verify(sourceForMove, Mockito.times(1)).moveTo(Mockito.any());
+ assertEquals(77L, moveResult.getBytesWrittenThisAction());
}
}
diff --git
a/plugins/transforms/processfiles/src/main/java/org/apache/hop/pipeline/transforms/processfiles/ProcessFiles.java
b/plugins/transforms/processfiles/src/main/java/org/apache/hop/pipeline/transforms/processfiles/ProcessFiles.java
index 5a29b25db5..af89338ec2 100644
---
a/plugins/transforms/processfiles/src/main/java/org/apache/hop/pipeline/transforms/processfiles/ProcessFiles.java
+++
b/plugins/transforms/processfiles/src/main/java/org/apache/hop/pipeline/transforms/processfiles/ProcessFiles.java
@@ -17,12 +17,18 @@
package org.apache.hop.pipeline.transforms.processfiles;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
import org.apache.commons.vfs2.FileObject;
import org.apache.commons.vfs2.FileSelectInfo;
import org.apache.commons.vfs2.FileSelector;
import org.apache.commons.vfs2.FileType;
+import org.apache.hop.core.Const;
import org.apache.hop.core.ResultFile;
import org.apache.hop.core.exception.HopException;
+import org.apache.hop.core.io.CountingInputStream;
+import org.apache.hop.core.io.CountingOutputStream;
import org.apache.hop.core.util.Utils;
import org.apache.hop.core.vfs.HopVfs;
import org.apache.hop.i18n.BaseMessages;
@@ -34,6 +40,8 @@ import org.apache.hop.pipeline.transform.TransformMeta;
public class ProcessFiles extends BaseTransform<ProcessFilesMeta,
ProcessFilesData> {
private static final Class<?> PKG = ProcessFilesMeta.class;
+ private static final int COPY_BUFFER_SIZE = 8192;
+
public ProcessFiles(
TransformMeta transformMeta,
ProcessFilesMeta meta,
@@ -90,7 +98,6 @@ public class ProcessFiles extends
BaseTransform<ProcessFilesMeta, ProcessFilesDa
}
if (meta.simulate && isBasic()) {
-
logBasic(BaseMessages.getString(PKG,
"ProcessFiles.Log.SimulationModeON"));
}
} // End If first
@@ -129,7 +136,8 @@ public class ProcessFiles extends
BaseTransform<ProcessFilesMeta, ProcessFilesDa
if (data.targetFile.exists()) {
if (isDetailed()) {
logDetailed(
- BaseMessages.getString(PKG,
"ProcessFiles.Log.TargetFileExists", targetFilename));
+ BaseMessages.getString(
+ PKG, "ProcessFiles.Log.TargetFileExists",
friendlyPath(targetFilename)));
}
// check if target is really a file otherwise it could overwrite a
complete folder by copy
// or move operations
@@ -154,6 +162,13 @@ public class ProcessFiles extends
BaseTransform<ProcessFilesMeta, ProcessFilesDa
parentFolder.toString()));
} else {
parentFolder.createFolder();
+ if (isDetailed()) {
+ logDetailed(
+ BaseMessages.getString(
+ PKG,
+ "ProcessFiles.Log.Detail.ParentFolderCreated",
+ friendlyPath(parentFolder.toString())));
+ }
}
}
if (parentFolder != null) {
@@ -162,77 +177,29 @@ public class ProcessFiles extends
BaseTransform<ProcessFilesMeta, ProcessFilesDa
}
}
+ if (isDetailed()) {
+ String targetForLog =
+ meta.getOperationType() == ProcessFilesMeta.OPERATION_TYPE_DELETE
+ ? "-"
+ : friendlyPath(targetFilename);
+ logDetailed(
+ BaseMessages.getString(
+ PKG,
+ "ProcessFiles.Log.Detail.RowHeader",
+ operationTypeLabel(),
+ friendlyPath(sourceFilename),
+ targetForLog));
+ }
+
switch (meta.getOperationType()) {
case ProcessFilesMeta.OPERATION_TYPE_COPY:
- if (((meta.isOverwriteTargetFile() && data.targetFile.exists())
- || !data.targetFile.exists())
- && !meta.simulate) {
- if (meta.isOverwriteTargetFile() && data.targetFile.exists())
- if (isBasic()) {
- logBasic(
- BaseMessages.getString(
- PKG,
- "ProcessFiles.Log.SourceFileCopied.TargetOverwritten",
- targetFilename));
- }
-
- // Better to delete the file before because. sometime, it's not
properly overwritten
- data.targetFile.delete();
- data.targetFile.copyFrom(data.sourceFile, new
TextOneToOneFileSelector());
-
- if (isDetailed()) {
- logDetailed(
- BaseMessages.getString(
- PKG, "ProcessFiles.Log.SourceFileCopied",
sourceFilename, targetFilename));
- }
- } else {
- if (isDetailed()) {
- logDetailed(
- BaseMessages.getString(
- PKG,
- "ProcessFiles.Log.TargetNotOverwritten",
- sourceFilename,
- targetFilename));
- }
- }
+ processCopyOperation(sourceFilename, targetFilename);
break;
case ProcessFilesMeta.OPERATION_TYPE_MOVE:
- if (((meta.isOverwriteTargetFile() && data.targetFile.exists())
- || !data.targetFile.exists())
- && !meta.simulate) {
- if (meta.isOverwriteTargetFile() && data.targetFile.exists())
- if (isBasic()) {
- logBasic(
- BaseMessages.getString(
- PKG,
"ProcessFiles.Log.SourceFileMoved.TargetOverwritten", targetFilename));
- }
- data.sourceFile.moveTo(HopVfs.getFileObject(targetFilename,
variables));
- if (isDetailed()) {
- logDetailed(
- BaseMessages.getString(
- PKG, "ProcessFiles.Log.SourceFileMoved", sourceFilename,
targetFilename));
- }
- } else {
- if (isDetailed()) {
- logDetailed(
- BaseMessages.getString(
- PKG,
- "ProcessFiles.Log.TargetNotOverwritten",
- sourceFilename,
- targetFilename));
- }
- }
+ processMoveOperation(sourceFilename, targetFilename);
break;
case ProcessFilesMeta.OPERATION_TYPE_DELETE:
- if (!meta.simulate && !data.sourceFile.delete()) {
- throw new HopException(
- BaseMessages.getString(
- PKG, "ProcessFiles.Error.CanNotDeleteFile",
data.sourceFile.toString()));
- }
- if (isDetailed()) {
- logDetailed(
- BaseMessages.getString(PKG,
"ProcessFiles.Log.SourceFileDeleted", sourceFilename));
- }
+ processDeleteOperation(sourceFilename);
break;
default:
break;
@@ -255,14 +222,17 @@ public class ProcessFiles extends
BaseTransform<ProcessFilesMeta, ProcessFilesDa
if (isDetailed()) {
logDetailed(
BaseMessages.getString(
- PKG, "ProcessFiles.Log.FilenameAddResult",
data.targetFile.toString()));
+ PKG,
+ "ProcessFiles.Log.FilenameAddResult",
+ friendlyPath(data.targetFile.toString())));
}
}
putRow(getInputRowMeta(), r); // copy row to possible alternate
rowset(s).
if (checkFeedback(getLinesRead()) && isBasic()) {
- logBasic(BaseMessages.getString(PKG, "ProcessFiles.LineNumber") +
getLinesRead());
+ logBasic(
+ BaseMessages.getString(PKG, "ProcessFiles.LineNumber",
Long.toString(getLinesRead())));
}
} catch (Exception e) {
boolean sendToErrorRow = false;
@@ -273,7 +243,7 @@ public class ProcessFiles extends
BaseTransform<ProcessFilesMeta, ProcessFilesDa
errorMessage = e.toString();
} else {
logError(
- BaseMessages.getString(PKG,
"ProcessFiles.ErrorInTransformRunning") + e.getMessage());
+ BaseMessages.getString(PKG,
"ProcessFiles.ErrorInTransformRunning", e.getMessage()), e);
setErrors(1);
stopAll();
setOutputDone(); // signal end to receiver(s)
@@ -288,6 +258,248 @@ public class ProcessFiles extends
BaseTransform<ProcessFilesMeta, ProcessFilesDa
return true;
}
+ private String operationTypeLabel() {
+ switch (meta.getOperationType()) {
+ case ProcessFilesMeta.OPERATION_TYPE_COPY:
+ return BaseMessages.getString(PKG,
"ProcessFilesMeta.operationType.Copy");
+ case ProcessFilesMeta.OPERATION_TYPE_MOVE:
+ return BaseMessages.getString(PKG,
"ProcessFilesMeta.operationType.Move");
+ case ProcessFilesMeta.OPERATION_TYPE_DELETE:
+ return BaseMessages.getString(PKG,
"ProcessFilesMeta.operationType.Delete");
+ default:
+ return "?";
+ }
+ }
+
+ /** Friendly URI for log lines (variables resolved). */
+ private String friendlyPath(String path) {
+ if (Utils.isEmpty(path)) {
+ return "";
+ }
+ try {
+ return HopVfs.getFriendlyURI(path, variables);
+ } catch (Exception e) {
+ return path;
+ }
+ }
+
+ private void processCopyOperation(String sourceFilename, String
targetFilename) throws Exception {
+ boolean targetFileExists =
+ data.targetFile.exists() && data.targetFile.getType() == FileType.FILE;
+ boolean shouldPerform =
+ !data.targetFile.exists() || (meta.isOverwriteTargetFile() &&
targetFileExists);
+
+ if (meta.simulate) {
+ if (shouldPerform && isDetailed()) {
+ logDetailed(
+ BaseMessages.getString(
+ PKG,
+ "ProcessFiles.Log.Detail.WouldCopy",
+ friendlyPath(sourceFilename),
+ friendlyPath(targetFilename)));
+ }
+ if (!shouldPerform && isDetailed()) {
+ logDetailed(
+ BaseMessages.getString(
+ PKG,
+ "ProcessFiles.Log.Detail.SimulateSkipCopyExists",
+ friendlyPath(targetFilename),
+ friendlyPath(sourceFilename)));
+ }
+ return;
+ }
+
+ if (shouldPerform) {
+ if (meta.isOverwriteTargetFile() && data.targetFile.exists() &&
isBasic()) {
+ logBasic(
+ BaseMessages.getString(
+ PKG, "ProcessFiles.Log.Basic.OverwriteWarningCopy",
friendlyPath(targetFilename)));
+ }
+ if (isDetailed()) {
+ logDetailed(
+ BaseMessages.getString(
+ PKG,
+ "ProcessFiles.Log.Detail.CopyStarted",
+ friendlyPath(sourceFilename),
+ friendlyPath(targetFilename)));
+ }
+ // Delete target first; some providers do not truncate correctly on
overwrite.
+ data.targetFile.delete();
+ if (isDataVolumeMetricEnabled()) {
+ copyFileTrackingVolume(data.sourceFile, data.targetFile);
+ } else {
+ data.targetFile.copyFrom(data.sourceFile, new
TextOneToOneFileSelector());
+ }
+ if (isBasic()) {
+ logBasic(
+ BaseMessages.getString(
+ PKG,
+ "ProcessFiles.Log.Basic.FileCopied",
+ friendlyPath(sourceFilename),
+ friendlyPath(targetFilename)));
+ }
+ if (isDetailed()) {
+ logDetailed(
+ BaseMessages.getString(
+ PKG,
+ "ProcessFiles.Log.Detail.FileCopied",
+ friendlyPath(sourceFilename),
+ friendlyPath(targetFilename)));
+ }
+ } else if (isDetailed()) {
+ logDetailed(
+ BaseMessages.getString(
+ PKG,
+ "ProcessFiles.Log.Detail.SkippedCopyExistsNoOverwrite",
+ friendlyPath(targetFilename),
+ friendlyPath(sourceFilename)));
+ }
+ }
+
+ private void processMoveOperation(String sourceFilename, String
targetFilename) throws Exception {
+ boolean targetFileExists =
+ data.targetFile.exists() && data.targetFile.getType() == FileType.FILE;
+ boolean shouldPerform =
+ !data.targetFile.exists() || (meta.isOverwriteTargetFile() &&
targetFileExists);
+
+ if (meta.simulate) {
+ if (shouldPerform && isDetailed()) {
+ logDetailed(
+ BaseMessages.getString(
+ PKG,
+ "ProcessFiles.Log.Detail.WouldMove",
+ friendlyPath(sourceFilename),
+ friendlyPath(targetFilename)));
+ }
+ if (!shouldPerform && isDetailed()) {
+ logDetailed(
+ BaseMessages.getString(
+ PKG,
+ "ProcessFiles.Log.Detail.SimulateSkipMoveExists",
+ friendlyPath(targetFilename),
+ friendlyPath(sourceFilename)));
+ }
+ return;
+ }
+
+ if (shouldPerform) {
+ if (meta.isOverwriteTargetFile() && data.targetFile.exists() &&
isBasic()) {
+ logBasic(
+ BaseMessages.getString(
+ PKG, "ProcessFiles.Log.Basic.OverwriteWarningMove",
friendlyPath(targetFilename)));
+ }
+ if (isDetailed()) {
+ logDetailed(
+ BaseMessages.getString(
+ PKG,
+ "ProcessFiles.Log.Detail.MoveStarted",
+ friendlyPath(sourceFilename),
+ friendlyPath(targetFilename)));
+ }
+ long movedBytes = 0;
+ if (isDataVolumeMetricEnabled()) {
+ movedBytes = safeContentSize(data.sourceFile);
+ }
+ data.sourceFile.moveTo(HopVfs.getFileObject(targetFilename, variables));
+ if (movedBytes > 0) {
+ dataVolumeIn = (dataVolumeIn != null ? dataVolumeIn : 0L) + movedBytes;
+ dataVolumeOut = (dataVolumeOut != null ? dataVolumeOut : 0L) +
movedBytes;
+ }
+ if (isBasic()) {
+ logBasic(
+ BaseMessages.getString(
+ PKG,
+ "ProcessFiles.Log.Basic.FileMoved",
+ friendlyPath(sourceFilename),
+ friendlyPath(targetFilename)));
+ }
+ if (isDetailed()) {
+ logDetailed(
+ BaseMessages.getString(
+ PKG,
+ "ProcessFiles.Log.Detail.FileMoved",
+ friendlyPath(sourceFilename),
+ friendlyPath(targetFilename)));
+ }
+ } else if (isDetailed()) {
+ logDetailed(
+ BaseMessages.getString(
+ PKG,
+ "ProcessFiles.Log.Detail.SkippedMoveExistsNoOverwrite",
+ friendlyPath(targetFilename),
+ friendlyPath(sourceFilename)));
+ }
+ }
+
+ private void processDeleteOperation(String sourceFilename) throws Exception {
+ if (meta.simulate) {
+ if (isDetailed()) {
+ logDetailed(
+ BaseMessages.getString(
+ PKG, "ProcessFiles.Log.Detail.WouldDelete",
friendlyPath(sourceFilename)));
+ }
+ return;
+ }
+ if (!data.sourceFile.delete()) {
+ throw new HopException(
+ BaseMessages.getString(
+ PKG, "ProcessFiles.Error.CanNotDeleteFile",
data.sourceFile.toString()));
+ }
+ if (isBasic()) {
+ logBasic(
+ BaseMessages.getString(
+ PKG, "ProcessFiles.Log.Basic.FileDeleted",
friendlyPath(sourceFilename)));
+ }
+ if (isDetailed()) {
+ logDetailed(
+ BaseMessages.getString(
+ PKG, "ProcessFiles.Log.Detail.FileDeleted",
friendlyPath(sourceFilename)));
+ }
+ }
+
+ private boolean isDataVolumeMetricEnabled() {
+ return getPipeline() != null
+ &&
Const.toBoolean(getPipeline().getVariable(Const.HOP_METRIC_DATA_VOLUME, "N"));
+ }
+
+ private void copyFileTrackingVolume(FileObject source, FileObject
destination)
+ throws IOException {
+ FileObject parent = destination.getParent();
+ if (parent != null && !parent.exists()) {
+ parent.createFolder();
+ }
+ long read;
+ long written;
+ try (InputStream rawIn = HopVfs.getInputStream(source);
+ CountingInputStream in = new CountingInputStream(rawIn);
+ OutputStream rawOut = HopVfs.getOutputStream(destination, false);
+ CountingOutputStream out = new CountingOutputStream(rawOut)) {
+ byte[] buffer = new byte[COPY_BUFFER_SIZE];
+ int len;
+ while ((len = in.read(buffer)) != -1) {
+ if (len > 0) {
+ out.write(buffer, 0, len);
+ }
+ }
+ out.flush();
+ read = in.getCount();
+ written = out.getCount();
+ }
+ dataVolumeIn = (dataVolumeIn != null ? dataVolumeIn : 0L) + read;
+ dataVolumeOut = (dataVolumeOut != null ? dataVolumeOut : 0L) + written;
+ }
+
+ private static long safeContentSize(FileObject file) {
+ try {
+ if (file != null && file.getType().hasContent()) {
+ return Math.max(0L, file.getContent().getSize());
+ }
+ } catch (Exception ignored) {
+ // metrics only
+ }
+ return 0L;
+ }
+
private class TextOneToOneFileSelector implements FileSelector {
@Override
public boolean includeFile(FileSelectInfo info) throws Exception {
diff --git
a/plugins/transforms/processfiles/src/main/resources/org/apache/hop/pipeline/transforms/processfiles/messages/messages_en_US.properties
b/plugins/transforms/processfiles/src/main/resources/org/apache/hop/pipeline/transforms/processfiles/messages/messages_en_US.properties
index 162b4ecb71..7abe95f679 100644
---
a/plugins/transforms/processfiles/src/main/resources/org/apache/hop/pipeline/transforms/processfiles/messages/messages_en_US.properties
+++
b/plugins/transforms/processfiles/src/main/resources/org/apache/hop/pipeline/transforms/processfiles/messages/messages_en_US.properties
@@ -24,8 +24,8 @@ ProcessFiles.Error.SourceFileNotFile=Source file [{0}] is not
a file (probably a
ProcessFiles.Error.TargetFileEmpty=Target filename is empty\!
ProcessFiles.Error.TargetFilenameFieldMissing=Target Filename is missing\!
ProcessFiles.Error.TargetFileNotFile=Target file [{0}] must be given as a file
(it is probably an existing folder)\!
-ProcessFiles.Error.TargetParentFolderNotExists=We can not fin target parent
folder [{0}]\!
-ProcessFiles.ErrorInTransformRunning=Because of an error, this transform
can''t continue\:
+ProcessFiles.Error.TargetParentFolderNotExists=Cannot find target parent
folder [{0}]. Enable ''Create target parent folder'' or create it before
running.
+ProcessFiles.ErrorInTransformRunning=Process files transform stopped: {0}
ProcessFiles.Exception.CouldnotFindField=Couldn''t find field ''{0}'' in row\!
ProcessFiles.Injection.AddResultFilenames=Add target filenames to result (Y/N)
ProcessFiles.Injection.CreateParentFolder=Create parent folder (Y/N)
@@ -34,17 +34,30 @@ ProcessFiles.Injection.OverwriteTargetFile=Overwrite target
files (Y/N)
ProcessFiles.Injection.Simulate=Turn on simulation mode (Y/N)
ProcessFiles.Injection.SourceFilenameField=Source filename fieldname
ProcessFiles.Injection.TargetFilenameField=Target filename fieldname
-ProcessFiles.LineNumber=linenr {0}
-ProcessFiles.Log.FileAddedResult=File was read by a Process files transform
-ProcessFiles.Log.FilenameAddResult=Filename [{0}] was added to result
filenames.
-ProcessFiles.Log.SimulationModeON=SIMULATION MODE ON \: WE WILL NOT
COPY/MOVE/DELETE FILES.
-ProcessFiles.Log.SourceFileCopied=Source file [{0}] was copied to [{1}]
-ProcessFiles.Log.SourceFileCopied.TargetOverwritten=Target file {0} will be
overwritten during copy operation.
-ProcessFiles.Log.SourceFileDeleted=Source file [{0}] was deleted.
-ProcessFiles.Log.SourceFileMoved=Source file [{0}] was moved to [{1}].
-ProcessFiles.Log.SourceFileMoved.TargetOverwritten=Target file {0} will be
overwritten during move operation.
-ProcessFiles.Log.TargetFileExists=Target file [{0}] exists.
-ProcessFiles.Log.TargetNotOverwritten=The file {1} was not overwritten by {0}.
+ProcessFiles.LineNumber=Process files: processed {0} input rows
+ProcessFiles.Log.Basic.FileCopied=Copied file {0} to {1}
+ProcessFiles.Log.Basic.FileDeleted=Deleted file {0}
+ProcessFiles.Log.Basic.FileMoved=Moved file {0} to {1}
+ProcessFiles.Log.Basic.OverwriteWarningCopy=Copy will replace existing target
file {0}
+ProcessFiles.Log.Basic.OverwriteWarningMove=Move will replace existing target
file {0}
+ProcessFiles.Log.Detail.CopyStarted=Starting copy: {0} -> {1}
+ProcessFiles.Log.Detail.FileCopied=Finished copy: {0} -> {1}
+ProcessFiles.Log.Detail.FileDeleted=Finished delete: {0}
+ProcessFiles.Log.Detail.FileMoved=Finished move: {0} -> {1}
+ProcessFiles.Log.Detail.MoveStarted=Starting move: {0} -> {1}
+ProcessFiles.Log.Detail.ParentFolderCreated=Created missing target parent
folder {0}
+ProcessFiles.Log.Detail.RowHeader=Row: operation={0}, source={1}, target={2}
+ProcessFiles.Log.Detail.SimulateSkipCopyExists=Simulation: skip copy — target
{0} exists and overwrite is off; source {1} unchanged
+ProcessFiles.Log.Detail.SimulateSkipMoveExists=Simulation: skip move — target
{0} exists and overwrite is off; source {1} unchanged
+ProcessFiles.Log.Detail.SkippedCopyExistsNoOverwrite=Skipped copy: target {0}
exists and overwrite is disabled; left source {1} unchanged
+ProcessFiles.Log.Detail.SkippedMoveExistsNoOverwrite=Skipped move: target {0}
exists and overwrite is disabled; left source {1} unchanged
+ProcessFiles.Log.Detail.WouldCopy=Simulation: would copy {0} to {1} (checks
passed)
+ProcessFiles.Log.Detail.WouldDelete=Simulation: would delete {0} (checks
passed)
+ProcessFiles.Log.Detail.WouldMove=Simulation: would move {0} to {1} (checks
passed)
+ProcessFiles.Log.FilenameAddResult=Added to pipeline result file list: {0}
+ProcessFiles.Log.FileAddedResult=File added to pipeline results by Process
files
+ProcessFiles.Log.SimulationModeON=Simulation mode on — this transform will not
copy, move, or delete files
+ProcessFiles.Log.TargetFileExists=Target path exists (file): {0}
ProcessFiles.Name=Process files
ProcessFilesDialog.AddResult.Label=Add target filename to result
ProcessFilesDialog.AddResult.Tooltip=Add target filename to result
diff --git
a/plugins/transforms/processfiles/src/main/resources/org/apache/hop/pipeline/transforms/processfiles/messages/messages_es_AR.properties
b/plugins/transforms/processfiles/src/main/resources/org/apache/hop/pipeline/transforms/processfiles/messages/messages_es_AR.properties
index 7caf98d981..c6c3dad7db 100644
---
a/plugins/transforms/processfiles/src/main/resources/org/apache/hop/pipeline/transforms/processfiles/messages/messages_es_AR.properties
+++
b/plugins/transforms/processfiles/src/main/resources/org/apache/hop/pipeline/transforms/processfiles/messages/messages_es_AR.properties
@@ -25,15 +25,12 @@ ProcessFiles.Error.SourceFileNotFile=El archivo fuente
[{0}] no es un archivo (p
ProcessFiles.Error.TargetFileEmpty=\u00A1Falta el nombre del archivo destino\!
ProcessFiles.Error.TargetFilenameFieldMissing=\u00A1Falta el nombre del
archivo destino\!
ProcessFiles.Error.TargetParentFolderNotExists=El directorio padre destino
[{0}] no puede ser encontrado.
-ProcessFiles.ErrorInTransformRunning=Este paso no puede continuar debido a un
error\:
+ProcessFiles.ErrorInTransformRunning=Este paso no puede continuar debido a un
error\: {0}
ProcessFiles.Exception.CouldnotFindField=El campo "{0}" no puede ser
encontrado en la fila.
ProcessFiles.LineNumber=Nro l\u00EDnea {0}
ProcessFiles.Log.FileAddedResult=El archivo fue le\u00EDdo por un paso
"Procesar archivos"
ProcessFiles.Log.FilenameAddResult=El archivo [{0}] fue a\u00F1adido al
listado de archvios resultado.
ProcessFiles.Log.SimulationModeON=MODO DE SIMULACI\u00D3N ACTIVADO\: NO SE
COPIAR\u00C1N/MOVER\u00C1N/ELIMINAR\u00C1N ARCHIVOS
-ProcessFiles.Log.SourceFileCopied=El archivo fuente [{0}] fue copiado a [{1}]
-ProcessFiles.Log.SourceFileDeleted=El archivo fuente [{0}] fue eliminado.
-ProcessFiles.Log.SourceFileMoved=El archivo fuente [{0}] fue movido a [{1}]
ProcessFiles.Log.TargetFileExists=El archivo destino [{0}] existe.
ProcessFilesDialog.AddResult.Label=A\u00F1adir el archivo destino al resultado
ProcessFilesDialog.AddResult.Tooltip=A\u00F1adir el archivo destino al listado
de archivos resultado
diff --git
a/plugins/transforms/processfiles/src/main/resources/org/apache/hop/pipeline/transforms/processfiles/messages/messages_fr_FR.properties
b/plugins/transforms/processfiles/src/main/resources/org/apache/hop/pipeline/transforms/processfiles/messages/messages_fr_FR.properties
index 3f8139c632..8f6543fd74 100644
---
a/plugins/transforms/processfiles/src/main/resources/org/apache/hop/pipeline/transforms/processfiles/messages/messages_fr_FR.properties
+++
b/plugins/transforms/processfiles/src/main/resources/org/apache/hop/pipeline/transforms/processfiles/messages/messages_fr_FR.properties
@@ -27,17 +27,13 @@ ProcessFiles.Error.TargetFileEmpty=Le fichier cible est
vide\!
ProcessFiles.Error.TargetFilenameFieldMissing=Le champ contenant le nom du
fichier cible n''a pas \u00E9t\u00E9 sp\u00E9cifi\u00E9\!
ProcessFiles.Error.TargetFileNotFile=[{0}] n''est pas un fichier (probablement
un r\u00E9pertoire)\!
ProcessFiles.Error.TargetParentFolderNotExists=Impossible de trouver le
r\u00E9pertoire parent cible [{0}]\!
-ProcessFiles.ErrorInTransformRunning=Apache Hop a rencontr\u00E9 une erreur
lors de l''ex\u00E9cution de cette transformation\:
+ProcessFiles.ErrorInTransformRunning=Apache Hop a rencontr\u00E9 une erreur
lors de l''ex\u00E9cution de cette transformation\: {0}
ProcessFiles.Exception.CouldnotFindField=Impossible de trouver le champ
''{0}'' dans le flux d''entr\u00E9e\!
ProcessFiles.LineNumber=N\u00B0Ligne {0}
ProcessFiles.Log.FileAddedResult=Ce fichier a \u00E9t\u00E9 lu par une
\u00E9tape ''Traitement fichiers''
ProcessFiles.Log.FilenameAddResult=Le nom de fichier [{0}] a \u00E9t\u00E9
ajout\u00E9 \u00E0 la liste des noms de fichiere li\u00E9e au r\u00E9sultat.
ProcessFiles.Log.SimulationModeON=LA SIMULATION A ETE ACTIVEE : LES FICHIERS
NE SERONT NI COPIER/DEPLACER/SUPPRIMER.
-ProcessFiles.Log.SourceFileCopied=Le fichier source [{0}] a \u00E9t\u00E9
copi\u00E9 vers [{1}]
-ProcessFiles.Log.SourceFileDeleted=Le ficher [{0}] a \u00E9t\u00E9 supprimer.
-ProcessFiles.Log.SourceFileMoved=Le fichier source [{0}] a \u00E9t\u00E9
d\u00E9plac\u00E9 vers [{1}]
ProcessFiles.Log.TargetFileExists=Le fichier cible [{0}] existe.
-ProcessFiles.Log.TargetNotOverwritten=Le fichier {1} n''a pas \u00E9t\u00E9
remplac\u00E9 par le fichier {0}.
ProcessFiles.Name=Traitement fichiers
ProcessFilesDialog.AddResult.Label=Ajouter nom fichier cible au r\u00E9sultat
ProcessFilesDialog.AddResult.Tooltip=Ajouter le nom du fichier cible\n\u00E0
la liste des noms de fichier li\u00E9 au r\u00E9sultat.
diff --git
a/plugins/transforms/processfiles/src/main/resources/org/apache/hop/pipeline/transforms/processfiles/messages/messages_it_IT.properties
b/plugins/transforms/processfiles/src/main/resources/org/apache/hop/pipeline/transforms/processfiles/messages/messages_it_IT.properties
index 8e8334068b..31b37429fe 100644
---
a/plugins/transforms/processfiles/src/main/resources/org/apache/hop/pipeline/transforms/processfiles/messages/messages_it_IT.properties
+++
b/plugins/transforms/processfiles/src/main/resources/org/apache/hop/pipeline/transforms/processfiles/messages/messages_it_IT.properties
@@ -26,17 +26,13 @@ ProcessFiles.Error.TargetFileEmpty=Il nome del file di
destinazione \u00E8 vuoto
ProcessFiles.Error.TargetFilenameFieldMissing=Manca il nome del file di
destinazione\!
ProcessFiles.Error.TargetFileNotFile=La destinazione [{0}] deve essere data
come un file (probabilmente \u00E8 una cartella esistente)\!
ProcessFiles.Error.TargetParentFolderNotExists=Impossibile trovare la cartella
padre per la posizione di destinazione [{0}]\!
-ProcessFiles.ErrorInTransformRunning=A causa di un errore, questo transform
non pu\u00F2 contuinuare\:
+ProcessFiles.ErrorInTransformRunning=A causa di un errore, questo transform
non pu\u00F2 contuinuare\: {0}
ProcessFiles.Exception.CouldnotFindField=Impossibile trovare il campo ''{0}''
nella riga\!
ProcessFiles.LineNumber=linea n\u00B0 {0}
ProcessFiles.Log.FileAddedResult=Il file \u00E8 stato letto dal transform
Process file
ProcessFiles.Log.FilenameAddResult=Il nome file [{0}] \u00E8 stato aggiunto al
nome file del risultato.
ProcessFiles.Log.SimulationModeON=MODALITA'' SIMULAZIONE ATTIVATA\: I FILE NON
VERRANNO COPIATI/SPOSTATI/CANCELLATI.
-ProcessFiles.Log.SourceFileCopied=Il file sorgente [{0}] \u00E8 stato copiato
in [{1}]
-ProcessFiles.Log.SourceFileDeleted=Il file sorgente [{0}] \u00E8 stato
cancellato.
-ProcessFiles.Log.SourceFileMoved=Il file sorgente [{0}] \u00E8 stato spostato
in [{1}]
ProcessFiles.Log.TargetFileExists=Il file di destinazione [{0}] esiste.
-ProcessFiles.Log.TargetNotOverwritten=Il file {1} non \u00E8 stato
sovrascritto da {0}.
ProcessFilesDialog.AddResult.Label=Aggiungi il nome file di destinazione al
risultato
ProcessFilesDialog.AddResult.Tooltip=Aggiungi il nome file di destinazione al
risultato
ProcessFilesDialog.CreateParentFolder.Label=Crea la cartella padre di
destinazione
diff --git
a/plugins/transforms/processfiles/src/main/resources/org/apache/hop/pipeline/transforms/processfiles/messages/messages_ja_JP.properties
b/plugins/transforms/processfiles/src/main/resources/org/apache/hop/pipeline/transforms/processfiles/messages/messages_ja_JP.properties
index 0fa19aec99..95565ddfea 100644
---
a/plugins/transforms/processfiles/src/main/resources/org/apache/hop/pipeline/transforms/processfiles/messages/messages_ja_JP.properties
+++
b/plugins/transforms/processfiles/src/main/resources/org/apache/hop/pipeline/transforms/processfiles/messages/messages_ja_JP.properties
@@ -26,15 +26,12 @@ ProcessFiles.Error.SourceFileNotFile=Source file [{0}] is
not a file (probably a
ProcessFiles.Error.TargetFileEmpty=Target filename is empty\!
ProcessFiles.Error.TargetFilenameFieldMissing=Target Filename is missing\!
ProcessFiles.Error.TargetParentFolderNotExists=We can not fin target parent
folder [{0}]\!
-ProcessFiles.ErrorInTransformRunning=Because of an error, this transform
can''t continue\:
+ProcessFiles.ErrorInTransformRunning=Because of an error, this transform
can''t continue\: {0}
ProcessFiles.Exception.CouldnotFindField=Couldn''t find field ''{0}'' in row\!
ProcessFiles.LineNumber=linenr {0}
ProcessFiles.Log.FileAddedResult=File was read by a Process files transform
ProcessFiles.Log.FilenameAddResult=Filename [{0}] was added to result
filenames.
ProcessFiles.Log.SimulationModeON=SIMULATION MODE ON \: WE WILL NOT
COPY/MOVE/DELETE FILES.
-ProcessFiles.Log.SourceFileCopied=Source file [{0}] was copied to [{1}]
-ProcessFiles.Log.SourceFileDeleted=Source file [{0}] was deleted.
-ProcessFiles.Log.SourceFileMoved=Source file [{0}] was moved to [{1}]
ProcessFiles.Log.TargetFileExists=Target file [{0}] exists.
ProcessFiles.Name=\u30D5\u30A1\u30A4\u30EB\u51E6\u7406
ProcessFilesDialog.AddResult.Label=\u30D5\u30A1\u30A4\u30EB\u540D\u3092\u7D50\u679C\u306B\u542B\u3080
diff --git
a/plugins/transforms/processfiles/src/main/resources/org/apache/hop/pipeline/transforms/processfiles/messages/messages_ko_KR.properties
b/plugins/transforms/processfiles/src/main/resources/org/apache/hop/pipeline/transforms/processfiles/messages/messages_ko_KR.properties
index d6b9fbc972..8f84d2657c 100644
---
a/plugins/transforms/processfiles/src/main/resources/org/apache/hop/pipeline/transforms/processfiles/messages/messages_ko_KR.properties
+++
b/plugins/transforms/processfiles/src/main/resources/org/apache/hop/pipeline/transforms/processfiles/messages/messages_ko_KR.properties
@@ -24,11 +24,9 @@ ProcessFiles.Error.SourceFileNotExist=\uC18C\uC2A4
\uD30C\uC77C [{0}]\uC744 \uCC
ProcessFiles.Error.SourceFileNotFile=\uC18C\uC2A4 \uD30C\uC77C [{0}]\uB294
\uD30C\uC77C\uC774 \uC544\uB2D9\uB2C8\uB2E4.(\uD3F4\uB354?)\!
ProcessFiles.Error.TargetFileEmpty=\uB300\uC0C1 \uD30C\uC77C\uC774\uB984\uC744
\uC785\uB825\uD558\uC2ED\uC2DC\uC624\!
ProcessFiles.Error.TargetFilenameFieldMissing=\uAC1C\uC0C1
\uD30C\uC77C\uC774\uB984\uC744 \uC785\uB825\uD558\uC2ED\uC2DC\uC624\!
-ProcessFiles.ErrorInTransformRunning=\uC624\uB958 \uB54C\uBB38\uC5D0,
\uACC4\uC18D\uD560 \uC218 \uC5C6\uC2B5\uB2C8\uB2E4\:
+ProcessFiles.ErrorInTransformRunning=\uC624\uB958 \uB54C\uBB38\uC5D0,
\uACC4\uC18D\uD560 \uC218 \uC5C6\uC2B5\uB2C8\uB2E4\: {0}
ProcessFiles.Exception.CouldnotFindField=\uB85C\uC6B0\uC5D0\uC11C \uD544\uB4DC
''{0}''\uC744 \uCC3E\uC744 \uC218 \uC5C6\uC2B5\uB2C8\uB2E4\!
ProcessFiles.LineNumber=\uC904\uBC88\uD638 {0}
-ProcessFiles.Log.SourceFileDeleted=\uC18C\uC2A4 \uD30C\uC77C [{0}] \uB97C
\uC0AD\uC81C\uD558\uC600\uC2B5\uB2C8\uB2E4.
-ProcessFiles.Log.SourceFileMoved=\uC18C\uC2A4 \uD30C\uC77C [{0}]\uB294
[{1}]\uB85C \uC774\uB3D9\uD558\uC600\uC2B5\uB2C8\uB2E4
ProcessFiles.Log.TargetFileExists=\uB300\uC0C1 \uD30C\uC77C
[{0}]\uC774(\uAC00) \uC874\uC7AC\uD569\uB2C8\uB2E4.
ProcessFilesDialog.AddResult.Label=\uACB0\uACFC\uC5D0 \uB300\uC0C1
\uD30C\uC77C\uC774\uB984 \uCD94\uAC00
ProcessFilesDialog.AddResult.Tooltip=\uACB0\uACFC\uC5D0 \uB300\uC0C1
\uD30C\uC77C\uC774\uB984 \uCD94\uAC00
diff --git
a/plugins/transforms/processfiles/src/main/resources/org/apache/hop/pipeline/transforms/processfiles/messages/messages_pt_BR.properties
b/plugins/transforms/processfiles/src/main/resources/org/apache/hop/pipeline/transforms/processfiles/messages/messages_pt_BR.properties
index e33188680f..fbbf8b9fd2 100644
---
a/plugins/transforms/processfiles/src/main/resources/org/apache/hop/pipeline/transforms/processfiles/messages/messages_pt_BR.properties
+++
b/plugins/transforms/processfiles/src/main/resources/org/apache/hop/pipeline/transforms/processfiles/messages/messages_pt_BR.properties
@@ -24,7 +24,7 @@ ProcessFiles.Error.SourceFileNotExist=Arquivo fonte[{0}]
n\u00E3o encontrado
ProcessFiles.Error.SourceFileNotFile=Fonte [{0}] n\u00E3o \u00E9 um arquivo (
provavelmente \u00E9 uma pasta ) !
ProcessFiles.Error.TargetFileEmpty=Nome de arquivo destino vazio!
ProcessFiles.Error.TargetFilenameFieldMissing=Nome de arquivo destino vazio!
-ProcessFiles.ErrorInTransformRunning=Por causa de um erro esta
transforma\u00E7\u00E3o n\u00E3o pode continuar:
+ProcessFiles.ErrorInTransformRunning=Por causa de um erro esta
transforma\u00E7\u00E3o n\u00E3o pode continuar: {0}
ProcessFiles.Exception.CouldnotFindField=N\u00E3o foi poss\u00EDvel encontrar
o campo ''{0}'' na linha !
ProcessFiles.Injection.AddResultFilenames=adicionar arquivos alvo ao resultado
ProcessFiles.Injection.CreateParentFolder=criar pasta superior
@@ -35,7 +35,6 @@ ProcessFiles.Injection.SourceFilenameField=nome do campo para
nome do arquivo de
ProcessFiles.Injection.TargetFilenameField=nome do campo para nome do arquivo
destino
ProcessFiles.LineNumber=N\u00FAmero da linha {0}
ProcessFiles.Log.FilenameAddResult=Nome de arquivo [{0}] foi adicionado ao
resultado.
-ProcessFiles.Log.SourceFileDeleted=Arquivo de origem [{0}] foi deletado
ProcessFiles.Log.TargetFileExists=arquivo destino [{0}] existe.
ProcessFiles.Name=Processar arquivos
ProcessFilesDialog.AddResult.Label=adicionar arquivo alvo ao resultado
@@ -65,11 +64,6 @@ ProcessFiles.Error.TargetFileNotFile=O arquivo alvo deve ser
dado como um arquiv
ProcessFiles.Error.TargetParentFolderNotExists=Não podemos achar a pasta dos
pais!
ProcessFiles.Log.FileAddedResult=O arquivo foi lido por um processo de
transformação.
ProcessFiles.Log.SimulationModeON=Não vamos copiar, mover ou eliminar arquivos.
-ProcessFiles.Log.SourceFileCopied=Arquivo fonte foi copiado para
-ProcessFiles.Log.SourceFileCopied.TargetOverwritten=O arquivo alvo será
substituído durante a operação de cópia.
-ProcessFiles.Log.SourceFileMoved=O arquivo fonte foi movido para.
-ProcessFiles.Log.SourceFileMoved.TargetOverwritten=O arquivo alvo será
substituído durante a operação.
-ProcessFiles.Log.TargetNotOverwritten=O arquivo não foi substituído por ele.
ProcessFilesDialog.Simulate.Tooltip=Se você selecionar esta opção, os arquivos
não serão copiados/movidos/eliminados.\r\nChecagem será feita nos arquivos
(existência, pasta superior) e mensagem exibida.
ProcessFilesMeta.CheckResult.SourceFileFieldMissing=O campo de nome de arquivo
fonte está ausente!
ProcessFilesMeta.CheckResult.SourceFileFieldOK=O campo Nome do arquivo fonte
foi especificado.
diff --git
a/plugins/transforms/processfiles/src/main/resources/org/apache/hop/pipeline/transforms/processfiles/messages/messages_zh_CN.properties
b/plugins/transforms/processfiles/src/main/resources/org/apache/hop/pipeline/transforms/processfiles/messages/messages_zh_CN.properties
index 638248faf7..c04f20032c 100644
---
a/plugins/transforms/processfiles/src/main/resources/org/apache/hop/pipeline/transforms/processfiles/messages/messages_zh_CN.properties
+++
b/plugins/transforms/processfiles/src/main/resources/org/apache/hop/pipeline/transforms/processfiles/messages/messages_zh_CN.properties
@@ -27,17 +27,13 @@
ProcessFiles.Error.TargetFileEmpty=\u76EE\u6807\u6587\u4EF6\u540D\u4E3A\u7A7A\!
ProcessFiles.Error.TargetFilenameFieldMissing=\u7F3A\u5931\u76EE\u6807\u6587\u4EF6\u540D\!
ProcessFiles.Error.TargetFileNotFile=\u6307\u5B9A\u76EE\u6807\u8DEF\u5F84
[{0}]
\u5FC5\u9700\u4E3A\u666E\u901A\u6587\u4EF6(\u53EF\u80FD\u5B58\u5728\u540C\u540D\u76EE\u5F55)\!
ProcessFiles.Error.TargetParentFolderNotExists=\u76EE\u6807\u8DEF\u5F84 [{0}]
\u7684\u4E0A\u7EA7\u76EE\u5F55\u4E0D\u5B58\u5728\!
-ProcessFiles.ErrorInTransformRunning=\u56E0\u51FA\u9519, \u7EC8\u6B62\:
+ProcessFiles.ErrorInTransformRunning=\u56E0\u51FA\u9519, \u7EC8\u6B62\: {0}
ProcessFiles.Exception.CouldnotFindField=\u5728\u6570\u636E\u884C\u4E2D\u627E\u4E0D\u5230\u5B57\u6BB5
"{0}"\!
ProcessFiles.LineNumber=\u884C\u53F7 {0}
ProcessFiles.Log.FileAddedResult=\u5DF2\u8BFB\u53D6\u6587\u4EF6
ProcessFiles.Log.FilenameAddResult=\u6587\u4EF6 [{0}]
\u5DF2\u6DFB\u52A0\u5230\u8FD4\u56DE\u6587\u4EF6\u5217\u8868
ProcessFiles.Log.SimulationModeON=\u5F00\u542F\u6A21\u62DF\u6A21\u5F0F\:
\u5E76\u4E0D\u4F1A\u771F\u6B63 \u590D\u5236/\u79FB\u52A8/\u5220\u9664
\u6587\u4EF6
-ProcessFiles.Log.SourceFileCopied=\u6E90\u6587\u4EF6 [{0}] \u590D\u5236\u5230
[{1}]
-ProcessFiles.Log.SourceFileDeleted=\u5DF2\u5220\u9664\u6E90\u6587\u4EF6 [{0}]
-ProcessFiles.Log.SourceFileMoved=\u6E90\u6587\u4EF6 [{0}] \u79FB\u52A8\u5230
[{1}]
ProcessFiles.Log.TargetFileExists=\u5B58\u5728\u76EE\u6807\u6587\u4EF6 [{0}]
-ProcessFiles.Log.TargetNotOverwritten=\u6587\u4EF6 {0}
\u6CA1\u6709\u8986\u76D6\u5230 {1}
ProcessFiles.Name=\u5904\u7406\u6587\u4EF6
ProcessFilesDialog.AddResult.Label=\u7ED3\u679C\u4E2D\u6DFB\u52A0\u76EE\u6807\u6587\u4EF6\u540D\:
ProcessFilesDialog.AddResult.Tooltip=\u5C06\u76EE\u6807\u6587\u4EF6\u6DFB\u52A0\u5230\u7ED3\u679C\u6587\u4EF6,
\u4FBF\u4E8E\u540E\u9762\u7684\u5904\u7406,
\u5982\u4F5C\u4E3A\u90AE\u4EF6\u9644\u4EF6.
diff --git
a/plugins/transforms/processfiles/src/test/java/org/apache/hop/pipeline/transforms/processfiles/ProcessFilesTest.java
b/plugins/transforms/processfiles/src/test/java/org/apache/hop/pipeline/transforms/processfiles/ProcessFilesTest.java
new file mode 100644
index 0000000000..6b0f0555f9
--- /dev/null
+++
b/plugins/transforms/processfiles/src/test/java/org/apache/hop/pipeline/transforms/processfiles/ProcessFilesTest.java
@@ -0,0 +1,393 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.pipeline.transforms.processfiles;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyLong;
+import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.ArgumentMatchers.nullable;
+import static org.mockito.Mockito.doNothing;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+import java.lang.reflect.Constructor;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import org.apache.hop.core.BlockingRowSet;
+import org.apache.hop.core.Const;
+import org.apache.hop.core.HopEnvironment;
+import org.apache.hop.core.exception.HopException;
+import org.apache.hop.core.logging.ILoggingObject;
+import org.apache.hop.core.row.IRowMeta;
+import org.apache.hop.core.row.RowMeta;
+import org.apache.hop.core.row.value.ValueMetaString;
+import org.apache.hop.junit.rules.RestoreHopEngineEnvironmentExtension;
+import org.apache.hop.pipeline.Pipeline;
+import org.apache.hop.pipeline.PipelineMeta;
+import org.apache.hop.pipeline.PipelineTestingUtil;
+import org.apache.hop.pipeline.transform.TransformMeta;
+import org.apache.hop.pipeline.transforms.mock.TransformMockHelper;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.RegisterExtension;
+import org.junit.jupiter.api.io.TempDir;
+
+class ProcessFilesTest {
+
+ private static final String SOURCE_FIELD = "source";
+ private static final String TARGET_FIELD = "target";
+
+ @RegisterExtension
+ static RestoreHopEngineEnvironmentExtension env = new
RestoreHopEngineEnvironmentExtension();
+
+ private TransformMockHelper<ProcessFilesMeta, ProcessFilesData> helper;
+
+ @BeforeAll
+ static void initHop() throws Exception {
+ HopEnvironment.init();
+ }
+
+ @BeforeEach
+ void setUp() {
+ helper =
+ new TransformMockHelper<>(
+ "ProcessFilesTest", ProcessFilesMeta.class,
ProcessFilesData.class);
+ when(helper.logChannelFactory.create(any(), any(ILoggingObject.class)))
+ .thenReturn(helper.iLogChannel);
+
when(helper.logChannelFactory.create(any())).thenReturn(helper.iLogChannel);
+ when(helper.pipeline.isRunning()).thenReturn(true);
+ }
+
+ @AfterEach
+ void tearDown() {
+ helper.cleanUp();
+ }
+
+ private ProcessFiles createTransform(ProcessFilesMeta meta, ProcessFilesData
data)
+ throws Exception {
+ Constructor<ProcessFiles> kons =
+ ProcessFiles.class.getConstructor(
+ TransformMeta.class,
+ ProcessFilesMeta.class,
+ ProcessFilesData.class,
+ int.class,
+ PipelineMeta.class,
+ Pipeline.class);
+ return kons.newInstance(
+ helper.transformMeta, meta, data, 0, helper.pipelineMeta,
helper.pipeline);
+ }
+
+ private RowMeta rowMeta() {
+ RowMeta rm = new RowMeta();
+ rm.addValueMeta(new ValueMetaString(SOURCE_FIELD));
+ rm.addValueMeta(new ValueMetaString(TARGET_FIELD));
+ return rm;
+ }
+
+ private static String vfsPath(Path path) {
+ return path.toUri().toString();
+ }
+
+ private void wireInput(ProcessFiles transform, RowMeta rm, Object[]... rows)
{
+ BlockingRowSet input = new BlockingRowSet(Math.max(16, rows.length + 1));
+ for (Object[] row : rows) {
+ input.putRow(rm, row);
+ }
+ input.setDone();
+ transform.setInputRowSets(new
ArrayList<>(Collections.singletonList(input)));
+ }
+
+ private ProcessFilesMeta baseMeta(int operationType) {
+ ProcessFilesMeta meta = new ProcessFilesMeta();
+ meta.setDefault();
+ meta.setSimulate(false);
+ meta.setOperationType(operationType);
+ meta.setSourceFilenameField(SOURCE_FIELD);
+ meta.setTargetFilenameField(TARGET_FIELD);
+ meta.setCreateParentFolder(false);
+ meta.setOverwriteTargetFile(false);
+ meta.setAddResultFilenames(false);
+ return meta;
+ }
+
+ @Test
+ void copy_writesTargetAndPassesRowThrough(@TempDir Path tempDir) throws
Exception {
+ Path source = tempDir.resolve("in.txt");
+ Path target = tempDir.resolve("out.txt");
+ Files.writeString(source, "hello-processfiles", StandardCharsets.UTF_8);
+
+ ProcessFilesMeta meta = baseMeta(ProcessFilesMeta.OPERATION_TYPE_COPY);
+ ProcessFiles transform = createTransform(meta, new ProcessFilesData());
+ RowMeta rm = rowMeta();
+ wireInput(transform, rm, new Object[] {vfsPath(source), vfsPath(target)});
+
+ List<Object[]> out = PipelineTestingUtil.execute(transform, 1, false);
+ PipelineTestingUtil.assertResult(
+ new Object[] {vfsPath(source), vfsPath(target)}, out.getFirst());
+
+ assertTrue(Files.exists(target));
+ assertEquals("hello-processfiles", Files.readString(target,
StandardCharsets.UTF_8));
+ assertTrue(Files.exists(source));
+ }
+
+ @Test
+ void copy_createsParentFolderWhenConfigured(@TempDir Path tempDir) throws
Exception {
+ Path source = tempDir.resolve("in.txt");
+ Path target = tempDir.resolve("nested/dir/out.txt");
+ Files.writeString(source, "x", StandardCharsets.UTF_8);
+
+ ProcessFilesMeta meta = baseMeta(ProcessFilesMeta.OPERATION_TYPE_COPY);
+ meta.setCreateParentFolder(true);
+ ProcessFiles transform = createTransform(meta, new ProcessFilesData());
+ RowMeta rm = rowMeta();
+ wireInput(transform, rm, new Object[] {vfsPath(source), vfsPath(target)});
+
+ PipelineTestingUtil.execute(transform, 1, false);
+
+ assertTrue(Files.isRegularFile(target));
+ assertEquals("x", Files.readString(target, StandardCharsets.UTF_8));
+ }
+
+ @Test
+ void copy_simulate_doesNotCreateTarget(@TempDir Path tempDir) throws
Exception {
+ Path source = tempDir.resolve("in.txt");
+ Path target = tempDir.resolve("out.txt");
+ Files.writeString(source, "only-source", StandardCharsets.UTF_8);
+
+ ProcessFilesMeta meta = baseMeta(ProcessFilesMeta.OPERATION_TYPE_COPY);
+ meta.setSimulate(true);
+ ProcessFiles transform = createTransform(meta, new ProcessFilesData());
+ RowMeta rm = rowMeta();
+ wireInput(transform, rm, new Object[] {vfsPath(source), vfsPath(target)});
+
+ PipelineTestingUtil.execute(transform, 1, false);
+
+ assertFalse(Files.exists(target));
+ }
+
+ @Test
+ void copy_skipsWhenTargetExistsAndOverwriteDisabled(@TempDir Path tempDir)
throws Exception {
+ Path source = tempDir.resolve("in.txt");
+ Path target = tempDir.resolve("out.txt");
+ Files.writeString(source, "new-content", StandardCharsets.UTF_8);
+ Files.writeString(target, "old-content", StandardCharsets.UTF_8);
+
+ ProcessFilesMeta meta = baseMeta(ProcessFilesMeta.OPERATION_TYPE_COPY);
+ meta.setOverwriteTargetFile(false);
+ ProcessFiles transform = createTransform(meta, new ProcessFilesData());
+ RowMeta rm = rowMeta();
+ wireInput(transform, rm, new Object[] {vfsPath(source), vfsPath(target)});
+
+ PipelineTestingUtil.execute(transform, 1, false);
+
+ assertEquals("old-content", Files.readString(target,
StandardCharsets.UTF_8));
+ }
+
+ @Test
+ void copy_overwritesWhenEnabled(@TempDir Path tempDir) throws Exception {
+ Path source = tempDir.resolve("in.txt");
+ Path target = tempDir.resolve("out.txt");
+ Files.writeString(source, "replacement", StandardCharsets.UTF_8);
+ Files.writeString(target, "stale", StandardCharsets.UTF_8);
+
+ ProcessFilesMeta meta = baseMeta(ProcessFilesMeta.OPERATION_TYPE_COPY);
+ meta.setOverwriteTargetFile(true);
+ ProcessFiles transform = createTransform(meta, new ProcessFilesData());
+ RowMeta rm = rowMeta();
+ wireInput(transform, rm, new Object[] {vfsPath(source), vfsPath(target)});
+
+ PipelineTestingUtil.execute(transform, 1, false);
+
+ assertEquals("replacement", Files.readString(target,
StandardCharsets.UTF_8));
+ }
+
+ @Test
+ void move_removesSourceAndWritesTarget(@TempDir Path tempDir) throws
Exception {
+ Path source = tempDir.resolve("move-src.txt");
+ Path target = tempDir.resolve("move-dst.txt");
+ Files.writeString(source, "moved-bytes", StandardCharsets.UTF_8);
+
+ ProcessFilesMeta meta = baseMeta(ProcessFilesMeta.OPERATION_TYPE_MOVE);
+ ProcessFiles transform = createTransform(meta, new ProcessFilesData());
+ RowMeta rm = rowMeta();
+ wireInput(transform, rm, new Object[] {vfsPath(source), vfsPath(target)});
+
+ PipelineTestingUtil.execute(transform, 1, false);
+
+ assertFalse(Files.exists(source));
+ assertEquals("moved-bytes", Files.readString(target,
StandardCharsets.UTF_8));
+ }
+
+ @Test
+ void delete_removesSource(@TempDir Path tempDir) throws Exception {
+ Path source = tempDir.resolve("del.txt");
+ Files.writeString(source, "bye", StandardCharsets.UTF_8);
+ Path dummyTarget = tempDir.resolve("ignored.txt");
+
+ ProcessFilesMeta meta = baseMeta(ProcessFilesMeta.OPERATION_TYPE_DELETE);
+ ProcessFiles transform = createTransform(meta, new ProcessFilesData());
+ RowMeta rm = rowMeta();
+ wireInput(transform, rm, new Object[] {vfsPath(source),
vfsPath(dummyTarget)});
+
+ PipelineTestingUtil.execute(transform, 1, false);
+
+ assertFalse(Files.exists(source));
+ }
+
+ @Test
+ void delete_simulate_leavesSource(@TempDir Path tempDir) throws Exception {
+ Path source = tempDir.resolve("keep.txt");
+ Files.writeString(source, "still-here", StandardCharsets.UTF_8);
+
+ ProcessFilesMeta meta = baseMeta(ProcessFilesMeta.OPERATION_TYPE_DELETE);
+ meta.setSimulate(true);
+ ProcessFiles transform = createTransform(meta, new ProcessFilesData());
+ RowMeta rm = rowMeta();
+ wireInput(transform, rm, new Object[] {vfsPath(source),
vfsPath(tempDir.resolve("dummy.txt"))});
+
+ PipelineTestingUtil.execute(transform, 1, false);
+
+ assertTrue(Files.exists(source));
+ }
+
+ @Test
+ void addResultFilenames_registersTargetAfterCopy(@TempDir Path tempDir)
throws Exception {
+ Path source = tempDir.resolve("in.txt");
+ Path target = tempDir.resolve("result.txt");
+ Files.writeString(source, "r", StandardCharsets.UTF_8);
+
+ ProcessFilesMeta meta = baseMeta(ProcessFilesMeta.OPERATION_TYPE_COPY);
+ meta.setAddResultFilenames(true);
+ ProcessFiles transform = createTransform(meta, new ProcessFilesData());
+ RowMeta rm = rowMeta();
+ wireInput(transform, rm, new Object[] {vfsPath(source), vfsPath(target)});
+
+ PipelineTestingUtil.execute(transform, 1, false);
+
+ assertEquals(1, transform.getResultFiles().size());
+
assertTrue(transform.getResultFiles().containsKey(target.toUri().toString()));
+ }
+
+ @Test
+ void copy_withDataVolumeVariable_tracksVolume(@TempDir Path tempDir) throws
Exception {
+ Path source = tempDir.resolve("in.txt");
+ Path target = tempDir.resolve("out.txt");
+ String payload = "metric-payload";
+ Files.writeString(source, payload, StandardCharsets.UTF_8);
+
+ helper.pipeline.setVariable(Const.HOP_METRIC_DATA_VOLUME, "Y");
+
+ ProcessFilesMeta meta = baseMeta(ProcessFilesMeta.OPERATION_TYPE_COPY);
+ ProcessFiles transform = createTransform(meta, new ProcessFilesData());
+ RowMeta rm = rowMeta();
+ wireInput(transform, rm, new Object[] {vfsPath(source), vfsPath(target)});
+
+ PipelineTestingUtil.execute(transform, 1, false);
+
+ assertEquals(payload.length(), (int) (long) transform.getDataVolumeIn());
+ assertEquals(payload.length(), (int) (long) transform.getDataVolumeOut());
+ }
+
+ @Test
+ void throwsWhenSourceFilenameFieldNotSet(@TempDir Path tempDir) throws
Exception {
+ Path source = tempDir.resolve("in.txt");
+ Path target = tempDir.resolve("out.txt");
+ Files.writeString(source, "x", StandardCharsets.UTF_8);
+
+ ProcessFilesMeta meta = baseMeta(ProcessFilesMeta.OPERATION_TYPE_COPY);
+ meta.setSourceFilenameField(null);
+ ProcessFiles transform = createTransform(meta, new ProcessFilesData());
+ RowMeta rm = rowMeta();
+ wireInput(transform, rm, new Object[] {vfsPath(source), vfsPath(target)});
+
+ assertThrows(HopException.class, () ->
PipelineTestingUtil.execute(transform, 1, false));
+ }
+
+ @Test
+ void missingParentWithoutCreateFolder_sendsRowToErrorWhenConfigured(@TempDir
Path tempDir)
+ throws Exception {
+ Path source = tempDir.resolve("in.txt");
+ Path target = tempDir.resolve("no_such_parent/out.txt");
+ Files.writeString(source, "x", StandardCharsets.UTF_8);
+
+ when(helper.transformMeta.isDoingErrorHandling()).thenReturn(true);
+
+ ProcessFilesMeta meta = baseMeta(ProcessFilesMeta.OPERATION_TYPE_COPY);
+ meta.setCreateParentFolder(false);
+ ProcessFiles transform = spy(createTransform(meta, new
ProcessFilesData()));
+ doNothing()
+ .when(transform)
+ .putError(
+ any(IRowMeta.class),
+ any(Object[].class),
+ anyLong(),
+ anyString(),
+ nullable(String.class),
+ anyString());
+
+ RowMeta rm = rowMeta();
+ wireInput(transform, rm, new Object[] {vfsPath(source), vfsPath(target)});
+
+ assertTrue(transform.processRow());
+ verify(transform)
+ .putError(
+ any(IRowMeta.class),
+ any(Object[].class),
+ anyLong(),
+ anyString(),
+ nullable(String.class),
+ anyString());
+ assertFalse(Files.exists(target));
+ }
+
+ @Test
+ void processesMultipleRows(@TempDir Path tempDir) throws Exception {
+ Path s1 = tempDir.resolve("a.txt");
+ Path t1 = tempDir.resolve("a.out");
+ Path s2 = tempDir.resolve("b.txt");
+ Path t2 = tempDir.resolve("b.out");
+ Files.writeString(s1, "1", StandardCharsets.UTF_8);
+ Files.writeString(s2, "2", StandardCharsets.UTF_8);
+
+ ProcessFilesMeta meta = baseMeta(ProcessFilesMeta.OPERATION_TYPE_COPY);
+ ProcessFiles transform = createTransform(meta, new ProcessFilesData());
+ RowMeta rm = rowMeta();
+ wireInput(
+ transform,
+ rm,
+ new Object[] {vfsPath(s1), vfsPath(t1)},
+ new Object[] {vfsPath(s2), vfsPath(t2)});
+
+ List<Object[]> out = PipelineTestingUtil.execute(transform, 2, false);
+ PipelineTestingUtil.assertResult(new Object[] {vfsPath(s1), vfsPath(t1)},
out.get(0));
+ PipelineTestingUtil.assertResult(new Object[] {vfsPath(s2), vfsPath(t2)},
out.get(1));
+ assertEquals("1", Files.readString(t1, StandardCharsets.UTF_8));
+ assertEquals("2", Files.readString(t2, StandardCharsets.UTF_8));
+ }
+}
