This is an automated email from the ASF dual-hosted git repository. davsclaus pushed a commit to branch fo in repository https://gitbox.apache.org/repos/asf/camel.git
commit 7587c6aca00a410c1dee318cc97c4264d63faf9a Author: Claus Ibsen <[email protected]> AuthorDate: Sun Dec 29 13:34:33 2024 +0100 CAMEL-17648: camel-file - Optimize file consumer when filtering file names. --- .../camel/component/file/azure/FilesConsumer.java | 12 ++++---- .../camel/component/file/remote/FtpConsumer.java | 32 ++++++++++++-------- .../camel/component/file/remote/RemoteFile.java | 3 +- .../camel/component/file/remote/SftpConsumer.java | 34 +++++++++++++--------- .../integration/SftpMoveWithOutMessageTest.java | 2 ++ 5 files changed, 51 insertions(+), 32 deletions(-) diff --git a/components/camel-azure/camel-azure-files/src/main/java/org/apache/camel/component/file/azure/FilesConsumer.java b/components/camel-azure/camel-azure-files/src/main/java/org/apache/camel/component/file/azure/FilesConsumer.java index d329a0452dd..0bca7138016 100644 --- a/components/camel-azure/camel-azure-files/src/main/java/org/apache/camel/component/file/azure/FilesConsumer.java +++ b/components/camel-azure/camel-azure-files/src/main/java/org/apache/camel/component/file/azure/FilesConsumer.java @@ -19,6 +19,7 @@ package org.apache.camel.component.file.azure; import java.util.Arrays; import java.util.Comparator; import java.util.List; +import java.util.function.Supplier; import com.azure.storage.file.share.models.ShareFileItem; import org.apache.camel.Message; @@ -33,6 +34,7 @@ import org.apache.camel.component.file.remote.RemoteFileConsumer; import org.apache.camel.util.FileUtil; import org.apache.camel.util.StringHelper; import org.apache.camel.util.URISupport; +import org.apache.camel.util.function.Suppliers; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -152,8 +154,8 @@ public class FilesConsumer extends RemoteFileConsumer<ShareFileItem> { int depth, ShareFileItem[] listedFileItems, ShareFileItem dir) { if (endpoint.isRecursive() && depth < endpoint.getMaxDepth()) { - var remote = asRemoteFile(path, dir); - if (isValidFile(remote, true, listedFileItems)) { + Supplier<GenericFile<ShareFileItem>> remote = Suppliers.memorize(() -> asRemoteFile(path, dir)); + if (isValidFile(remote, dir.getName(), remote.get().getAbsoluteFilePath(), true, listedFileItems)) { String dirName = dir.getName(); String dirPath = FilesPath.concat(path, dirName); boolean canPollMore = doSafePollSubDirectory(dirPath, dirName, polledFiles, depth); @@ -169,9 +171,9 @@ public class FilesConsumer extends RemoteFileConsumer<ShareFileItem> { String path, List<GenericFile<ShareFileItem>> polledFiles, int depth, ShareFileItem[] listedFileItems, ShareFileItem file) { if (depth >= endpoint.getMinDepth()) { - var remote = asRemoteFile(path, file); - if (isValidFile(remote, false, listedFileItems)) { - polledFiles.add(remote); + Supplier<GenericFile<ShareFileItem>> remote = Suppliers.memorize(() -> asRemoteFile(path, file)); + if (isValidFile(remote, file.getName(), remote.get().getAbsoluteFilePath(), false, listedFileItems)) { + polledFiles.add(remote.get()); } } } diff --git a/components/camel-ftp/src/main/java/org/apache/camel/component/file/remote/FtpConsumer.java b/components/camel-ftp/src/main/java/org/apache/camel/component/file/remote/FtpConsumer.java index 127aa045759..73169e1c794 100644 --- a/components/camel-ftp/src/main/java/org/apache/camel/component/file/remote/FtpConsumer.java +++ b/components/camel-ftp/src/main/java/org/apache/camel/component/file/remote/FtpConsumer.java @@ -19,6 +19,7 @@ package org.apache.camel.component.file.remote; import java.util.Arrays; import java.util.Comparator; import java.util.List; +import java.util.function.Supplier; import org.apache.camel.Exchange; import org.apache.camel.Message; @@ -34,6 +35,7 @@ import org.apache.camel.util.ObjectHelper; import org.apache.camel.util.StringHelper; import org.apache.camel.util.TimeUtils; import org.apache.camel.util.URISupport; +import org.apache.camel.util.function.Suppliers; import org.apache.commons.net.ftp.FTPClient; import org.apache.commons.net.ftp.FTPFile; import org.slf4j.Logger; @@ -181,14 +183,17 @@ public class FtpConsumer extends RemoteFileConsumer<FTPFile> { private boolean handleDirectory( String absolutePath, List<GenericFile<FTPFile>> fileList, int depth, FTPFile[] files, FTPFile file) { - RemoteFile<FTPFile> remote = asRemoteFile(absolutePath, file, getEndpoint().getCharset()); - if (endpoint.isRecursive() && depth < endpoint.getMaxDepth() && isValidFile(remote, true, files)) { - // recursive scan and add the sub files and folders - String subDirectory = file.getName(); - String path = ObjectHelper.isNotEmpty(absolutePath) ? absolutePath + "/" + subDirectory : subDirectory; - boolean canPollMore = pollSubDirectory(path, subDirectory, fileList, depth); - if (!canPollMore) { - return true; + if (endpoint.isRecursive() && depth < endpoint.getMaxDepth()) { + Supplier<GenericFile<FTPFile>> remote + = Suppliers.memorize(() -> asRemoteFile(absolutePath, file, getEndpoint().getCharset())); + if (isValidFile(remote, file.getName(), remote.get().getAbsoluteFilePath(), true, files)) { + // recursive scan and add the sub files and folders + String subDirectory = file.getName(); + String path = ObjectHelper.isNotEmpty(absolutePath) ? absolutePath + "/" + subDirectory : subDirectory; + boolean canPollMore = pollSubDirectory(path, subDirectory, fileList, depth); + if (!canPollMore) { + return true; + } } } return false; @@ -196,10 +201,13 @@ public class FtpConsumer extends RemoteFileConsumer<FTPFile> { private void handleFile( String absolutePath, List<GenericFile<FTPFile>> fileList, int depth, FTPFile[] files, FTPFile file) { - RemoteFile<FTPFile> remote = asRemoteFile(absolutePath, file, getEndpoint().getCharset()); - if (depth >= endpoint.getMinDepth() && isValidFile(remote, false, files)) { - // matched file so add - fileList.add(remote); + if (depth >= endpoint.getMinDepth()) { + Supplier<GenericFile<FTPFile>> remote + = Suppliers.memorize(() -> asRemoteFile(absolutePath, file, getEndpoint().getCharset())); + if (isValidFile(remote, file.getName(), remote.get().getAbsoluteFilePath(), false, files)) { + // matched file so add + fileList.add(remote.get()); + } } } diff --git a/components/camel-ftp/src/main/java/org/apache/camel/component/file/remote/RemoteFile.java b/components/camel-ftp/src/main/java/org/apache/camel/component/file/remote/RemoteFile.java index e2c539128ba..c7787bb4f4f 100644 --- a/components/camel-ftp/src/main/java/org/apache/camel/component/file/remote/RemoteFile.java +++ b/components/camel-ftp/src/main/java/org/apache/camel/component/file/remote/RemoteFile.java @@ -64,7 +64,7 @@ public class RemoteFile<T> extends GenericFile<T> implements Cloneable { @Override protected boolean isAbsolute(String name) { - if (name.length() > 0) { + if (!name.isEmpty()) { return name.charAt(0) == '/' || name.charAt(0) == '\\'; } return false; @@ -79,7 +79,6 @@ public class RemoteFile<T> extends GenericFile<T> implements Cloneable { public void copyFromPopulateAdditional(GenericFile<T> source, GenericFile<T> result) { RemoteFile<?> remoteSource = (RemoteFile<?>) source; RemoteFile<?> remoteResult = (RemoteFile<?>) result; - remoteResult.setHostname(remoteSource.getHostname()); } diff --git a/components/camel-ftp/src/main/java/org/apache/camel/component/file/remote/SftpConsumer.java b/components/camel-ftp/src/main/java/org/apache/camel/component/file/remote/SftpConsumer.java index 56c10fec3c8..7e9503a93cd 100644 --- a/components/camel-ftp/src/main/java/org/apache/camel/component/file/remote/SftpConsumer.java +++ b/components/camel-ftp/src/main/java/org/apache/camel/component/file/remote/SftpConsumer.java @@ -19,6 +19,7 @@ package org.apache.camel.component.file.remote; import java.util.Arrays; import java.util.Comparator; import java.util.List; +import java.util.function.Supplier; import com.jcraft.jsch.ChannelSftp; import com.jcraft.jsch.SftpException; @@ -33,6 +34,7 @@ import org.apache.camel.util.FileUtil; import org.apache.camel.util.ObjectHelper; import org.apache.camel.util.StringHelper; import org.apache.camel.util.URISupport; +import org.apache.camel.util.function.Suppliers; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -124,7 +126,7 @@ public class SftpConsumer extends RemoteFileConsumer<SftpRemoteFile> { dirName = FileUtil.stripTrailingSeparator(dirName); // compute dir depending on stepwise is enabled or not - String dir = null; + String dir; if (isStepwise()) { dir = ObjectHelper.isNotEmpty(dirName) ? dirName : absolutePath; operations.changeCurrentDirectory(dir); @@ -160,24 +162,30 @@ public class SftpConsumer extends RemoteFileConsumer<SftpRemoteFile> { } if (file.isDirectory()) { - RemoteFile<SftpRemoteFile> remote = asRemoteFile(absolutePath, file, getEndpoint().getCharset()); - if (endpoint.isRecursive() && depth < endpoint.getMaxDepth() && isValidFile(remote, true, files)) { - // recursive scan and add the sub files and folders - String subDirectory = file.getFilename(); - String path = ObjectHelper.isNotEmpty(absolutePath) ? absolutePath + "/" + subDirectory : subDirectory; - boolean canPollMore = pollSubDirectory(path, subDirectory, fileList, depth); - if (!canPollMore) { - return false; + if (endpoint.isRecursive() && depth < endpoint.getMaxDepth()) { + Supplier<GenericFile<SftpRemoteFile>> remote + = Suppliers.memorize(() -> asRemoteFile(absolutePath, file, getEndpoint().getCharset())); + if (isValidFile(remote, file.getFilename(), remote.get().getAbsoluteFilePath(), true, files)) { + // recursive scan and add the sub files and folders + String subDirectory = file.getFilename(); + String path = ObjectHelper.isNotEmpty(absolutePath) ? absolutePath + "/" + subDirectory : subDirectory; + boolean canPollMore = pollSubDirectory(path, subDirectory, fileList, depth); + if (!canPollMore) { + return false; + } } } // we cannot use file.getAttrs().isLink on Windows, so we dont // invoke the method // just assuming its a file we should poll } else { - RemoteFile<SftpRemoteFile> remote = asRemoteFile(absolutePath, file, getEndpoint().getCharset()); - if (depth >= endpoint.getMinDepth() && isValidFile(remote, false, files)) { - // matched file so add - fileList.add(remote); + if (depth >= endpoint.getMinDepth()) { + Supplier<GenericFile<SftpRemoteFile>> remote + = Suppliers.memorize(() -> asRemoteFile(absolutePath, file, getEndpoint().getCharset())); + if (isValidFile(remote, file.getFilename(), remote.get().getAbsoluteFilePath(), false, files)) { + // matched file so add + fileList.add(remote.get()); + } } } } diff --git a/components/camel-ftp/src/test/java/org/apache/camel/component/file/remote/sftp/integration/SftpMoveWithOutMessageTest.java b/components/camel-ftp/src/test/java/org/apache/camel/component/file/remote/sftp/integration/SftpMoveWithOutMessageTest.java index 1f01c1d386a..a529cc97cd8 100644 --- a/components/camel-ftp/src/test/java/org/apache/camel/component/file/remote/sftp/integration/SftpMoveWithOutMessageTest.java +++ b/components/camel-ftp/src/test/java/org/apache/camel/component/file/remote/sftp/integration/SftpMoveWithOutMessageTest.java @@ -24,6 +24,7 @@ import org.apache.camel.Processor; import org.apache.camel.ProducerTemplate; import org.apache.camel.builder.RouteBuilder; import org.apache.camel.support.DefaultMessage; +import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Timeout; import org.junit.jupiter.api.condition.EnabledIf; @@ -36,6 +37,7 @@ import static org.junit.jupiter.api.Assertions.assertTrue; * Test that the existence of a outMessage in an exchange will not break the move-file post-processing */ @EnabledIf(value = "org.apache.camel.test.infra.ftp.services.embedded.SftpUtil#hasRequiredAlgorithms('src/test/resources/hostkey.pem')") +@Disabled public class SftpMoveWithOutMessageTest extends SftpServerTestSupport { @Timeout(value = 30)
