This is an automated email from the ASF dual-hosted git repository. davsclaus pushed a commit to branch fo in repository https://gitbox.apache.org/repos/asf/camel.git
commit 1e2211825c69c433bec135c24132577d71e99ebc Author: Claus Ibsen <[email protected]> AuthorDate: Sun Dec 29 13:51:50 2024 +0100 CAMEL-17648: camel-file - Optimize file consumer when filtering file names. --- .../camel/component/file/azure/FilesConsumer.java | 6 ++-- .../camel/component/file/remote/FtpConsumer.java | 33 ++++++++------------ .../camel/component/file/remote/FtpUtils.java | 35 ++++++++++++++++++++-- .../camel/component/file/remote/SftpConsumer.java | 28 ++++++++--------- 4 files changed, 61 insertions(+), 41 deletions(-) diff --git a/components/camel-azure/camel-azure-files/src/main/java/org/apache/camel/component/file/azure/FilesConsumer.java b/components/camel-azure/camel-azure-files/src/main/java/org/apache/camel/component/file/azure/FilesConsumer.java index 0bca7138016..081d4c6c355 100644 --- a/components/camel-azure/camel-azure-files/src/main/java/org/apache/camel/component/file/azure/FilesConsumer.java +++ b/components/camel-azure/camel-azure-files/src/main/java/org/apache/camel/component/file/azure/FilesConsumer.java @@ -155,7 +155,8 @@ public class FilesConsumer extends RemoteFileConsumer<ShareFileItem> { if (endpoint.isRecursive() && depth < endpoint.getMaxDepth()) { Supplier<GenericFile<ShareFileItem>> remote = Suppliers.memorize(() -> asRemoteFile(path, dir)); - if (isValidFile(remote, dir.getName(), remote.get().getAbsoluteFilePath(), true, listedFileItems)) { + String absoluteFilePath = FilesPath.concat(path, dir.getName()); + if (isValidFile(remote, dir.getName(), absoluteFilePath, true, listedFileItems)) { String dirName = dir.getName(); String dirPath = FilesPath.concat(path, dirName); boolean canPollMore = doSafePollSubDirectory(dirPath, dirName, polledFiles, depth); @@ -172,7 +173,8 @@ public class FilesConsumer extends RemoteFileConsumer<ShareFileItem> { ShareFileItem[] listedFileItems, ShareFileItem file) { if (depth >= endpoint.getMinDepth()) { Supplier<GenericFile<ShareFileItem>> remote = Suppliers.memorize(() -> asRemoteFile(path, file)); - if (isValidFile(remote, file.getName(), remote.get().getAbsoluteFilePath(), false, listedFileItems)) { + String absoluteFilePath = FilesPath.concat(path, file.getName()); + if (isValidFile(remote, file.getName(), absoluteFilePath, false, listedFileItems)) { polledFiles.add(remote.get()); } } diff --git a/components/camel-ftp/src/main/java/org/apache/camel/component/file/remote/FtpConsumer.java b/components/camel-ftp/src/main/java/org/apache/camel/component/file/remote/FtpConsumer.java index 73169e1c794..1112ed48109 100644 --- a/components/camel-ftp/src/main/java/org/apache/camel/component/file/remote/FtpConsumer.java +++ b/components/camel-ftp/src/main/java/org/apache/camel/component/file/remote/FtpConsumer.java @@ -184,9 +184,12 @@ public class FtpConsumer extends RemoteFileConsumer<FTPFile> { private boolean handleDirectory( String absolutePath, List<GenericFile<FTPFile>> fileList, int depth, FTPFile[] files, FTPFile file) { if (endpoint.isRecursive() && depth < endpoint.getMaxDepth()) { + // calculate the absolute file path using util class + String absoluteFilePath + = FtpUtils.absoluteFilePath((FtpConfiguration) endpoint.getConfiguration(), absolutePath, file.getName()); Supplier<GenericFile<FTPFile>> remote - = Suppliers.memorize(() -> asRemoteFile(absolutePath, file, getEndpoint().getCharset())); - if (isValidFile(remote, file.getName(), remote.get().getAbsoluteFilePath(), true, files)) { + = Suppliers.memorize(() -> asRemoteFile(absolutePath, absoluteFilePath, file, getEndpoint().getCharset())); + if (isValidFile(remote, file.getName(), absoluteFilePath, true, files)) { // recursive scan and add the sub files and folders String subDirectory = file.getName(); String path = ObjectHelper.isNotEmpty(absolutePath) ? absolutePath + "/" + subDirectory : subDirectory; @@ -202,9 +205,12 @@ public class FtpConsumer extends RemoteFileConsumer<FTPFile> { private void handleFile( String absolutePath, List<GenericFile<FTPFile>> fileList, int depth, FTPFile[] files, FTPFile file) { if (depth >= endpoint.getMinDepth()) { + // calculate the absolute file path using util class + String absoluteFilePath + = FtpUtils.absoluteFilePath((FtpConfiguration) endpoint.getConfiguration(), absolutePath, file.getName()); Supplier<GenericFile<FTPFile>> remote - = Suppliers.memorize(() -> asRemoteFile(absolutePath, file, getEndpoint().getCharset())); - if (isValidFile(remote, file.getName(), remote.get().getAbsoluteFilePath(), false, files)) { + = Suppliers.memorize(() -> asRemoteFile(absolutePath, absoluteFilePath, file, getEndpoint().getCharset())); + if (isValidFile(remote, file.getName(), absoluteFilePath, false, files)) { // matched file so add fileList.add(remote.get()); } @@ -302,7 +308,7 @@ public class FtpConsumer extends RemoteFileConsumer<FTPFile> { return super.ignoreCannotRetrieveFile(name, exchange, cause); } - private RemoteFile<FTPFile> asRemoteFile(String absolutePath, FTPFile file, String charset) { + private RemoteFile<FTPFile> asRemoteFile(String absolutePath, String absoluteFilePath, FTPFile file, String charset) { RemoteFile<FTPFile> answer = new RemoteFile<>(); answer.setCharset(charset); @@ -319,23 +325,10 @@ public class FtpConsumer extends RemoteFileConsumer<FTPFile> { // absolute or relative path boolean absolute = FileUtil.hasLeadingSeparator(absolutePath); answer.setAbsolute(absolute); - - // create a pseudo absolute name - String dir = FileUtil.stripTrailingSeparator(absolutePath); - String fileName = file.getName(); - if (((FtpConfiguration) endpoint.getConfiguration()).isHandleDirectoryParserAbsoluteResult()) { - fileName = FtpUtils.extractDirNameFromAbsolutePath(file.getName()); - } - String absoluteFileName = FileUtil.stripLeadingSeparator(dir + "/" + fileName); - // if absolute start with a leading separator otherwise let it be - // relative - if (absolute) { - absoluteFileName = "/" + absoluteFileName; - } - answer.setAbsoluteFilePath(absoluteFileName); + answer.setAbsoluteFilePath(absoluteFilePath); // the relative filename, skip the leading endpoint configured path - String relativePath = StringHelper.after(absoluteFileName, endpointPath); + String relativePath = StringHelper.after(absoluteFilePath, endpointPath); // skip leading / relativePath = FileUtil.stripLeadingSeparator(relativePath); answer.setRelativeFilePath(relativePath); diff --git a/components/camel-ftp/src/main/java/org/apache/camel/component/file/remote/FtpUtils.java b/components/camel-ftp/src/main/java/org/apache/camel/component/file/remote/FtpUtils.java index ec84ed12952..f39f1d36b16 100644 --- a/components/camel-ftp/src/main/java/org/apache/camel/component/file/remote/FtpUtils.java +++ b/components/camel-ftp/src/main/java/org/apache/camel/component/file/remote/FtpUtils.java @@ -126,9 +126,7 @@ public final class FtpUtils { * Checks whether directory used in ftp/ftps/sftp endpoint URI is relative. Absolute path will be converted to * relative path and a WARN will be printed. * - * @see <a href="http://camel.apache.org/ftp2.html";>FTP/SFTP/FTPS Component</a> - * @param ftpComponent - * @param configuration + * @see <a href="http://camel.apache.org/ftp2.html";>FTP/SFTP/FTPS Component</a> */ public static void ensureRelativeFtpDirectory(Component ftpComponent, RemoteFileConfiguration configuration) { if (FileUtil.hasLeadingSeparator(configuration.getDirectoryName())) { @@ -141,4 +139,35 @@ public final class FtpUtils { } } + public static String absoluteFilePath(FtpConfiguration configuration, String absolutePath, String name) { + boolean absolute = FileUtil.hasLeadingSeparator(absolutePath); + // create a pseudo absolute name + String dir = FileUtil.stripTrailingSeparator(absolutePath); + String fileName = name; + if (configuration.isHandleDirectoryParserAbsoluteResult()) { + fileName = FtpUtils.extractDirNameFromAbsolutePath(name); + } + String absoluteFileName = FileUtil.stripLeadingSeparator(dir + "/" + fileName); + // if absolute start with a leading separator otherwise let it be + // relative + if (absolute) { + absoluteFileName = "/" + absoluteFileName; + } + return absoluteFileName; + } + + public static String absoluteFilePath(String absolutePath, String name) { + boolean absolute = FileUtil.hasLeadingSeparator(absolutePath); + + // create a pseudo absolute name + String dir = FileUtil.stripTrailingSeparator(absolutePath); + String absoluteFileName = FileUtil.stripLeadingSeparator(dir + "/" + name); + // if absolute start with a leading separator otherwise let it be + // relative + if (absolute) { + absoluteFileName = "/" + absoluteFileName; + } + return absoluteFileName; + } + } diff --git a/components/camel-ftp/src/main/java/org/apache/camel/component/file/remote/SftpConsumer.java b/components/camel-ftp/src/main/java/org/apache/camel/component/file/remote/SftpConsumer.java index 7e9503a93cd..0c7f4dad1e5 100644 --- a/components/camel-ftp/src/main/java/org/apache/camel/component/file/remote/SftpConsumer.java +++ b/components/camel-ftp/src/main/java/org/apache/camel/component/file/remote/SftpConsumer.java @@ -163,9 +163,11 @@ public class SftpConsumer extends RemoteFileConsumer<SftpRemoteFile> { if (file.isDirectory()) { if (endpoint.isRecursive() && depth < endpoint.getMaxDepth()) { + String absoluteFilePath = FtpUtils.absoluteFilePath(absolutePath, file.getFilename()); Supplier<GenericFile<SftpRemoteFile>> remote - = Suppliers.memorize(() -> asRemoteFile(absolutePath, file, getEndpoint().getCharset())); - if (isValidFile(remote, file.getFilename(), remote.get().getAbsoluteFilePath(), true, files)) { + = Suppliers.memorize( + () -> asRemoteFile(absolutePath, absoluteFilePath, file, getEndpoint().getCharset())); + if (isValidFile(remote, file.getFilename(), absoluteFilePath, true, files)) { // recursive scan and add the sub files and folders String subDirectory = file.getFilename(); String path = ObjectHelper.isNotEmpty(absolutePath) ? absolutePath + "/" + subDirectory : subDirectory; @@ -180,9 +182,11 @@ public class SftpConsumer extends RemoteFileConsumer<SftpRemoteFile> { // just assuming its a file we should poll } else { if (depth >= endpoint.getMinDepth()) { + String absoluteFilePath = FtpUtils.absoluteFilePath(absolutePath, file.getFilename()); Supplier<GenericFile<SftpRemoteFile>> remote - = Suppliers.memorize(() -> asRemoteFile(absolutePath, file, getEndpoint().getCharset())); - if (isValidFile(remote, file.getFilename(), remote.get().getAbsoluteFilePath(), false, files)) { + = Suppliers.memorize( + () -> asRemoteFile(absolutePath, absoluteFilePath, file, getEndpoint().getCharset())); + if (isValidFile(remote, file.getFilename(), absoluteFilePath, false, files)) { // matched file so add fileList.add(remote.get()); } @@ -261,7 +265,8 @@ public class SftpConsumer extends RemoteFileConsumer<SftpRemoteFile> { return super.ignoreCannotRetrieveFile(name, exchange, cause); } - private RemoteFile<SftpRemoteFile> asRemoteFile(String absolutePath, SftpRemoteFile file, String charset) { + private RemoteFile<SftpRemoteFile> asRemoteFile( + String absolutePath, String absoluteFilePath, SftpRemoteFile file, String charset) { RemoteFile<SftpRemoteFile> answer = new RemoteFile<>(); answer.setCharset(charset); @@ -276,19 +281,10 @@ public class SftpConsumer extends RemoteFileConsumer<SftpRemoteFile> { // absolute or relative path boolean absolute = FileUtil.hasLeadingSeparator(absolutePath); answer.setAbsolute(absolute); - - // create a pseudo absolute name - String dir = FileUtil.stripTrailingSeparator(absolutePath); - String absoluteFileName = FileUtil.stripLeadingSeparator(dir + "/" + file.getFilename()); - // if absolute start with a leading separator otherwise let it be - // relative - if (absolute) { - absoluteFileName = "/" + absoluteFileName; - } - answer.setAbsoluteFilePath(absoluteFileName); + answer.setAbsoluteFilePath(absoluteFilePath); // the relative filename, skip the leading endpoint configured path - String relativePath = StringHelper.after(absoluteFileName, endpointPath); + String relativePath = StringHelper.after(absoluteFilePath, endpointPath); // skip trailing / relativePath = FileUtil.stripLeadingSeparator(relativePath); answer.setRelativeFilePath(relativePath);
