This is an automated email from the ASF dual-hosted git repository. davsclaus pushed a commit to branch fo in repository https://gitbox.apache.org/repos/asf/camel.git
commit 4695413062a74fa6c0c20591dbef4d68c4c835b7 Author: Claus Ibsen <[email protected]> AuthorDate: Sun Dec 29 12:56:58 2024 +0100 CAMEL-17648: camel-file - Optimize file consumer when filtering file names. --- .../apache/camel/component/file/FileConsumer.java | 11 ++--- .../camel/component/file/GenericFileConsumer.java | 55 +++++++++++----------- 2 files changed, 32 insertions(+), 34 deletions(-) diff --git a/components/camel-file/src/main/java/org/apache/camel/component/file/FileConsumer.java b/components/camel-file/src/main/java/org/apache/camel/component/file/FileConsumer.java index 890231d9117..9c82e6c43d3 100644 --- a/components/camel-file/src/main/java/org/apache/camel/component/file/FileConsumer.java +++ b/components/camel-file/src/main/java/org/apache/camel/component/file/FileConsumer.java @@ -183,19 +183,19 @@ public class FileConsumer extends GenericFileConsumer<File> implements ResumeAwa } protected boolean isValidFile(Supplier<GenericFile<File>> gf, File file, boolean isDirectory, File[] files) { - if (!isMatched(file, isDirectory, files)) { + if (!isMatched(file, isDirectory)) { LOG.trace("File did not match. Will skip this file: {}", file); return false; } // optimized check done continue to use general check - return super.isValidFile(gf.get(), isDirectory, files); + return super.isValidFile(gf, file.getName(), file.getAbsolutePath(), isDirectory, files); } /** * Optimized check for is valid that uses java.io.File objects only, as creating the GenericFile object has overhead * when polling from file systems that contains a lot of files. */ - private boolean isMatched(File file, boolean isDirectory, File[] files) { + private boolean isMatched(File file, boolean isDirectory) { String name = file.getName(); if (!isMatchedHiddenFile(file, isDirectory)) { @@ -438,9 +438,8 @@ public class FileConsumer extends GenericFileConsumer<File> implements ResumeAwa } @Override - protected boolean isMatchedHiddenFile(GenericFile<File> file, boolean isDirectory) { + protected boolean isMatchedHiddenFile(Supplier<GenericFile<File>> file, String name, boolean isDirectory) { if (isDirectory) { - String name = file.getFileNameOnly(); if (!name.startsWith(".")) { return true; } @@ -450,7 +449,7 @@ public class FileConsumer extends GenericFileConsumer<File> implements ResumeAwa if (getEndpoint().isIncludeHiddenFiles()) { return true; } else { - return super.isMatchedHiddenFile(file, isDirectory); + return super.isMatchedHiddenFile(file, name, isDirectory); } } diff --git a/components/camel-file/src/main/java/org/apache/camel/component/file/GenericFileConsumer.java b/components/camel-file/src/main/java/org/apache/camel/component/file/GenericFileConsumer.java index 6cbe9d2ee43..e868f5a58bc 100644 --- a/components/camel-file/src/main/java/org/apache/camel/component/file/GenericFileConsumer.java +++ b/components/camel-file/src/main/java/org/apache/camel/component/file/GenericFileConsumer.java @@ -22,6 +22,7 @@ import java.util.Deque; import java.util.LinkedList; import java.util.List; import java.util.Queue; +import java.util.function.Supplier; import java.util.regex.Pattern; import org.apache.camel.CamelContextAware; @@ -580,13 +581,15 @@ public abstract class GenericFileConsumer<T> extends ScheduledBatchPollingConsum * Strategy for validating if the given remote file should be included or not * * @param file the file + * @param name the file name * @param isDirectory whether the file is a directory or a file * @param files files in the directory * @return <tt>true</tt> to include the file, <tt>false</tt> to skip it */ - protected boolean isValidFile(GenericFile<T> file, boolean isDirectory, T[] files) { - if (!isMatched(file, isDirectory, files)) { - LOG.trace("File did not match. Will skip this file: {}", file); + protected boolean isValidFile( + Supplier<GenericFile<T>> file, String name, String absoluteFilePath, boolean isDirectory, T[] files) { + if (!isMatched(file, name, absoluteFilePath, isDirectory, files)) { + LOG.trace("File did not match. Will skip this file: {}", name); return false; } @@ -596,10 +599,9 @@ public abstract class GenericFileConsumer<T> extends ScheduledBatchPollingConsum } // check if file is already in progress - String absoluteFilePath = file.getAbsoluteFilePath(); if (endpoint.getInProgressRepository().contains(absoluteFilePath)) { if (LOG.isTraceEnabled()) { - LOG.trace("Skipping as file is already in progress: {}", file.getFileName()); + LOG.trace("Skipping as file is already in progress: {}", name); } return false; } @@ -607,7 +609,7 @@ public abstract class GenericFileConsumer<T> extends ScheduledBatchPollingConsum // if it is a file then check we have the file in the idempotent registry // already if (Boolean.TRUE.equals(endpoint.isIdempotent())) { - if (notUnique(file)) { + if (notUnique(file, absoluteFilePath)) { return false; } } @@ -618,13 +620,13 @@ public abstract class GenericFileConsumer<T> extends ScheduledBatchPollingConsum return endpoint.getInProgressRepository().add(absoluteFilePath); } - private boolean notUnique(GenericFile<T> file) { + private boolean notUnique(Supplier<GenericFile<T>> file, String absoluteFilePath) { boolean answer = false; // use absolute file path as default key, but evaluate if an // expression key was configured - String key = file.getAbsoluteFilePath(); + String key = absoluteFilePath; if (endpoint.getIdempotentKey() != null) { - Exchange dummy = endpoint.createExchange(file); + Exchange dummy = endpoint.createExchange(file.get()); key = endpoint.getIdempotentKey().evaluate(dummy, String.class); LOG.trace("Evaluated idempotentKey: {} for file: {}", key, file); } @@ -648,9 +650,7 @@ public abstract class GenericFileConsumer<T> extends ScheduledBatchPollingConsum * <li>Starting with a dot (hidden)</li> * </ul> */ - protected boolean isMatchedHiddenFile(GenericFile<T> file, boolean isDirectory) { - String name = file.getFileNameOnly(); - + protected boolean isMatchedHiddenFile(Supplier<GenericFile<T>> file, String name, boolean isDirectory) { // folders/names starting with dot is always skipped (eg. ".", ".camel", // ".camelLock") if (name.startsWith(".")) { @@ -678,15 +678,15 @@ public abstract class GenericFileConsumer<T> extends ScheduledBatchPollingConsum * And then <tt>true</tt> for directories. * * @param file the file + * @param name the file name * @param isDirectory whether the file is a directory or a file * @param files files in the directory * @return <tt>true</tt> if the file is matched, <tt>false</tt> if not */ - protected boolean isMatched(GenericFile<T> file, boolean isDirectory, T[] files) { - String name = file.getFileNameOnly(); - + protected boolean isMatched( + Supplier<GenericFile<T>> file, String name, String absoluteFilePath, boolean isDirectory, T[] files) { // this has already been pre-checked - if (!isMatchedHiddenFile(file, isDirectory)) { + if (!isMatchedHiddenFile(file, name, isDirectory)) { // folders/names starting with dot is always skipped (eg. ".", ".camel", // ".camelLock") return false; @@ -698,13 +698,13 @@ public abstract class GenericFileConsumer<T> extends ScheduledBatchPollingConsum } if (endpoint.getFilter() != null) { - if (!endpoint.getFilter().accept(file)) { + if (!endpoint.getFilter().accept(file.get())) { return false; } } if (endpoint.getAntFilter() != null) { - if (!endpoint.getAntFilter().accept(file)) { + if (!endpoint.getAntFilter().accept(file.get())) { return false; } } @@ -712,7 +712,7 @@ public abstract class GenericFileConsumer<T> extends ScheduledBatchPollingConsum if (isDirectory && endpoint.getFilterDirectory() != null) { // create a dummy exchange as Exchange is needed for expression // evaluation - Exchange dummy = endpoint.createExchange(file); + Exchange dummy = endpoint.createExchange(file.get()); boolean matches = endpoint.getFilterDirectory().matches(dummy); if (!matches) { return false; @@ -725,13 +725,13 @@ public abstract class GenericFileConsumer<T> extends ScheduledBatchPollingConsum } // this has already been pre-checked - if (!isPreMatched() && hasInclusionsOrExclusions(file, name)) { + if (!isPreMatched() && hasInclusionsOrExclusions(name)) { return false; } if (endpoint.getFileName() != null) { // create a dummy exchange as Exchange is needed for expression evaluation - Exchange dummy = endpoint.createExchange(file); + Exchange dummy = endpoint.createExchange(file.get()); String result = evaluateFileExpression(dummy); if (result != null) { if (!name.equals(result)) { @@ -742,7 +742,7 @@ public abstract class GenericFileConsumer<T> extends ScheduledBatchPollingConsum if (endpoint.getFilterFile() != null) { // create a dummy exchange as Exchange is needed for expression evaluation - Exchange dummy = endpoint.createExchange(file); + Exchange dummy = endpoint.createExchange(file.get()); boolean matches = endpoint.getFilterFile().matches(dummy); if (!matches) { return false; @@ -753,16 +753,16 @@ public abstract class GenericFileConsumer<T> extends ScheduledBatchPollingConsum // file exists if (endpoint.getDoneFileName() != null) { // done file must be in same path as the file - String doneFileName = endpoint.createDoneFileName(file.getAbsoluteFilePath()); + String doneFileName = endpoint.createDoneFileName(absoluteFilePath); StringHelper.notEmpty(doneFileName, "doneFileName", endpoint); // is it a done file name? - if (endpoint.isDoneFile(file.getFileNameOnly())) { + if (endpoint.isDoneFile(name)) { LOG.trace("Skipping done file: {}", file); return false; } - if (!isMatched(file, doneFileName, files)) { + if (!isMatched(file.get(), doneFileName, files)) { return false; } } @@ -770,15 +770,15 @@ public abstract class GenericFileConsumer<T> extends ScheduledBatchPollingConsum return true; } - private boolean hasInclusionsOrExclusions(GenericFile<T> file, String name) { + private boolean hasInclusionsOrExclusions(String name) { // exclude take precedence over include if (excludePattern != null) { if (excludePattern.matcher(name).matches()) { return true; } } + String fname = name.toLowerCase(); if (excludeExt != null) { - String fname = file.getFileName().toLowerCase(); if (hasExtExlusions(fname)) { return true; } @@ -789,7 +789,6 @@ public abstract class GenericFileConsumer<T> extends ScheduledBatchPollingConsum } } if (includeExt != null) { - String fname = file.getFileName().toLowerCase(); if (hasExtInclusions(fname)) { return true; }
