This is an automated email from the ASF dual-hosted git repository.
davsclaus pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/camel.git
The following commit(s) were added to refs/heads/main by this push:
new 96725b787a60 CAMEL-25905: camel-file - the checksum of
checksumFileAlgorithm is calculated from the bytes written to the file (#28429)
96725b787a60 is described below
commit 96725b787a6076489f5eced72f66e8633df3153a
Author: allthingssecurity <[email protected]>
AuthorDate: Sun Oct 25 13:01:09 2026 +0530
CAMEL-25905: camel-file - the checksum of checksumFileAlgorithm is
calculated from the bytes written to the file (#28429)
Co-Authored-By: Claude Opus 5.5 <[email protected]>
Signed-off-by: Claus Ibsen <[email protected]>
---
.../camel/component/file/FileOperations.java | 77 +++++++++++--------
.../FileProducerChecksumOfWrittenFileTest.java | 88 ++++++++++++++++++++++
.../ROOT/pages/camel-4x-upgrade-guide-4_24.adoc | 11 +++
3 files changed, 143 insertions(+), 33 deletions(-)
diff --git
a/components/camel-file/src/main/java/org/apache/camel/component/file/FileOperations.java
b/components/camel-file/src/main/java/org/apache/camel/component/file/FileOperations.java
index 9d3d2395c1d6..2d99e5f682db 100644
---
a/components/camel-file/src/main/java/org/apache/camel/component/file/FileOperations.java
+++
b/components/camel-file/src/main/java/org/apache/camel/component/file/FileOperations.java
@@ -22,6 +22,8 @@ import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
+import java.io.OutputStream;
+import java.io.OutputStreamWriter;
import java.io.Reader;
import java.io.Writer;
import java.nio.Buffer;
@@ -29,13 +31,14 @@ import java.nio.ByteBuffer;
import java.nio.channels.FileChannel;
import java.nio.channels.SeekableByteChannel;
import java.nio.charset.Charset;
+import java.nio.charset.CharsetEncoder;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.StandardCopyOption;
import java.nio.file.StandardOpenOption;
import java.nio.file.attribute.PosixFilePermission;
import java.nio.file.attribute.PosixFilePermissions;
-import java.security.DigestInputStream;
+import java.security.DigestOutputStream;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.util.Date;
@@ -346,31 +349,26 @@ public class FileOperations implements
GenericFileOperations<File> {
}
}
- // if we use checksum them we need to use an input stream to
calculate while writing the file
- DigestInputStream dis = null;
+ // the checksum is calculated from the bytes written to the file,
which are not the bytes of the body
+ // when they are encoded with the charset of the endpoint or
followed by the appendChars
+ MessageDigest digest = null;
if (endpoint.getChecksumFileAlgorithm() != null) {
- MessageDigest digest =
MessageDigest.getInstance(endpoint.getChecksumFileAlgorithm());
- // prefer FileInputStream over general stream
- InputStream is =
exchange.getMessage().getBody(FileInputStream.class);
- if (is == null) {
- is = mandatoryBodyStream(exchange, exchange.getMessage(),
file);
- }
- dis = new DigestInputStream(is, digest);
+ digest =
MessageDigest.getInstance(endpoint.getChecksumFileAlgorithm());
}
if (charset != null) {
// charset configured so we must use a reader so we can write
// with encoding
- handleReaderAsFileSource(exchange, file, charset, dis);
- } else if (dis == null && exchange.getIn().getBody() instanceof
String) {
+ handleReaderAsFileSource(exchange, file, charset, digest);
+ } else if (digest == null && exchange.getIn().getBody() instanceof
String) {
// If the body is a string, write it directly
handleStringAsFileSource(exchange, file);
} else {
// fallback and use stream based
- handleStreamAsFileSource(exchange, file, dis);
+ handleStreamAsFileSource(exchange, file, digest);
}
- if (dis != null) {
- String hash =
StringHelper.bytesToHex(dis.getMessageDigest().digest());
+ if (digest != null) {
+ String hash = StringHelper.bytesToHex(digest.digest());
exchange.getMessage().setHeader(FileConstants.FILE_CHECKSUM,
hash);
}
@@ -423,12 +421,17 @@ public class FileOperations implements
GenericFileOperations<File> {
return true;
}
- private void handleStreamAsFileSource(Exchange exchange, File file,
InputStream is)
+ private void handleStreamAsFileSource(Exchange exchange, File file,
MessageDigest digest)
throws InvalidPayloadException, IOException {
+ InputStream is = null;
+ if (digest != null) {
+ // prefer FileInputStream over general stream
+ is = exchange.getMessage().getBody(FileInputStream.class);
+ }
if (is == null) {
is = mandatoryBodyStream(exchange, exchange.getIn(), file);
}
- writeFileByStream(is, file);
+ writeFileByStream(is, file, digest);
}
private void handleStringAsFileSource(Exchange exchange, File file) throws
IOException {
@@ -436,24 +439,18 @@ public class FileOperations implements
GenericFileOperations<File> {
writeFileByString(stringBody, file);
}
- private void handleReaderAsFileSource(Exchange exchange, File file, String
charset, InputStream is)
+ private void handleReaderAsFileSource(Exchange exchange, File file, String
charset, MessageDigest digest)
throws InvalidPayloadException, IOException {
- Reader in = null;
- if (is == null) {
- in =
exchange.getContext().getTypeConverter().tryConvertTo(Reader.class, exchange,
- exchange.getIn().getBody());
- }
+ Reader in =
exchange.getContext().getTypeConverter().tryConvertTo(Reader.class, exchange,
+ exchange.getIn().getBody());
if (in == null) {
// okay no direct reader conversion, so use an input stream
// (which a lot can be converted as)
- if (is == null) {
- is = mandatoryBodyStream(exchange, exchange.getIn(), file);
- }
- in = new InputStreamReader(is);
+ in = new InputStreamReader(mandatoryBodyStream(exchange,
exchange.getIn(), file));
}
// buffer the reader
in = IOHelper.buffered(in);
- writeFileByReaderWithCharset(in, file, charset);
+ writeFileByReaderWithCharset(in, file, charset, digest);
}
private boolean handleFileAsFileSource(Exchange exchange, File file, File
source) throws IOException {
@@ -597,7 +594,7 @@ public class FileOperations implements
GenericFileOperations<File> {
}
}
- private void writeFileByStream(InputStream in, File target) throws
IOException {
+ private void writeFileByStream(InputStream in, File target, MessageDigest
digest) throws IOException {
try (SeekableByteChannel out = prepareOutputFileChannel(target)) {
LOG.debug("Using InputStream to write file: {}", target);
int size = endpoint.getBufferSize();
@@ -610,12 +607,19 @@ public class FileOperations implements
GenericFileOperations<File> {
}
out.write(byteBuffer);
((Buffer) byteBuffer).clear();
+ if (digest != null) {
+ digest.update(buffer, 0, bytesRead);
+ }
}
boolean append = endpoint.getFileExist() ==
GenericFileExist.Append;
if (append && endpoint.getAppendChars() != null) {
- byteBuffer =
ByteBuffer.wrap(endpoint.getAppendChars().getBytes());
+ byte[] appendBytes = endpoint.getAppendChars().getBytes();
+ byteBuffer = ByteBuffer.wrap(appendBytes);
out.write(byteBuffer);
+ if (digest != null) {
+ digest.update(appendBytes);
+ }
Buffer buf = byteBuffer;
buf.clear();
}
@@ -624,10 +628,17 @@ public class FileOperations implements
GenericFileOperations<File> {
}
}
- private void writeFileByReaderWithCharset(Reader in, File target, String
charset) throws IOException {
+ private void writeFileByReaderWithCharset(Reader in, File target, String
charset, MessageDigest digest)
+ throws IOException {
boolean append = endpoint.getFileExist() == GenericFileExist.Append;
- try (Writer out = Files.newBufferedWriter(target.toPath(),
Charset.forName(charset), StandardOpenOption.WRITE,
- append ? StandardOpenOption.APPEND :
StandardOpenOption.TRUNCATE_EXISTING, StandardOpenOption.CREATE)) {
+ // an encoder that reports characters the charset cannot encode, as
Files.newBufferedWriter does
+ CharsetEncoder encoder = Charset.forName(charset).newEncoder();
+ OutputStream os = Files.newOutputStream(target.toPath(),
StandardOpenOption.WRITE,
+ append ? StandardOpenOption.APPEND :
StandardOpenOption.TRUNCATE_EXISTING, StandardOpenOption.CREATE);
+ if (digest != null) {
+ os = new DigestOutputStream(os, digest);
+ }
+ try (Writer out = IOHelper.buffered(new OutputStreamWriter(os,
encoder))) {
LOG.debug("Using Reader to write file: {} with charset: {}",
target, charset);
int size = endpoint.getBufferSize();
IOHelper.copy(in, out, size);
diff --git
a/core/camel-core/src/test/java/org/apache/camel/component/file/FileProducerChecksumOfWrittenFileTest.java
b/core/camel-core/src/test/java/org/apache/camel/component/file/FileProducerChecksumOfWrittenFileTest.java
new file mode 100644
index 000000000000..68e29b381ebb
--- /dev/null
+++
b/core/camel-core/src/test/java/org/apache/camel/component/file/FileProducerChecksumOfWrittenFileTest.java
@@ -0,0 +1,88 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.camel.component.file;
+
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.security.MessageDigest;
+
+import org.apache.camel.ContextTestSupport;
+import org.apache.camel.Exchange;
+import org.apache.camel.util.StringHelper;
+import org.junit.jupiter.api.Test;
+
+import static org.junit.jupiter.api.Assertions.assertArrayEquals;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+/**
+ * The checksum of checksumFileAlgorithm is the checksum of the file that has
been written, also when the producer
+ * encodes the body with its charset or adds the appendChars, and adding a
checksum does not change how the body is
+ * decoded.
+ */
+class FileProducerChecksumOfWrittenFileTest extends ContextTestSupport {
+
+ private static final String TEXT = "Grüße aus Köln";
+
+ private String sha256OfFile(String name) throws Exception {
+ byte[] data = Files.readAllBytes(testFile(name));
+ return
StringHelper.bytesToHex(MessageDigest.getInstance("SHA-256").digest(data));
+ }
+
+ private String checksumFile(String name) throws Exception {
+ return Files.readString(testFile(name + ".sha256"));
+ }
+
+ @Test
+ void transcodedFileHasItsOwnChecksumAndContent() throws Exception {
+ // the bytes of an ISO-8859-1 file, as a file consumer with
charset=ISO-8859-1 hands them over
+ Exchange out =
template.request(fileUri("?charset=UTF-8&checksumFileAlgorithm=sha256"),
exchange -> {
+ exchange.setProperty(Exchange.CHARSET_NAME, "ISO-8859-1");
+ exchange.getIn().setHeader(Exchange.FILE_NAME, "transcoded.txt");
+
exchange.getIn().setBody(TEXT.getBytes(StandardCharsets.ISO_8859_1));
+ });
+
+ assertArrayEquals(TEXT.getBytes(StandardCharsets.UTF_8),
Files.readAllBytes(testFile("transcoded.txt")));
+ assertEquals(sha256OfFile("transcoded.txt"),
checksumFile("transcoded.txt"));
+ assertEquals(sha256OfFile("transcoded.txt"),
out.getMessage().getHeader(FileConstants.FILE_CHECKSUM));
+ }
+
+ @Test
+ void stringWrittenWithCharsetHasTheChecksumOfTheFile() throws Exception {
+
template.sendBodyAndHeader(fileUri("?charset=ISO-8859-1&checksumFileAlgorithm=sha256"),
TEXT,
+ Exchange.FILE_NAME, "latin1.txt");
+
+ assertArrayEquals(TEXT.getBytes(StandardCharsets.ISO_8859_1),
Files.readAllBytes(testFile("latin1.txt")));
+ assertEquals(sha256OfFile("latin1.txt"), checksumFile("latin1.txt"));
+ }
+
+ @Test
+ void appendCharsArePartOfTheChecksum() throws Exception {
+
template.sendBodyAndHeader(fileUri("?fileExist=Append&appendChars=\\n&checksumFileAlgorithm=sha256"),
+ "Hello World", Exchange.FILE_NAME, "appended.txt");
+
+ assertEquals("Hello World\n",
Files.readString(testFile("appended.txt")));
+ assertEquals(sha256OfFile("appended.txt"),
checksumFile("appended.txt"));
+ }
+
+ @Test
+ void bodyWithoutCharsetKeepsItsChecksum() throws Exception {
+ template.sendBodyAndHeader(fileUri("?checksumFileAlgorithm=sha256"),
TEXT, Exchange.FILE_NAME, "plain.txt");
+
+ assertArrayEquals(TEXT.getBytes(StandardCharsets.UTF_8),
Files.readAllBytes(testFile("plain.txt")));
+ assertEquals(sha256OfFile("plain.txt"), checksumFile("plain.txt"));
+ }
+}
diff --git
a/docs/user-manual/modules/ROOT/pages/camel-4x-upgrade-guide-4_24.adoc
b/docs/user-manual/modules/ROOT/pages/camel-4x-upgrade-guide-4_24.adoc
index 597f745d7393..7d2ae9d195d6 100644
--- a/docs/user-manual/modules/ROOT/pages/camel-4x-upgrade-guide-4_24.adoc
+++ b/docs/user-manual/modules/ROOT/pages/camel-4x-upgrade-guide-4_24.adoc
@@ -328,6 +328,17 @@ The generated Endpoint DSL header accessors on the Swift
`HeaderNameBuilder` are
`openstackSwiftVersionsLocation()`, `xContainerRead()` ->
`openstackSwiftContainerRead()`, `xContainerWrite()` ->
`openstackSwiftContainerWrite()`).
+=== camel-file - the checksum of checksumFileAlgorithm is the checksum of the
written file
+
+With `checksumFileAlgorithm`, the file producer calculated the checksum from
the body before it was written. With
+`charset` set, it also decoded that body with the default charset of the JVM
instead of the charset of the exchange.
+So with `charset`, the checksum (the `CamelFileChecksum` header and the
checksum file) did not match the written file
+as soon as the encoding changed a byte, a body read with another charset (such
as the bytes of a file consumed with
+`charset=ISO-8859-1`) was written with its non-ASCII characters replaced, and
a String body could fail with an
+`UnmappableCharacterException`; with `fileExist=Append` and `appendChars`, the
checksum left out the appended
+characters. The checksum is now calculated from the bytes written to the file,
and the body is decoded as it is
+without `checksumFileAlgorithm`. Checksums of such files differ from the ones
written before.
+
=== camel-pqc - an unknown hybridKdfAlgorithm is rejected
The hybrid KEM operations now fail with an `IllegalArgumentException` when
`hybridKdfAlgorithm` is not one of
