martin-g commented on code in PR #3625:
URL: https://github.com/apache/avro/pull/3625#discussion_r2690697321
##########
lang/java/avro/src/main/java/org/apache/avro/file/DeflateCodec.java:
##########
@@ -38,6 +39,20 @@
public class DeflateCodec extends Codec {
private static final int DEFAULT_BUFFER_SIZE = 8192;
+ private static final String MAX_DECOMPRESS_LENGTH_PROPERTY =
"org.apache.avro.limits.decompress.maxLength";
+ private static final long DEFAULT_MAX_DECOMPRESS_LENGTH = 200L * 1024 *
1024; // 200MB default limit
+
+ private static long getMaxDecompressLength() {
+ String prop = System.getProperty(MAX_DECOMPRESS_LENGTH_PROPERTY);
+ if (prop != null) {
+ try {
+ return Long.parseLong(prop);
+ } catch (NumberFormatException e) {
+ // Use default
Review Comment:
This probably should be logged as a WARNING.
##########
lang/java/avro/src/main/java/org/apache/avro/file/DeflateCodec.java:
##########
@@ -38,6 +39,20 @@
public class DeflateCodec extends Codec {
private static final int DEFAULT_BUFFER_SIZE = 8192;
+ private static final String MAX_DECOMPRESS_LENGTH_PROPERTY =
"org.apache.avro.limits.decompress.maxLength";
+ private static final long DEFAULT_MAX_DECOMPRESS_LENGTH = 200L * 1024 *
1024; // 200MB default limit
+
+ private static long getMaxDecompressLength() {
+ String prop = System.getProperty(MAX_DECOMPRESS_LENGTH_PROPERTY);
+ if (prop != null) {
+ try {
+ return Long.parseLong(prop);
Review Comment:
This will also accept a negative and 0 as values which are not very sensible.
Probably these should be reported earlier here ?!
##########
lang/java/avro/src/main/java/org/apache/avro/file/DeflateCodec.java:
##########
@@ -78,10 +93,32 @@ public ByteBuffer compress(ByteBuffer data) throws
IOException {
@Override
public ByteBuffer decompress(ByteBuffer data) throws IOException {
+ long maxLength = getMaxDecompressLength();
NonCopyingByteArrayOutputStream baos = new
NonCopyingByteArrayOutputStream(DEFAULT_BUFFER_SIZE);
- try (OutputStream outputStream = new InflaterOutputStream(baos,
getInflater())) {
- outputStream.write(data.array(), computeOffset(data), data.remaining());
+ byte[] buffer = new byte[DEFAULT_BUFFER_SIZE];
+ long totalBytes = 0;
+
+ Inflater inflater = getInflater();
+ inflater.setInput(data.array(), computeOffset(data), data.remaining());
+
+ try {
+ while (!inflater.finished()) {
+ int len = inflater.inflate(buffer);
+ if (len == 0 && inflater.needsInput()) {
+ break;
+ }
+ totalBytes += len;
+ if (totalBytes > maxLength) {
+ throw new AvroRuntimeException(
Review Comment:
```suggestion
throw new IOException(
```
The method declares that it throws IOException when there are problems.
##########
lang/java/avro/src/main/java/org/apache/avro/file/DeflateCodec.java:
##########
@@ -78,10 +93,32 @@ public ByteBuffer compress(ByteBuffer data) throws
IOException {
@Override
public ByteBuffer decompress(ByteBuffer data) throws IOException {
+ long maxLength = getMaxDecompressLength();
NonCopyingByteArrayOutputStream baos = new
NonCopyingByteArrayOutputStream(DEFAULT_BUFFER_SIZE);
- try (OutputStream outputStream = new InflaterOutputStream(baos,
getInflater())) {
- outputStream.write(data.array(), computeOffset(data), data.remaining());
+ byte[] buffer = new byte[DEFAULT_BUFFER_SIZE];
+ long totalBytes = 0;
+
+ Inflater inflater = getInflater();
+ inflater.setInput(data.array(), computeOffset(data), data.remaining());
+
+ try {
+ while (!inflater.finished()) {
+ int len = inflater.inflate(buffer);
+ if (len == 0 && inflater.needsInput()) {
+ break;
+ }
+ totalBytes += len;
+ if (totalBytes > maxLength) {
+ throw new AvroRuntimeException(
+ "Decompressed size " + totalBytes + " exceeds maximum allowed
size " + maxLength
Review Comment:
```suggestion
"Decompressed size " + totalBytes + " (bytes) exceeds maximum
allowed size " + maxLength
```
##########
lang/java/avro/src/main/java/org/apache/avro/file/DeflateCodec.java:
##########
@@ -78,10 +93,32 @@ public ByteBuffer compress(ByteBuffer data) throws
IOException {
@Override
public ByteBuffer decompress(ByteBuffer data) throws IOException {
+ long maxLength = getMaxDecompressLength();
Review Comment:
There is no need to call this method on every `decompress()`.
You can read it once in a `static {...}` block and reuse it.
##########
lang/java/avro/src/main/java/org/apache/avro/file/DeflateCodec.java:
##########
@@ -78,10 +93,32 @@ public ByteBuffer compress(ByteBuffer data) throws
IOException {
@Override
public ByteBuffer decompress(ByteBuffer data) throws IOException {
+ long maxLength = getMaxDecompressLength();
NonCopyingByteArrayOutputStream baos = new
NonCopyingByteArrayOutputStream(DEFAULT_BUFFER_SIZE);
- try (OutputStream outputStream = new InflaterOutputStream(baos,
getInflater())) {
- outputStream.write(data.array(), computeOffset(data), data.remaining());
+ byte[] buffer = new byte[DEFAULT_BUFFER_SIZE];
+ long totalBytes = 0;
+
+ Inflater inflater = getInflater();
+ inflater.setInput(data.array(), computeOffset(data), data.remaining());
+
+ try {
+ while (!inflater.finished()) {
+ int len = inflater.inflate(buffer);
+ if (len == 0 && inflater.needsInput()) {
+ break;
+ }
+ totalBytes += len;
+ if (totalBytes > maxLength) {
+ throw new AvroRuntimeException(
+ "Decompressed size " + totalBytes + " exceeds maximum allowed
size " + maxLength
+ + ". This can be configured by setting the system property "
+ MAX_DECOMPRESS_LENGTH_PROPERTY);
Review Comment:
```suggestion
+ ". This can be configured by setting the system property
'" + MAX_DECOMPRESS_LENGTH_PROPERTY + "'");
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
