This is an automated email from the ASF dual-hosted git repository. HTHou pushed a commit to branch codex/cherry-pick-16724-dev-1.3 in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit ba266caf644a4152b7efeeca8d32143bd6195e9a Author: HTHou <[email protected]> AuthorDate: Mon Nov 10 14:42:35 2025 +0800 Add thrift max frame size calculate logic --- .../rpc/TCompressedElasticFramedTransport.java | 8 +-- .../apache/iotdb/rpc/TElasticFramedTransport.java | 80 +++++++++++++++++----- .../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 9 ++- .../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 4 -- .../conf/iotdb-system.properties.template | 4 +- .../apache/iotdb/commons/conf/IoTDBConstant.java | 1 - 6 files changed, 71 insertions(+), 35 deletions(-) diff --git a/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/TCompressedElasticFramedTransport.java b/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/TCompressedElasticFramedTransport.java index 8579f46a3cb..07768dd98bd 100644 --- a/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/TCompressedElasticFramedTransport.java +++ b/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/TCompressedElasticFramedTransport.java @@ -44,13 +44,7 @@ public abstract class TCompressedElasticFramedTransport extends TElasticFramedTr protected void readFrame() throws TTransportException { underlying.readAll(i32buf, 0, 4); int size = TFramedTransport.decodeFrameSize(i32buf); - - if (size < 0) { - close(); - throw new TTransportException( - TTransportException.CORRUPTED_DATA, "Read a negative frame size (" + size + ")!"); - } - + checkFrameSize(size); readBuffer.fill(underlying, size); RpcStat.readCompressedBytes.addAndGet(size); try { diff --git a/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/TElasticFramedTransport.java b/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/TElasticFramedTransport.java index d98914ce934..2524e8ae4cd 100644 --- a/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/TElasticFramedTransport.java +++ b/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/TElasticFramedTransport.java @@ -174,29 +174,71 @@ public class TElasticFramedTransport extends TTransport { protected void readFrame() throws TTransportException { underlying.readAll(i32buf, 0, 4); int size = TFramedTransport.decodeFrameSize(i32buf); + checkFrameSize(size); + readBuffer.fill(underlying, size); + } - if (size < 0) { - close(); - throw new TTransportException( - TTransportException.CORRUPTED_DATA, "Read a negative frame size (" + size + ")!"); + protected void checkFrameSize(int size) throws TTransportException { + final int HTTP_GET_SIGNATURE = 0x47455420; // "GET " + final int HTTP_POST_SIGNATURE = 0x504F5354; // "POST" + final int TLS_MIN_VERSION = 0x160300; + final int TLS_MAX_VERSION = 0x160303; + final int TLS_LENGTH_HIGH_MAX = 0x02; + + FrameError error = null; + if (size == HTTP_GET_SIGNATURE || size == HTTP_POST_SIGNATURE) { + error = FrameError.HTTP_REQUEST; + } else { + int high24 = size >>> 8; + if (high24 >= TLS_MIN_VERSION + && high24 <= TLS_MAX_VERSION + && (i32buf[3] & 0xFF) <= TLS_LENGTH_HIGH_MAX) { + error = FrameError.TLS_REQUEST; + } else if (size < 0) { + error = FrameError.NEGATIVE_FRAME_SIZE; + } else if (size > thriftMaxFrameSize) { + error = FrameError.FRAME_SIZE_EXCEEDED; + } } - if (size > thriftMaxFrameSize) { - close(); - if (size == 1195725856L || size == 1347375956L) { - // if someone sends HTTP GET/POST to this port, the size will be read as the following - throw new TTransportException( - TTransportException.CORRUPTED_DATA, - "Singular frame size (" - + size - + ") detected, you may be sending HTTP GET/POST requests to the Thrift-RPC port, please confirm that you are using the right port"); - } else { - throw new TTransportException( - TTransportException.CORRUPTED_DATA, - "Frame size (" + size + ") larger than protect max size (" + thriftMaxFrameSize + ")!"); - } + if (error == null) { + return; + } + + SocketAddress remoteAddress = null; + if (underlying instanceof TSocket) { + remoteAddress = ((TSocket) underlying).getSocket().getRemoteSocketAddress(); + } + String remoteInfo = (remoteAddress == null) ? "" : " from " + remoteAddress; + close(); + + error.throwException(size, remoteInfo, thriftMaxFrameSize); + } + + private enum FrameError { + HTTP_REQUEST( + "Singular frame size (%d) detected, you may be sending HTTP GET/POST%s " + + "requests to the Thrift-RPC port, please confirm that you are using the right port"), + TLS_REQUEST( + "Singular frame size (%d) detected, you may be sending TLS ClientHello " + + "requests%s to the Non-SSL Thrift-RPC port, please confirm that you are using " + + "the right configuration"), + NEGATIVE_FRAME_SIZE("Read a negative frame size (%d)%s!"), + FRAME_SIZE_EXCEEDED("Frame size (%d) larger than protect max size (%d)%s!"); + + private final String messageFormat; + + FrameError(String messageFormat) { + this.messageFormat = messageFormat; + } + + void throwException(int size, String remoteInfo, int maxSize) throws TTransportException { + String message = + (this == FRAME_SIZE_EXCEEDED) + ? String.format(messageFormat, size, maxSize, remoteInfo) + : String.format(messageFormat, size, remoteInfo); + throw new TTransportException(TTransportException.CORRUPTED_DATA, message); } - readBuffer.fill(underlying, size); } protected void checkWriteFrameSize(int size) throws TTransportException { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java index 989dc757464..af9e40ffa73 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java @@ -868,7 +868,7 @@ public class IoTDBConfig { private float udfCollectorMemoryBudgetInMB = (float) (1.0 / 3 * udfMemoryBudgetInMB); /** Unit: byte */ - private int thriftMaxFrameSize = 536870912; + private int thriftMaxFrameSize = 0; private int thriftDefaultBufferSize = RpcUtils.THRIFT_DEFAULT_BUF_CAPACITY; @@ -2784,7 +2784,12 @@ public class IoTDBConfig { } public void setThriftMaxFrameSize(int thriftMaxFrameSize) { - this.thriftMaxFrameSize = thriftMaxFrameSize; + this.thriftMaxFrameSize = + thriftMaxFrameSize <= 0 + ? Math.min( + 64 * 1024 * 1024, + (int) Math.min(Runtime.getRuntime().maxMemory() / 64, Integer.MAX_VALUE)) + : thriftMaxFrameSize; BaseRpcTransportFactory.setThriftMaxFrameSize(this.thriftMaxFrameSize); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java index c755381e88d..e965611016d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java @@ -861,10 +861,6 @@ public class IoTDBDescriptor { properties.getProperty( "dn_thrift_max_frame_size", String.valueOf(conf.getThriftMaxFrameSize())))); - if (conf.getThriftMaxFrameSize() < IoTDBConstant.LEFT_SIZE_IN_REQUEST * 2) { - conf.setThriftMaxFrameSize(IoTDBConstant.LEFT_SIZE_IN_REQUEST * 2); - } - conf.setThriftDefaultBufferSize( Integer.parseInt( properties.getProperty( diff --git a/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template b/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template index 018551d889a..e3b1e4da1bb 100644 --- a/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template +++ b/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template @@ -509,10 +509,10 @@ dn_rpc_min_concurrent_client_num=1 # Datatype: int dn_rpc_max_concurrent_client_num=1000 -# thrift max frame size, 512MB by default +# thrift max frame size in bytes, When = 0, use min(64MB, datanode heap memory / 64) # effectiveMode: restart # Datatype: int -dn_thrift_max_frame_size=536870912 +dn_thrift_max_frame_size=0 # thrift init buffer size # effectiveMode: restart diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/IoTDBConstant.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/IoTDBConstant.java index 18ac41409ba..f9f69f486e6 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/IoTDBConstant.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/IoTDBConstant.java @@ -268,7 +268,6 @@ public class IoTDBConstant { public static final String MQTT_MAX_MESSAGE_SIZE = "mqtt_max_message_size"; // thrift - public static final int LEFT_SIZE_IN_REQUEST = 4 * 1024 * 1024; public static final int DEFAULT_FETCH_SIZE = 5000; public static final int DEFAULT_CONNECTION_TIMEOUT_MS = 0;
