This is an automated email from the ASF dual-hosted git repository. haonan pushed a commit to branch codex/rest-api in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 89c23b385c5d5ebaf6a559d472a51fefa6ae9de3 Author: HTHou <[email protected]> AuthorDate: Wed Apr 15 10:25:37 2026 +0800 Limit REST write request size and return 413 on overflow --- .../exception/RequestLimitExceededException.java | 25 +++++ .../protocol/filter/RequestSizeLimitFilter.java | 98 ++++++++++++++++++ .../rest/protocol/handler/RequestLimitChecker.java | 56 +++++++++++ .../table/v1/handler/ExceptionHandler.java | 10 +- .../table/v1/handler/RequestValidationHandler.java | 8 ++ .../protocol/table/v1/impl/RestApiServiceImpl.java | 16 ++- .../rest/protocol/v1/handler/ExceptionHandler.java | 10 +- .../v1/handler/RequestValidationHandler.java | 23 +++++ .../protocol/v1/impl/GrafanaApiServiceImpl.java | 12 ++- .../rest/protocol/v1/impl/RestApiServiceImpl.java | 12 ++- .../rest/protocol/v2/handler/ExceptionHandler.java | 10 +- .../v2/handler/RequestValidationHandler.java | 37 +++++++ .../protocol/v2/impl/GrafanaApiServiceImpl.java | 12 ++- .../rest/protocol/v2/impl/RestApiServiceImpl.java | 20 +++- .../handler/RequestValidationLimitTest.java | 112 +++++++++++++++++++++ .../iotdb/db/conf/rest/IoTDBRestServiceConfig.java | 44 ++++++++ .../db/conf/rest/IoTDBRestServiceDescriptor.java | 17 ++++ .../src/test/resources/iotdb-common.properties | 12 +++ .../src/test/resources/iotdb-system.properties | 12 +++ .../conf/iotdb-system.properties.template | 20 ++++ 20 files changed, 545 insertions(+), 21 deletions(-) diff --git a/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/exception/RequestLimitExceededException.java b/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/exception/RequestLimitExceededException.java new file mode 100644 index 00000000000..ab56ec4bb03 --- /dev/null +++ b/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/exception/RequestLimitExceededException.java @@ -0,0 +1,25 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.iotdb.rest.protocol.exception; + +public class RequestLimitExceededException extends IllegalArgumentException { + + public RequestLimitExceededException(String message) { + super(message); + } +} diff --git a/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/filter/RequestSizeLimitFilter.java b/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/filter/RequestSizeLimitFilter.java new file mode 100644 index 00000000000..c04c7a0c4b3 --- /dev/null +++ b/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/filter/RequestSizeLimitFilter.java @@ -0,0 +1,98 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.iotdb.rest.protocol.filter; + +import org.apache.iotdb.db.conf.rest.IoTDBRestServiceDescriptor; + +import javax.ws.rs.WebApplicationException; +import javax.ws.rs.container.ContainerRequestContext; +import javax.ws.rs.container.ContainerRequestFilter; +import javax.ws.rs.container.PreMatching; +import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; +import javax.ws.rs.ext.Provider; + +import java.io.FilterInputStream; +import java.io.IOException; +import java.io.InputStream; + +@Provider +@PreMatching +public class RequestSizeLimitFilter implements ContainerRequestFilter { + + @Override + public void filter(ContainerRequestContext requestContext) { + long maxBodySize = + IoTDBRestServiceDescriptor.getInstance().getConfig().getRestMaxRequestBodySizeInBytes(); + if (maxBodySize <= 0) { + return; + } + + int contentLength = requestContext.getLength(); + if (contentLength > maxBodySize) { + requestContext.abortWith(buildPayloadTooLargeResponse(maxBodySize)); + return; + } + + requestContext.setEntityStream( + new LimitedInputStream(requestContext.getEntityStream(), maxBodySize)); + } + + private static Response buildPayloadTooLargeResponse(long maxBodySize) { + return Response.status(413) + .type(MediaType.TEXT_PLAIN_TYPE) + .entity("REST request body exceeds limit " + maxBodySize + " bytes") + .build(); + } + + private static class LimitedInputStream extends FilterInputStream { + + private final long maxBodySize; + private long bytesRead; + + private LimitedInputStream(InputStream in, long maxBodySize) { + super(in); + this.maxBodySize = maxBodySize; + } + + @Override + public int read() throws IOException { + int result = super.read(); + if (result != -1) { + incrementBytesRead(1); + } + return result; + } + + @Override + public int read(byte[] b, int off, int len) throws IOException { + int result = super.read(b, off, len); + if (result > 0) { + incrementBytesRead(result); + } + return result; + } + + private void incrementBytesRead(int increment) { + bytesRead += increment; + if (bytesRead > maxBodySize) { + throw new WebApplicationException(buildPayloadTooLargeResponse(maxBodySize)); + } + } + } +} diff --git a/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/handler/RequestLimitChecker.java b/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/handler/RequestLimitChecker.java new file mode 100644 index 00000000000..085b0176843 --- /dev/null +++ b/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/handler/RequestLimitChecker.java @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.iotdb.rest.protocol.handler; + +import org.apache.iotdb.db.conf.rest.IoTDBRestServiceConfig; +import org.apache.iotdb.db.conf.rest.IoTDBRestServiceDescriptor; +import org.apache.iotdb.rest.protocol.exception.RequestLimitExceededException; + +public class RequestLimitChecker { + + private RequestLimitChecker() {} + + public static void checkRowCount(String requestName, int rowCount) { + int maxRows = getConfig().getRestMaxInsertRows(); + if (maxRows > 0 && rowCount > maxRows) { + throw new RequestLimitExceededException( + String.format("%s row count %d exceeds limit %d", requestName, rowCount, maxRows)); + } + } + + public static void checkColumnCount(String requestName, int columnCount) { + int maxColumns = getConfig().getRestMaxInsertColumns(); + if (maxColumns > 0 && columnCount > maxColumns) { + throw new RequestLimitExceededException( + String.format( + "%s column count %d exceeds limit %d", requestName, columnCount, maxColumns)); + } + } + + public static void checkValueCount(String requestName, long valueCount) { + long maxValues = getConfig().getRestMaxInsertValues(); + if (maxValues > 0 && valueCount > maxValues) { + throw new RequestLimitExceededException( + String.format("%s value count %d exceeds limit %d", requestName, valueCount, maxValues)); + } + } + + private static IoTDBRestServiceConfig getConfig() { + return IoTDBRestServiceDescriptor.getInstance().getConfig(); + } +} diff --git a/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/table/v1/handler/ExceptionHandler.java b/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/table/v1/handler/ExceptionHandler.java index 61e64465a06..5d9b78a6cfe 100644 --- a/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/table/v1/handler/ExceptionHandler.java +++ b/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/table/v1/handler/ExceptionHandler.java @@ -28,6 +28,7 @@ import org.apache.iotdb.db.exception.query.QueryProcessException; import org.apache.iotdb.db.exception.sql.SemanticException; import org.apache.iotdb.db.exception.sql.StatementAnalyzeException; import org.apache.iotdb.db.queryengine.plan.relational.sql.parser.ParsingException; +import org.apache.iotdb.rest.protocol.exception.RequestLimitExceededException; import org.apache.iotdb.rest.protocol.model.ExecutionStatus; import org.apache.iotdb.rpc.TSStatusCode; @@ -46,7 +47,10 @@ public class ExceptionHandler { public static ExecutionStatus tryCatchException(Exception e) { ExecutionStatus responseResult = new ExecutionStatus(); - if (e instanceof QueryProcessException) { + if (e instanceof RequestLimitExceededException) { + responseResult.setMessage(e.getMessage()); + responseResult.setCode(413); + } else if (e instanceof QueryProcessException) { responseResult.setMessage(e.getMessage()); responseResult.setCode(((QueryProcessException) e).getErrorCode()); } else if (e instanceof DatabaseNotSetException) { @@ -92,4 +96,8 @@ public class ExceptionHandler { LOGGER.warn(e.getMessage(), e); return responseResult; } + + public static int getHttpStatus(Exception e) { + return e instanceof RequestLimitExceededException ? 413 : Status.OK.getStatusCode(); + } } diff --git a/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/table/v1/handler/RequestValidationHandler.java b/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/table/v1/handler/RequestValidationHandler.java index fc546212caf..ae127460bf8 100644 --- a/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/table/v1/handler/RequestValidationHandler.java +++ b/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/table/v1/handler/RequestValidationHandler.java @@ -17,6 +17,7 @@ package org.apache.iotdb.rest.protocol.table.v1.handler; +import org.apache.iotdb.rest.protocol.handler.RequestLimitChecker; import org.apache.iotdb.rest.protocol.table.v1.model.InsertTabletRequest; import org.apache.iotdb.rest.protocol.table.v1.model.SQL; @@ -62,6 +63,13 @@ public class RequestValidationHandler { errorMessages.add("values and timestamps should have the same size"); } + int rowCount = insertTabletRequest.getTimestamps().size(); + int columnCount = insertTabletRequest.getColumnNames().size(); + RequestLimitChecker.checkRowCount("table insertTablet request", rowCount); + RequestLimitChecker.checkColumnCount("table insertTablet request", columnCount); + RequestLimitChecker.checkValueCount( + "table insertTablet request", (long) rowCount * columnCount); + for (int i = 0; i < insertTabletRequest.getDataTypes().size(); i++) { String dataType = insertTabletRequest.getDataTypes().get(i); if (isDataType(dataType)) { diff --git a/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/table/v1/impl/RestApiServiceImpl.java b/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/table/v1/impl/RestApiServiceImpl.java index b35e9d27f59..753ef436c66 100644 --- a/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/table/v1/impl/RestApiServiceImpl.java +++ b/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/table/v1/impl/RestApiServiceImpl.java @@ -109,7 +109,9 @@ public class RestApiServiceImpl extends RestApiService { return res; } } catch (Exception e) { - return Response.ok().entity(ExceptionHandler.tryCatchException(e)).build(); + return Response.status(ExceptionHandler.getHttpStatus(e)) + .entity(ExceptionHandler.tryCatchException(e)) + .build(); } finally { if (queryId != null) { COORDINATOR.cleanupQueryExecution(queryId); @@ -132,7 +134,9 @@ public class RestApiServiceImpl extends RestApiService { return executeQueryInternal(sql, statement, clientSession, relationSqlParser); } catch (Exception e) { - return Response.ok().entity(ExceptionHandler.tryCatchException(e)).build(); + return Response.status(ExceptionHandler.getHttpStatus(e)) + .entity(ExceptionHandler.tryCatchException(e)) + .build(); } finally { long costTime = System.nanoTime() - startTime; Optional.ofNullable(statement) @@ -174,7 +178,9 @@ public class RestApiServiceImpl extends RestApiService { return responseGenerateHelper(result); } catch (Exception e) { - return Response.ok().entity(ExceptionHandler.tryCatchException(e)).build(); + return Response.status(ExceptionHandler.getHttpStatus(e)) + .entity(ExceptionHandler.tryCatchException(e)) + .build(); } finally { long costTime = System.nanoTime() - startTime; Optional.ofNullable(insertTabletStatement) @@ -231,7 +237,9 @@ public class RestApiServiceImpl extends RestApiService { } return responseGenerateHelper(result); } catch (Exception e) { - return Response.ok().entity(ExceptionHandler.tryCatchException(e)).build(); + return Response.status(ExceptionHandler.getHttpStatus(e)) + .entity(ExceptionHandler.tryCatchException(e)) + .build(); } finally { long costTime = System.nanoTime() - startTime; Optional.ofNullable(statement) diff --git a/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/v1/handler/ExceptionHandler.java b/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/v1/handler/ExceptionHandler.java index 6edcb57ab4d..22301ff28ec 100644 --- a/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/v1/handler/ExceptionHandler.java +++ b/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/v1/handler/ExceptionHandler.java @@ -27,6 +27,7 @@ import org.apache.iotdb.db.exception.metadata.DatabaseNotSetException; import org.apache.iotdb.db.exception.query.QueryProcessException; import org.apache.iotdb.db.exception.sql.SemanticException; import org.apache.iotdb.db.exception.sql.StatementAnalyzeException; +import org.apache.iotdb.rest.protocol.exception.RequestLimitExceededException; import org.apache.iotdb.rest.protocol.v1.model.ExecutionStatus; import org.apache.iotdb.rpc.TSStatusCode; @@ -45,7 +46,10 @@ public class ExceptionHandler { public static ExecutionStatus tryCatchException(Exception e) { ExecutionStatus responseResult = new ExecutionStatus(); - if (e instanceof QueryProcessException) { + if (e instanceof RequestLimitExceededException) { + responseResult.setMessage(e.getMessage()); + responseResult.setCode(413); + } else if (e instanceof QueryProcessException) { responseResult.setMessage(e.getMessage()); responseResult.setCode(((QueryProcessException) e).getErrorCode()); } else if (e instanceof DatabaseNotSetException) { @@ -88,4 +92,8 @@ public class ExceptionHandler { LOGGER.warn(e.getMessage(), e); return responseResult; } + + public static int getHttpStatus(Exception e) { + return e instanceof RequestLimitExceededException ? 413 : Status.OK.getStatusCode(); + } } diff --git a/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/v1/handler/RequestValidationHandler.java b/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/v1/handler/RequestValidationHandler.java index dc81322cc78..518217070c4 100644 --- a/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/v1/handler/RequestValidationHandler.java +++ b/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/v1/handler/RequestValidationHandler.java @@ -17,12 +17,14 @@ package org.apache.iotdb.rest.protocol.v1.handler; +import org.apache.iotdb.rest.protocol.handler.RequestLimitChecker; import org.apache.iotdb.rest.protocol.v1.model.ExpressionRequest; import org.apache.iotdb.rest.protocol.v1.model.InsertTabletRequest; import org.apache.iotdb.rest.protocol.v1.model.SQL; import org.apache.tsfile.external.commons.lang3.Validate; +import java.util.List; import java.util.Objects; public class RequestValidationHandler { @@ -40,8 +42,29 @@ public class RequestValidationHandler { Objects.requireNonNull(insertTabletRequest.getTimestamps(), "timestamps should not be null"); Objects.requireNonNull(insertTabletRequest.getIsAligned(), "isAligned should not be null"); Objects.requireNonNull(insertTabletRequest.getDeviceId(), "deviceId should not be null"); + Objects.requireNonNull(insertTabletRequest.getMeasurements(), "measurements should not be null"); Objects.requireNonNull(insertTabletRequest.getDataTypes(), "dataTypes should not be null"); Objects.requireNonNull(insertTabletRequest.getValues(), "values should not be null"); + + if (insertTabletRequest.getMeasurements().size() != insertTabletRequest.getDataTypes().size()) { + throw new IllegalArgumentException("measurements and dataTypes should have the same size"); + } + if (insertTabletRequest.getValues().size() != insertTabletRequest.getDataTypes().size()) { + throw new IllegalArgumentException("values and dataTypes should have the same size"); + } + + int rowCount = insertTabletRequest.getTimestamps().size(); + int columnCount = insertTabletRequest.getMeasurements().size(); + RequestLimitChecker.checkRowCount("insertTablet request", rowCount); + RequestLimitChecker.checkColumnCount("insertTablet request", columnCount); + RequestLimitChecker.checkValueCount("insertTablet request", (long) rowCount * columnCount); + + for (List<Object> column : insertTabletRequest.getValues()) { + if (column.size() != rowCount) { + throw new IllegalArgumentException( + "Each value column should have the same size as timestamps"); + } + } } public static void validateExpressionRequest(ExpressionRequest expressionRequest) { diff --git a/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/v1/impl/GrafanaApiServiceImpl.java b/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/v1/impl/GrafanaApiServiceImpl.java index 0db4cd06c66..8322599fc26 100644 --- a/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/v1/impl/GrafanaApiServiceImpl.java +++ b/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/v1/impl/GrafanaApiServiceImpl.java @@ -133,7 +133,9 @@ public class GrafanaApiServiceImpl extends GrafanaApiService { return QueryDataSetHandler.fillGrafanaVariablesResult(queryExecution, statement); } } catch (Exception e) { - return Response.ok().entity(ExceptionHandler.tryCatchException(e)).build(); + return Response.status(ExceptionHandler.getHttpStatus(e)) + .entity(ExceptionHandler.tryCatchException(e)) + .build(); } finally { if (queryId != null) { COORDINATOR.cleanupQueryExecution(queryId); @@ -206,7 +208,9 @@ public class GrafanaApiServiceImpl extends GrafanaApiService { } } } catch (Exception e) { - return Response.ok().entity(ExceptionHandler.tryCatchException(e)).build(); + return Response.status(ExceptionHandler.getHttpStatus(e)) + .entity(ExceptionHandler.tryCatchException(e)) + .build(); } finally { if (queryId != null) { COORDINATOR.cleanupQueryExecution(queryId); @@ -269,7 +273,9 @@ public class GrafanaApiServiceImpl extends GrafanaApiService { return QueryDataSetHandler.fillGrafanaNodesResult(null); } } catch (Exception e) { - return Response.ok().entity(ExceptionHandler.tryCatchException(e)).build(); + return Response.status(ExceptionHandler.getHttpStatus(e)) + .entity(ExceptionHandler.tryCatchException(e)) + .build(); } finally { if (queryId != null) { COORDINATOR.cleanupQueryExecution(queryId); diff --git a/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/v1/impl/RestApiServiceImpl.java b/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/v1/impl/RestApiServiceImpl.java index 329ac47034b..55c48a4bae9 100644 --- a/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/v1/impl/RestApiServiceImpl.java +++ b/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/v1/impl/RestApiServiceImpl.java @@ -149,7 +149,9 @@ public class RestApiServiceImpl extends RestApiService { .build(); } catch (Exception e) { finish = true; - return Response.ok().entity(ExceptionHandler.tryCatchException(e)).build(); + return Response.status(ExceptionHandler.getHttpStatus(e)) + .entity(ExceptionHandler.tryCatchException(e)) + .build(); } finally { long costTime = System.nanoTime() - startTime; Optional.ofNullable(statement) @@ -230,7 +232,9 @@ public class RestApiServiceImpl extends RestApiService { } } catch (Exception e) { finish = true; - return Response.ok().entity(ExceptionHandler.tryCatchException(e)).build(); + return Response.status(ExceptionHandler.getHttpStatus(e)) + .entity(ExceptionHandler.tryCatchException(e)) + .build(); } finally { long costTime = System.nanoTime() - startTime; Optional.ofNullable(statement) @@ -301,7 +305,9 @@ public class RestApiServiceImpl extends RestApiService { .message(result.status.getMessage())) .build(); } catch (Exception e) { - return Response.ok().entity(ExceptionHandler.tryCatchException(e)).build(); + return Response.status(ExceptionHandler.getHttpStatus(e)) + .entity(ExceptionHandler.tryCatchException(e)) + .build(); } finally { long costTime = System.nanoTime() - startTime; Optional.ofNullable(insertTabletStatement) diff --git a/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/v2/handler/ExceptionHandler.java b/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/v2/handler/ExceptionHandler.java index 931ec0cec00..23155368d0d 100644 --- a/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/v2/handler/ExceptionHandler.java +++ b/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/v2/handler/ExceptionHandler.java @@ -27,6 +27,7 @@ import org.apache.iotdb.db.exception.metadata.DatabaseNotSetException; import org.apache.iotdb.db.exception.query.QueryProcessException; import org.apache.iotdb.db.exception.sql.SemanticException; import org.apache.iotdb.db.exception.sql.StatementAnalyzeException; +import org.apache.iotdb.rest.protocol.exception.RequestLimitExceededException; import org.apache.iotdb.rest.protocol.model.ExecutionStatus; import org.apache.iotdb.rpc.TSStatusCode; @@ -45,7 +46,10 @@ public class ExceptionHandler { public static ExecutionStatus tryCatchException(Exception e) { ExecutionStatus responseResult = new ExecutionStatus(); - if (e instanceof QueryProcessException) { + if (e instanceof RequestLimitExceededException) { + responseResult.setMessage(e.getMessage()); + responseResult.setCode(413); + } else if (e instanceof QueryProcessException) { responseResult.setMessage(e.getMessage()); responseResult.setCode(((QueryProcessException) e).getErrorCode()); } else if (e instanceof DatabaseNotSetException) { @@ -88,4 +92,8 @@ public class ExceptionHandler { LOGGER.warn(e.getMessage(), e); return responseResult; } + + public static int getHttpStatus(Exception e) { + return e instanceof RequestLimitExceededException ? 413 : Status.OK.getStatusCode(); + } } diff --git a/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/v2/handler/RequestValidationHandler.java b/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/v2/handler/RequestValidationHandler.java index 5f999b3ddcc..76cd07085ad 100644 --- a/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/v2/handler/RequestValidationHandler.java +++ b/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/v2/handler/RequestValidationHandler.java @@ -17,6 +17,7 @@ package org.apache.iotdb.rest.protocol.v2.handler; +import org.apache.iotdb.rest.protocol.handler.RequestLimitChecker; import org.apache.iotdb.rest.protocol.v2.model.ExpressionRequest; import org.apache.iotdb.rest.protocol.v2.model.InsertRecordsRequest; import org.apache.iotdb.rest.protocol.v2.model.InsertTabletRequest; @@ -56,6 +57,23 @@ public class RequestValidationHandler { Objects.requireNonNull( insertTabletRequest.getMeasurements(), "measurements should not be null"); Objects.requireNonNull(insertTabletRequest.getValues(), "values should not be null"); + if (insertTabletRequest.getMeasurements().size() != insertTabletRequest.getDataTypes().size()) { + throw new IllegalArgumentException("measurements and data_types should have the same size"); + } + if (insertTabletRequest.getValues().size() != insertTabletRequest.getDataTypes().size()) { + throw new IllegalArgumentException("values and data_types should have the same size"); + } + int rowCount = insertTabletRequest.getTimestamps().size(); + int columnCount = insertTabletRequest.getMeasurements().size(); + RequestLimitChecker.checkRowCount("insertTablet request", rowCount); + RequestLimitChecker.checkColumnCount("insertTablet request", columnCount); + RequestLimitChecker.checkValueCount("insertTablet request", (long) rowCount * columnCount); + for (List<Object> column : insertTabletRequest.getValues()) { + if (column.size() != rowCount) { + throw new IllegalArgumentException( + "Each value column should have the same size as timestamps"); + } + } List<String> errorMessages = new ArrayList<>(); String device = insertTabletRequest.getDevice(); for (int i = 0; i < insertTabletRequest.getMeasurements().size(); i++) { @@ -80,10 +98,28 @@ public class RequestValidationHandler { Objects.requireNonNull(insertRecordsRequest.getValuesList(), "values_list should not be null"); Objects.requireNonNull( insertRecordsRequest.getMeasurementsList(), "measurements_list should not be null"); + int rowCount = insertRecordsRequest.getDevices().size(); + if (insertRecordsRequest.getTimestamps().size() != rowCount + || insertRecordsRequest.getMeasurementsList().size() != rowCount + || insertRecordsRequest.getDataTypesList().size() != rowCount + || insertRecordsRequest.getValuesList().size() != rowCount) { + throw new IllegalArgumentException( + "devices, timestamps, measurements_list, data_types_list and values_list should have the same size"); + } + RequestLimitChecker.checkRowCount("insertRecords request", rowCount); List<String> errorMessages = new ArrayList<>(); + long valueCount = 0; for (int i = 0; i < insertRecordsRequest.getDataTypesList().size(); i++) { String device = insertRecordsRequest.getDevices().get(i); List<String> measurements = insertRecordsRequest.getMeasurementsList().get(i); + List<String> dataTypes = insertRecordsRequest.getDataTypesList().get(i); + List<Object> values = insertRecordsRequest.getValuesList().get(i); + if (measurements.size() != dataTypes.size() || values.size() != dataTypes.size()) { + throw new IllegalArgumentException( + "Each insertRecords row should have the same number of measurements, data types and values"); + } + RequestLimitChecker.checkColumnCount("insertRecords request", measurements.size()); + valueCount += values.size(); for (int c = 0; c < insertRecordsRequest.getDataTypesList().get(i).size(); c++) { String dataType = insertRecordsRequest.getDataTypesList().get(i).get(c); String measurement = measurements.get(c); @@ -93,6 +129,7 @@ public class RequestValidationHandler { } } } + RequestLimitChecker.checkValueCount("insertRecords request", valueCount); if (!errorMessages.isEmpty()) { throw new RuntimeException(String.join(",", errorMessages)); } diff --git a/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/v2/impl/GrafanaApiServiceImpl.java b/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/v2/impl/GrafanaApiServiceImpl.java index 5b120b9c1d7..882d049f1dd 100644 --- a/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/v2/impl/GrafanaApiServiceImpl.java +++ b/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/v2/impl/GrafanaApiServiceImpl.java @@ -133,7 +133,9 @@ public class GrafanaApiServiceImpl extends GrafanaApiService { return QueryDataSetHandler.fillGrafanaVariablesResult(queryExecution, statement); } } catch (Exception e) { - return Response.ok().entity(ExceptionHandler.tryCatchException(e)).build(); + return Response.status(ExceptionHandler.getHttpStatus(e)) + .entity(ExceptionHandler.tryCatchException(e)) + .build(); } finally { if (queryId != null) { COORDINATOR.cleanupQueryExecution(queryId); @@ -206,7 +208,9 @@ public class GrafanaApiServiceImpl extends GrafanaApiService { } } } catch (Exception e) { - return Response.ok().entity(ExceptionHandler.tryCatchException(e)).build(); + return Response.status(ExceptionHandler.getHttpStatus(e)) + .entity(ExceptionHandler.tryCatchException(e)) + .build(); } finally { if (queryId != null) { COORDINATOR.cleanupQueryExecution(queryId); @@ -269,7 +273,9 @@ public class GrafanaApiServiceImpl extends GrafanaApiService { return QueryDataSetHandler.fillGrafanaNodesResult(null); } } catch (Exception e) { - return Response.ok().entity(ExceptionHandler.tryCatchException(e)).build(); + return Response.status(ExceptionHandler.getHttpStatus(e)) + .entity(ExceptionHandler.tryCatchException(e)) + .build(); } finally { if (queryId != null) { COORDINATOR.cleanupQueryExecution(queryId); diff --git a/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/v2/impl/RestApiServiceImpl.java b/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/v2/impl/RestApiServiceImpl.java index f6c42533a62..3ecce38e7c5 100644 --- a/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/v2/impl/RestApiServiceImpl.java +++ b/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/v2/impl/RestApiServiceImpl.java @@ -210,7 +210,9 @@ public class RestApiServiceImpl extends RestApiService { } catch (Exception e) { finish = true; t = e; - return Response.ok().entity(ExceptionHandler.tryCatchException(e)).build(); + return Response.status(ExceptionHandler.getHttpStatus(e)) + .entity(ExceptionHandler.tryCatchException(e)) + .build(); } finally { long endTime = System.nanoTime(); long costTime = endTime - startTime; @@ -307,7 +309,9 @@ public class RestApiServiceImpl extends RestApiService { return responseGenerateHelper(result); } catch (Exception e) { finish = true; - return Response.ok().entity(ExceptionHandler.tryCatchException(e)).build(); + return Response.status(ExceptionHandler.getHttpStatus(e)) + .entity(ExceptionHandler.tryCatchException(e)) + .build(); } finally { long costTime = System.nanoTime() - startTime; if (statement != null) @@ -388,7 +392,9 @@ public class RestApiServiceImpl extends RestApiService { } } catch (Exception e) { finish = true; - return Response.ok().entity(ExceptionHandler.tryCatchException(e)).build(); + return Response.status(ExceptionHandler.getHttpStatus(e)) + .entity(ExceptionHandler.tryCatchException(e)) + .build(); } finally { long costTime = System.nanoTime() - startTime; Optional.ofNullable(statement) @@ -438,7 +444,9 @@ public class RestApiServiceImpl extends RestApiService { return responseGenerateHelper(result); } catch (Exception e) { - return Response.ok().entity(ExceptionHandler.tryCatchException(e)).build(); + return Response.status(ExceptionHandler.getHttpStatus(e)) + .entity(ExceptionHandler.tryCatchException(e)) + .build(); } finally { long costTime = System.nanoTime() - startTime; Optional.ofNullable(insertRowsStatement) @@ -492,7 +500,9 @@ public class RestApiServiceImpl extends RestApiService { false); return responseGenerateHelper(result); } catch (Exception e) { - return Response.ok().entity(ExceptionHandler.tryCatchException(e)).build(); + return Response.status(ExceptionHandler.getHttpStatus(e)) + .entity(ExceptionHandler.tryCatchException(e)) + .build(); } finally { long costTime = System.nanoTime() - startTime; Optional.ofNullable(insertTabletStatement) diff --git a/external-service-impl/rest/src/test/java/org/apache/iotdb/rest/protocol/handler/RequestValidationLimitTest.java b/external-service-impl/rest/src/test/java/org/apache/iotdb/rest/protocol/handler/RequestValidationLimitTest.java new file mode 100644 index 00000000000..9da02ecfa98 --- /dev/null +++ b/external-service-impl/rest/src/test/java/org/apache/iotdb/rest/protocol/handler/RequestValidationLimitTest.java @@ -0,0 +1,112 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.iotdb.rest.protocol.handler; + +import org.apache.iotdb.db.conf.rest.IoTDBRestServiceConfig; +import org.apache.iotdb.db.conf.rest.IoTDBRestServiceDescriptor; +import org.apache.iotdb.rest.protocol.exception.RequestLimitExceededException; + +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.util.Arrays; +import java.util.Collections; + +public class RequestValidationLimitTest { + + private IoTDBRestServiceConfig config; + private int originalMaxInsertRows; + private int originalMaxInsertColumns; + private long originalMaxInsertValues; + + @Before + public void setUp() { + config = IoTDBRestServiceDescriptor.getInstance().getConfig(); + originalMaxInsertRows = config.getRestMaxInsertRows(); + originalMaxInsertColumns = config.getRestMaxInsertColumns(); + originalMaxInsertValues = config.getRestMaxInsertValues(); + } + + @After + public void tearDown() { + config.setRestMaxInsertRows(originalMaxInsertRows); + config.setRestMaxInsertColumns(originalMaxInsertColumns); + config.setRestMaxInsertValues(originalMaxInsertValues); + } + + @Test(expected = RequestLimitExceededException.class) + public void testV1InsertTabletRejectsTooManyRows() { + config.setRestMaxInsertRows(2); + + org.apache.iotdb.rest.protocol.v1.model.InsertTabletRequest request = + new org.apache.iotdb.rest.protocol.v1.model.InsertTabletRequest(); + request.setDeviceId("root.sg.d1"); + request.setIsAligned(false); + request.setMeasurements(Collections.singletonList("s1")); + request.setDataTypes(Collections.singletonList("INT64")); + request.setTimestamps(Arrays.asList(1L, 2L, 3L)); + request.setValues(Collections.singletonList(Arrays.asList(1L, 2L, 3L))); + + org.apache.iotdb.rest.protocol.v1.handler.RequestValidationHandler.validateInsertTabletRequest( + request); + } + + @Test(expected = RequestLimitExceededException.class) + public void testV2InsertRecordsRejectsTooManyValues() { + config.setRestMaxInsertRows(10); + config.setRestMaxInsertColumns(10); + config.setRestMaxInsertValues(2); + + org.apache.iotdb.rest.protocol.v2.model.InsertRecordsRequest request = + new org.apache.iotdb.rest.protocol.v2.model.InsertRecordsRequest(); + request.setIsAligned(false); + request.setDevices(Arrays.asList("root.sg.d1", "root.sg.d2")); + request.setTimestamps(Arrays.asList(1L, 2L)); + request.setMeasurementsList( + Arrays.asList( + Arrays.asList("s1", "s2"), + Collections.singletonList("s1"))); + request.setDataTypesList( + Arrays.asList( + Arrays.asList("INT64", "INT64"), + Collections.singletonList("INT64"))); + request.setValuesList(Arrays.asList(Arrays.asList(1L, 2L), Collections.singletonList(3L))); + + org.apache.iotdb.rest.protocol.v2.handler.RequestValidationHandler.validateInsertRecordsRequest( + request); + } + + @Test(expected = RequestLimitExceededException.class) + public void testTableInsertTabletRejectsTooManyColumns() { + config.setRestMaxInsertColumns(1); + + org.apache.iotdb.rest.protocol.table.v1.model.InsertTabletRequest request = + new org.apache.iotdb.rest.protocol.table.v1.model.InsertTabletRequest(); + request.setDatabase("db"); + request.setTable("t1"); + request.setColumnNames(Arrays.asList("tag1", "s1")); + request.setColumnCategories(Arrays.asList("TAG", "FIELD")); + request.setDataTypes(Arrays.asList("STRING", "INT64")); + request.setTimestamps(Collections.singletonList(1L)); + request.setValues(Collections.singletonList(Arrays.asList("a", 1L))); + + org.apache.iotdb.rest.protocol.table.v1.handler.RequestValidationHandler + .validateInsertTabletRequest(request); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/rest/IoTDBRestServiceConfig.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/rest/IoTDBRestServiceConfig.java index 64c0f65fe30..2e2acd2e4d9 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/rest/IoTDBRestServiceConfig.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/rest/IoTDBRestServiceConfig.java @@ -59,6 +59,18 @@ public class IoTDBRestServiceConfig { private int restQueryDefaultRowSizeLimit = 10000; + /** Maximum accepted REST request body size in bytes. */ + private long restMaxRequestBodySizeInBytes = 16 * 1024 * 1024L; + + /** Maximum row count accepted by a single REST write request. */ + private int restMaxInsertRows = 100000; + + /** Maximum column count accepted by a single REST write request. */ + private int restMaxInsertColumns = 1024; + + /** Maximum value cell count accepted by a single REST write request. */ + private long restMaxInsertValues = 1000000L; + /** Is client authentication required. */ private boolean clientAuth = false; @@ -173,4 +185,36 @@ public class IoTDBRestServiceConfig { public void setRestQueryDefaultRowSizeLimit(int restQueryDefaultRowSizeLimit) { this.restQueryDefaultRowSizeLimit = restQueryDefaultRowSizeLimit; } + + public long getRestMaxRequestBodySizeInBytes() { + return restMaxRequestBodySizeInBytes; + } + + public void setRestMaxRequestBodySizeInBytes(long restMaxRequestBodySizeInBytes) { + this.restMaxRequestBodySizeInBytes = restMaxRequestBodySizeInBytes; + } + + public int getRestMaxInsertRows() { + return restMaxInsertRows; + } + + public void setRestMaxInsertRows(int restMaxInsertRows) { + this.restMaxInsertRows = restMaxInsertRows; + } + + public int getRestMaxInsertColumns() { + return restMaxInsertColumns; + } + + public void setRestMaxInsertColumns(int restMaxInsertColumns) { + this.restMaxInsertColumns = restMaxInsertColumns; + } + + public long getRestMaxInsertValues() { + return restMaxInsertValues; + } + + public void setRestMaxInsertValues(long restMaxInsertValues) { + this.restMaxInsertValues = restMaxInsertValues; + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/rest/IoTDBRestServiceDescriptor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/rest/IoTDBRestServiceDescriptor.java index c4f9d131de9..124c0fbbb7c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/rest/IoTDBRestServiceDescriptor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/rest/IoTDBRestServiceDescriptor.java @@ -91,6 +91,23 @@ public class IoTDBRestServiceDescriptor { trimProperties.getProperty( "rest_query_default_row_size_limit", Integer.toString(conf.getRestQueryDefaultRowSizeLimit())))); + conf.setRestMaxRequestBodySizeInBytes( + Long.parseLong( + trimProperties.getProperty( + "rest_max_request_body_size_in_bytes", + Long.toString(conf.getRestMaxRequestBodySizeInBytes())))); + conf.setRestMaxInsertRows( + Integer.parseInt( + trimProperties.getProperty( + "rest_max_insert_rows", Integer.toString(conf.getRestMaxInsertRows())))); + conf.setRestMaxInsertColumns( + Integer.parseInt( + trimProperties.getProperty( + "rest_max_insert_columns", Integer.toString(conf.getRestMaxInsertColumns())))); + conf.setRestMaxInsertValues( + Long.parseLong( + trimProperties.getProperty( + "rest_max_insert_values", Long.toString(conf.getRestMaxInsertValues())))); conf.setEnableSwagger( Boolean.parseBoolean( trimProperties.getProperty( diff --git a/iotdb-core/datanode/src/test/resources/iotdb-common.properties b/iotdb-core/datanode/src/test/resources/iotdb-common.properties index 1053eaafa2d..74c5a2c70aa 100644 --- a/iotdb-core/datanode/src/test/resources/iotdb-common.properties +++ b/iotdb-core/datanode/src/test/resources/iotdb-common.properties @@ -33,6 +33,18 @@ enable_rest_service=true # the default row limit to a REST query response when the rowSize parameter is not given in request # rest_query_default_row_size_limit=10000 +# Maximum REST request body size in bytes +# rest_max_request_body_size_in_bytes=16777216 + +# Maximum rows accepted by a single REST write request +# rest_max_insert_rows=100000 + +# Maximum columns accepted by a single REST write request +# rest_max_insert_columns=1024 + +# Maximum values accepted by a single REST write request +# rest_max_insert_values=1000000 + # is SSL enabled # enable_https=false diff --git a/iotdb-core/datanode/src/test/resources/iotdb-system.properties b/iotdb-core/datanode/src/test/resources/iotdb-system.properties index ce0ecff34f3..4e542beba4f 100644 --- a/iotdb-core/datanode/src/test/resources/iotdb-system.properties +++ b/iotdb-core/datanode/src/test/resources/iotdb-system.properties @@ -50,6 +50,18 @@ enable_rest_service=true # the default row limit to a REST query response when the rowSize parameter is not given in request # rest_query_default_row_size_limit=10000 +# Maximum REST request body size in bytes +# rest_max_request_body_size_in_bytes=16777216 + +# Maximum rows accepted by a single REST write request +# rest_max_insert_rows=100000 + +# Maximum columns accepted by a single REST write request +# rest_max_insert_columns=1024 + +# Maximum values accepted by a single REST write request +# rest_max_insert_values=1000000 + # is SSL enabled # enable_https=false diff --git a/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template b/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template index 378a6226cbf..18bcc038b89 100644 --- a/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template +++ b/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template @@ -584,6 +584,26 @@ enable_swagger=false # Datatype: int rest_query_default_row_size_limit=10000 +# Maximum REST request body size in bytes. Set to 0 or a negative value to disable the limit. +# effectiveMode: restart +# Datatype: long +rest_max_request_body_size_in_bytes=16777216 + +# Maximum rows accepted by a single REST write request. Set to 0 or a negative value to disable the limit. +# effectiveMode: restart +# Datatype: int +rest_max_insert_rows=100000 + +# Maximum columns accepted by a single REST write request. Set to 0 or a negative value to disable the limit. +# effectiveMode: restart +# Datatype: int +rest_max_insert_columns=1024 + +# Maximum values accepted by a single REST write request. Set to 0 or a negative value to disable the limit. +# effectiveMode: restart +# Datatype: long +rest_max_insert_values=1000000 + # Is client authentication required # effectiveMode: restart # Datatype: boolean
