This is an automated email from the ASF dual-hosted git repository.
rong pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new 02b64d2 [IOTDB-2313] GC overhead limit when use "select * from
root.**" (#4778)
02b64d2 is described below
commit 02b64d213987975683d99c0718dfc363d401c434
Author: CloudWise-Lukemiao
<[email protected]>
AuthorDate: Fri Jan 14 14:46:13 2022 +0800
[IOTDB-2313] GC overhead limit when use "select * from root.**" (#4778)
Co-authored-by: Steve Yurong Su <[email protected]>
---
.../Communication-Service-Protocol/RestService.md | 45 ++++++++++++----------
openapi/src/main/openapi3/iotdb-rest.yaml | 2 +
.../assembly/resources/conf/iotdb-rest.properties | 5 ++-
.../iotdb/db/conf/rest/IoTDBRestServiceConfig.java | 10 +++++
.../db/conf/rest/IoTDBRestServiceDescriptor.java | 5 +++
.../protocol/rest/handler/QueryDataSetHandler.java | 23 ++++++++++-
.../protocol/rest/impl/GrafanaApiServiceImpl.java | 2 +-
.../db/protocol/rest/impl/RestApiServiceImpl.java | 26 +++++++++++--
server/src/test/resources/iotdb-rest.properties | 5 ++-
session/src/test/resources/iotdb-rest.properties | 5 ++-
10 files changed, 98 insertions(+), 30 deletions(-)
diff --git a/docs/zh/UserGuide/Communication-Service-Protocol/RestService.md
b/docs/zh/UserGuide/Communication-Service-Protocol/RestService.md
index 37dbdd7..46b99db 100644
--- a/docs/zh/UserGuide/Communication-Service-Protocol/RestService.md
+++ b/docs/zh/UserGuide/Communication-Service-Protocol/RestService.md
@@ -81,6 +81,7 @@ $ curl -H "Authorization:Basic cm9vdDpyb2901"
http://127.0.0.1:18080/ping
|参数名称 |参数类型 |是否必填|参数描述|
| ------------ | ------------ | ------------ |------------ |
| sql | string | 是 | |
+| rowLimit | integer | 否 | 一次查询能返回的结果集的最大行数。<br />如果不设置该参数，将使用配置文件的
`rest_query_default_row_size_limit` 作为默认值。<br />当返回结果集的行数超出限制时，将返回状态码 `411`。 |
请求示例:
```shell
@@ -206,63 +207,65 @@ enable_rest_service=true
rest_service_port=18080
```
-* REST Service 是否开启 SSL 配置，将 `enable_https` 设置为 `true` 以启用该模块，而将 `false`
设置为禁用该模块。默认情况下，该值为 `false`。
+* 一次查询能返回的结果集最大行数。当返回结果集的行数超出参数限制时，您只会得到在行数范围内的结果集，且将得到状态码`411`。
```properties
-enable_https=false
+rest_query_default_row_size_limit=10000
```
-* keyStore 所在路径（非必填）
+* 缓存客户登录信息的过期时间（用于加速用户鉴权的速度，单位为秒，默认是8个小时）
```properties
-key_store_path=
+cache_expire=28800
```
-
-* keyStore 密码（非必填）
+* 缓存中存储的最大用户数量（默认是100）
```properties
-key_store_pwd=
+cache_max_num=100
```
-
-* trustStore 所在路径（非必填）
+* 缓存初始容量（默认是10）
```properties
-trust_store_path=
+cache_init_num=10
```
-* trustStore 密码（非必填）
+* REST Service 是否开启 SSL 配置，将 `enable_https` 设置为 `true` 以启用该模块，而将 `false`
设置为禁用该模块。默认情况下，该值为 `false`。
```properties
-trust_store_pwd=
+enable_https=false
```
-
-* SSL 超时时间，单位为秒
+* keyStore 所在路径（非必填）
```properties
-idle_timeout=5000
+key_store_path=
```
-* 缓存客户登录信息的过期时间（用于加速用户鉴权的速度，单位为秒，默认是8个小时）
+* keyStore 密码（非必填）
```properties
-cache_expire=28800
+key_store_pwd=
```
-* 缓存中存储的最大用户数量（默认是100）
+* trustStore 所在路径（非必填）
```properties
-cache_max_num=100
+trust_store_path=
```
-* 缓存初始容量（默认是10）
+* trustStore 密码（非必填）
```properties
-cache_init_num=10
+trust_store_pwd=
```
+* SSL 超时时间，单位为秒
+
+```properties
+idle_timeout=5000
+```
diff --git a/openapi/src/main/openapi3/iotdb-rest.yaml
b/openapi/src/main/openapi3/iotdb-rest.yaml
index 85397c3..bcc238c 100644
--- a/openapi/src/main/openapi3/iotdb-rest.yaml
+++ b/openapi/src/main/openapi3/iotdb-rest.yaml
@@ -139,6 +139,8 @@ components:
properties:
sql:
type: string
+ rowLimit:
+ type: Integer
InsertTabletRequest:
title: InsertTabletRequest
diff --git a/server/src/assembly/resources/conf/iotdb-rest.properties
b/server/src/assembly/resources/conf/iotdb-rest.properties
index 6f9d59f..c942a7f 100644
--- a/server/src/assembly/resources/conf/iotdb-rest.properties
+++ b/server/src/assembly/resources/conf/iotdb-rest.properties
@@ -27,7 +27,10 @@
# the binding port of the REST service
# rest_service_port=18080
-# the expiration time of the user login infomation cache (in seconds)
+# the default row limit to a REST query response when the rowSize parameter is
not given in request
+# rest_query_default_row_size_limit=10000
+
+# the expiration time of the user login information cache (in seconds)
# cache_expire_in_seconds=28800
# maximum number of users can be stored in the user login cache.
diff --git
a/server/src/main/java/org/apache/iotdb/db/conf/rest/IoTDBRestServiceConfig.java
b/server/src/main/java/org/apache/iotdb/db/conf/rest/IoTDBRestServiceConfig.java
index 80ef8d4..a5b101d 100644
---
a/server/src/main/java/org/apache/iotdb/db/conf/rest/IoTDBRestServiceConfig.java
+++
b/server/src/main/java/org/apache/iotdb/db/conf/rest/IoTDBRestServiceConfig.java
@@ -53,6 +53,8 @@ public class IoTDBRestServiceConfig {
/** init number of users stored in cache */
private int cacheInitNum = 10;
+ private int restQueryFetchSize = 10000;
+
public String getTrustStorePwd() {
return trustStorePwd;
}
@@ -140,4 +142,12 @@ public class IoTDBRestServiceConfig {
public void setCacheInitNum(int cacheInitNum) {
this.cacheInitNum = cacheInitNum;
}
+
+ public int getRestQueryDefaultRowSizeLimit() {
+ return restQueryFetchSize;
+ }
+
+ public void setRestQueryDefaultRowSizeLimit(int restQueryFetchSize) {
+ this.restQueryFetchSize = restQueryFetchSize;
+ }
}
diff --git
a/server/src/main/java/org/apache/iotdb/db/conf/rest/IoTDBRestServiceDescriptor.java
b/server/src/main/java/org/apache/iotdb/db/conf/rest/IoTDBRestServiceDescriptor.java
index 848c64e..86c3f42 100644
---
a/server/src/main/java/org/apache/iotdb/db/conf/rest/IoTDBRestServiceDescriptor.java
+++
b/server/src/main/java/org/apache/iotdb/db/conf/rest/IoTDBRestServiceDescriptor.java
@@ -65,6 +65,11 @@ public class IoTDBRestServiceDescriptor {
Integer.parseInt(
properties.getProperty(
"rest_service_port",
Integer.toString(conf.getRestServicePort()))));
+ conf.setRestQueryDefaultRowSizeLimit(
+ Integer.parseInt(
+ properties.getProperty(
+ "rest_query_default_row_size_limit",
+ Integer.toString(conf.getRestQueryDefaultRowSizeLimit()))));
conf.setEnableHttps(
Boolean.parseBoolean(
diff --git
a/server/src/main/java/org/apache/iotdb/db/protocol/rest/handler/QueryDataSetHandler.java
b/server/src/main/java/org/apache/iotdb/db/protocol/rest/handler/QueryDataSetHandler.java
index af746e3..8e8ffc4 100644
---
a/server/src/main/java/org/apache/iotdb/db/protocol/rest/handler/QueryDataSetHandler.java
+++
b/server/src/main/java/org/apache/iotdb/db/protocol/rest/handler/QueryDataSetHandler.java
@@ -17,10 +17,12 @@
package org.apache.iotdb.db.protocol.rest.handler;
+import org.apache.iotdb.db.protocol.rest.model.ExecutionStatus;
import org.apache.iotdb.db.qp.physical.PhysicalPlan;
import org.apache.iotdb.db.qp.physical.crud.QueryPlan;
import org.apache.iotdb.db.qp.physical.sys.ShowChildPathsPlan;
import org.apache.iotdb.db.query.expression.ResultColumn;
+import org.apache.iotdb.rpc.TSStatusCode;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
import org.apache.iotdb.tsfile.read.common.Field;
import org.apache.iotdb.tsfile.read.common.RowRecord;
@@ -37,7 +39,11 @@ public class QueryDataSetHandler {
private QueryDataSetHandler() {}
- public static Response fillDateSet(QueryDataSet dataSet, QueryPlan
queryPlan) {
+ /**
+ * @param actualRowSizeLimit max number of rows to return. no limit when
actualRowSizeLimit <= 0.
+ */
+ public static Response fillDateSet(
+ QueryDataSet dataSet, QueryPlan queryPlan, final int actualRowSizeLimit)
{
org.apache.iotdb.db.protocol.rest.model.QueryDataSet queryDataSet =
new org.apache.iotdb.db.protocol.rest.model.QueryDataSet();
@@ -52,7 +58,20 @@ public class QueryDataSetHandler {
dataSetIndexes[i] =
sourcePathToQueryDataSetIndex.get(resultColumn.getResultColumnName());
}
+ int fetched = 0;
while (dataSet.hasNext()) {
+ if (0 < actualRowSizeLimit && actualRowSizeLimit <= fetched) {
+ return Response.ok()
+ .entity(
+ new ExecutionStatus()
+ .code(TSStatusCode.QUERY_PROCESS_ERROR.getStatusCode())
+ .message(
+ String.format(
+ "Dataset row size exceeded the given max row
size (%d)",
+ actualRowSizeLimit)))
+ .build();
+ }
+
RowRecord rowRecord = dataSet.next();
List<Field> fields = rowRecord.getFields();
@@ -69,6 +88,8 @@ public class QueryDataSetHandler {
: field.getObjectValue(field.getDataType()));
}
}
+
+ ++fetched;
}
} catch (IOException e) {
return
Response.ok().entity(ExceptionHandler.tryCatchException(e)).build();
diff --git
a/server/src/main/java/org/apache/iotdb/db/protocol/rest/impl/GrafanaApiServiceImpl.java
b/server/src/main/java/org/apache/iotdb/db/protocol/rest/impl/GrafanaApiServiceImpl.java
index 21bb8fb..e377375 100644
---
a/server/src/main/java/org/apache/iotdb/db/protocol/rest/impl/GrafanaApiServiceImpl.java
+++
b/server/src/main/java/org/apache/iotdb/db/protocol/rest/impl/GrafanaApiServiceImpl.java
@@ -157,7 +157,7 @@ public class GrafanaApiServiceImpl extends
GrafanaApiService {
QueryDataSet queryDataSet =
serviceProvider.createQueryDataSet(
queryContext, physicalPlan, IoTDBConstant.DEFAULT_FETCH_SIZE);
- return QueryDataSetHandler.fillDateSet(queryDataSet, (QueryPlan)
physicalPlan);
+ return QueryDataSetHandler.fillDateSet(queryDataSet, (QueryPlan)
physicalPlan, 0);
} finally {
ServiceProvider.SESSION_MANAGER.releaseQueryResourceNoExceptions(queryId);
}
diff --git
a/server/src/main/java/org/apache/iotdb/db/protocol/rest/impl/RestApiServiceImpl.java
b/server/src/main/java/org/apache/iotdb/db/protocol/rest/impl/RestApiServiceImpl.java
index 7624515..0549668 100644
---
a/server/src/main/java/org/apache/iotdb/db/protocol/rest/impl/RestApiServiceImpl.java
+++
b/server/src/main/java/org/apache/iotdb/db/protocol/rest/impl/RestApiServiceImpl.java
@@ -18,6 +18,7 @@
package org.apache.iotdb.db.protocol.rest.impl;
import org.apache.iotdb.db.conf.IoTDBConstant;
+import org.apache.iotdb.db.conf.rest.IoTDBRestServiceDescriptor;
import org.apache.iotdb.db.exception.query.QueryProcessException;
import org.apache.iotdb.db.protocol.rest.RestApiService;
import org.apache.iotdb.db.protocol.rest.handler.AuthorizationHandler;
@@ -46,12 +47,17 @@ public class RestApiServiceImpl extends RestApiService {
public static ServiceProvider serviceProvider = IoTDB.serviceProvider;
- protected final Planner planner;
+ private final Planner planner;
private final AuthorizationHandler authorizationHandler;
+ private final Integer defaultQueryRowLimit;
+
public RestApiServiceImpl() throws QueryProcessException {
- this.authorizationHandler = new AuthorizationHandler(serviceProvider);
planner = serviceProvider.getPlanner();
+ authorizationHandler = new AuthorizationHandler(serviceProvider);
+
+ defaultQueryRowLimit =
+
IoTDBRestServiceDescriptor.getInstance().getConfig().getRestQueryDefaultRowSizeLimit();
}
@Override
@@ -94,12 +100,24 @@ public class RestApiServiceImpl extends RestApiService {
.message(TSStatusCode.EXECUTE_STATEMENT_ERROR.name()))
.build();
}
+ QueryPlan queryPlan = (QueryPlan) physicalPlan;
- Response response = authorizationHandler.checkAuthority(securityContext,
physicalPlan);
+ Response response = authorizationHandler.checkAuthority(securityContext,
queryPlan);
if (response != null) {
return response;
}
+ // set max row limit to avoid OOM
+ Integer rowLimitInRequest = sql.getRowLimit();
+ if (rowLimitInRequest == null) {
+ rowLimitInRequest = defaultQueryRowLimit;
+ }
+ int rowLimitInQueryPlan = queryPlan.getRowLimit();
+ if (rowLimitInQueryPlan <= 0) {
+ rowLimitInQueryPlan = defaultQueryRowLimit;
+ }
+ final int actualRowSizeLimit = Math.min(rowLimitInRequest,
rowLimitInQueryPlan);
+
final long queryId =
QueryResourceManager.getInstance().assignQueryId(true);
try {
QueryContext queryContext =
@@ -112,7 +130,7 @@ public class RestApiServiceImpl extends RestApiService {
QueryDataSet queryDataSet =
serviceProvider.createQueryDataSet(
queryContext, physicalPlan, IoTDBConstant.DEFAULT_FETCH_SIZE);
- return QueryDataSetHandler.fillDateSet(queryDataSet, (QueryPlan)
physicalPlan);
+ return QueryDataSetHandler.fillDateSet(queryDataSet, queryPlan,
actualRowSizeLimit);
} finally {
ServiceProvider.SESSION_MANAGER.releaseQueryResourceNoExceptions(queryId);
}
diff --git a/server/src/test/resources/iotdb-rest.properties
b/server/src/test/resources/iotdb-rest.properties
index a807831..75d0ae1 100644
--- a/server/src/test/resources/iotdb-rest.properties
+++ b/server/src/test/resources/iotdb-rest.properties
@@ -27,7 +27,10 @@ enable_rest_service=true
# the binding port of the REST service
# rest_service_port=18080
-# the expiration time of the user login infomation cache (in seconds)
+# the default row limit to a REST query response when the rowSize parameter is
not given in request
+# rest_query_default_row_size_limit=10000
+
+# the expiration time of the user login information cache (in seconds)
# cache_expire_in_seconds=28800
# maximum number of users can be stored in the user login cache.
diff --git a/session/src/test/resources/iotdb-rest.properties
b/session/src/test/resources/iotdb-rest.properties
index a807831..75d0ae1 100644
--- a/session/src/test/resources/iotdb-rest.properties
+++ b/session/src/test/resources/iotdb-rest.properties
@@ -27,7 +27,10 @@ enable_rest_service=true
# the binding port of the REST service
# rest_service_port=18080
-# the expiration time of the user login infomation cache (in seconds)
+# the default row limit to a REST query response when the rowSize parameter is
not given in request
+# rest_query_default_row_size_limit=10000
+
+# the expiration time of the user login information cache (in seconds)
# cache_expire_in_seconds=28800
# maximum number of users can be stored in the user login cache.
