This is an automated email from the ASF dual-hosted git repository.
xxyu pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 116718225ba01ad239ba0924e507503bdeb2e562
Author: lixiang <447399...@qq.com>
AuthorDate: Tue Mar 14 10:55:48 2023 +0800
KYLIN-5556 Add a new meothod to load tables with a threshold
---
.../kylin/rest/controller/NAdminController.java | 1 +
.../kylin/rest/response/LoadTableResponse.java | 3 +
.../org/apache/kylin/common/KylinConfigBase.java | 4 +
.../common/exception/code/ErrorCodeServer.java | 1 +
.../src/main/resources/kylin-defaults0.properties | 3 +
.../resources/kylin_error_msg_conf_cn.properties | 1 +
.../resources/kylin_error_msg_conf_en.properties | 1 +
.../main/resources/kylin_errorcode_conf.properties | 1 +
.../apache/kylin/rest/service/TableExtService.java | 109 ++++++++++++++++++++-
.../kylin/rest/service/TableExtServiceTest.java | 45 +++++++++
.../kylin/rest/controller/NTableController.java | 22 +----
.../rest/controller/NTableControllerTest.java | 18 ++--
12 files changed, 178 insertions(+), 31 deletions(-)
diff --git
a/src/common-server/src/main/java/org/apache/kylin/rest/controller/NAdminController.java
b/src/common-server/src/main/java/org/apache/kylin/rest/controller/NAdminController.java
index c2f26bd8aa..421fe1b009 100644
---
a/src/common-server/src/main/java/org/apache/kylin/rest/controller/NAdminController.java
+++
b/src/common-server/src/main/java/org/apache/kylin/rest/controller/NAdminController.java
@@ -78,6 +78,7 @@ public class NAdminController extends NBasicController {
propertyKeys.add("kylin.source.ddl.hive.enabled");
propertyKeys.add("kylin.source.ddl.logical-view.database");
propertyKeys.add("kylin.storage.check-quota-enabled");
+ propertyKeys.add("kylin.table.load-threshold-enabled");
// add second storage
if
(StringUtils.isNotEmpty(KylinConfig.getInstanceFromEnv().getSecondStorage())) {
diff --git
a/src/common-service/src/main/java/org/apache/kylin/rest/response/LoadTableResponse.java
b/src/common-service/src/main/java/org/apache/kylin/rest/response/LoadTableResponse.java
index fcf5a8b069..a59720f1cf 100644
---
a/src/common-service/src/main/java/org/apache/kylin/rest/response/LoadTableResponse.java
+++
b/src/common-service/src/main/java/org/apache/kylin/rest/response/LoadTableResponse.java
@@ -22,6 +22,7 @@ import java.io.Serializable;
import java.util.HashSet;
import java.util.Set;
+import com.fasterxml.jackson.annotation.JsonIgnore;
import com.fasterxml.jackson.annotation.JsonProperty;
import lombok.Getter;
@@ -35,5 +36,7 @@ public class LoadTableResponse implements Serializable {
private Set<String> loaded = new HashSet<>();
@JsonProperty("failed")
private Set<String> failed = new HashSet<>();
+ @JsonIgnore
+ private Set<String> needRealSampling = new HashSet<>();
}
diff --git
a/src/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
b/src/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index f887fbd30a..0c5465d87b 100644
--- a/src/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/src/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -3894,4 +3894,8 @@ public abstract class KylinConfigBase implements
Serializable {
public int getJobHLLMaxReducerNumber() {
return
Integer.parseInt(getOptional("kylin.index.costbased.hll-max-reducer-number",
"1"));
}
+
+ public boolean isTableLoadThresholdEnabled() {
+ return
Boolean.parseBoolean(getOptional("kylin.table.load-threshold-enabled", TRUE));
+ }
}
diff --git
a/src/core-common/src/main/java/org/apache/kylin/common/exception/code/ErrorCodeServer.java
b/src/core-common/src/main/java/org/apache/kylin/common/exception/code/ErrorCodeServer.java
index 67cf57afed..5fab5c855c 100644
---
a/src/core-common/src/main/java/org/apache/kylin/common/exception/code/ErrorCodeServer.java
+++
b/src/core-common/src/main/java/org/apache/kylin/common/exception/code/ErrorCodeServer.java
@@ -63,6 +63,7 @@ public enum ErrorCodeServer implements ErrorCodeProducer {
TABLE_RELOAD_MODEL_RETRY("KE-010007204"),
TABLE_RELOAD_HAVING_NOT_FINAL_JOB("KE-010007208"),
EXCLUDED_TABLE_REQUEST_NOT_ALLOWED("KE-010007301"),
+ ONCE_LOAD_TABLE_LIMIT("KE-010007011"),
// 10032XXX add job result
JOB_CREATE_CHECK_MULTI_PARTITION_EMPTY("KE-010032201"),
diff --git a/src/core-common/src/main/resources/kylin-defaults0.properties
b/src/core-common/src/main/resources/kylin-defaults0.properties
index 9aee2acdda..f60a75a2df 100644
--- a/src/core-common/src/main/resources/kylin-defaults0.properties
+++ b/src/core-common/src/main/resources/kylin-defaults0.properties
@@ -434,6 +434,9 @@
kylin.security.user-password-encoder=org.apache.kylin.rest.security.CachedBCrypt
kylin.model.recommendation-page-size=500
kylin.model.dimension-measure-name.max-length=300
+# table
+kylin.table.load-threshold-enabled=true
+
# recommend
kylin.model.suggest-model-sql-limit=200
diff --git
a/src/core-common/src/main/resources/kylin_error_msg_conf_cn.properties
b/src/core-common/src/main/resources/kylin_error_msg_conf_cn.properties
index 11f10b5af9..0314410ab1 100644
--- a/src/core-common/src/main/resources/kylin_error_msg_conf_cn.properties
+++ b/src/core-common/src/main/resources/kylin_error_msg_conf_cn.properties
@@ -64,6 +64,7 @@ KE-010022219=当前 Segments 所包含的分区不一致，请先构建分区并
KE-010007204=源表 %1$s 中列 %2$s 的数据类型发生变更。请从模型 %3$s 中删除该列，或修改该列的数据类型。
KE-010007208=当前暂不可重载表。存在运行中的任务，任务对象为： %s。请等任务完成后再重载，或手动终止任务。
KE-010007301=获取屏蔽列设置失败，项目 %s 没有打开屏蔽列开关。
+KE-010007011=一次最多可加载 1000 张表，请修改后重新提交。
## 100322XX job
KE-010032201=无法添加任务，子分区值为空。请检查后重试。
diff --git
a/src/core-common/src/main/resources/kylin_error_msg_conf_en.properties
b/src/core-common/src/main/resources/kylin_error_msg_conf_en.properties
index 028f593bce..df7ed31b2a 100644
--- a/src/core-common/src/main/resources/kylin_error_msg_conf_en.properties
+++ b/src/core-common/src/main/resources/kylin_error_msg_conf_en.properties
@@ -63,6 +63,7 @@ KE-010022219=The partitions included in the selected segments
are not consistent
KE-010007204=The data type of column %2$s from the source table %1$s has
changed. Please remove the column from model %3$s, or modify the data type.
KE-010007208=The table metadata can't be reloaded now. There are ongoing jobs
with the following target subjects(s): %s. Please try reloading until all the
jobs are completed, or manually discard the jobs.
KE-010007301=Fail to fetch setting of excluded columns, because project %s is
not enable excluded columns.
+KE-010007011=Up to 1000 tables could be loaded per time, please modify and
resubmit.
## 100322XX job
KE-010032201=Can't add the job, as the subpartition value is empty. Please
check and try again.
diff --git a/src/core-common/src/main/resources/kylin_errorcode_conf.properties
b/src/core-common/src/main/resources/kylin_errorcode_conf.properties
index 091be9bf2b..170a66723f 100644
--- a/src/core-common/src/main/resources/kylin_errorcode_conf.properties
+++ b/src/core-common/src/main/resources/kylin_errorcode_conf.properties
@@ -62,6 +62,7 @@ KE-010022217
KE-010007204
KE-010007208
KE-010007301
+KE-010007011
## 100322XX job
KE-010032201
diff --git
a/src/datasource-service/src/main/java/org/apache/kylin/rest/service/TableExtService.java
b/src/datasource-service/src/main/java/org/apache/kylin/rest/service/TableExtService.java
index cb13bf5944..8765c10247 100644
---
a/src/datasource-service/src/main/java/org/apache/kylin/rest/service/TableExtService.java
+++
b/src/datasource-service/src/main/java/org/apache/kylin/rest/service/TableExtService.java
@@ -20,6 +20,7 @@ package org.apache.kylin.rest.service;
import static
org.apache.kylin.common.exception.ServerErrorCode.INVALID_TABLE_NAME;
import static
org.apache.kylin.common.exception.code.ErrorCodeServer.EXCLUDED_TABLE_REQUEST_NOT_ALLOWED;
+import static
org.apache.kylin.common.exception.code.ErrorCodeServer.ONCE_LOAD_TABLE_LIMIT;
import java.io.IOException;
import java.security.PrivilegedExceptionAction;
@@ -34,6 +35,7 @@ import java.util.Objects;
import java.util.Set;
import java.util.stream.Collectors;
+import org.apache.commons.lang3.ArrayUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.kylin.common.KylinConfig;
@@ -41,6 +43,7 @@ import org.apache.kylin.common.exception.KylinException;
import org.apache.kylin.common.msg.MsgPicker;
import org.apache.kylin.common.util.HadoopUtil;
import org.apache.kylin.common.util.Pair;
+import org.apache.kylin.common.util.StringHelper;
import org.apache.kylin.metadata.model.ColumnDesc;
import org.apache.kylin.metadata.model.ISourceAware;
import org.apache.kylin.metadata.model.NTableMetadataManager;
@@ -54,12 +57,14 @@ import org.apache.kylin.metadata.view.LogicalViewManager;
import org.apache.kylin.rest.aspect.Transaction;
import org.apache.kylin.rest.request.S3TableExtInfo;
import org.apache.kylin.rest.request.TableExclusionRequest;
+import org.apache.kylin.rest.request.TableLoadRequest;
import org.apache.kylin.rest.request.UpdateAWSTableExtDescRequest;
import org.apache.kylin.rest.response.DataResult;
import org.apache.kylin.rest.response.ExcludedColumnResponse;
import org.apache.kylin.rest.response.ExcludedTableDetailResponse;
import org.apache.kylin.rest.response.ExcludedTableResponse;
import org.apache.kylin.rest.response.LoadTableResponse;
+import org.apache.kylin.rest.response.TableNameResponse;
import org.apache.kylin.rest.response.UpdateAWSTableExtDescResponse;
import org.apache.kylin.rest.security.KerberosLoginManager;
import org.apache.kylin.rest.util.AclEvaluate;
@@ -82,6 +87,8 @@ public class TableExtService extends BasicService {
public static final int DEFAULT_EXCLUDED_COLUMN_SIZE = 15;
+ public static final int ONCE_LOAD_TABLE_LIMIT_COUNT = 1000;
+
@Autowired
@Qualifier("tableService")
private TableService tableService;
@@ -89,13 +96,79 @@ public class TableExtService extends BasicService {
@Autowired
private AclEvaluate aclEvaluate;
- public LoadTableResponse loadDbTables(String[] dbTables, String project,
boolean isDb) throws Exception {
+ public LoadTableResponse loadTablesWithShortCircuit(TableLoadRequest
request) throws Exception {
+ String project = request.getProject();
aclEvaluate.checkProjectWritePermission(project);
- Map<String, Set<String>> dbTableMap = classifyDbTables(dbTables, isDb);
Set<String> existDbs =
Sets.newHashSet(tableService.getSourceDbNames(project));
- LoadTableResponse tableResponse = new LoadTableResponse();
+ int count = 0;
+ int dbSize = ArrayUtils.isNotEmpty(request.getDatabases()) ?
request.getDatabases().length : 0;
+ int tableSize = ArrayUtils.isNotEmpty(request.getTables()) ?
request.getTables().length : 0;
+ KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
+ boolean thresholdEnabled = kylinConfig.isTableLoadThresholdEnabled();
+ checkThreshold(thresholdEnabled, tableSize);
+
+ LoadTableResponse tableResponseFromDB = null;
+ List<Pair<TableDesc, TableExtDesc>> canLoadTablesFromDB = null;
+ Map<String, Set<String>> dbs = new HashMap<>();
+ if (dbSize > 0) {
+ tableResponseFromDB = new LoadTableResponse();
+ StringHelper.toUpperCaseArray(request.getDatabases(),
request.getDatabases());
+ dbs = classifyDbTables(request.getDatabases(), true);
+ Pair<List<Pair<TableDesc, TableExtDesc>>, Integer> pair =
findCanLoadTables(dbs, project,
+ true, tableResponseFromDB, existDbs);
+ canLoadTablesFromDB = pair.getFirst();
+ count = pair.getSecond();
+ checkThreshold(thresholdEnabled, count);
+ }
+
+ LoadTableResponse tableResponse = null;
+ List<Pair<TableDesc, TableExtDesc>> canLoadTables = null;
+ if (tableSize > 0) {
+ StringHelper.toUpperCaseArray(request.getTables(),
request.getTables());
+ Map<String, Set<String>> tables =
classifyDbTables(request.getTables(), false);
+ excludeTableFromFormalDB(dbs, tables);
+ tableResponse = new LoadTableResponse();
+ Pair<List<Pair<TableDesc, TableExtDesc>>, Integer> pair =
findCanLoadTables(tables, project,
+ false, tableResponse, existDbs);
+ canLoadTables = pair.getFirst();
+ count = pair.getSecond() + count;
+ checkThreshold(thresholdEnabled, count);
+ }
+
+ LoadTableResponse loadTableResponse = new LoadTableResponse();
+ if (tableResponseFromDB != null && !canLoadTablesFromDB.isEmpty()) {
+ innerLoadTables(project, tableResponseFromDB, canLoadTablesFromDB);
+
loadTableResponse.getFailed().addAll(tableResponseFromDB.getFailed());
+
loadTableResponse.getLoaded().addAll(tableResponseFromDB.getLoaded());
+
loadTableResponse.getNeedRealSampling().addAll(tableResponseFromDB.getNeedRealSampling());
+ }
+
+ if (tableResponse != null && !canLoadTables.isEmpty()) {
+ innerLoadTables(project, tableResponse, canLoadTables);
+ loadTableResponse.getFailed().addAll(tableResponse.getFailed());
+ loadTableResponse.getLoaded().addAll(tableResponse.getLoaded());
+
loadTableResponse.getNeedRealSampling().addAll(tableResponse.getNeedRealSampling());
+ }
+ return loadTableResponse;
+ }
+
+ private void checkThreshold(boolean thresholdEnabled, int count) {
+ if (thresholdEnabled && count > ONCE_LOAD_TABLE_LIMIT_COUNT) {
+ throw new KylinException(ONCE_LOAD_TABLE_LIMIT);
+ }
+ }
+
+ private void excludeTableFromFormalDB(Map<String, Set<String>> dbs,
Map<String, Set<String>> tables) {
+ if (dbs.size() > 0) {
+ tables.entrySet().removeIf(entry ->
dbs.containsKey(entry.getKey()));
+ }
+ }
+
+ public Pair<List<Pair<TableDesc, TableExtDesc>>, Integer>
findCanLoadTables(Map<String, Set<String>> dbTables, String project,
+ boolean isDb, LoadTableResponse tableResponse, Set<String>
existDbs) throws Exception {
List<Pair<TableDesc, TableExtDesc>> canLoadTables =
Lists.newArrayList();
- for (Map.Entry<String, Set<String>> entry : dbTableMap.entrySet()) {
+ List<TableNameResponse> responseAll = Lists.newArrayList();
+ for (Map.Entry<String, Set<String>> entry : dbTables.entrySet()) {
String db = entry.getKey();
Set<String> tableSet = entry.getValue();
if (!existDbs.contains(db)) {
@@ -117,10 +190,31 @@ public class TableExtService extends BasicService {
}
String[] tables = existTables.stream().map(table -> db + "." +
table).toArray(String[]::new);
- if (tables.length > 0){
+ if (tables.length > 0) {
filterAccessTables(tables, canLoadTables, tableResponse,
project);
}
+
+ List<TableNameResponse> response =
tableService.getHiveTableNameResponses(project, db, "");
+ response.forEach(t -> t.setTableName(db + "." + t.getTableName()));
+ responseAll.addAll(response);
}
+ return new Pair<>(canLoadTables, getTableCount(responseAll,
canLoadTables));
+ }
+
+ private int getTableCount(List<TableNameResponse> responseAll,
+ List<Pair<TableDesc, TableExtDesc>> canLoadTables) {
+ List<String> loaded =
responseAll.stream().filter(TableNameResponse::isLoaded)
+
.map(TableNameResponse::getTableName).collect(Collectors.toList());
+ return (int) canLoadTables.stream().filter(t ->
!loaded.contains(t.getFirst().getIdentity())).count();
+ }
+
+ public LoadTableResponse loadDbTables(String[] dbTables, String project,
boolean isDb) throws Exception {
+ aclEvaluate.checkProjectWritePermission(project);
+ Set<String> existDbs =
Sets.newHashSet(tableService.getSourceDbNames(project));
+ LoadTableResponse tableResponse = new LoadTableResponse();
+ Map<String, Set<String>> tables = classifyDbTables(dbTables, isDb);
+ List<Pair<TableDesc, TableExtDesc>> canLoadTables =
findCanLoadTables(tables, project, isDb, tableResponse,
+ existDbs).getFirst();
if (!canLoadTables.isEmpty()) {
return innerLoadTables(project, tableResponse, canLoadTables);
}
@@ -260,7 +354,9 @@ public class TableExtService extends BasicService {
loadTables.forEach(pair -> {
String tableName = pair.getFirst().getIdentity();
boolean success = true;
+ boolean realLoaded = false;
if (tableManager.getTableDesc(tableName) == null) {
+ realLoaded = true;
try {
loadTable(pair.getFirst(), pair.getSecond(), project);
} catch (Exception ex) {
@@ -270,6 +366,9 @@ public class TableExtService extends BasicService {
}
Set<String> targetSet = success ? tableResponse.getLoaded() :
tableResponse.getFailed();
targetSet.add(tableName);
+ if (success && realLoaded) {
+ tableResponse.getNeedRealSampling().add(tableName);
+ }
});
return tableResponse;
}, project, 1);
diff --git
a/src/datasource-service/src/test/java/org/apache/kylin/rest/service/TableExtServiceTest.java
b/src/datasource-service/src/test/java/org/apache/kylin/rest/service/TableExtServiceTest.java
index 5d7dfa889e..8b0440b854 100644
---
a/src/datasource-service/src/test/java/org/apache/kylin/rest/service/TableExtServiceTest.java
+++
b/src/datasource-service/src/test/java/org/apache/kylin/rest/service/TableExtServiceTest.java
@@ -23,6 +23,7 @@ import java.util.ArrayList;
import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;
+import java.util.stream.IntStream;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.exception.KylinException;
@@ -36,6 +37,7 @@ import org.apache.kylin.metadata.model.TableExtDesc;
import org.apache.kylin.rest.constant.Constant;
import org.apache.kylin.rest.request.S3TableExtInfo;
import org.apache.kylin.rest.request.TableExclusionRequest;
+import org.apache.kylin.rest.request.TableLoadRequest;
import org.apache.kylin.rest.request.UpdateAWSTableExtDescRequest;
import org.apache.kylin.rest.response.ExcludedColumnResponse;
import org.apache.kylin.rest.response.ExcludedTableDetailResponse;
@@ -539,6 +541,49 @@ public class TableExtServiceTest extends
NLocalFileMetadataTestCase {
}
}
+ @Test
+ public void testLoadTablesWithShortCircuit() throws Exception {
+ List<Pair<TableDesc, TableExtDesc>> lt1000 = mockTablePair(8, "TB");
+
Mockito.doReturn(lt1000).when(tableService).extractTableMeta(Mockito.any(),
Mockito.any());
+ TableLoadRequest request = new TableLoadRequest();
+ request.setDatabases(new String[]{"DEFAULT"});
+ request.setProject("default");
+ LoadTableResponse lt1000response =
tableExtService.loadTablesWithShortCircuit(request);
+ Assert.assertEquals(8, lt1000response.getFailed().size());
+
+ List<Pair<TableDesc, TableExtDesc>> gt1000 = mockTablePair(1001, "TB");
+
Mockito.doReturn(gt1000).when(tableService).extractTableMeta(Mockito.any(),
Mockito.any());
+ Assert.assertThrows(KylinException.class, () ->
tableExtService.loadTablesWithShortCircuit(request));
+
+ request.setTables(mockInputDBOrTable());
+ Assert.assertThrows(KylinException.class, () ->
tableExtService.loadTablesWithShortCircuit(request));
+
+ request.setTables(new String[]{"TEST_KYLIN_FACT"});
+ Assert.assertThrows(KylinException.class, () ->
tableExtService.loadTablesWithShortCircuit(request));
+
+ request.setDatabases(null);
+ gt1000.forEach(t ->
Mockito.doNothing().when(tableExtService).loadTable(t.getFirst(), t.getSecond(),
+ "default"));
+
Mockito.doReturn(gt1000).when(tableService).extractTableMeta(Mockito.any(),
Mockito.any());
+ Assert.assertThrows(KylinException.class, () ->
tableExtService.loadTablesWithShortCircuit(request));
+
+ request.setDatabases(null);
+ request.setTables(new String[]{"TEST_KYLIN_FACT"});
+ List<Pair<TableDesc, TableExtDesc>> table8 = mockTablePair(8, "TB");
+
Mockito.doReturn(table8).when(tableService).extractTableMeta(Mockito.any(),
Mockito.any());
+ LoadTableResponse response1 =
tableExtService.loadTablesWithShortCircuit(request);
+ Assert.assertEquals(8, response1.getFailed().size());
+
+ request.setDatabases(new String[]{"DEFAULT"});
+ request.setTables(new String[]{"TEST_KYLIN_FACT"});
+ LoadTableResponse response2 =
tableExtService.loadTablesWithShortCircuit(request);
+ Assert.assertEquals(8, response2.getFailed().size());
+ }
+
+ private String[] mockInputDBOrTable() {
+ return IntStream.range(0, 1000).mapToObj(t -> "TB" +
t).toArray(String[]::new);
+ }
+
private List<Pair<TableDesc, TableExtDesc>> mockTablePair(int size, String
tableName) {
List<Pair<TableDesc, TableExtDesc>> result = new ArrayList<>();
for (int i = 0; i < size; i++) {
diff --git
a/src/metadata-server/src/main/java/org/apache/kylin/rest/controller/NTableController.java
b/src/metadata-server/src/main/java/org/apache/kylin/rest/controller/NTableController.java
index b685328a42..ff35be5737 100644
---
a/src/metadata-server/src/main/java/org/apache/kylin/rest/controller/NTableController.java
+++
b/src/metadata-server/src/main/java/org/apache/kylin/rest/controller/NTableController.java
@@ -42,7 +42,6 @@ import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.exception.KylinException;
import org.apache.kylin.common.msg.MsgPicker;
import org.apache.kylin.common.util.Pair;
-import org.apache.kylin.common.util.StringHelper;
import org.apache.kylin.metadata.model.TableDesc;
import org.apache.kylin.metadata.project.NProjectManager;
import org.apache.kylin.rest.request.AWSTableLoadRequest;
@@ -230,26 +229,11 @@ public class NTableController extends NBasicController {
throw new KylinException(EMPTY_PARAMETER, "You should select at
least one table or database to load!!");
}
- LoadTableResponse loadTableResponse = new LoadTableResponse();
- if (ArrayUtils.isNotEmpty(tableLoadRequest.getTables())) {
- StringHelper.toUpperCaseArray(tableLoadRequest.getTables(),
tableLoadRequest.getTables());
- LoadTableResponse loadByTable =
tableExtService.loadDbTables(tableLoadRequest.getTables(),
- tableLoadRequest.getProject(), false);
- loadTableResponse.getFailed().addAll(loadByTable.getFailed());
- loadTableResponse.getLoaded().addAll(loadByTable.getLoaded());
- }
-
- if (ArrayUtils.isNotEmpty(tableLoadRequest.getDatabases())) {
- StringHelper.toUpperCaseArray(tableLoadRequest.getDatabases(),
tableLoadRequest.getDatabases());
- LoadTableResponse loadByDb =
tableExtService.loadDbTables(tableLoadRequest.getDatabases(),
- tableLoadRequest.getProject(), true);
- loadTableResponse.getFailed().addAll(loadByDb.getFailed());
- loadTableResponse.getLoaded().addAll(loadByDb.getLoaded());
- }
+ LoadTableResponse loadTableResponse =
tableExtService.loadTablesWithShortCircuit(tableLoadRequest);
- if (!loadTableResponse.getLoaded().isEmpty() &&
Boolean.TRUE.equals(tableLoadRequest.getNeedSampling())) {
+ if (!loadTableResponse.getNeedRealSampling().isEmpty() &&
Boolean.TRUE.equals(tableLoadRequest.getNeedSampling())) {
TableSamplingService.checkSamplingRows(tableLoadRequest.getSamplingRows());
- tableSamplingService.sampling(loadTableResponse.getLoaded(),
tableLoadRequest.getProject(),
+
tableSamplingService.sampling(loadTableResponse.getNeedRealSampling(),
tableLoadRequest.getProject(),
tableLoadRequest.getSamplingRows(),
tableLoadRequest.getPriority(), tableLoadRequest.getYarnQueue(),
tableLoadRequest.getTag());
}
diff --git
a/src/metadata-server/src/test/java/org/apache/kylin/rest/controller/NTableControllerTest.java
b/src/metadata-server/src/test/java/org/apache/kylin/rest/controller/NTableControllerTest.java
index a1f41efe75..34e87639cc 100644
---
a/src/metadata-server/src/test/java/org/apache/kylin/rest/controller/NTableControllerTest.java
+++
b/src/metadata-server/src/test/java/org/apache/kylin/rest/controller/NTableControllerTest.java
@@ -300,6 +300,8 @@ public class NTableControllerTest extends
NLocalFileMetadataTestCase {
.thenReturn(loadTableResponse);
Mockito.when(tableExtService.loadDbTables(tableLoadRequest.getDatabases(),
"default", true))
.thenReturn(loadTableResponse);
+
Mockito.when(tableExtService.loadTablesWithShortCircuit(tableLoadRequest))
+ .thenReturn(loadTableResponse);
}
@Test
@@ -313,6 +315,8 @@ public class NTableControllerTest extends
NLocalFileMetadataTestCase {
{
final TableLoadRequest tableLoadRequest = mockLoadTableRequest();
initMockito(loadTableResponse, tableLoadRequest);
+ tableLoadRequest.setNeedSampling(false);
+ tableLoadRequest.setSamplingRows(0);
mockMvc.perform(MockMvcRequestBuilders.post("/api/tables") //
.contentType(MediaType.APPLICATION_JSON) //
.content(JsonUtil.writeValueAsString(tableLoadRequest)) //
@@ -333,9 +337,10 @@ public class NTableControllerTest extends
NLocalFileMetadataTestCase {
TableLoadRequest request = new TableLoadRequest();
request.setDatabases(databasesUppercase);
request.setTables(tablesUppercase);
- request.setNeedSampling(false);
request.setProject(project);
initMockito(loadTableResponse, request);
+ request.setNeedSampling(false);
+ request.setSamplingRows(0);
request.setDatabases(databasesMixTure);
request.setTables(tablesMixTure);
@@ -344,8 +349,7 @@ public class NTableControllerTest extends
NLocalFileMetadataTestCase {
.content(JsonUtil.writeValueAsString(request)) //
.accept(MediaType.parseMediaType(APPLICATION_JSON)))
.andExpect(MockMvcResultMatchers.status().isOk());
- Mockito.verify(tableExtService,
Mockito.times(1)).loadDbTables(tablesUppercase, project, false);
- Mockito.verify(tableExtService,
Mockito.times(1)).loadDbTables(databasesUppercase, project, true);
+ Mockito.verify(tableExtService,
Mockito.times(1)).loadTablesWithShortCircuit(request);
request.setDatabases(databasesLowercase);
request.setTables(tablesLowercase);
@@ -354,8 +358,7 @@ public class NTableControllerTest extends
NLocalFileMetadataTestCase {
.content(JsonUtil.writeValueAsString(request)) //
.accept(MediaType.parseMediaType(APPLICATION_JSON)))
.andExpect(MockMvcResultMatchers.status().isOk());
- Mockito.verify(tableExtService,
Mockito.times(2)).loadDbTables(tablesUppercase, project, false);
- Mockito.verify(tableExtService,
Mockito.times(2)).loadDbTables(databasesUppercase, project, true);
+ Mockito.verify(tableExtService,
Mockito.times(1)).loadTablesWithShortCircuit(request);
request.setDatabases(databasesUppercase);
request.setTables(tablesUppercase);
@@ -364,8 +367,7 @@ public class NTableControllerTest extends
NLocalFileMetadataTestCase {
.content(JsonUtil.writeValueAsString(request)) //
.accept(MediaType.parseMediaType(APPLICATION_JSON)))
.andExpect(MockMvcResultMatchers.status().isOk());
- Mockito.verify(tableExtService,
Mockito.times(3)).loadDbTables(tablesUppercase, project, false);
- Mockito.verify(tableExtService,
Mockito.times(3)).loadDbTables(databasesUppercase, project, true);
+ Mockito.verify(tableExtService,
Mockito.times(1)).loadTablesWithShortCircuit(request);
}
}
@@ -649,6 +651,7 @@ public class NTableControllerTest extends
NLocalFileMetadataTestCase {
Set<String> loaded = Sets.newHashSet("default.test_kylin_fact");
LoadTableResponse loadTableResponse = new LoadTableResponse();
loadTableResponse.setLoaded(loaded);
+ loadTableResponse.setNeedRealSampling(loaded);
final TableLoadRequest tableLoadRequest = mockLoadTableRequest();
tableLoadRequest.setNeedSampling(true);
tableLoadRequest.setSamplingRows(200);
@@ -670,6 +673,7 @@ public class NTableControllerTest extends
NLocalFileMetadataTestCase {
Set<String> loaded = Sets.newHashSet("default.test_kylin_fact");
LoadTableResponse loadTableResponse = new LoadTableResponse();
loadTableResponse.setLoaded(loaded);
+ loadTableResponse.setNeedRealSampling(loaded);
final TableLoadRequest tableLoadRequest = mockLoadTableRequest();
tableLoadRequest.setNeedSampling(true);
tableLoadRequest.setSamplingRows(30_000_000);
