This is an automated email from the ASF dual-hosted git repository. xxyu pushed a commit to branch kylin5 in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 0f0f149bd5b77571c6fa10ce2dc5dea78e57bb99 Author: Guoliang Sun <guoliang....@kyligence.io> AuthorDate: Fri Dec 9 13:58:06 2022 +0800 KYLIN-5440 Follow up, optimize the speed of three datasource API --- .../kylin/rest/controller/NBasicController.java | 9 ++ .../org/apache/kylin/rest/response/DataResult.java | 7 ++ .../apache/kylin/rest/service/BasicService.java | 12 +++ .../kylin/rest/service/SnapshotSupporter.java | 6 +- .../rest/controller/NBasicControllerTest.java | 19 ++++ .../kylin/rest/controller/SnapshotController.java | 8 +- .../rest/controller/SnapshotControllerTest.java | 9 +- .../apache/kylin/rest/service/SnapshotService.java | 104 ++++++++++-------- .../kylin/rest/service/SnapshotServiceTest.java | 106 ++++++++++++++++++- .../kylin/rest/request/TableDescRequest.java | 88 ++++++++++++++++ .../apache/kylin/rest/service/TableService.java | 117 ++++++++++++--------- .../org/apache/kylin/rest/util/TableUtils.java | 17 +++ .../rest/service/StreamingTableServiceTest.java | 14 ++- .../kylin/rest/controller/NTableController.java | 40 +++---- .../rest/controller/open/OpenTableController.java | 9 +- .../rest/controller/v2/NTableControllerV2.java | 11 +- .../rest/controller/NTableControllerTest.java | 17 ++- .../rest/controller/NTableControllerV2Test.java | 7 +- .../controller/open/OpenTableControllerTest.java | 23 ++-- .../kylin/rest/service/TableServiceTest.java | 90 ++++++++++------ 20 files changed, 540 insertions(+), 173 deletions(-) diff --git a/src/common-service/src/main/java/org/apache/kylin/rest/controller/NBasicController.java b/src/common-service/src/main/java/org/apache/kylin/rest/controller/NBasicController.java index 884a807128..ab39cab22d 100644 --- a/src/common-service/src/main/java/org/apache/kylin/rest/controller/NBasicController.java +++ b/src/common-service/src/main/java/org/apache/kylin/rest/controller/NBasicController.java @@ -83,11 +83,13 @@ import org.apache.kylin.common.msg.MsgPicker; import org.apache.kylin.common.persistence.transaction.TransactionException; import org.apache.kylin.common.util.DateFormat; import org.apache.kylin.common.util.JsonUtil; +import org.apache.kylin.common.util.Pair; import org.apache.kylin.job.constant.JobStatusEnum; import org.apache.kylin.job.dao.ExecutablePO; import org.apache.kylin.job.execution.JobTypeEnum; import org.apache.kylin.metadata.model.NDataModel; import org.apache.kylin.metadata.model.NDataModelManager; +import org.apache.kylin.metadata.model.TableDesc; import org.apache.kylin.metadata.project.NProjectManager; import org.apache.kylin.metadata.project.ProjectInstance; import org.apache.kylin.metadata.streaming.KafkaConfigManager; @@ -378,6 +380,13 @@ public class NBasicController { return data; } + public Map<String, Object> setCustomDataResponse(String name, Pair<List<TableDesc>, Integer> result, int offset, int limit) { + Map<String, Object> data = new HashMap<>(); + data.put(name, PagingUtil.cutPage(result.getFirst(), offset, limit)); + data.put("size", result.getSecond()); + return data; + } + public List<?> getDataNoEnvelopeResponse(List<?> result, int offset, int limit) { return PagingUtil.cutPage(result, offset, limit); } diff --git a/src/common-service/src/main/java/org/apache/kylin/rest/response/DataResult.java b/src/common-service/src/main/java/org/apache/kylin/rest/response/DataResult.java index b7533cb581..86aee70e2d 100644 --- a/src/common-service/src/main/java/org/apache/kylin/rest/response/DataResult.java +++ b/src/common-service/src/main/java/org/apache/kylin/rest/response/DataResult.java @@ -21,6 +21,7 @@ package org.apache.kylin.rest.response; import java.util.Collection; import java.util.List; +import org.apache.kylin.common.util.Pair; import org.apache.kylin.rest.util.PagingUtil; import com.fasterxml.jackson.annotation.JsonProperty; @@ -65,4 +66,10 @@ public class DataResult<T extends Collection> { public static <E> DataResult<List<E>> get(List<E> data, int offset, int limit) { return get(PagingUtil.cutPage(data, offset, limit), data, offset, limit); } + + public static <E> DataResult<List<E>> getCustom(Pair<List<E>, Integer> objWithActualSize, int offset, int limit) { + // objWithActualSize's data cannot be null + return new DataResult<>(PagingUtil.cutPage(objWithActualSize.getFirst(), offset, limit), + objWithActualSize.getSecond(), offset, limit); + } } diff --git a/src/common-service/src/main/java/org/apache/kylin/rest/service/BasicService.java b/src/common-service/src/main/java/org/apache/kylin/rest/service/BasicService.java index 46a74acd81..b349943496 100644 --- a/src/common-service/src/main/java/org/apache/kylin/rest/service/BasicService.java +++ b/src/common-service/src/main/java/org/apache/kylin/rest/service/BasicService.java @@ -34,6 +34,7 @@ import org.apache.kylin.common.persistence.transaction.BroadcastEventReadyNotifi import org.apache.kylin.common.util.JsonUtil; import org.apache.kylin.metadata.epoch.EpochManager; import org.apache.kylin.metadata.project.EnhancedUnitOfWork; +import org.apache.kylin.common.util.Pair; import org.apache.kylin.metadata.project.NProjectManager; import org.apache.kylin.metadata.streaming.DataParserManager; import org.apache.kylin.rest.response.EnvelopeResponse; @@ -140,6 +141,17 @@ public abstract class BasicService { return true; } + public Pair<String, String> checkDatabaseAndTable(String table) { + if (table == null) + table = ""; + String database = null; + if (table.contains(".")) { + database = table.split("\\.", 2)[0].trim(); + table = table.split("\\.", 2)[1].trim(); + } + return Pair.newPair(database, table); + } + protected void initDefaultParser(String project) { if (getManager(DataParserManager.class, project).isInitialized()) { return; diff --git a/src/common-service/src/main/java/org/apache/kylin/rest/service/SnapshotSupporter.java b/src/common-service/src/main/java/org/apache/kylin/rest/service/SnapshotSupporter.java index b1052be4fb..5936dd20ef 100644 --- a/src/common-service/src/main/java/org/apache/kylin/rest/service/SnapshotSupporter.java +++ b/src/common-service/src/main/java/org/apache/kylin/rest/service/SnapshotSupporter.java @@ -21,10 +21,12 @@ package org.apache.kylin.rest.service; import java.util.List; import java.util.Set; +import org.apache.kylin.common.util.Pair; import org.apache.kylin.rest.constant.SnapshotStatus; import org.apache.kylin.rest.response.SnapshotInfoResponse; public interface SnapshotSupporter { - List<SnapshotInfoResponse> getProjectSnapshots(String project, String table, Set<SnapshotStatus> statusFilter, - Set<Boolean> partitionFilter, String sortBy, boolean isReversed); + Pair<List<SnapshotInfoResponse>, Integer> getProjectSnapshots(String project, String table, + Set<SnapshotStatus> statusFilter, Set<Boolean> partitionFilter, String sortBy, boolean isReversed, + Pair<Integer, Integer> offsetAndLimit); } diff --git a/src/common-service/src/test/java/org/apache/kylin/rest/controller/NBasicControllerTest.java b/src/common-service/src/test/java/org/apache/kylin/rest/controller/NBasicControllerTest.java index e36df02ccb..e436db2066 100644 --- a/src/common-service/src/test/java/org/apache/kylin/rest/controller/NBasicControllerTest.java +++ b/src/common-service/src/test/java/org/apache/kylin/rest/controller/NBasicControllerTest.java @@ -36,13 +36,16 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.Locale; +import java.util.Map; import org.apache.commons.lang3.StringUtils; import org.apache.kylin.common.exception.KylinException; import org.apache.kylin.common.msg.Message; import org.apache.kylin.common.msg.MsgPicker; import org.apache.kylin.common.util.NLocalFileMetadataTestCase; +import org.apache.kylin.common.util.Pair; import org.apache.kylin.metadata.model.PartitionDesc; +import org.apache.kylin.metadata.model.TableDesc; import org.apache.kylin.rest.controller.fixture.FixtureController; import org.apache.kylin.rest.exception.ForbiddenException; import org.apache.kylin.rest.exception.NotFoundException; @@ -292,4 +295,20 @@ public class NBasicControllerTest extends NLocalFileMetadataTestCase { () -> nBasicController.checkNonNegativeIntegerArg("id", -1)); } + @Test + public void testSetCustomDataResponse() { + TableDesc tableDesc = new TableDesc(); + tableDesc.setName("table1"); + Map<String, Object> mockDataResponse = nBasicController.setCustomDataResponse("table", + Pair.newPair(Collections.singletonList(tableDesc), 3), 0, 10); + Assert.assertNotNull(mockDataResponse); + Object tableData = mockDataResponse.get("table"); + if (tableData instanceof List<?>) { + for (Object tableDatum : (List<?>) tableData) { + Assert.assertEquals("table1", ((TableDesc)tableDatum).getName().toLowerCase(Locale.ROOT)); + } + } + Assert.assertEquals(3, mockDataResponse.get("size")); + } + } diff --git a/src/data-loading-server/src/main/java/org/apache/kylin/rest/controller/SnapshotController.java b/src/data-loading-server/src/main/java/org/apache/kylin/rest/controller/SnapshotController.java index f0030909e5..24bf3ccac4 100644 --- a/src/data-loading-server/src/main/java/org/apache/kylin/rest/controller/SnapshotController.java +++ b/src/data-loading-server/src/main/java/org/apache/kylin/rest/controller/SnapshotController.java @@ -31,6 +31,7 @@ import java.util.Set; import org.apache.commons.lang.exception.ExceptionUtils; import org.apache.kylin.common.KylinConfig; import org.apache.kylin.common.exception.KylinException; +import org.apache.kylin.common.util.Pair; import org.apache.kylin.metadata.project.NProjectManager; import org.apache.kylin.metadata.project.ProjectInstance; import org.apache.kylin.rest.constant.SnapshotStatus; @@ -221,9 +222,10 @@ public class SnapshotController extends BaseController { } catch (NoSuchFieldException e) { throw new KylinException(SORT_BY_FIELD_NOT_EXIST, sortBy); } - List<SnapshotInfoResponse> responses = snapshotService.getProjectSnapshots(project, table, statusFilter, - partitionFilter, sortBy, isReversed); - return new EnvelopeResponse<>(KylinException.CODE_SUCCESS, DataResult.get(responses, offset, limit), ""); + Pair<List<SnapshotInfoResponse>, Integer> snapshotsAndSize = snapshotService.getProjectSnapshots(project, table, + statusFilter, partitionFilter, sortBy, isReversed, Pair.newPair(offset, limit)); + return new EnvelopeResponse<>(KylinException.CODE_SUCCESS, + new DataResult<>(snapshotsAndSize.getFirst(), snapshotsAndSize.getSecond(), offset, limit), ""); } @ApiOperation(value = "getTables", tags = { "AI" }, notes = "get all tables with or without snapshot") diff --git a/src/data-loading-server/src/test/java/org/apache/kylin/rest/controller/SnapshotControllerTest.java b/src/data-loading-server/src/test/java/org/apache/kylin/rest/controller/SnapshotControllerTest.java index d3a2d3feee..924e8a08b4 100644 --- a/src/data-loading-server/src/test/java/org/apache/kylin/rest/controller/SnapshotControllerTest.java +++ b/src/data-loading-server/src/test/java/org/apache/kylin/rest/controller/SnapshotControllerTest.java @@ -25,6 +25,7 @@ import java.util.Map; import java.util.Set; import org.apache.kylin.common.util.JsonUtil; +import org.apache.kylin.common.util.Pair; import org.apache.kylin.rest.constant.Constant; import org.apache.kylin.common.util.NLocalFileMetadataTestCase; import org.apache.kylin.rest.constant.SnapshotStatus; @@ -235,8 +236,8 @@ public class SnapshotControllerTest extends NLocalFileMetadataTestCase { Set<SnapshotStatus> statusFilter = Sets.newHashSet(); String sortBy = "last_modified_time"; boolean isReversed = true; - Mockito.doAnswer(x -> null).when(snapshotService).getProjectSnapshots(project, table, statusFilter, - Sets.newHashSet(), sortBy, isReversed); + Mockito.doAnswer(x -> Pair.newPair(null, 10)).when(snapshotService).getProjectSnapshots(project, table, statusFilter, + Sets.newHashSet(), sortBy, isReversed, Pair.newPair(0, 10)); mockMvc.perform(MockMvcRequestBuilders.get("/api/snapshots").param("project", project) .contentType(MediaType.APPLICATION_JSON).accept(MediaType.parseMediaType(APPLICATION_PUBLIC_JSON))) .andExpect(MockMvcResultMatchers.status().isOk()); @@ -251,8 +252,8 @@ public class SnapshotControllerTest extends NLocalFileMetadataTestCase { Set<SnapshotStatus> statusFilter = Sets.newHashSet(); String sortBy = "UNKNOWN"; boolean isReversed = true; - Mockito.doAnswer(x -> null).when(snapshotService).getProjectSnapshots(project, table, statusFilter, null, - sortBy, isReversed); + Mockito.doAnswer(x -> Pair.newPair(null, 10)).when(snapshotService).getProjectSnapshots(project, table, statusFilter, null, + sortBy, isReversed, Pair.newPair(0, 10)); final MvcResult mvcResult = mockMvc .perform(MockMvcRequestBuilders.get("/api/snapshots").param("project", project).param("sort_by", sortBy) .contentType(MediaType.APPLICATION_JSON) diff --git a/src/data-loading-service/src/main/java/org/apache/kylin/rest/service/SnapshotService.java b/src/data-loading-service/src/main/java/org/apache/kylin/rest/service/SnapshotService.java index aeca5521a4..431758773d 100644 --- a/src/data-loading-service/src/main/java/org/apache/kylin/rest/service/SnapshotService.java +++ b/src/data-loading-service/src/main/java/org/apache/kylin/rest/service/SnapshotService.java @@ -79,6 +79,7 @@ import java.util.Locale; import java.util.Map; import java.util.Optional; import java.util.Set; +import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -94,6 +95,7 @@ import static org.apache.kylin.common.exception.code.ErrorCodeServer.REQUEST_PAR import static org.apache.kylin.job.execution.JobTypeEnum.SNAPSHOT_BUILD; import static org.apache.kylin.job.execution.JobTypeEnum.SNAPSHOT_REFRESH; import static org.apache.kylin.rest.constant.SnapshotStatus.BROKEN; +import static org.apache.kylin.rest.util.TableUtils.calculateTableSize; @Component("snapshotService") public class SnapshotService extends BasicService implements SnapshotSupporter { @@ -437,34 +439,72 @@ public class SnapshotService extends BasicService implements SnapshotSupporter { } @Override - public List<SnapshotInfoResponse> getProjectSnapshots(String project, String table, - Set<SnapshotStatus> statusFilter, Set<Boolean> partitionFilter, String sortBy, boolean isReversed) { + public Pair<List<SnapshotInfoResponse>, Integer> getProjectSnapshots(String project, String table, + Set<SnapshotStatus> statusFilter, Set<Boolean> partitionFilter, String sortBy, boolean isReversed, + Pair<Integer, Integer> offsetAndLimit) { checkSnapshotManualManagement(project); aclEvaluate.checkProjectReadPermission(project); NTableMetadataManager nTableMetadataManager = getManager(NTableMetadataManager.class, project); val execManager = NExecutableManager.getInstance(getConfig(), project); List<AbstractExecutable> executables = execManager.listExecByJobTypeAndStatus(ExecutableState::isRunning, SNAPSHOT_BUILD, SNAPSHOT_REFRESH); - if (table == null) - table = ""; - String database = null; - if (table.contains(".")) { - database = table.split("\\.", 2)[0].trim(); - table = table.split("\\.", 2)[1].trim(); - } - - final String finalTable = table; - final String finalDatabase = database; + Pair<String, String> databaseAndTable = checkDatabaseAndTable(table); Set<String> groups = getCurrentUserGroups(); boolean canUseACLGreenChannel = AclPermissionUtil.canUseACLGreenChannel(project, groups); + Set<String> finalAuthorizedTables = getAclAuthorizedTables(project, canUseACLGreenChannel); + + // Adjust the operation of adding SnapshotInfoResponse and then removing it to + // first remove the tableDesc that does not meet the conditions, and then add SnapshotInfoResponse + List<TableDesc> tables = getFilteredTables(nTableMetadataManager, databaseAndTable, canUseACLGreenChannel, + finalAuthorizedTables, executables, statusFilter, partitionFilter); + + List<SnapshotInfoResponse> response = new ArrayList<>(); + // Here we keep the actual size of tableSnapshots and process only a portion of the data based on paging + final int returnTableSize = calculateTableSize(offsetAndLimit.getFirst(), offsetAndLimit.getSecond()); + final int actualTableSize = tables.size(); + AtomicInteger satisfiedTableSize = new AtomicInteger(); + + tables.forEach(tableDesc -> { + if (satisfiedTableSize.get() == returnTableSize) { + return; + } + TableExtDesc tableExtDesc = nTableMetadataManager.getOrCreateTableExt(tableDesc); + Pair<Integer, Integer> countPair = getModelCount(tableDesc); + response.add(new SnapshotInfoResponse(tableDesc, tableExtDesc, tableDesc.getSnapshotTotalRows(), countPair.getFirst(), + countPair.getSecond(), getSnapshotJobStatus(tableDesc, executables), + getForbiddenColumns(tableDesc))); + satisfiedTableSize.getAndIncrement(); + }); + + sortBy = StringUtils.isEmpty(sortBy) ? "last_modified_time" : sortBy; + if ("last_modified_time".equalsIgnoreCase(sortBy) && isReversed) { + // The reverse order here needs to be cut from the beginning to the end, otherwise the initial data is always returned + response.sort(SnapshotInfoResponse::compareTo); + return Pair.newPair(PagingUtil.cutPage(response, 0, offsetAndLimit.getSecond()), actualTableSize); + } else { + // Here the positive order needs to be cut from the offset position backwards + Comparator<SnapshotInfoResponse> comparator = BasicService.propertyComparator(sortBy, !isReversed); + response.sort(comparator); + return Pair.newPair(PagingUtil.cutPage(response, offsetAndLimit.getFirst(), offsetAndLimit.getSecond()), actualTableSize); + } + } + + public Set<String> getAclAuthorizedTables(String project, boolean canUseACLGreenChannel) { Set<String> authorizedTables = new HashSet<>(); if (!canUseACLGreenChannel) { authorizedTables = getAuthorizedTables(project, getManager(AclTCRManager.class, project)); } - Set<String> finalAuthorizedTables = authorizedTables; - List<TableDesc> tables = nTableMetadataManager.listAllTables().stream().filter(tableDesc -> { + return authorizedTables; + } + + public List<TableDesc> getFilteredTables(NTableMetadataManager nTableMetadataManager, + Pair<String, String> databaseAndTable, boolean canUseACLGreenChannel, Set<String> finalAuthorizedTables, + List<AbstractExecutable> executables, Set<SnapshotStatus> statusFilter, Set<Boolean> partitionFilter) { + String finalDatabase = databaseAndTable.getFirst(); + String finalTable = databaseAndTable.getSecond(); + return nTableMetadataManager.listAllTables().stream().filter(tableDesc -> { if (StringUtils.isEmpty(finalDatabase)) { return true; } @@ -482,36 +522,16 @@ public class SnapshotService extends BasicService implements SnapshotSupporter { if (canUseACLGreenChannel) { return true; } - return finalAuthorizedTables.contains(tableDesc.getIdentity()); - }).filter(tableDesc -> hasLoadedSnapshot(tableDesc, executables)).collect(Collectors.toList()); - - List<SnapshotInfoResponse> response = new ArrayList<>(); - tables.forEach(tableDesc -> { - TableExtDesc tableExtDesc = nTableMetadataManager.getOrCreateTableExt(tableDesc); - Pair<Integer, Integer> countPair = getModelCount(tableDesc); - response.add(new SnapshotInfoResponse(tableDesc, tableExtDesc, tableDesc.getSnapshotTotalRows(), countPair.getFirst(), - countPair.getSecond(), getSnapshotJobStatus(tableDesc, executables), - getForbiddenColumns(tableDesc))); - }); - - if (!statusFilter.isEmpty()) { - response.removeIf(res -> !statusFilter.contains(res.getStatus())); - } - if (partitionFilter.size() == 1) { + }).filter(tableDesc -> hasLoadedSnapshot(tableDesc, executables) + ).filter(tableDesc -> statusFilter.isEmpty() || statusFilter.contains(getSnapshotJobStatus(tableDesc, executables)) + ).filter(tableDesc -> { + if (partitionFilter.size() != 1) { + return true; + } boolean isPartition = partitionFilter.iterator().next(); - response.removeIf(res -> isPartition == (res.getSelectPartitionCol() == null)); - } - - sortBy = StringUtils.isEmpty(sortBy) ? "last_modified_time" : sortBy; - if ("last_modified_time".equalsIgnoreCase(sortBy) && isReversed) { - response.sort(SnapshotInfoResponse::compareTo); - } else { - Comparator<SnapshotInfoResponse> comparator = BasicService.propertyComparator(sortBy, !isReversed); - response.sort(comparator); - } - - return response; + return isPartition != (tableDesc.getSelectedSnapshotPartitionCol() == null); + }).collect(Collectors.toList()); } private Pair<Integer, Integer> getModelCount(TableDesc tableDesc) { diff --git a/src/data-loading-service/src/test/java/org/apache/kylin/rest/service/SnapshotServiceTest.java b/src/data-loading-service/src/test/java/org/apache/kylin/rest/service/SnapshotServiceTest.java index 73334f8b0f..16032ee326 100644 --- a/src/data-loading-service/src/test/java/org/apache/kylin/rest/service/SnapshotServiceTest.java +++ b/src/data-loading-service/src/test/java/org/apache/kylin/rest/service/SnapshotServiceTest.java @@ -485,12 +485,12 @@ public class SnapshotServiceTest extends NLocalFileMetadataTestCase { SecurityContextHolder.getContext() .setAuthentication(new TestingAuthenticationToken("testuser", "testuser", Constant.ROLE_MODELER)); List<SnapshotInfoResponse> responses = snapshotService.getProjectSnapshots(PROJECT, tablePattern, statusFilter, - Sets.newHashSet(), sortBy, true); + Sets.newHashSet(), sortBy, true, Pair.newPair(0, 10)).getFirst(); Assert.assertEquals(0, responses.size()); SecurityContextHolder.getContext() .setAuthentication(new TestingAuthenticationToken("ADMIN", "ADMIN", Constant.ROLE_ADMIN)); responses = snapshotService.getProjectSnapshots(PROJECT, tablePattern, statusFilter, Sets.newHashSet(), sortBy, - true); + true, Pair.newPair(0, 2)).getFirst(); SnapshotInfoResponse response = responses.get(0); Assert.assertEquals(2, responses.size()); Assert.assertEquals("SSB", response.getDatabase()); @@ -498,6 +498,102 @@ public class SnapshotServiceTest extends NLocalFileMetadataTestCase { responses.stream().map(SnapshotInfoResponse::getTable).collect(Collectors.toSet())); } + @Test + public void testGetProjectSnapshotsReturn() { + enableSnapshotManualManagement(); + setSnapshotPath("SSB.LINEORDER", "some_path"); + setSnapshotPath("SSB.P_LINEORDER", "some_path"); + getTestConfig().setProperty("kylin.query.security.acl-tcr-enabled", "true"); + SecurityContextHolder.getContext() + .setAuthentication(new TestingAuthenticationToken("ADMIN", "ADMIN", Constant.ROLE_ADMIN)); + + // default sort + Pair<List<SnapshotInfoResponse>, Integer> projectSnapshots = snapshotService.getProjectSnapshots(PROJECT, "SSB", + Sets.newHashSet(SnapshotStatus.ONLINE), Sets.newHashSet(), "", true, + Pair.newPair(0, 1)); + Assert.assertEquals(1, projectSnapshots.getFirst().size()); + Assert.assertEquals(2, projectSnapshots.getSecond().intValue()); + + // default sort but reverse + projectSnapshots = snapshotService.getProjectSnapshots(PROJECT, "SSB", + Sets.newHashSet(SnapshotStatus.ONLINE), Sets.newHashSet(), "", false, + Pair.newPair(0, 1)); + Assert.assertEquals(1, projectSnapshots.getFirst().size()); + Assert.assertEquals(2, projectSnapshots.getSecond().intValue()); + + // sort by table + projectSnapshots = snapshotService.getProjectSnapshots(PROJECT, "SSB", + Sets.newHashSet(SnapshotStatus.ONLINE), Sets.newHashSet(), "table", true, + Pair.newPair(0, 1)); + Assert.assertEquals(1, projectSnapshots.getFirst().size()); + Assert.assertEquals(2, projectSnapshots.getSecond().intValue()); + + // sort by table not reverse + projectSnapshots = snapshotService.getProjectSnapshots(PROJECT, "SSB", + Sets.newHashSet(SnapshotStatus.ONLINE), Sets.newHashSet(), "table", false, + Pair.newPair(0, 1)); + Assert.assertEquals(1, projectSnapshots.getFirst().size()); + Assert.assertEquals(2, projectSnapshots.getSecond().intValue()); + } + + @Test + public void testGetProjectSnapshotsFilter() { + enableSnapshotManualManagement(); + setSnapshotPath("SSB.LINEORDER", "some_path"); + setSnapshotPath("SSB.P_LINEORDER", "some_path"); + getTestConfig().setProperty("kylin.query.security.acl-tcr-enabled", "true"); + SecurityContextHolder.getContext() + .setAuthentication(new TestingAuthenticationToken("ADMIN", "ADMIN", Constant.ROLE_ADMIN)); + + // status empty + Pair<List<SnapshotInfoResponse>, Integer> projectSnapshots = snapshotService.getProjectSnapshots(PROJECT, "SSB", + Sets.newHashSet(), Sets.newHashSet(), "", true, + Pair.newPair(0, 1)); + Assert.assertEquals(1, projectSnapshots.getFirst().size()); + + // sort by table and status broken + projectSnapshots = snapshotService.getProjectSnapshots(PROJECT, "SSB", + Sets.newHashSet(SnapshotStatus.BROKEN), Sets.newHashSet(), "table", true, + Pair.newPair(0, 1)); + Assert.assertEquals(0, projectSnapshots.getFirst().size()); + + // partitionFilter false and tableSelectedSnapshotPartitionCol is null + projectSnapshots = snapshotService.getProjectSnapshots(PROJECT, "SSB", + Sets.newHashSet(), Sets.newHashSet(false), "table", true, + Pair.newPair(0, 1)); + Assert.assertEquals(1, projectSnapshots.getFirst().size()); + + // partitionFilter false and tableSelectedSnapshotPartitionCol is not null + String partColName = "LO_ORDERKEY"; + String tableName = "SSB.LINEORDER"; + snapshotService.configSnapshotPartitionCol(PROJECT, + ImmutableMap.<String, String> builder().put(tableName, partColName).build()); + TableDesc tableDesc = NTableMetadataManager.getInstance(getTestConfig(), PROJECT).getTableDesc(tableName); + Assert.assertEquals(partColName, tableDesc.getSelectedSnapshotPartitionCol()); + projectSnapshots = snapshotService.getProjectSnapshots(PROJECT, "SSB", + Sets.newHashSet(), Sets.newHashSet(false), "table", true, + Pair.newPair(0, 1)); + Assert.assertEquals(1, projectSnapshots.getFirst().size()); + + // partitionFilter true and tableSelectedSnapshotPartitionCol is null + projectSnapshots = snapshotService.getProjectSnapshots(PROJECT, "SSB", + Sets.newHashSet(), Sets.newHashSet(true), "table", true, + Pair.newPair(0, 1)); + Assert.assertEquals(1, projectSnapshots.getFirst().size()); + + // partitionFilter true and tableSelectedSnapshotPartitionCol is not null + projectSnapshots = snapshotService.getProjectSnapshots(PROJECT, "SSB", + Sets.newHashSet(), Sets.newHashSet(true), "table", true, + Pair.newPair(0, 1)); + Assert.assertEquals(1, projectSnapshots.getFirst().size()); + } + + @Test + public void testCheckDatabaseAndTable() { + Pair<String, String> tableAndDatabase = Pair.newPair("SSB", "CUSTOM"); + Assert.assertEquals(tableAndDatabase, snapshotService.checkDatabaseAndTable("SSB.CUSTOM")); + } + @Test public void testGetTables() { enableSnapshotManualManagement(); @@ -669,7 +765,8 @@ public class SnapshotServiceTest extends NLocalFileMetadataTestCase { return null; }, PROJECT); List<SnapshotInfoResponse> responses = snapshotService.getProjectSnapshots(PROJECT, null, - Sets.newHashSet(SnapshotStatus.BROKEN), Sets.newHashSet(), null, true); + Sets.newHashSet(SnapshotStatus.BROKEN), Sets.newHashSet(), null, true, + Pair.newPair(0, 10)).getFirst(); Assert.assertEquals(1, responses.size()); } @@ -729,7 +826,8 @@ public class SnapshotServiceTest extends NLocalFileMetadataTestCase { return null; }, PROJECT); List<SnapshotInfoResponse> responses = snapshotService.getProjectSnapshots(PROJECT, tableName, - Sets.newHashSet(SnapshotStatus.BROKEN), Sets.newHashSet(), null, true); + Sets.newHashSet(SnapshotStatus.BROKEN), Sets.newHashSet(), null, true, + Pair.newPair(0, 10)).getFirst(); Assert.assertEquals(1, responses.size()); Assert.assertEquals(10, responses.get(0).getUsage()); } diff --git a/src/datasource-service/src/main/java/org/apache/kylin/rest/request/TableDescRequest.java b/src/datasource-service/src/main/java/org/apache/kylin/rest/request/TableDescRequest.java new file mode 100644 index 0000000000..522cff00e3 --- /dev/null +++ b/src/datasource-service/src/main/java/org/apache/kylin/rest/request/TableDescRequest.java @@ -0,0 +1,88 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kylin.rest.request; + +import com.fasterxml.jackson.annotation.JsonProperty; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; +import org.apache.kylin.common.util.Pair; + +import java.util.List; + +@NoArgsConstructor +@AllArgsConstructor +@Data +public class TableDescRequest { + + @JsonProperty(required = true) + private String project; + + private String table; + + private String database; + + @JsonProperty(value = "ext") + private boolean withExt; + + @JsonProperty(value = "is_fuzzy", defaultValue = "false") + private boolean isFuzzy; + + @JsonProperty(value = "page_offset", defaultValue = "0") + private Integer offset; + + @JsonProperty(value = "page_size", defaultValue = "10") + private Integer limit; + + @JsonProperty(value = "source_type", defaultValue = "9") + private List<Integer> sourceType; + + @JsonProperty(value = "with_excluded", defaultValue = "true") + private boolean withExcluded; + + public TableDescRequest(String project, boolean withExt, String table, String database, boolean isFuzzy, List<Integer> sourceType) { + this.project = project; + this.withExt = withExt; + this.table = table; + this.database = database; + this.isFuzzy = isFuzzy; + this.sourceType = sourceType; + } + + public TableDescRequest(String project, String table, Integer offset, Integer limit, boolean withExcluded, List<Integer> sourceType) { + this.project = project; + this.table = table; + this.offset = offset; + this.limit = limit; + this.withExcluded = withExcluded; + this.sourceType = sourceType; + } + + public TableDescRequest(String project, String table, String database, boolean withExt, boolean isFuzzy, + Pair<Integer, Integer> offsetAndLimit, List<Integer> sourceType) { + this.project = project; + this.table = table; + this.database = database; + this.withExt = withExt; + this.isFuzzy = isFuzzy; + this.offset = offsetAndLimit.getFirst(); + this.limit = offsetAndLimit.getSecond(); + this.sourceType = sourceType; + } +} diff --git a/src/datasource-service/src/main/java/org/apache/kylin/rest/service/TableService.java b/src/datasource-service/src/main/java/org/apache/kylin/rest/service/TableService.java index 436a030671..950469188d 100644 --- a/src/datasource-service/src/main/java/org/apache/kylin/rest/service/TableService.java +++ b/src/datasource-service/src/main/java/org/apache/kylin/rest/service/TableService.java @@ -34,6 +34,7 @@ import static org.apache.kylin.common.exception.code.ErrorCodeServer.TABLE_RELOA import static org.apache.kylin.common.exception.code.ErrorCodeServer.TABLE_RELOAD_MODEL_RETRY; import static org.apache.kylin.job.execution.JobTypeEnum.SNAPSHOT_BUILD; import static org.apache.kylin.job.execution.JobTypeEnum.SNAPSHOT_REFRESH; +import static org.apache.kylin.rest.util.TableUtils.calculateTableSize; import java.io.File; import java.io.IOException; @@ -139,6 +140,7 @@ import org.apache.kylin.rest.constant.JobInfoEnum; import org.apache.kylin.rest.request.AutoMergeRequest; import org.apache.kylin.rest.request.DateRangeRequest; import org.apache.kylin.rest.request.S3TableExtInfo; +import org.apache.kylin.rest.request.TableDescRequest; import org.apache.kylin.rest.response.AutoMergeConfigResponse; import org.apache.kylin.rest.response.BatchLoadTableResponse; import org.apache.kylin.rest.response.EnvelopeResponse; @@ -226,44 +228,43 @@ public class TableService extends BasicService { @Autowired private ClusterManager clusterManager; - public List<TableDesc> getTableDescByType(String project, boolean withExt, final String tableName, - final String database, boolean isFuzzy, int sourceType) throws IOException { - return getTableDesc(project, withExt, tableName, database, isFuzzy).stream() - .filter(tableDesc -> sourceType == tableDesc.getSourceType()).collect(Collectors.toList()); + public Pair<List<TableDesc>, Integer> getTableDesc(String project, boolean withExt, final String table, final String database, + boolean isFuzzy, List<Integer> sourceType, int returnTableSize) throws IOException { + TableDescRequest internalTableDescRequest = new TableDescRequest(project, withExt, table, database, isFuzzy, sourceType); + return getTableDesc(internalTableDescRequest, returnTableSize); } - public List<TableDesc> getTableDescByTypes(String project, boolean withExt, final String tableName, - final String database, boolean isFuzzy, List<Integer> sourceType) throws IOException { - return getTableDesc(project, withExt, tableName, database, isFuzzy).stream() - .filter(tableDesc -> sourceType.contains(tableDesc.getSourceType())).collect(Collectors.toList()); - } - - public List<TableDesc> getTableDesc(String project, boolean withExt, final String tableName, final String database, - boolean isFuzzy) throws IOException { - aclEvaluate.checkProjectReadPermission(project); + public Pair<List<TableDesc>, Integer> getTableDesc(TableDescRequest tableDescRequest, int returnTableSize) throws IOException { + aclEvaluate.checkProjectReadPermission(tableDescRequest.getProject()); boolean streamingEnabled = getConfig().streamingEnabled(); - NTableMetadataManager nTableMetadataManager = getManager(NTableMetadataManager.class, project); + NTableMetadataManager nTableMetadataManager = getManager(NTableMetadataManager.class, tableDescRequest.getProject()); List<TableDesc> tables = Lists.newArrayList(); //get table not fuzzy,can use getTableDesc(tableName) - if (StringUtils.isNotEmpty(tableName) && !isFuzzy) { - val tableDesc = nTableMetadataManager.getTableDesc(database + "." + tableName); + if (StringUtils.isNotEmpty(tableDescRequest.getTable()) && !tableDescRequest.isFuzzy()) { + val tableDesc = nTableMetadataManager.getTableDesc(tableDescRequest.getDatabase() + "." + tableDescRequest.getTable()); if (tableDesc != null && tableDesc.isAccessible(streamingEnabled)) tables.add(tableDesc); } else { tables.addAll(nTableMetadataManager.listAllTables().stream().filter(tableDesc -> { - if (StringUtils.isEmpty(database)) { + if (StringUtils.isEmpty(tableDescRequest.getDatabase())) { return true; } - return tableDesc.getDatabase().equalsIgnoreCase(database); + return tableDesc.getDatabase().equalsIgnoreCase(tableDescRequest.getDatabase()); }).filter(tableDesc -> { - if (StringUtils.isEmpty(tableName)) { + if (StringUtils.isEmpty(tableDescRequest.getTable())) { return true; } - return tableDesc.getName().toLowerCase(Locale.ROOT).contains(tableName.toLowerCase(Locale.ROOT)); + return tableDesc.getName().toLowerCase(Locale.ROOT).contains(tableDescRequest.getTable().toLowerCase(Locale.ROOT)); + }).filter(tableDesc -> { + // Advance the logic of filtering the table by sourceType to here + if (!tableDescRequest.getSourceType().isEmpty()) { + return tableDescRequest.getSourceType().contains(tableDesc.getSourceType()); + } + return true; }).filter(table -> table.isAccessible(streamingEnabled)).sorted(this::compareTableDesc) .collect(Collectors.toList())); } - return getTablesResponse(tables, project, withExt); + return getTablesResponse(tables, tableDescRequest.getProject(), tableDescRequest.isWithExt(), returnTableSize); } public int compareTableDesc(TableDesc table1, TableDesc table2) { @@ -417,7 +418,10 @@ public class TableService extends BasicService { return tableNameResponses; } - private TableDescResponse getTableResponse(TableDesc table, String project) { + private TableDescResponse getTableResponse(TableDesc table, String project, boolean withExt) { + if (!withExt) { + return new TableDescResponse(table); + } TableDescResponse tableDescResponse = new TableDescResponse(table); TableExtDesc tableExtDesc = getManager(NTableMetadataManager.class, project).getTableExtIfExists(table); if (table.isKafkaTable()) { @@ -447,7 +451,7 @@ public class TableService extends BasicService { return tableDescResponse; } - private List<TableDesc> getTablesResponse(List<TableDesc> tables, String project, boolean withExt) { + private Pair<List<TableDesc>, Integer> getTablesResponse(List<TableDesc> tables, String project, boolean withExt, int returnTableSize) { List<TableDesc> descs = new ArrayList<>(); val projectManager = getManager(NProjectManager.class); val groups = getCurrentUserGroups(); @@ -458,23 +462,23 @@ public class TableService extends BasicService { List<NDataModel> healthyModels = projectManager.listHealthyModels(project); Set<String> extPermissionSet = accessService.getUserNormalExtPermissions(project); boolean hasDataQueryPermission = extPermissionSet.contains(ExternalAclProvider.DATA_QUERY); + int satisfiedTableSize = 0; for (val originTable : tables) { + // New judgment logic, when the total size of tables meet the current size of paging directly after the exit + // Also, if the processing is not finished, the total size of tables is returned + if (satisfiedTableSize == returnTableSize) { + return Pair.newPair(descs, tables.size()); + } TableDesc table = getAuthorizedTableDesc(project, isAclGreen, originTable, aclTCRS); if (Objects.isNull(table)) { continue; } - TableDescResponse tableDescResponse; + TableDescResponse tableDescResponse = getTableResponse(table, project, withExt); List<NDataModel> modelsUsingTable = healthyModels.stream() // .filter(model -> model.containsTable(table)).collect(Collectors.toList()); List<NDataModel> modelsUsingRootTable = healthyModels.stream() // .filter(model -> model.isRootFactTable(table)).collect(Collectors.toList()); - if (withExt) { - tableDescResponse = getTableResponse(table, project); - } else { - tableDescResponse = new TableDescResponse(table); - } - TableExtDesc tableExtDesc = getManager(NTableMetadataManager.class, project).getTableExtIfExists(table); if (tableExtDesc != null) { tableDescResponse.setTotalRecords(tableExtDesc.getTotalRows()); @@ -496,9 +500,9 @@ public class TableService extends BasicService { tableDescResponse.setForeignKey(tableColumnType.getSecond()); tableDescResponse.setPrimaryKey(tableColumnType.getFirst()); descs.add(tableDescResponse); + satisfiedTableSize++; } - - return descs; + return Pair.newPair(descs, descs.size()); } @VisibleForTesting @@ -1780,22 +1784,23 @@ public class TableService extends BasicService { return loadedDatabases; } - public interface ProjectTablesFilter { - List process(String database, String table) throws Exception; + public NInitTablesResponse getProjectTables(String project, String table, int offset, int limit, + boolean withExcluded, boolean useHiveDatabase, List<Integer> sourceType) throws Exception { + TableDescRequest internalTableDescRequest = new TableDescRequest(project, table, offset, limit, withExcluded, sourceType); + return getProjectTables(internalTableDescRequest, useHiveDatabase); } - public NInitTablesResponse getProjectTables(String project, String table, int offset, int limit, - boolean withExcluded, boolean useHiveDatabase, ProjectTablesFilter projectTablesFilter) throws Exception { + public NInitTablesResponse getProjectTables(TableDescRequest tableDescRequest, boolean useHiveDatabase) throws Exception { + String project = tableDescRequest.getProject(); aclEvaluate.checkProjectReadPermission(project); NInitTablesResponse response = new NInitTablesResponse(); - logger.debug("only get project tables of excluded: {}", withExcluded); - if (table == null) - table = ""; - String exceptDatabase = null; - if (table.contains(".")) { - exceptDatabase = table.split("\\.", 2)[0].trim(); - table = table.split("\\.", 2)[1].trim(); - } + logger.debug("only get project tables of excluded: {}", tableDescRequest.isWithExcluded()); + + Pair<String, String> databaseAndTable = checkDatabaseAndTable(tableDescRequest.getTable()); + String exceptDatabase = databaseAndTable.getFirst(); + String table = databaseAndTable.getSecond(); + String notAllowedModifyTableName = table; + Collection<String> databases = useHiveDatabase ? getSourceDbNames(project) : getLoadedDatabases(project); val projectInstance = getManager(NProjectManager.class).getProject(project); List<String> tableFilterList = DataSourceState.getInstance().getHiveFilterList(projectInstance); @@ -1804,15 +1809,29 @@ public class TableService extends BasicService { || (!tableFilterList.isEmpty() && !tableFilterList.contains(database))) { continue; } - List<?> tables; + // we may temporarily change the table name, but later to change back + // Avoid affecting the next loop and causing logic errors if (exceptDatabase == null && database.toLowerCase(Locale.ROOT).contains(table.toLowerCase(Locale.ROOT))) { - tables = projectTablesFilter.process(database, ""); + table = ""; + } + tableDescRequest.setDatabase(database); + tableDescRequest.setTable(table); + Pair<List<?>, Integer> objWithActualSize = new Pair<>(); + if (tableDescRequest.getSourceType().isEmpty()) { + // This means request api for showProjectTableNames + List<TableNameResponse> hiveTableNameResponses = getHiveTableNameResponses(project, database, table); + objWithActualSize.setFirst(hiveTableNameResponses); + objWithActualSize.setSecond(hiveTableNameResponses.size()); } else { - tables = projectTablesFilter.process(database, table); + int returnTableSize = calculateTableSize(tableDescRequest.getOffset(), tableDescRequest.getLimit()); + Pair<List<TableDesc>, Integer> tableDescWithActualSize = getTableDesc(tableDescRequest, returnTableSize); + objWithActualSize.setFirst(tableDescWithActualSize.getFirst()); + objWithActualSize.setSecond(tableDescWithActualSize.getSecond()); } - List<?> tablePage = PagingUtil.cutPage(tables, offset, limit); + table = notAllowedModifyTableName; + List<?> tablePage = PagingUtil.cutPage(objWithActualSize.getFirst(), tableDescRequest.getOffset(), tableDescRequest.getLimit()); if (!tablePage.isEmpty()) { - response.putDatabase(database, tables.size(), tablePage); + response.putDatabase(database, objWithActualSize.getSecond(), tablePage); } } return response; diff --git a/src/datasource-service/src/main/java/org/apache/kylin/rest/util/TableUtils.java b/src/datasource-service/src/main/java/org/apache/kylin/rest/util/TableUtils.java index 4e44101e67..c2ed932052 100644 --- a/src/datasource-service/src/main/java/org/apache/kylin/rest/util/TableUtils.java +++ b/src/datasource-service/src/main/java/org/apache/kylin/rest/util/TableUtils.java @@ -42,4 +42,21 @@ public class TableUtils { } } + /** + * <p> + * Calculate the number of valid tables to be returned based on pageOffset and pageSize + * Note: Tables will be filtered under certain conditions, but the final result must still be the number of valid tables, + * unless all tables have been processed. + * For example: + * the first page: pageOffset 0, pageSize 7, return 0 * 7 + 7 = 7 + * The second page: pageOffset 1, pageSize 7, return 1 * 7 + 7 = 14 + * </p> + * + * @param pageOffset page offset + * @param pageSize page size + * @return Number of valid tables + */ + public static int calculateTableSize(int pageOffset, int pageSize) { + return pageOffset * pageSize + pageSize; + } } diff --git a/src/datasource-service/src/test/java/org/apache/kylin/rest/service/StreamingTableServiceTest.java b/src/datasource-service/src/test/java/org/apache/kylin/rest/service/StreamingTableServiceTest.java index 75f774865f..a5f92508b0 100644 --- a/src/datasource-service/src/test/java/org/apache/kylin/rest/service/StreamingTableServiceTest.java +++ b/src/datasource-service/src/test/java/org/apache/kylin/rest/service/StreamingTableServiceTest.java @@ -48,6 +48,7 @@ import org.junit.Assert; import org.junit.Before; import org.junit.Rule; import org.junit.Test; +import org.junit.rules.TemporaryFolder; import org.mockito.InjectMocks; import org.mockito.Mock; import org.mockito.Mockito; @@ -59,6 +60,8 @@ import org.springframework.test.util.ReflectionTestUtils; import lombok.val; public class StreamingTableServiceTest extends NLocalFileMetadataTestCase { + @Rule + public TemporaryFolder temporaryFolder = new TemporaryFolder(); @Mock private AclUtil aclUtil = Mockito.spy(AclUtil.class); @@ -115,7 +118,7 @@ public class StreamingTableServiceTest extends NLocalFileMetadataTestCase { val prj = prjManager.getProject(PROJECT); val copy = prjManager.copyForWrite(prj); prjManager.updateProject(copy); - Mockito.when(userService.listSuperAdminUsers()).thenReturn(Collections.singletonList("admin")); + Mockito.when(userService.listSuperAdminUsers()).thenReturn(Arrays.asList("admin")); Mockito.when(userAclService.hasUserAclPermissionInProject(Mockito.anyString(), Mockito.anyString())) .thenReturn(false); @@ -135,8 +138,12 @@ public class StreamingTableServiceTest extends NLocalFileMetadataTestCase { @Test public void testInnerReloadTable() { + val database = "SSB"; + + val config = getTestConfig(); try { - val tableDescList = tableService.getTableDesc(PROJECT, true, "P_LINEORDER_STR", "SSB", false); + val tableDescList = tableService.getTableDesc(PROJECT, true, "P_LINEORDER_STR", database, false, + Collections.emptyList(), 10).getFirst(); Assert.assertEquals(1, tableDescList.size()); val tableDesc = tableDescList.get(0); val tableExtDesc = tableService.getOrCreateTableExt(PROJECT, tableDesc); @@ -152,7 +159,8 @@ public class StreamingTableServiceTest extends NLocalFileMetadataTestCase { val database = "DEFAULT"; try { - val tableDescList = tableService.getTableDesc(PROJECT, true, "", database, true); + val tableDescList = tableService.getTableDesc(PROJECT, true, "", database, true, + Collections.emptyList(), 10).getFirst(); Assert.assertEquals(2, tableDescList.size()); val tableDesc = tableDescList.get(0); val tableExtDesc = tableService.getOrCreateTableExt(PROJECT, tableDesc); diff --git a/src/metadata-server/src/main/java/org/apache/kylin/rest/controller/NTableController.java b/src/metadata-server/src/main/java/org/apache/kylin/rest/controller/NTableController.java index 128a9ebb21..9466eb6374 100644 --- a/src/metadata-server/src/main/java/org/apache/kylin/rest/controller/NTableController.java +++ b/src/metadata-server/src/main/java/org/apache/kylin/rest/controller/NTableController.java @@ -24,9 +24,10 @@ import static org.apache.kylin.common.exception.ServerErrorCode.EMPTY_PARAMETER; import static org.apache.kylin.common.exception.ServerErrorCode.INVALID_TABLE_NAME; import static org.apache.kylin.common.exception.ServerErrorCode.INVALID_TABLE_REFRESH_PARAMETER; import static org.apache.kylin.common.exception.code.ErrorCodeServer.PROJECT_NOT_EXIST; +import static org.apache.kylin.rest.util.TableUtils.calculateTableSize; import java.io.IOException; -import java.util.ArrayList; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -40,6 +41,7 @@ import org.apache.commons.lang.StringUtils; import org.apache.kylin.common.KylinConfig; import org.apache.kylin.common.exception.KylinException; import org.apache.kylin.common.msg.MsgPicker; +import org.apache.kylin.common.util.Pair; import org.apache.kylin.common.util.StringUtil; import org.apache.kylin.metadata.model.TableDesc; import org.apache.kylin.metadata.project.NProjectManager; @@ -48,6 +50,7 @@ import org.apache.kylin.rest.request.AutoMergeRequest; import org.apache.kylin.rest.request.PartitionKeyRequest; import org.apache.kylin.rest.request.PushDownModeRequest; import org.apache.kylin.rest.request.ReloadTableRequest; +import org.apache.kylin.rest.request.TableDescRequest; import org.apache.kylin.rest.request.TableExclusionRequest; import org.apache.kylin.rest.request.TableLoadRequest; import org.apache.kylin.rest.request.TopTableRequest; @@ -114,7 +117,7 @@ public class NTableController extends NBasicController { "AI" }, notes = "Update Param: is_fuzzy, page_offset, page_size; Update Response: no format!") @GetMapping(value = "", produces = { HTTP_VND_APACHE_KYLIN_JSON }) @ResponseBody - public EnvelopeResponse getTableDesc(@RequestParam(value = "ext", required = false) boolean withExt, + public EnvelopeResponse<Map<String, Object>> getTableDesc(@RequestParam(value = "ext", required = false) boolean withExt, @RequestParam(value = "project") String project, @RequestParam(value = "table", required = false) String table, @RequestParam(value = "database", required = false) String database, @@ -125,11 +128,16 @@ public class NTableController extends NBasicController { throws IOException { checkProjectName(project); - List<TableDesc> tableDescs = new ArrayList<>(); - - tableDescs.addAll(tableService.getTableDescByType(project, withExt, table, database, isFuzzy, sourceType)); - return new EnvelopeResponse<>(KylinException.CODE_SUCCESS, getDataResponse("tables", tableDescs, offset, limit), - ""); + // In addition to the tables that have been processed, the actual size of tables should be returned, + // so that the front-end UI knows whether to show more presses to be loaded + int returnTableSize = calculateTableSize(offset, limit); + TableDescRequest tableDescRequest = new TableDescRequest(project, table, database, withExt, isFuzzy, + Pair.newPair(offset, limit), Collections.singletonList(sourceType)); + Pair<List<TableDesc>, Integer> tableDescWithActualSize = tableService.getTableDesc(tableDescRequest, returnTableSize); + // Finally, the results are processed based on the paging parameters and returned to the front-end UI, + // where the results table to be processed each time is getting longer as the number of paging increases + Map<String, Object> mockDataResponse = setCustomDataResponse("tables", tableDescWithActualSize, offset, limit); + return new EnvelopeResponse<>(KylinException.CODE_SUCCESS, mockDataResponse, ""); } @ApiOperation(value = "getProjectTables", tags = { "AI" }, notes = "Update Param: is_fuzzy, page_offset, page_size") @@ -146,13 +154,10 @@ public class NTableController extends NBasicController { @RequestParam(value = "with_excluded", required = false, defaultValue = "true") boolean withExcluded, @RequestParam(value = "source_type", required = false, defaultValue = "9") List<Integer> sourceType) throws Exception { - - String projectName = checkProjectName(project); - NInitTablesResponse projectTables = tableService.getProjectTables(projectName, table, offset, limit, - withExcluded, false, (databaseName, tableName) -> { - return tableService.getTableDescByTypes(projectName, withExt, tableName, databaseName, isFuzzy, - sourceType); - }); + checkProjectName(project); + TableDescRequest tableDescRequest = new TableDescRequest(project, table, "", withExt, isFuzzy, + offset, limit, sourceType, withExcluded); + NInitTablesResponse projectTables = tableService.getProjectTables(tableDescRequest, false); return new EnvelopeResponse<>(KylinException.CODE_SUCCESS, projectTables, ""); } @@ -346,10 +351,9 @@ public class NTableController extends NBasicController { @RequestParam(value = "page_offset", required = false, defaultValue = "0") Integer offset, @RequestParam(value = "page_size", required = false, defaultValue = "10") Integer limit) throws Exception { String projectName = checkProjectName(project); - return new EnvelopeResponse<>(KylinException.CODE_SUCCESS, - tableService.getProjectTables(projectName, table, offset, limit, true, true, (databaseName, - tableName) -> tableService.getHiveTableNameResponses(projectName, databaseName, tableName)), - ""); + NInitTablesResponse data = tableService.getProjectTables(projectName, table, offset, limit, true, + true, Collections.emptyList()); + return new EnvelopeResponse<>(KylinException.CODE_SUCCESS, data, ""); } @ApiOperation(value = "getTablesAndColumns", tags = { diff --git a/src/metadata-server/src/main/java/org/apache/kylin/rest/controller/open/OpenTableController.java b/src/metadata-server/src/main/java/org/apache/kylin/rest/controller/open/OpenTableController.java index b511f565ae..a135f4dcaa 100644 --- a/src/metadata-server/src/main/java/org/apache/kylin/rest/controller/open/OpenTableController.java +++ b/src/metadata-server/src/main/java/org/apache/kylin/rest/controller/open/OpenTableController.java @@ -22,8 +22,10 @@ import static org.apache.kylin.common.exception.ServerErrorCode.INVALID_TABLE_NA import static org.apache.kylin.common.exception.ServerErrorCode.UNSUPPORTED_DATA_SOURCE_TYPE; import static org.apache.kylin.common.exception.ServerErrorCode.UNSUPPORTED_STREAMING_OPERATION; import static org.apache.kylin.common.exception.code.ErrorCodeServer.JOB_SAMPLING_RANGE_INVALID; +import static org.apache.kylin.rest.util.TableUtils.calculateTableSize; import java.io.IOException; +import java.util.Collections; import java.util.List; import java.util.Locale; @@ -119,10 +121,11 @@ public class OpenTableController extends NBasicController { throw new KylinException(UNSUPPORTED_STREAMING_OPERATION, MsgPicker.getMsg().getStreamingOperationNotSupport()); } - List<TableDesc> result = tableService.getTableDescByType(project, withExt, + int returnTableSize = calculateTableSize(offset, limit); + Pair<List<TableDesc>, Integer> tableDescWithActualSize = tableService.getTableDesc(project, withExt, StringUtils.upperCase(table, Locale.ROOT), StringUtils.upperCase(database, Locale.ROOT), isFuzzy, - sourceType); - return new EnvelopeResponse<>(KylinException.CODE_SUCCESS, DataResult.get(result, offset, limit), ""); + Collections.singletonList(sourceType), returnTableSize); + return new EnvelopeResponse<>(KylinException.CODE_SUCCESS, DataResult.getCustom(tableDescWithActualSize, offset, limit), ""); } @ApiOperation(value = "loadTables", tags = { "AI" }) diff --git a/src/metadata-server/src/main/java/org/apache/kylin/rest/controller/v2/NTableControllerV2.java b/src/metadata-server/src/main/java/org/apache/kylin/rest/controller/v2/NTableControllerV2.java index 0c195a84f4..1b4b462b63 100644 --- a/src/metadata-server/src/main/java/org/apache/kylin/rest/controller/v2/NTableControllerV2.java +++ b/src/metadata-server/src/main/java/org/apache/kylin/rest/controller/v2/NTableControllerV2.java @@ -19,12 +19,15 @@ package org.apache.kylin.rest.controller.v2; import static org.apache.kylin.common.constant.HttpConstant.HTTP_VND_APACHE_KYLIN_V2_JSON; import static org.apache.kylin.common.exception.ServerErrorCode.UNSUPPORTED_STREAMING_OPERATION; +import static org.apache.kylin.rest.util.TableUtils.calculateTableSize; import java.io.IOException; +import java.util.Collections; import java.util.List; import org.apache.kylin.common.exception.KylinException; import org.apache.kylin.common.msg.MsgPicker; +import org.apache.kylin.common.util.Pair; import org.apache.kylin.metadata.model.ISourceAware; import org.apache.kylin.metadata.model.TableDesc; import org.apache.kylin.rest.controller.NBasicController; @@ -65,9 +68,11 @@ public class NTableControllerV2 extends NBasicController { throw new KylinException(UNSUPPORTED_STREAMING_OPERATION, MsgPicker.getMsg().getStreamingOperationNotSupport()); } - List<TableDesc> result = tableService.getTableDescByType(project, withExt, table, database, isFuzzy, - sourceType); - return new EnvelopeResponse<>(KylinException.CODE_SUCCESS, DataResult.get(result, offset, limit).getValue(), + int returnTableSize = calculateTableSize(offset, limit); + Pair<List<TableDesc>, Integer> tableDescWithActualSize = tableService.getTableDesc(project, withExt, table, database, + isFuzzy, Collections.singletonList(sourceType), returnTableSize); + return new EnvelopeResponse<>(KylinException.CODE_SUCCESS, + DataResult.getCustom(tableDescWithActualSize, offset, limit).getValue(), ""); } } \ No newline at end of file diff --git a/src/metadata-server/src/test/java/org/apache/kylin/rest/controller/NTableControllerTest.java b/src/metadata-server/src/test/java/org/apache/kylin/rest/controller/NTableControllerTest.java index 3301be1d06..ad10f6f199 100644 --- a/src/metadata-server/src/test/java/org/apache/kylin/rest/controller/NTableControllerTest.java +++ b/src/metadata-server/src/test/java/org/apache/kylin/rest/controller/NTableControllerTest.java @@ -24,6 +24,7 @@ import static org.apache.kylin.common.exception.code.ErrorCodeServer.JOB_SAMPLIN import java.nio.charset.StandardCharsets; import java.util.ArrayList; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Set; @@ -32,6 +33,7 @@ import org.apache.commons.lang.StringUtils; import org.apache.kylin.common.exception.KylinException; import org.apache.kylin.common.util.JsonUtil; import org.apache.kylin.common.util.NLocalFileMetadataTestCase; +import org.apache.kylin.common.util.Pair; import org.apache.kylin.common.util.StringUtil; import org.apache.kylin.metadata.model.TableDesc; import org.apache.kylin.rest.constant.Constant; @@ -41,6 +43,7 @@ import org.apache.kylin.rest.request.PartitionKeyRequest; import org.apache.kylin.rest.request.PushDownModeRequest; import org.apache.kylin.rest.request.ReloadTableRequest; import org.apache.kylin.rest.request.S3TableExtInfo; +import org.apache.kylin.rest.request.TableDescRequest; import org.apache.kylin.rest.request.TableExclusionRequest; import org.apache.kylin.rest.request.TableLoadRequest; import org.apache.kylin.rest.request.TopTableRequest; @@ -149,8 +152,11 @@ public class NTableControllerTest extends NLocalFileMetadataTestCase { @Test public void testGetTableDesc() throws Exception { - Mockito.when(tableService.getTableDesc("default", false, "", "DEFAULT", true)) // - .thenReturn(mockTables()); + TableDescRequest mockTableDescRequest = new TableDescRequest("default", "", "DEFAULT", false, + true, Pair.newPair(0, 10), Collections.singletonList(9)); + + Mockito.when(tableService.getTableDesc(mockTableDescRequest, 10)).thenReturn(Pair.newPair(mockTables(), 10)); + mockMvc.perform(MockMvcRequestBuilders.get("/api/tables") // .contentType(MediaType.APPLICATION_JSON) // .param("ext", "false") // @@ -181,8 +187,11 @@ public class NTableControllerTest extends NLocalFileMetadataTestCase { @Test public void testGetTableDescWithName() throws Exception { - Mockito.when(tableService.getTableDesc("default", true, "TEST_KYLIN_FACT", "DEFAULT", false)) - .thenReturn(mockTables()); + TableDescRequest mockTableDescRequest = new TableDescRequest("default", "TEST_KYLIN_FACT", "DEFAULT", false, + false, Pair.newPair(0, 10), Collections.singletonList(9)); + + Mockito.when(tableService.getTableDesc(mockTableDescRequest, 10)).thenReturn(Pair.newPair(mockTables(), 10)); + mockMvc.perform(MockMvcRequestBuilders.get("/api/tables") // .contentType(MediaType.APPLICATION_JSON) // .param("withExt", "false") // diff --git a/src/metadata-server/src/test/java/org/apache/kylin/rest/controller/NTableControllerV2Test.java b/src/metadata-server/src/test/java/org/apache/kylin/rest/controller/NTableControllerV2Test.java index 6e1b3e608f..2392584e6c 100644 --- a/src/metadata-server/src/test/java/org/apache/kylin/rest/controller/NTableControllerV2Test.java +++ b/src/metadata-server/src/test/java/org/apache/kylin/rest/controller/NTableControllerV2Test.java @@ -21,9 +21,11 @@ import static org.apache.kylin.common.constant.HttpConstant.HTTP_VND_APACHE_KYLI import java.nio.charset.StandardCharsets; import java.util.ArrayList; +import java.util.Collections; import java.util.List; import org.apache.kylin.common.util.NLocalFileMetadataTestCase; +import org.apache.kylin.common.util.Pair; import org.apache.kylin.metadata.model.TableDesc; import org.apache.kylin.rest.constant.Constant; import org.apache.kylin.rest.controller.v2.NTableControllerV2; @@ -81,8 +83,9 @@ public class NTableControllerV2Test extends NLocalFileMetadataTestCase { @Test public void testGetTableDesc() throws Exception { - Mockito.when(tableService.getTableDesc("default", false, "", "DEFAULT", true)) // - .thenReturn(mockTables()); + Mockito.when(tableService.getTableDesc("default", false, "", "DEFAULT", true, Collections.singletonList(9), 10)) // + .thenReturn(Pair.newPair(mockTables(), 10)); + mockMvc.perform(MockMvcRequestBuilders.get("/api/tables") // .contentType(MediaType.APPLICATION_JSON) // .param("ext", "false") // diff --git a/src/metadata-server/src/test/java/org/apache/kylin/rest/controller/open/OpenTableControllerTest.java b/src/metadata-server/src/test/java/org/apache/kylin/rest/controller/open/OpenTableControllerTest.java index 6d56ebe1c2..1c2242f8d5 100644 --- a/src/metadata-server/src/test/java/org/apache/kylin/rest/controller/open/OpenTableControllerTest.java +++ b/src/metadata-server/src/test/java/org/apache/kylin/rest/controller/open/OpenTableControllerTest.java @@ -21,6 +21,7 @@ import static org.apache.kylin.common.constant.HttpConstant.HTTP_VND_APACHE_KYLI import java.nio.charset.StandardCharsets; import java.util.ArrayList; +import java.util.Collections; import java.util.List; import java.util.Map; @@ -122,6 +123,9 @@ public class OpenTableControllerTest extends NLocalFileMetadataTestCase { String tableName = "TEST_KYLIN_FACT"; String database = "DEFAULT"; + Mockito.when(tableService.getTableDesc(project, true, tableName, database, false, Collections.singletonList(9), 10)) + .thenReturn(Pair.newPair(Collections.singletonList(new TableDesc()), 10)); + mockMvc.perform(MockMvcRequestBuilders.get("/api/tables") // .contentType(MediaType.APPLICATION_JSON) // .param("project", project).param("table", tableName).param("database", database) @@ -136,6 +140,9 @@ public class OpenTableControllerTest extends NLocalFileMetadataTestCase { String tableName = "P_LINEORDER_STR"; String database = "SSB"; + Mockito.when(tableService.getTableDesc(project, true, tableName, database, false, Collections.singletonList(1), 10)) + .thenReturn(Pair.newPair(Collections.singletonList(new TableDesc()), 10)); + mockMvc.perform(MockMvcRequestBuilders.get("/api/tables") // .contentType(MediaType.APPLICATION_JSON) // .param("project", project).param("table", tableName).param("database", database) @@ -154,29 +161,33 @@ public class OpenTableControllerTest extends NLocalFileMetadataTestCase { String databaseLowercase = "ssb"; String databaseUppercase = "SSB"; + Mockito.when(tableService.getTableDesc(project, true, tableNameUppercase, databaseUppercase, false, + Collections.singletonList(9), 10)) + .thenReturn(Pair.newPair(Collections.singletonList(new TableDesc()), 10)); + mockMvc.perform(MockMvcRequestBuilders.get("/api/tables") // .contentType(MediaType.APPLICATION_JSON) // .param("project", project).param("table", tableNameMixture).param("database", databaseMixture) .accept(MediaType.parseMediaType(HTTP_VND_APACHE_KYLIN_V4_PUBLIC_JSON))) // .andExpect(MockMvcResultMatchers.status().isOk()); - Mockito.verify(tableService, Mockito.times(1)).getTableDescByType(project, true, tableNameUppercase, - databaseUppercase, false, 9); + Mockito.verify(tableService, Mockito.times(1)).getTableDesc(project, true, tableNameUppercase, + databaseUppercase, false, Collections.singletonList(9), 10); mockMvc.perform(MockMvcRequestBuilders.get("/api/tables") // .contentType(MediaType.APPLICATION_JSON) // .param("project", project).param("table", tableNameLowerCase).param("database", databaseLowercase) .accept(MediaType.parseMediaType(HTTP_VND_APACHE_KYLIN_V4_PUBLIC_JSON))) // .andExpect(MockMvcResultMatchers.status().isOk()); - Mockito.verify(tableService, Mockito.times(2)).getTableDescByType(project, true, tableNameUppercase, - databaseUppercase, false, 9); + Mockito.verify(tableService, Mockito.times(2)).getTableDesc(project, true, tableNameUppercase, + databaseUppercase, false, Collections.singletonList(9), 10); mockMvc.perform(MockMvcRequestBuilders.get("/api/tables") // .contentType(MediaType.APPLICATION_JSON) // .param("project", project).param("table", tableNameUppercase).param("database", databaseUppercase) .accept(MediaType.parseMediaType(HTTP_VND_APACHE_KYLIN_V4_PUBLIC_JSON))) // .andExpect(MockMvcResultMatchers.status().isOk()); - Mockito.verify(tableService, Mockito.times(3)).getTableDescByType(project, true, tableNameUppercase, - databaseUppercase, false, 9); + Mockito.verify(tableService, Mockito.times(3)).getTableDesc(project, true, tableNameUppercase, + databaseUppercase, false, Collections.singletonList(9), 10); } } diff --git a/src/modeling-service/src/test/java/org/apache/kylin/rest/service/TableServiceTest.java b/src/modeling-service/src/test/java/org/apache/kylin/rest/service/TableServiceTest.java index a6580e25b4..4f47d6cc3c 100644 --- a/src/modeling-service/src/test/java/org/apache/kylin/rest/service/TableServiceTest.java +++ b/src/modeling-service/src/test/java/org/apache/kylin/rest/service/TableServiceTest.java @@ -205,18 +205,26 @@ public class TableServiceTest extends CSVSourceTestCase { @Test public void testGetTableDesc() throws IOException { - List<TableDesc> tableDesc = tableService.getTableDesc("default", true, "", "DEFAULT", true); + List<Integer> sourceType = new ArrayList<>(); + sourceType.add(1); // Kafka table + sourceType.add(9); // Hive table + List<TableDesc> tableDesc = tableService.getTableDesc("default", true, "", "DEFAULT", true, + sourceType, 12).getFirst(); Assert.assertEquals(12, tableDesc.size()); - List<TableDesc> tableDesc2 = tableService.getTableDesc("default", true, "TEST_COUNTRY", "DEFAULT", false); + List<TableDesc> tableDesc2 = tableService.getTableDesc("default", true, "TEST_COUNTRY", "DEFAULT", false, + sourceType, 10).getFirst(); Assert.assertEquals(1, tableDesc2.size()); - List<TableDesc> tables3 = tableService.getTableDesc("default", true, "", "", true); + List<TableDesc> tables3 = tableService.getTableDesc("default", true, "", "", true, + sourceType, 100).getFirst(); Assert.assertEquals(21, tables3.size()); - List<TableDesc> tables = tableService.getTableDesc("default", true, "TEST_KYLIN_FACT", "DEFAULT", true); + List<TableDesc> tables = tableService.getTableDesc("default", true, "TEST_KYLIN_FACT", "DEFAULT", true, + sourceType, 10).getFirst(); Assert.assertEquals("TEST_KYLIN_FACT", tables.get(0).getName()); Assert.assertEquals(5633024, ((TableDescResponse) tables.get(0)).getStorageSize()); Assert.assertEquals(0, ((TableDescResponse) tables.get(0)).getTotalRecords()); - List<TableDesc> table2 = tableService.getTableDesc("default", true, "country", "DEFAULT", true); + List<TableDesc> table2 = tableService.getTableDesc("default", true, "country", "DEFAULT", true, + sourceType, 10).getFirst(); Assert.assertEquals("TEST_COUNTRY", table2.get(0).getName()); Assert.assertEquals(0L, ((TableDescResponse) table2.get(0)).getStorageSize()); @@ -225,21 +233,25 @@ public class TableServiceTest extends CSVSourceTestCase { countryTable.setLastSnapshotPath("cannot/find/it"); manager.updateTableDesc(countryTable); - table2 = tableService.getTableDesc("default", true, "country", "DEFAULT", true); + table2 = tableService.getTableDesc("default", true, "country", "DEFAULT", true, + sourceType, 10).getFirst(); Assert.assertEquals("TEST_COUNTRY", table2.get(0).getName()); Assert.assertEquals(0L, ((TableDescResponse) table2.get(0)).getStorageSize()); // get a not existing table desc - tableDesc = tableService.getTableDesc("default", true, "not_exist_table", "DEFAULT", false); + tableDesc = tableService.getTableDesc("default", true, "not_exist_table", "DEFAULT", false, + sourceType, 10).getFirst(); Assert.assertEquals(0, tableDesc.size()); - tableDesc = tableService.getTableDesc("streaming_test", true, "", "DEFAULT", true); + tableDesc = tableService.getTableDesc("streaming_test", true, "", "DEFAULT", true, + sourceType, 10).getFirst(); Assert.assertEquals(2, tableDesc.size()); val tableMetadataManager = getInstance(getTestConfig(), "streaming_test"); var tableDesc1 = tableMetadataManager.getTableDesc("DEFAULT.SSB_TOPIC"); Assert.assertTrue(tableDesc1.isAccessible(getTestConfig().streamingEnabled())); getTestConfig().setProperty("kylin.streaming.enabled", "false"); - tableDesc = tableService.getTableDesc("streaming_test", true, "", "DEFAULT", true); + tableDesc = tableService.getTableDesc("streaming_test", true, "", "DEFAULT", true, + sourceType, 10).getFirst(); Assert.assertEquals(0, tableDesc.size()); // check kafka table Assert.assertFalse(tableDesc1.isAccessible(getTestConfig().streamingEnabled())); @@ -277,7 +289,8 @@ public class TableServiceTest extends CSVSourceTestCase { Assert.assertEquals(1, newTableExt.getAllColumnStats().size()); // call api to check tableDescResponse has the correct value - final List<TableDesc> tables = tableService.getTableDesc("newten", true, "TEST_COUNTRY", "DEFAULT", true); + final List<TableDesc> tables = tableService.getTableDesc("newten", true, "TEST_COUNTRY", "DEFAULT", true, + Collections.emptyList(), 10).getFirst(); Assert.assertEquals(1, tables.size()); Assert.assertTrue(tables.get(0) instanceof TableDescResponse); TableDescResponse t = (TableDescResponse) tables.get(0); @@ -320,7 +333,8 @@ public class TableServiceTest extends CSVSourceTestCase { Mockito.when(userAclService.hasUserAclPermissionInProject(Mockito.anyString(), Mockito.anyString())) .thenReturn(false); - List<TableDesc> tableExtList = tableService.getTableDesc("newten", true, "TEST_COUNTRY", "DEFAULT", true); + List<TableDesc> tableExtList = tableService.getTableDesc("newten", true, "TEST_COUNTRY", "DEFAULT", + true, Collections.emptyList(), 10).getFirst(); Assert.assertEquals(0, ((TableDescResponse) tableExtList.get(0)).getSamplingRows().size()); SecurityContextHolder.getContext() .setAuthentication(new TestingAuthenticationToken("ADMIN", "ADMIN", Constant.ROLE_ADMIN)); @@ -392,7 +406,8 @@ public class TableServiceTest extends CSVSourceTestCase { Assert.assertEquals("float", confirmedTableDesc.getColumns()[2].getDatatype()); // call api to check tableDescResponse has the correct value - final List<TableDesc> tables = tableService.getTableDesc("newten", true, "TEST_COUNTRY", "DEFAULT", true); + final List<TableDesc> tables = tableService.getTableDesc("newten", true, "TEST_COUNTRY", "DEFAULT", true, + Collections.emptyList(), 10).getFirst(); Assert.assertEquals(1, tables.size()); Assert.assertTrue(tables.get(0) instanceof TableDescResponse); TableDescResponse t = (TableDescResponse) tables.get(0); @@ -480,7 +495,8 @@ public class TableServiceTest extends CSVSourceTestCase { @Test public void testLoadTableToProject() throws IOException { - List<TableDesc> tables = tableService.getTableDesc("default", true, "TEST_COUNTRY", "DEFAULT", true); + List<TableDesc> tables = tableService.getTableDesc("default", true, "TEST_COUNTRY", "DEFAULT", true, + Collections.emptyList(), 10).getFirst(); TableDesc nTableDesc = new TableDesc(tables.get(0)); TableExtDesc tableExt = new TableExtDesc(); tableExt.setIdentity("DEFAULT.TEST_COUNTRY"); @@ -493,7 +509,8 @@ public class TableServiceTest extends CSVSourceTestCase { public void testLoadTableToProjectWithS3Role() throws IOException { getTestConfig().setProperty("kylin.env.use-dynamic-S3-role-credential-in-table", "true"); assert !SparderEnv.getSparkSession().conf().contains(String.format(S3AUtil.ROLE_ARN_KEY_FORMAT, "testbucket")); - List<TableDesc> tables = tableService.getTableDesc("default", true, "TEST_COUNTRY", "DEFAULT", true); + List<TableDesc> tables = tableService.getTableDesc("default", true, "TEST_COUNTRY", "DEFAULT", true, + Collections.emptyList(), 10).getFirst(); TableDesc nTableDesc = new TableDesc(tables.get(0)); TableExtDesc tableExt = new TableExtDesc(); tableExt.setIdentity("DEFAULT.TEST_COUNTRY"); @@ -757,7 +774,8 @@ public class TableServiceTest extends CSVSourceTestCase { private void testSetPartitionKeyWithoutException() throws Exception { tableService.setPartitionKey("DEFAULT.TEST_KYLIN_FACT", "default", "CAL_DT", "yyyy-MM-dd"); - List<TableDesc> tables = tableService.getTableDesc("default", false, "", "DEFAULT", true); + List<TableDesc> tables = tableService.getTableDesc("default", false, "", "DEFAULT", true, + Collections.emptyList(), 10).getFirst(); //test set fact and table list order by fact Assert.assertTrue(tables.get(0).getName().equals("TEST_KYLIN_FACT") && tables.get(0).isIncrementLoading()); } @@ -904,7 +922,8 @@ public class TableServiceTest extends CSVSourceTestCase { public void testSetTop() throws IOException { TopTableRequest topTableRequest = mockTopTableRequest(); tableService.setTop(topTableRequest.getTable(), topTableRequest.getProject(), topTableRequest.isTop()); - List<TableDesc> tables = tableService.getTableDesc("default", false, "", "DEFAULT", true); + List<TableDesc> tables = tableService.getTableDesc("default", false, "", "DEFAULT", true, + Collections.emptyList(), 10).getFirst(); Assert.assertTrue(tables.get(0).isTop()); } @@ -1118,47 +1137,52 @@ public class TableServiceTest extends CSVSourceTestCase { @Test public void testGetProjectTables() throws Exception { NInitTablesResponse response; + overwriteSystemProp("kylin.source.load-hive-tablename-enabled", "false"); + response = tableService.getProjectTables("default", "SSB.SS", 0, 14, true, true, - (databaseName, tableName) -> tableService.getTableNameResponses("default", databaseName, tableName)); + Collections.emptyList()); Assert.assertEquals(0, response.getDatabases().size()); response = tableService.getProjectTables("default", "SSB.CU", 0, 14, true, true, - (databaseName, tableName) -> tableService.getTableNameResponses("default", databaseName, tableName)); + Collections.emptyList()); Assert.assertEquals(1, response.getDatabases().size()); Assert.assertEquals(2, response.getDatabases().get(0).getTables().size()); response = tableService.getProjectTables("default", "", 0, 14, true, true, - (databaseName, tableName) -> tableService.getTableNameResponses("default", databaseName, tableName)); + Collections.emptyList()); Assert.assertEquals(3, response.getDatabases().size()); Assert.assertEquals(21, response.getDatabases().get(0).getTables().size() + response.getDatabases().get(1).getTables().size() + response.getDatabases().get(2).getTables().size()); response = tableService.getProjectTables("default", "TEST", 0, 14, true, true, - (databaseName, tableName) -> tableService.getTableNameResponses("default", databaseName, tableName)); + Collections.emptyList()); Assert.assertEquals(2, response.getDatabases().size()); Assert.assertEquals(13, response.getDatabases().get(0).getTables().size() + response.getDatabases().get(1).getTables().size()); response = tableService.getProjectTables("default", "EDW.", 0, 14, true, true, - (databaseName, tableName) -> tableService.getTableNameResponses("default", databaseName, tableName)); + Collections.emptyList()); Assert.assertEquals(1, response.getDatabases().size()); Assert.assertEquals(3, response.getDatabases().get(0).getTables().size()); response = tableService.getProjectTables("default", "EDW.", 0, 14, true, false, - (databaseName, tableName) -> tableService.getTableDesc("default", true, tableName, databaseName, true)); + Collections.emptyList()); Assert.assertEquals(1, response.getDatabases().size()); Assert.assertEquals(3, response.getDatabases().get(0).getTables().size()); response = tableService.getProjectTables("default", "DEFAULT.TEST_ORDER", 0, 14, true, false, - (databaseName, tableName) -> tableService.getTableDesc("default", true, tableName, databaseName, true)); + Collections.emptyList()); Assert.assertEquals(1, response.getDatabases().size()); Assert.assertEquals(1, response.getDatabases().get(0).getTables().size()); response = tableService.getProjectTables("default", ".TEST_ORDER", 0, 14, true, false, - (databaseName, tableName) -> tableService.getTableDesc("default", true, tableName, databaseName, true)); + Collections.emptyList()); Assert.assertEquals(0, response.getDatabases().size()); + response = tableService.getProjectTables("default", "", 0, 14, true, true, + Collections.singletonList(9)); + Assert.assertEquals(3, response.getDatabases().size()); } @Test @@ -1383,7 +1407,8 @@ public class TableServiceTest extends CSVSourceTestCase { tableExt.setJodID("949afe5d-0221-420f-92db-cdd91cb31ac8"); tableMgr.mergeAndUpdateTableExt(oldExtDesc, tableExt); - List<TableDesc> tables = tableService.getTableDesc("newten", true, "TEST_COUNTRY", "DEFAULT", true); + List<TableDesc> tables = tableService.getTableDesc("newten", true, "TEST_COUNTRY", "DEFAULT", true, + Collections.emptyList(), 10).getFirst(); Assert.assertEquals(1, tables.size()); Assert.assertEquals("949afe5d-0221-420f-92db-cdd91cb31ac8", ((TableDescResponse) tables.get(0)).getJodID()); @@ -1412,15 +1437,18 @@ public class TableServiceTest extends CSVSourceTestCase { public void testGetTableDescByType() { String project = "streaming_test"; try { - val tableDescs = tableService.getTableDescByType(project, true, "", "default", true, 1); + val tableDescs = tableService.getTableDesc(project, true, "", "default", true, + Collections.singletonList(1), 10).getFirst(); Assert.assertNotNull(tableDescs); - val tableDescs1 = tableService.getTableDescByType(project, true, "P_LINEORDER_STREAMING", "ssb", true, 1); + val tableDescs1 = tableService.getTableDesc(project, true, "P_LINEORDER_STREAMING", "ssb", true, + Collections.singletonList(1), 10).getFirst(); Assert.assertEquals(1, tableDescs1.size()); val tableDesc1 = tableDescs1.get(0); Assert.assertEquals(tableDesc1.getTableAlias(), tableDesc1.getKafkaConfig().getBatchTable()); - val tableDescs2 = tableService.getTableDescByType(project, true, "LINEORDER_HIVE", "SSB", false, 9); + val tableDescs2 = tableService.getTableDesc(project, true, "LINEORDER_HIVE", "SSB", false, + Collections.singletonList(9), 10).getFirst(); Assert.assertEquals(1, tableDescs2.size()); val tableDesc2 = tableDescs2.get(0); Assert.assertEquals(tableDesc2.getTableAlias(), tableDesc2.getIdentity()); @@ -1435,7 +1463,8 @@ public class TableServiceTest extends CSVSourceTestCase { String project = "streaming_test"; try { List<Integer> sourceTypes = Arrays.asList(1, 9); - val tableDescs2 = tableService.getTableDescByTypes(project, true, "", "SSB", false, sourceTypes); + val tableDescs2 = tableService.getTableDesc(project, true, "", "SSB", false, + sourceTypes, 10).getFirst(); assert tableDescs2.stream().anyMatch(tableDesc -> tableDesc.getSourceType() == 1); assert tableDescs2.stream().anyMatch(tableDesc -> tableDesc.getSourceType() == 9); } catch (Exception e) { @@ -1520,7 +1549,8 @@ public class TableServiceTest extends CSVSourceTestCase { tableExt.setColumnStats(Lists.newArrayList(col1)); tableMgr.mergeAndUpdateTableExt(oldExtDesc, tableExt); - final List<TableDesc> tables = tableService.getTableDesc("newten", true, "TEST_COUNTRY", "DEFAULT", true); + final List<TableDesc> tables = tableService.getTableDesc("newten", true, "TEST_COUNTRY", "DEFAULT", true, + Collections.emptyList(), 10).getFirst(); Assert.assertEquals(1, tables.size()); Assert.assertTrue(tables.get(0) instanceof TableDescResponse); TableDescResponse t = (TableDescResponse) tables.get(0);