This is an automated email from the ASF dual-hosted git repository. xxyu pushed a commit to branch kylin5 in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 1174339344bdf720713047f19fd8df6ca1f7acc7 Author: Hang Jia <754332...@qq.com> AuthorDate: Sat Oct 22 18:00:55 2022 +0800 KYLIN-5333 V2 version api is forward compatible -- Model/CUBE/data source/BUILD API --- .../metadata/querymeta/TableMetaWithType.java | 11 ++ .../kylin/rest/controller/v2/JobControllerV2.java | 54 +++++++++- .../rest/controller/v2/JobControllerV2Test.java | 62 +++++++++++- .../kylin/rest/response/ExecutableResponse.java | 68 +++++++++---- .../org/apache/kylin/rest/service/JobService.java | 21 ++++ .../kylin/rest/response/TableDescResponse.java | 17 ++++ .../apache/kylin/rest/service/JobServiceTest.java | 101 ++++++++++++++++--- .../rest/controller/v2/NModelControllerV2.java | 21 ++++ .../rest/controller/v2/NTableControllerV2.java | 73 ++++++++++++++ .../rest/controller/NModelControllerV2Test.java | 22 +++- .../rest/controller/NTableControllerV2Test.java | 111 +++++++++++++++++++++ .../kylin/rest/response/NDataModelOldParams.java | 38 +++++++ .../apache/kylin/rest/service/ModelService.java | 1 + .../kylin/rest/service/TableServiceTest.java | 35 +++++++ .../rest/controller/v2/NQueryMetaController.java | 10 +- .../apache/kylin/rest/service/QueryService.java | 25 +++++ .../kylin/rest/service/QueryServiceTest.java | 41 ++++++++ 17 files changed, 669 insertions(+), 42 deletions(-) diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/querymeta/TableMetaWithType.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/querymeta/TableMetaWithType.java index 25c8f4b55e..64eaaf65ba 100644 --- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/querymeta/TableMetaWithType.java +++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/querymeta/TableMetaWithType.java @@ -20,6 +20,7 @@ package org.apache.kylin.metadata.querymeta; import java.io.Serializable; import java.util.HashSet; +import java.util.stream.Collectors; import lombok.NoArgsConstructor; @@ -53,6 +54,16 @@ public class TableMetaWithType extends TableMeta { TYPE = new HashSet<tableTypeEnum>(); } + public static TableMetaWithType ofColumnMeta(TableMeta tableMeta) { + TableMetaWithType tableMetaWithType = new TableMetaWithType(tableMeta.getTABLE_CAT(), + tableMeta.getTABLE_SCHEM(), tableMeta.getTABLE_NAME(), tableMeta.getTABLE_TYPE(), + tableMeta.getREMARKS(), tableMeta.getTYPE_CAT(), tableMeta.getTYPE_SCHEM(), tableMeta.getTYPE_NAME(), + tableMeta.getSELF_REFERENCING_COL_NAME(), tableMeta.getREF_GENERATION()); + tableMetaWithType.setColumns( + tableMeta.getColumns().stream().map(ColumnMetaWithType::ofColumnMeta).collect(Collectors.toList())); + return tableMetaWithType; + } + public HashSet<tableTypeEnum> getTYPE() { return TYPE; } diff --git a/src/data-loading-server/src/main/java/org/apache/kylin/rest/controller/v2/JobControllerV2.java b/src/data-loading-server/src/main/java/org/apache/kylin/rest/controller/v2/JobControllerV2.java index 592b3be80e..acc02883b4 100644 --- a/src/data-loading-server/src/main/java/org/apache/kylin/rest/controller/v2/JobControllerV2.java +++ b/src/data-loading-server/src/main/java/org/apache/kylin/rest/controller/v2/JobControllerV2.java @@ -25,11 +25,14 @@ import java.util.List; import java.util.Map; import java.util.Objects; +import org.apache.commons.lang3.StringUtils; +import org.apache.kylin.common.KylinVersion; import org.apache.kylin.common.exception.KylinException; import org.apache.kylin.job.constant.JobActionEnum; import org.apache.kylin.job.constant.JobStatusEnum; import org.apache.kylin.rest.controller.BaseController; import org.apache.kylin.rest.request.JobFilter; +import org.apache.kylin.rest.response.DataResult; import org.apache.kylin.rest.response.EnvelopeResponse; import org.apache.kylin.rest.response.ExecutableResponse; import org.apache.kylin.rest.service.JobService; @@ -53,6 +56,7 @@ import io.swagger.annotations.ApiOperation; public class JobControllerV2 extends BaseController { private static final String JOB_ID_ARG_NAME = "jobId"; + private static final String STEP_ID_ARG_NAME = "stepId"; @Autowired @Qualifier("jobService") @@ -79,12 +83,17 @@ public class JobControllerV2 extends BaseController { @RequestParam(value = "status", required = false, defaultValue = "") Integer[] status, @RequestParam(value = "timeFilter") Integer timeFilter, @RequestParam(value = "jobName", required = false) String jobName, - @RequestParam(value = "projectName") String project, + @RequestParam(value = "projectName", required = false) String project, @RequestParam(value = "key", required = false) String key, @RequestParam(value = "pageOffset", required = false, defaultValue = "0") Integer pageOffset, @RequestParam(value = "pageSize", required = false, defaultValue = "10") Integer pageSize, @RequestParam(value = "sortBy", required = false, defaultValue = "last_modified") String sortBy, + @RequestParam(value = "sortby", required = false) String sortby, //param for 3x @RequestParam(value = "reverse", required = false, defaultValue = "true") Boolean reverse) { + // 3x default last_modify + if (!StringUtils.isEmpty(sortby) && !"last_modify".equals(sortby)) { + sortBy = sortby; + } checkNonNegativeIntegerArg("pageOffset", pageOffset); checkNonNegativeIntegerArg("pageSize", pageSize); List<String> statuses = Lists.newArrayList(); @@ -100,9 +109,50 @@ public class JobControllerV2 extends BaseController { JobFilter jobFilter = new JobFilter(statuses, Objects.isNull(jobName) ? Lists.newArrayList() : Lists.newArrayList(jobName), timeFilter, null, key, project, sortBy, reverse); - List<ExecutableResponse> executables = jobService.listJobs(jobFilter); + List<ExecutableResponse> executables = null; + if (!StringUtils.isEmpty(project)) { + executables = jobService.listJobs(jobFilter); + } else { + DataResult<List<ExecutableResponse>> dataResult = jobService.listGlobalJobs(jobFilter, 0, + Integer.MAX_VALUE); + if (dataResult != null) { + executables = dataResult.getValue(); + } + } executables = jobService.addOldParams(executables); + executables.forEach( + executableResponse -> executableResponse.setVersion(KylinVersion.getCurrentVersion().toString())); Map<String, Object> result = getDataResponse("jobs", executables, pageOffset, pageSize); return new EnvelopeResponse<>(KylinException.CODE_SUCCESS, result, ""); } + + @ApiOperation(value = "getJob", tags = { "DW" }) + @GetMapping(value = "/{jobId}") + @ResponseBody + public EnvelopeResponse<ExecutableResponse> getJob(@PathVariable(value = "jobId") String jobId) { + checkRequiredArg(JOB_ID_ARG_NAME, jobId); + ExecutableResponse jobInstance = jobService.getJobInstance(jobId); + List<ExecutableResponse> executables = Lists.newArrayList(jobInstance); + executables = jobService.addOldParams(executables); + if (executables != null && executables.size() != 0) { + jobInstance = executables.get(0); + } + if (jobInstance != null) { + jobInstance.setVersion(KylinVersion.getCurrentVersion().toString()); + } + return new EnvelopeResponse<>(KylinException.CODE_SUCCESS, jobInstance, ""); + } + + @ApiOperation(value = "getJobOutput", tags = { "DW" }) + @GetMapping(value = "/{job_id:.+}/steps/{step_id:.+}/output") + @ResponseBody + public EnvelopeResponse<Map<String, Object>> getJobOutput(@PathVariable("job_id") String jobId, + @PathVariable("step_id") String stepId) { + String project = jobService.getProjectByJobId(jobId); + checkProjectName(project); + Map<String, Object> result = jobService.getStepOutput(project, jobId, stepId); + result.put(JOB_ID_ARG_NAME, jobId); + result.put(STEP_ID_ARG_NAME, stepId); + return new EnvelopeResponse<>(KylinException.CODE_SUCCESS, result, ""); + } } diff --git a/src/data-loading-server/src/test/java/org/apache/kylin/rest/controller/v2/JobControllerV2Test.java b/src/data-loading-server/src/test/java/org/apache/kylin/rest/controller/v2/JobControllerV2Test.java index 1fe66e13cc..a2ed9db17d 100644 --- a/src/data-loading-server/src/test/java/org/apache/kylin/rest/controller/v2/JobControllerV2Test.java +++ b/src/data-loading-server/src/test/java/org/apache/kylin/rest/controller/v2/JobControllerV2Test.java @@ -23,13 +23,17 @@ import static org.apache.kylin.common.constant.HttpConstant.HTTP_VND_APACHE_KYLI import java.util.ArrayList; import java.util.List; +import org.apache.kylin.common.util.NLocalFileMetadataTestCase; import org.apache.kylin.job.constant.JobActionEnum; import org.apache.kylin.rest.constant.Constant; import org.apache.kylin.rest.request.JobFilter; +import org.apache.kylin.rest.request.JobUpdateRequest; +import org.apache.kylin.rest.response.DataResult; import org.apache.kylin.rest.response.ExecutableResponse; import org.apache.kylin.rest.service.JobService; import org.apache.kylin.rest.util.AclEvaluate; import org.apache.kylin.rest.util.AclUtil; +import org.junit.After; import org.junit.Before; import org.junit.Test; import org.mockito.InjectMocks; @@ -48,7 +52,7 @@ import org.springframework.test.web.servlet.setup.MockMvcBuilders; import com.google.common.collect.Lists; -public class JobControllerV2Test { +public class JobControllerV2Test extends NLocalFileMetadataTestCase { private MockMvc mockMvc; @@ -75,6 +79,12 @@ public class JobControllerV2Test { SecurityContextHolder.getContext().setAuthentication(authentication); ReflectionTestUtils.setField(aclEvaluate, "aclUtil", aclUtil); ReflectionTestUtils.setField(jobService, "aclEvaluate", aclEvaluate); + createTestMetadata(); + } + + @After + public void tearDown() { + cleanupTestMetadata(); } @Test @@ -107,7 +117,54 @@ public class JobControllerV2Test { .andExpect(MockMvcResultMatchers.status().isOk()).andReturn(); Mockito.verify(jobControllerV2).getJobList(new Integer[] { 0 }, 1, "", "default", null, 0, 10, "last_modified", - true); + null, true); + } + + @Test + public void testGetJobsWithoutProjectAndSortby() throws Exception { + List<ExecutableResponse> jobs = new ArrayList<>(); + List<String> jobNames = Lists.newArrayList(); + JobFilter jobFilter = new JobFilter(Lists.newArrayList(), jobNames, 4, null, null, null, "job_name", true); + Mockito.when(jobService.listGlobalJobs(jobFilter, 0, Integer.MAX_VALUE)) + .thenReturn(new DataResult<>(jobs, 0, 0, 0)); + mockMvc.perform( + MockMvcRequestBuilders.get("/api/jobs").contentType(MediaType.APPLICATION_JSON).param("timeFilter", "4") + .param("sortby", "job_name").accept(MediaType.parseMediaType(HTTP_VND_APACHE_KYLIN_V2_JSON))) + .andExpect(MockMvcResultMatchers.status().isOk()).andReturn(); + + Mockito.verify(jobControllerV2).getJobList(new Integer[0], 4, null, null, null, 0, 10, "last_modified", + "job_name", true); + } + + @Test + public void testGetJob() throws Exception { + mockJobUpdateRequest(); + String jobId = "e1ad7bb0-522e-456a-859d-2eab1df448de"; + mockMvc.perform(MockMvcRequestBuilders.get("/api/jobs/{jobId}", jobId) + .accept(MediaType.parseMediaType(HTTP_VND_APACHE_KYLIN_V2_JSON))) + .andExpect(MockMvcResultMatchers.status().isOk()).andReturn(); + + Mockito.verify(jobControllerV2).getJob(jobId); + } + + @Test + public void testGetJobOutput() throws Exception { + mockJobUpdateRequest(); + String jobId = "e1ad7bb0-522e-456a-859d-2eab1df448de"; + Mockito.when(jobService.getProjectByJobId(jobId)).thenReturn("default"); + mockMvc.perform(MockMvcRequestBuilders.get("/api/jobs/{job_id:.+}/steps/{step_id:.+}/output", jobId, jobId) + .accept(MediaType.parseMediaType(HTTP_VND_APACHE_KYLIN_V2_JSON))) + .andExpect(MockMvcResultMatchers.status().isOk()).andReturn(); + + Mockito.verify(jobControllerV2).getJobOutput(jobId, jobId); + } + + private JobUpdateRequest mockJobUpdateRequest() { + JobUpdateRequest jobUpdateRequest = new JobUpdateRequest(); + jobUpdateRequest.setProject("default"); + jobUpdateRequest.setAction("RESUME"); + jobUpdateRequest.setJobIds(Lists.newArrayList("e1ad7bb0-522e-456a-859d-2eab1df448de")); + return jobUpdateRequest; } @Test @@ -147,7 +204,6 @@ public class JobControllerV2Test { .accept(MediaType.parseMediaType(HTTP_VND_APACHE_KYLIN_V2_JSON))) .andExpect(MockMvcResultMatchers.status().isOk()); - } } diff --git a/src/data-loading-service/src/main/java/org/apache/kylin/rest/response/ExecutableResponse.java b/src/data-loading-service/src/main/java/org/apache/kylin/rest/response/ExecutableResponse.java index a9612ad681..4a0ea0f381 100644 --- a/src/data-loading-service/src/main/java/org/apache/kylin/rest/response/ExecutableResponse.java +++ b/src/data-loading-service/src/main/java/org/apache/kylin/rest/response/ExecutableResponse.java @@ -18,23 +18,17 @@ package org.apache.kylin.rest.response; -import com.clearspring.analytics.util.Lists; -import com.fasterxml.jackson.annotation.JsonManagedReference; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.annotation.JsonUnwrapped; -import com.google.common.collect.Maps; -import org.apache.kylin.engine.spark.job.NSparkSnapshotJob; -import org.apache.kylin.engine.spark.job.NTableSamplingJob; -import lombok.AllArgsConstructor; -import lombok.Getter; -import lombok.NoArgsConstructor; -import lombok.Setter; -import lombok.SneakyThrows; -import lombok.val; -import lombok.var; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang.StringUtils; import org.apache.kylin.common.KylinConfig; import org.apache.kylin.common.util.JsonUtil; +import org.apache.kylin.engine.spark.job.NSparkSnapshotJob; +import org.apache.kylin.engine.spark.job.NTableSamplingJob; import org.apache.kylin.job.SecondStorageCleanJobUtil; import org.apache.kylin.job.constant.JobStatusEnum; import org.apache.kylin.job.execution.AbstractExecutable; @@ -49,9 +43,19 @@ import org.apache.kylin.metadata.model.NTableMetadataManager; import org.apache.kylin.metadata.model.SegmentStatusEnumToDisplay; import org.apache.kylin.metadata.model.TableDesc; -import java.util.List; -import java.util.Map; -import java.util.Optional; +import com.clearspring.analytics.util.Lists; +import com.fasterxml.jackson.annotation.JsonManagedReference; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonUnwrapped; +import com.google.common.collect.Maps; + +import lombok.AllArgsConstructor; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; +import lombok.SneakyThrows; +import lombok.val; +import lombok.var; @Setter @Getter @@ -111,6 +115,36 @@ public class ExecutableResponse implements Comparable<ExecutableResponse> { private static final String SNAPSHOT_FULL_RANGE = "FULL"; private static final String SNAPSHOT_INC_RANGE = "INC"; + @JsonProperty("version") + protected String version; + + @JsonProperty("related_segment") + public String getRelatedSegment() { + return CollectionUtils.isEmpty(targetSegments) ? "" : String.join(",", targetSegments); + } + + @JsonProperty("progress") + public double getProgress() { + int completedStepCount = 0; + + for (ExecutableStepResponse step : this.getSteps()) { + if (step.getStatus().equals(JobStatusEnum.FINISHED)) { + completedStepCount++; + } + } + if (steps.isEmpty()) { + return 0.0; + } + return 100.0 * completedStepCount / steps.size(); + } + + public List<ExecutableStepResponse> getSteps() { + if (steps == null) { + steps = Collections.emptyList(); + } + return steps; + } + private static ExecutableResponse newInstance(AbstractExecutable abstractExecutable) { ExecutableResponse executableResponse = new ExecutableResponse(); executableResponse.setDataRangeEnd(abstractExecutable.getDataRangeEnd()); diff --git a/src/data-loading-service/src/main/java/org/apache/kylin/rest/service/JobService.java b/src/data-loading-service/src/main/java/org/apache/kylin/rest/service/JobService.java index 60bc38c812..22ea2b774a 100644 --- a/src/data-loading-service/src/main/java/org/apache/kylin/rest/service/JobService.java +++ b/src/data-loading-service/src/main/java/org/apache/kylin/rest/service/JobService.java @@ -32,6 +32,7 @@ import java.util.Calendar; import java.util.Collections; import java.util.Comparator; import java.util.Date; +import java.util.HashMap; import java.util.List; import java.util.Locale; import java.util.Map; @@ -1097,6 +1098,26 @@ public class JobService extends BasicService implements JobSupporter { return executableManager.getOutputFromHDFSByJobId(jobId, stepId).getVerboseMsg(); } + public Map<String, Object> getStepOutput(String project, String jobId, String stepId) { + aclEvaluate.checkProjectOperationPermission(project); + val executableManager = getManager(NExecutableManager.class, project); + Output output = executableManager.getOutputFromHDFSByJobId(jobId, stepId); + Map<String, Object> result = new HashMap<>(); + result.put("cmd_output", output.getVerboseMsg()); + + Map<String, String> info = output.getExtra(); + List<String> servers = Lists.newArrayList(); + if (info != null && info.get("nodes") != null) { + servers = Lists.newArrayList(info.get("nodes").split(",")); + } + List<String> nodes = servers.stream().map(server -> { + String[] split = server.split(":"); + return split[0] + ":" + split[1]; + }).collect(Collectors.toList()); + result.put("nodes", nodes); + return result; + } + @SneakyThrows public InputStream getAllJobOutput(String project, String jobId, String stepId) { aclEvaluate.checkProjectOperationPermission(project); diff --git a/src/datasource-service/src/main/java/org/apache/kylin/rest/response/TableDescResponse.java b/src/datasource-service/src/main/java/org/apache/kylin/rest/response/TableDescResponse.java index df6c6798b8..5519386bb6 100644 --- a/src/datasource-service/src/main/java/org/apache/kylin/rest/response/TableDescResponse.java +++ b/src/datasource-service/src/main/java/org/apache/kylin/rest/response/TableDescResponse.java @@ -30,6 +30,7 @@ import org.apache.kylin.metadata.model.SegmentRange; import org.apache.kylin.metadata.model.TableDesc; import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.collect.Maps; import lombok.Getter; import lombok.Setter; @@ -79,6 +80,22 @@ public class TableDescResponse extends TableDesc { } } + @JsonProperty(value = "cardinality", access = JsonProperty.Access.READ_ONLY) + public Map<String, Long> getCardinality() { + Map<String, Long> cardinality = Maps.newHashMapWithExpectedSize(extColumns.length); + for (ColumnDescResponse extColumn : extColumns) { + if (extColumn.getCardinality() != null) { + cardinality.put(extColumn.getName(), extColumn.getCardinality()); + } + } + return cardinality; + } + + @JsonProperty(value = "is_transactional", access = JsonProperty.Access.READ_ONLY) + public boolean getTransactionalV2() { + return super.isTransactional(); + } + @Getter @Setter public class ColumnDescResponse extends ColumnDesc { diff --git a/src/job-service/src/test/java/org/apache/kylin/rest/service/JobServiceTest.java b/src/job-service/src/test/java/org/apache/kylin/rest/service/JobServiceTest.java index 49bf1d5a9a..e29d23a875 100644 --- a/src/job-service/src/test/java/org/apache/kylin/rest/service/JobServiceTest.java +++ b/src/job-service/src/test/java/org/apache/kylin/rest/service/JobServiceTest.java @@ -1206,14 +1206,14 @@ public class JobServiceTest extends NLocalFileMetadataTestCase { private long getCreateTime(String name) { switch (name) { - case "1": - return 1560324101000L; - case "2": - return 1560324102000L; - case "3": - return 1560324103000L; - default: - return 0L; + case "1": + return 1560324101000L; + case "2": + return 1560324102000L; + case "3": + return 1560324103000L; + default: + return 0L; } } @@ -1331,8 +1331,8 @@ public class JobServiceTest extends NLocalFileMetadataTestCase { String sampleLog = ""; try (InputStream allJobOutput = jobService.getAllJobOutput("default", "e1ad7bb0-522e-456a-859d-2eab1df448de", "e1ad7bb0-522e-456a-859d-2eab1df448de"); - BufferedReader reader = new BufferedReader( - new InputStreamReader(allJobOutput, Charset.defaultCharset()))) { + BufferedReader reader = new BufferedReader( + new InputStreamReader(allJobOutput, Charset.defaultCharset()))) { String line; StringBuilder sampleData = new StringBuilder(); @@ -1581,14 +1581,14 @@ public class JobServiceTest extends NLocalFileMetadataTestCase { // testGetProjectNameAndJobStepId_NotContains String yarnAppId1 = "application"; overwriteSystemProp("kylin.engine.spark.cluster-manager-class-name", sparkClusterManagerName); - Assert.assertThrows("Async profiler status error, yarnAppId entered incorrectly, please try again.", KylinException.class, - () -> jobService.getProjectNameAndJobStepId(yarnAppId1)); + Assert.assertThrows("Async profiler status error, yarnAppId entered incorrectly, please try again.", + KylinException.class, () -> jobService.getProjectNameAndJobStepId(yarnAppId1)); // testGetProjectNameAndJobStepId_LengthError String yarnAppId2 = "application_"; overwriteSystemProp("kylin.engine.spark.cluster-manager-class-name", sparkClusterManagerName); - Assert.assertThrows("Async profiler status error, yarnAppId entered incorrectly, please try again.", KylinException.class, - () -> jobService.getProjectNameAndJobStepId(yarnAppId2)); + Assert.assertThrows("Async profiler status error, yarnAppId entered incorrectly, please try again.", + KylinException.class, () -> jobService.getProjectNameAndJobStepId(yarnAppId2)); // testGetProjectNameAndJobStepId_NotContainsJob String yarnAppId = "application_1554187389076_-1"; @@ -1803,4 +1803,77 @@ public class JobServiceTest extends NLocalFileMetadataTestCase { } Assert.assertEquals("", errorMsg); } + + @Test + public void testGetStepOutput() { + String jobId = "e1ad7bb0-522e-456a-859d-2eab1df448de"; + NExecutableManager manager = NExecutableManager.getInstance(jobService.getConfig(), "default"); + ExecutableOutputPO executableOutputPO = new ExecutableOutputPO(); + Map<String, String> info = Maps.newHashMap(); + info.put("nodes", "localhost:7070:all"); + executableOutputPO.setInfo(info); + manager.updateJobOutputToHDFS(KylinConfig.getInstanceFromEnv().getJobTmpOutputStorePath("default", jobId), + executableOutputPO); + + Map<String, Object> result = Maps.newHashMap(); + result.put("nodes", Lists.newArrayList("localhost:7070")); + result.put("cmd_output", null); + Assert.assertEquals(result, jobService.getStepOutput("default", jobId, jobId)); + + executableOutputPO.setInfo(null); + manager.updateJobOutputToHDFS(KylinConfig.getInstanceFromEnv().getJobTmpOutputStorePath("default", jobId), + executableOutputPO); + + result = Maps.newHashMap(); + result.put("nodes", Lists.newArrayList()); + result.put("cmd_output", null); + Assert.assertEquals(result, jobService.getStepOutput("default", jobId, jobId)); + + info = Maps.newHashMap(); + executableOutputPO.setInfo(info); + manager.updateJobOutputToHDFS(KylinConfig.getInstanceFromEnv().getJobTmpOutputStorePath("default", jobId), + executableOutputPO); + + result = Maps.newHashMap(); + result.put("nodes", Lists.newArrayList()); + result.put("cmd_output", null); + Assert.assertEquals(result, jobService.getStepOutput("default", jobId, jobId)); + } + + @Test + public void testExecutableResponse() throws Exception { + val modelManager = mock(NDataModelManager.class); + + Mockito.when(modelService.getManager(NDataModelManager.class, "default")).thenReturn(modelManager); + NDataModel nDataModel = mock(NDataModel.class); + Mockito.when(modelManager.getDataModelDesc(Mockito.anyString())).thenReturn(nDataModel); + + NExecutableManager executableManager = Mockito.spy(NExecutableManager.getInstance(getTestConfig(), "default")); + Mockito.when(jobService.getManager(NExecutableManager.class, "default")).thenReturn(executableManager); + val mockJobs = mockDetailJobs(false); + Mockito.when(executableManager.getAllJobs(Mockito.anyLong(), Mockito.anyLong())).thenReturn(mockJobs); + for (ExecutablePO po : mockJobs) { + AbstractExecutable exe = executableManager.fromPO(po); + Mockito.when(executableManager.getJob(po.getId())).thenReturn(exe); + } + getTestConfig().setProperty("kylin.streaming.enabled", "false"); + // test size + List<String> jobNames = Lists.newArrayList(); + JobFilter jobFilter = new JobFilter(Lists.newArrayList(), jobNames, 4, "", "", "default", "", true); + List<ExecutableResponse> jobs = jobService.listJobs(jobFilter); + List<ExecutableResponse> executableResponses = jobService.addOldParams(jobs); + ExecutableResponse executable = executableResponses.get(0); + Assert.assertEquals("", executable.getRelatedSegment()); + Assert.assertEquals(0, executable.getProgress(), 0); + executable.getSteps().get(0).setStatus(JobStatusEnum.FINISHED); + Assert.assertEquals(33, executable.getProgress(), 1); + executable.setSteps(null); + String uuid = UUID.randomUUID().toString(); + executable.setTargetSegments(Lists.newArrayList(uuid)); + Assert.assertEquals(0.0, executable.getProgress(), 0); + Assert.assertEquals(uuid, executable.getRelatedSegment()); + executable.setTargetSegments(Collections.emptyList()); + Assert.assertEquals(0.0, executable.getProgress(), 0); + Assert.assertEquals("", executable.getRelatedSegment()); + } } diff --git a/src/metadata-server/src/main/java/org/apache/kylin/rest/controller/v2/NModelControllerV2.java b/src/metadata-server/src/main/java/org/apache/kylin/rest/controller/v2/NModelControllerV2.java index 7552b476d5..224b68c6a8 100644 --- a/src/metadata-server/src/main/java/org/apache/kylin/rest/controller/v2/NModelControllerV2.java +++ b/src/metadata-server/src/main/java/org/apache/kylin/rest/controller/v2/NModelControllerV2.java @@ -18,8 +18,10 @@ package org.apache.kylin.rest.controller.v2; import static org.apache.kylin.common.constant.HttpConstant.HTTP_VND_APACHE_KYLIN_V2_JSON; +import static org.apache.kylin.common.exception.code.ErrorCodeServer.MODEL_ID_NOT_EXIST; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; import java.util.Map; @@ -32,11 +34,13 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.stereotype.Controller; import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PathVariable; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.ResponseBody; import com.google.common.collect.Lists; +import com.google.common.collect.Maps; import io.swagger.annotations.ApiOperation; @@ -68,4 +72,21 @@ public class NModelControllerV2 extends NBasicController { return new EnvelopeResponse<>(KylinException.CODE_SUCCESS, modelResponse, ""); } + @ApiOperation(value = "getModelDesc", tags = { "AI" }) + @GetMapping(value = "/{projectName}/{modelName}", produces = { HTTP_VND_APACHE_KYLIN_V2_JSON }) + @ResponseBody + public EnvelopeResponse<Map<String, Object>> getModelDesc(@PathVariable("projectName") String project, + @PathVariable("modelName") String modelAlias) { + checkProjectName(project); + List<NDataModel> models = new ArrayList<>( + modelService.getModels(modelAlias, project, true, null, Lists.newArrayList(), "last_modify", true)); + if (models.size() == 0) { + throw new KylinException(MODEL_ID_NOT_EXIST, modelAlias); + } + models = modelService.addOldParams(project, models); + + HashMap<String, Object> modelResponse = new HashMap<>(); + modelResponse.put("model", models.size() == 0 ? Maps.newHashMap() : models.get(0)); + return new EnvelopeResponse<>(KylinException.CODE_SUCCESS, modelResponse, ""); + } } diff --git a/src/metadata-server/src/main/java/org/apache/kylin/rest/controller/v2/NTableControllerV2.java b/src/metadata-server/src/main/java/org/apache/kylin/rest/controller/v2/NTableControllerV2.java new file mode 100644 index 0000000000..0c195a84f4 --- /dev/null +++ b/src/metadata-server/src/main/java/org/apache/kylin/rest/controller/v2/NTableControllerV2.java @@ -0,0 +1,73 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kylin.rest.controller.v2; + +import static org.apache.kylin.common.constant.HttpConstant.HTTP_VND_APACHE_KYLIN_V2_JSON; +import static org.apache.kylin.common.exception.ServerErrorCode.UNSUPPORTED_STREAMING_OPERATION; + +import java.io.IOException; +import java.util.List; + +import org.apache.kylin.common.exception.KylinException; +import org.apache.kylin.common.msg.MsgPicker; +import org.apache.kylin.metadata.model.ISourceAware; +import org.apache.kylin.metadata.model.TableDesc; +import org.apache.kylin.rest.controller.NBasicController; +import org.apache.kylin.rest.response.DataResult; +import org.apache.kylin.rest.response.EnvelopeResponse; +import org.apache.kylin.rest.service.TableService; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Controller; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.ResponseBody; + +import io.swagger.annotations.ApiOperation; + +@Controller +@RequestMapping(value = "/api/tables", produces = { HTTP_VND_APACHE_KYLIN_V2_JSON }) +public class NTableControllerV2 extends NBasicController { + + @Autowired + private TableService tableService; + + @ApiOperation(value = "getTableDesc", tags = { "AI" }) + @GetMapping(value = "") + @ResponseBody + public EnvelopeResponse<List<TableDesc>> getTableDesc(@RequestParam(value = "project") String project, + @RequestParam(value = "table", required = false) String table, + @RequestParam(value = "database", required = false) String database, + @RequestParam(value = "is_fuzzy", required = false, defaultValue = "false") boolean isFuzzy, + @RequestParam(value = "ext", required = false, defaultValue = "true") boolean withExt, + @RequestParam(value = "page_offset", required = false, defaultValue = "0") Integer offset, + @RequestParam(value = "page_size", required = false, defaultValue = "10") Integer limit, + @RequestParam(value = "source_type", required = false, defaultValue = "9") Integer sourceType) + throws IOException { + checkProjectName(project); + checkNonNegativeIntegerArg("page_offset", offset); + if (sourceType == ISourceAware.ID_STREAMING) { + throw new KylinException(UNSUPPORTED_STREAMING_OPERATION, + MsgPicker.getMsg().getStreamingOperationNotSupport()); + } + List<TableDesc> result = tableService.getTableDescByType(project, withExt, table, database, isFuzzy, + sourceType); + return new EnvelopeResponse<>(KylinException.CODE_SUCCESS, DataResult.get(result, offset, limit).getValue(), + ""); + } +} \ No newline at end of file diff --git a/src/metadata-server/src/test/java/org/apache/kylin/rest/controller/NModelControllerV2Test.java b/src/metadata-server/src/test/java/org/apache/kylin/rest/controller/NModelControllerV2Test.java index cf6a0ced47..f7f0dfe2fe 100644 --- a/src/metadata-server/src/test/java/org/apache/kylin/rest/controller/NModelControllerV2Test.java +++ b/src/metadata-server/src/test/java/org/apache/kylin/rest/controller/NModelControllerV2Test.java @@ -23,9 +23,9 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.List; -import org.apache.kylin.rest.constant.Constant; import org.apache.kylin.common.util.NLocalFileMetadataTestCase; import org.apache.kylin.metadata.model.NDataModel; +import org.apache.kylin.rest.constant.Constant; import org.apache.kylin.rest.controller.v2.NModelControllerV2; import org.apache.kylin.rest.response.NDataModelResponse; import org.apache.kylin.rest.response.RelatedModelResponse; @@ -47,6 +47,8 @@ import org.springframework.test.web.servlet.request.MockMvcRequestBuilders; import org.springframework.test.web.servlet.result.MockMvcResultMatchers; import org.springframework.test.web.servlet.setup.MockMvcBuilders; +import com.google.common.collect.Lists; + public class NModelControllerV2Test extends NLocalFileMetadataTestCase { private MockMvc mockMvc; @@ -139,4 +141,22 @@ public class NModelControllerV2Test extends NLocalFileMetadataTestCase { Mockito.verify(nModelControllerV2).getModels("", true, "default", 0, 10, "last_modify", true); } + @Test + public void testGetModelDesc() throws Exception { + Mockito.when(modelService.getModels("model1", "default", true, null, Lists.newArrayList(), "last_modify", true)) + .thenReturn(mockModelDesc()); + mockMvc.perform(MockMvcRequestBuilders.get("/api/models/default/model1") + .accept(MediaType.parseMediaType(HTTP_VND_APACHE_KYLIN_V2_JSON))) + .andExpect(MockMvcResultMatchers.status().isOk()).andReturn(); + Mockito.verify(nModelControllerV2).getModelDesc("default", "model1"); + } + + private List<NDataModelResponse> mockModelDesc() { + final List<NDataModelResponse> models = new ArrayList<>(); + NDataModel model = new NDataModel(); + model.setAlias("model1"); + model.setUuid("model1"); + models.add(new NDataModelResponse(model)); + return models; + } } diff --git a/src/metadata-server/src/test/java/org/apache/kylin/rest/controller/NTableControllerV2Test.java b/src/metadata-server/src/test/java/org/apache/kylin/rest/controller/NTableControllerV2Test.java new file mode 100644 index 0000000000..6e1b3e608f --- /dev/null +++ b/src/metadata-server/src/test/java/org/apache/kylin/rest/controller/NTableControllerV2Test.java @@ -0,0 +1,111 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kylin.rest.controller; + +import static org.apache.kylin.common.constant.HttpConstant.HTTP_VND_APACHE_KYLIN_V2_JSON; + +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; + +import org.apache.kylin.common.util.NLocalFileMetadataTestCase; +import org.apache.kylin.metadata.model.TableDesc; +import org.apache.kylin.rest.constant.Constant; +import org.apache.kylin.rest.controller.v2.NTableControllerV2; +import org.apache.kylin.rest.service.TableService; +import org.junit.After; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.MockitoAnnotations; +import org.springframework.http.MediaType; +import org.springframework.security.authentication.TestingAuthenticationToken; +import org.springframework.security.core.Authentication; +import org.springframework.security.core.context.SecurityContextHolder; +import org.springframework.test.web.servlet.MockMvc; +import org.springframework.test.web.servlet.request.MockMvcRequestBuilders; +import org.springframework.test.web.servlet.result.MockMvcResultMatchers; +import org.springframework.test.web.servlet.setup.MockMvcBuilders; + +public class NTableControllerV2Test extends NLocalFileMetadataTestCase { + + private static final String APPLICATION_JSON = HTTP_VND_APACHE_KYLIN_V2_JSON; + + private MockMvc mockMvc; + + @Rule + public ExpectedException thrown = ExpectedException.none(); + + @Mock + private TableService tableService; + + @InjectMocks + private NTableControllerV2 nTableControllerV2 = Mockito.spy(new NTableControllerV2()); + + private final Authentication authentication = new TestingAuthenticationToken("ADMIN", "ADMIN", Constant.ROLE_ADMIN); + + @Before + public void setup() { + MockitoAnnotations.initMocks(this); + + mockMvc = MockMvcBuilders.standaloneSetup(nTableControllerV2).defaultRequest(MockMvcRequestBuilders.get("/")) + .defaultResponseCharacterEncoding(StandardCharsets.UTF_8).build(); + + SecurityContextHolder.getContext().setAuthentication(authentication); + createTestMetadata(); + } + + @After + public void tearDown() { + cleanupTestMetadata(); + } + + @Test + public void testGetTableDesc() throws Exception { + Mockito.when(tableService.getTableDesc("default", false, "", "DEFAULT", true)) // + .thenReturn(mockTables()); + mockMvc.perform(MockMvcRequestBuilders.get("/api/tables") // + .contentType(MediaType.APPLICATION_JSON) // + .param("ext", "false") // + .param("project", "default") // + .param("table", "") // + .param("database", "DEFAULT") // + .param("page_offset", "0") // + .param("page_size", "10") // + .param("is_fuzzy", "true") // + .accept(MediaType.parseMediaType(APPLICATION_JSON))) // + .andExpect(MockMvcResultMatchers.status().isOk()).andReturn(); + + Mockito.verify(nTableControllerV2).getTableDesc("default", "", "DEFAULT", true, false, 0, 10, 9); + } + + private List<TableDesc> mockTables() { + final List<TableDesc> tableDescs = new ArrayList<>(); + TableDesc tableDesc = new TableDesc(); + tableDesc.setName("table1"); + tableDescs.add(tableDesc); + TableDesc tableDesc2 = new TableDesc(); + tableDesc2.setName("table2"); + tableDescs.add(tableDesc2); + return tableDescs; + } +} \ No newline at end of file diff --git a/src/modeling-service/src/main/java/org/apache/kylin/rest/response/NDataModelOldParams.java b/src/modeling-service/src/main/java/org/apache/kylin/rest/response/NDataModelOldParams.java index 39f80e7dd2..dda95a2a64 100644 --- a/src/modeling-service/src/main/java/org/apache/kylin/rest/response/NDataModelOldParams.java +++ b/src/modeling-service/src/main/java/org/apache/kylin/rest/response/NDataModelOldParams.java @@ -18,11 +18,18 @@ package org.apache.kylin.rest.response; import java.io.Serializable; +import java.util.ArrayList; +import java.util.HashMap; import java.util.List; +import java.util.Map; +import java.util.Set; +import org.apache.commons.collections.CollectionUtils; import org.apache.kylin.metadata.model.JoinTableDesc; +import org.apache.kylin.metadata.model.ModelDimensionDesc; import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.collect.Sets; import lombok.Getter; import lombok.Setter; @@ -50,4 +57,35 @@ public class NDataModelOldParams implements Serializable { @JsonProperty("project") private String projectName; + + @JsonProperty("dimensions") + private List<ModelDimensionDesc> dimensions; + + public void setDimensions(List<NDataModelResponse.SimplifiedNamedColumn> simplifiedDimensions) { + if (CollectionUtils.isEmpty(simplifiedDimensions)) { + return; + } + + Map<String, Set<String>> dimCandidate = new HashMap<>(); + for (NDataModelResponse.SimplifiedNamedColumn dimension : simplifiedDimensions) { + String[] tableColumn = dimension.getAliasDotColumn().split("\\."); + String table = tableColumn[0]; + String column = tableColumn[1]; + addCandidate(dimCandidate, table, column); + } + + List<ModelDimensionDesc> dims = new ArrayList<>(); + for (Map.Entry<String, Set<String>> dimensionEntry : dimCandidate.entrySet()) { + ModelDimensionDesc dimension = new ModelDimensionDesc(); + dimension.setTable(dimensionEntry.getKey()); + dimension.setColumns(dimensionEntry.getValue().toArray(new String[0])); + dims.add(dimension); + } + this.dimensions = dims; + } + + private static void addCandidate(Map<String, Set<String>> tblColMap, String table, String column) { + tblColMap.computeIfAbsent(table, k -> Sets.newHashSet()); + tblColMap.get(table).add(column); + } } diff --git a/src/modeling-service/src/main/java/org/apache/kylin/rest/service/ModelService.java b/src/modeling-service/src/main/java/org/apache/kylin/rest/service/ModelService.java index 7fcf9f610c..07cab15306 100644 --- a/src/modeling-service/src/main/java/org/apache/kylin/rest/service/ModelService.java +++ b/src/modeling-service/src/main/java/org/apache/kylin/rest/service/ModelService.java @@ -359,6 +359,7 @@ public class ModelService extends AbstractModelService implements TableModelSupp if (model instanceof NDataModelResponse) { oldParams.setProjectName(model.getProject()); oldParams.setSizeKB(((NDataModelResponse) model).getStorage() / 1024); + oldParams.setDimensions(((NDataModelResponse) model).getNamedColumns()); ((NDataModelResponse) model).setOldParams(oldParams); } else if (model instanceof RelatedModelResponse) { ((RelatedModelResponse) model).setOldParams(oldParams); diff --git a/src/modeling-service/src/test/java/org/apache/kylin/rest/service/TableServiceTest.java b/src/modeling-service/src/test/java/org/apache/kylin/rest/service/TableServiceTest.java index bde1acaccc..ff8a49d330 100644 --- a/src/modeling-service/src/test/java/org/apache/kylin/rest/service/TableServiceTest.java +++ b/src/modeling-service/src/test/java/org/apache/kylin/rest/service/TableServiceTest.java @@ -1459,4 +1459,39 @@ public class TableServiceTest extends CSVSourceTestCase { new ArrayList<>()); Assert.assertThrows(KylinException.class, func); } + + @Test + public void testTableDescResponseV2() throws IOException { + final String tableIdentity = "DEFAULT.TEST_COUNTRY"; + final NTableMetadataManager tableMgr = NTableMetadataManager.getInstance(getTestConfig(), "newten"); + final TableDesc tableDesc = tableMgr.getTableDesc(tableIdentity); + final TableExtDesc oldExtDesc = tableMgr.getOrCreateTableExt(tableDesc); + // mock table ext desc + TableExtDesc tableExt = new TableExtDesc(oldExtDesc); + tableExt.setIdentity(tableIdentity); + TableExtDesc.ColumnStats col1 = new TableExtDesc.ColumnStats(); + col1.setCardinality(100); + col1.setTableExtDesc(tableExt); + col1.setColumnName(tableDesc.getColumns()[0].getName()); + col1.setMinValue("America"); + col1.setMaxValue("Zimbabwe"); + col1.setNullCount(0); + tableExt.setColumnStats(Lists.newArrayList(col1)); + tableMgr.mergeAndUpdateTableExt(oldExtDesc, tableExt); + + final List<TableDesc> tables = tableService.getTableDesc("newten", true, "TEST_COUNTRY", "DEFAULT", true); + Assert.assertEquals(1, tables.size()); + Assert.assertTrue(tables.get(0) instanceof TableDescResponse); + TableDescResponse t = (TableDescResponse) tables.get(0); + Map<String, Long> cardinality = t.getCardinality(); + for (int i = 0; i < t.getExtColumns().length; i++) { + if (t.getExtColumns()[i].getCardinality() != null) { + Assert.assertEquals(cardinality.get(t.getExtColumns()[i].getName()), + t.getExtColumns()[i].getCardinality()); + } + } + Assert.assertEquals(t.getTransactionalV2(), t.isTransactional()); + t.setTransactional(true); + Assert.assertEquals(t.getTransactionalV2(), t.isTransactional()); + } } diff --git a/src/query-server/src/main/java/org/apache/kylin/rest/controller/v2/NQueryMetaController.java b/src/query-server/src/main/java/org/apache/kylin/rest/controller/v2/NQueryMetaController.java index 6227fa46bb..d93c93f8c5 100644 --- a/src/query-server/src/main/java/org/apache/kylin/rest/controller/v2/NQueryMetaController.java +++ b/src/query-server/src/main/java/org/apache/kylin/rest/controller/v2/NQueryMetaController.java @@ -22,10 +22,10 @@ import static org.apache.kylin.common.constant.HttpConstant.HTTP_VND_APACHE_KYLI import java.util.List; -import org.apache.kylin.metadata.querymeta.TableMeta; +import org.apache.kylin.metadata.querymeta.TableMetaWithType; +import org.apache.kylin.rest.controller.NBasicController; import org.apache.kylin.rest.request.MetaRequest; import org.apache.kylin.rest.service.QueryService; -import org.apache.kylin.rest.controller.NBasicController; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.RequestMapping; @@ -56,11 +56,11 @@ public class NQueryMetaController extends NBasicController { @GetMapping(value = "/tables_and_columns", produces = { "application/json", HTTP_VND_APACHE_KYLIN_V2_JSON }) @ResponseBody @Deprecated - public List<TableMeta> getMetadataForDriver(MetaRequest metaRequest) { + public List<TableMetaWithType> getMetadataForDriver(MetaRequest metaRequest) { if (metaRequest.getCube() == null || metaRequest.getCube().isEmpty()) { - return queryService.getMetadata(metaRequest.getProject()); + return queryService.getMetadataAddType(metaRequest.getProject(), null); } else { - return queryService.getMetadata(metaRequest.getProject(), metaRequest.getCube()); + return queryService.getMetadataAddType(metaRequest.getProject(), metaRequest.getCube()); } } } diff --git a/src/query-service/src/main/java/org/apache/kylin/rest/service/QueryService.java b/src/query-service/src/main/java/org/apache/kylin/rest/service/QueryService.java index c3cbe877cb..69d75265df 100644 --- a/src/query-service/src/main/java/org/apache/kylin/rest/service/QueryService.java +++ b/src/query-service/src/main/java/org/apache/kylin/rest/service/QueryService.java @@ -1633,4 +1633,29 @@ public class QueryService extends BasicService implements CacheSignatureQuerySup queryContext.setModelPriorities(QueryModelPriorities.getModelPrioritiesFromComment(sql)); } + public List<TableMetaWithType> getMetadataAddType(String project, String modelAlias) { + List<TableMeta> tableMetas = getMetadata(project, modelAlias); + + Map<TableMetaIdentify, TableMetaWithType> tableMap = Maps.newLinkedHashMap(); + Map<ColumnMetaIdentify, ColumnMetaWithType> columnMap = Maps.newLinkedHashMap(); + + for (TableMeta tableMeta : tableMetas) { + TableMetaWithType tblMeta = TableMetaWithType.ofColumnMeta(tableMeta); + tableMap.put(new TableMetaIdentify(tblMeta.getTABLE_SCHEM(), tblMeta.getTABLE_NAME()), tblMeta); + + for (ColumnMeta columnMeta : tblMeta.getColumns()) { + columnMap.put(new ColumnMetaIdentify(columnMeta.getTABLE_SCHEM(), columnMeta.getTABLE_NAME(), + columnMeta.getCOLUMN_NAME()), (ColumnMetaWithType) columnMeta); + } + } + + List<NDataModel> models = getModels(project, modelAlias); + + for (NDataModel model : models) { + clarifyTblTypeToFactOrLookup(model, tableMap); + clarifyPkFkCols(model, columnMap); + } + + return Lists.newArrayList(tableMap.values()); + } } diff --git a/src/query-service/src/test/java/org/apache/kylin/rest/service/QueryServiceTest.java b/src/query-service/src/test/java/org/apache/kylin/rest/service/QueryServiceTest.java index 1e08a7d9df..5011e83b98 100644 --- a/src/query-service/src/test/java/org/apache/kylin/rest/service/QueryServiceTest.java +++ b/src/query-service/src/test/java/org/apache/kylin/rest/service/QueryServiceTest.java @@ -754,6 +754,47 @@ public class QueryServiceTest extends NLocalFileMetadataTestCase { } } + @Test + public void testGetMetadataAddType() throws Exception { + List<TableMetaWithType> tableMetasAddType = queryService.getMetadataAddType("default", null); + List<TableMeta> tableMetas = queryService.getMetadata("default", null); + List<TableMeta> tablesV2 = Lists.newLinkedList(); + for (TableMetaWithType t : tableMetasAddType) { + TableMeta tableMeta = new TableMeta(t.getTABLE_CAT(), t.getTABLE_SCHEM(), t.getTABLE_NAME(), + t.getTABLE_TYPE(), t.getREMARKS(), t.getTYPE_CAT(), t.getTYPE_SCHEM(), t.getTYPE_NAME(), + t.getSELF_REFERENCING_COL_NAME(), t.getREF_GENERATION()); + tableMeta.setColumns(t.getColumns().stream() + .map(c -> new ColumnMeta(c.getTABLE_CAT(), c.getTABLE_SCHEM(), c.getTABLE_NAME(), + c.getCOLUMN_NAME(), c.getDATA_TYPE(), c.getTYPE_NAME(), c.getCOLUMN_SIZE(), + c.getBUFFER_LENGTH(), c.getDECIMAL_DIGITS(), c.getNUM_PREC_RADIX(), c.getNULLABLE(), + c.getREMARKS(), c.getCOLUMN_DEF(), c.getSQL_DATA_TYPE(), c.getSQL_DATETIME_SUB(), + c.getCHAR_OCTET_LENGTH(), c.getORDINAL_POSITION(), c.getIS_NULLABLE(), c.getSCOPE_CATLOG(), + c.getSCOPE_SCHEMA(), c.getSCOPE_TABLE(), c.getSOURCE_DATA_TYPE(), c.getIS_AUTOINCREMENT())) + .collect(Collectors.toList())); + tablesV2.add(tableMeta); + } + Assert.assertEquals(JsonUtil.writeValueAsString(tablesV2), JsonUtil.writeValueAsString(tableMetas)); + + tableMetasAddType = queryService.getMetadataAddType("default", "test_bank"); + tableMetas = queryService.getMetadata("default", "test_bank"); + tablesV2 = Lists.newLinkedList(); + for (TableMetaWithType t : tableMetasAddType) { + TableMeta tableMeta = new TableMeta(t.getTABLE_CAT(), t.getTABLE_SCHEM(), t.getTABLE_NAME(), + t.getTABLE_TYPE(), t.getREMARKS(), t.getTYPE_CAT(), t.getTYPE_SCHEM(), t.getTYPE_NAME(), + t.getSELF_REFERENCING_COL_NAME(), t.getREF_GENERATION()); + tableMeta.setColumns(t.getColumns().stream() + .map(c -> new ColumnMeta(c.getTABLE_CAT(), c.getTABLE_SCHEM(), c.getTABLE_NAME(), + c.getCOLUMN_NAME(), c.getDATA_TYPE(), c.getTYPE_NAME(), c.getCOLUMN_SIZE(), + c.getBUFFER_LENGTH(), c.getDECIMAL_DIGITS(), c.getNUM_PREC_RADIX(), c.getNULLABLE(), + c.getREMARKS(), c.getCOLUMN_DEF(), c.getSQL_DATA_TYPE(), c.getSQL_DATETIME_SUB(), + c.getCHAR_OCTET_LENGTH(), c.getORDINAL_POSITION(), c.getIS_NULLABLE(), c.getSCOPE_CATLOG(), + c.getSCOPE_SCHEMA(), c.getSCOPE_TABLE(), c.getSOURCE_DATA_TYPE(), c.getIS_AUTOINCREMENT())) + .collect(Collectors.toList())); + tablesV2.add(tableMeta); + } + Assert.assertEquals(JsonUtil.writeValueAsString(tablesV2), JsonUtil.writeValueAsString(tableMetas)); + } + @Test public void testExposedColumnsProjectConfigByModel() throws Exception { NProjectManager projectManager = NProjectManager.getInstance(getTestConfig());