This is an automated email from the ASF dual-hosted git repository. nju_yaho pushed a commit to tag ebay-3.1.0-release-20200701 in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 6c5c4551609a354c99cbec670dc398061ce111f7 Author: Zhong, Yanghong <nju_y...@apache.org> AuthorDate: Thu Jun 18 17:28:41 2020 +0800 KYLIN-4579 Add fat API to create cube & delete cube --- .../kylin/rest/controller/CubeController.java | 142 ++++++++++++++++++++- .../kylin/rest/request/CubeWithAllRequest.java | 97 ++++++++++++++ .../org/apache/kylin/rest/service/CubeService.java | 12 +- 3 files changed, 245 insertions(+), 6 deletions(-) diff --git a/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java b/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java index af3e9b1..baedcd9 100644 --- a/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java +++ b/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java @@ -29,6 +29,7 @@ import java.util.Map; import java.util.Set; import java.util.TreeMap; import java.util.TreeSet; +import java.util.UUID; import javax.servlet.http.HttpServletResponse; @@ -36,6 +37,7 @@ import org.apache.commons.lang.StringUtils; import org.apache.kylin.common.util.JsonUtil; import org.apache.kylin.common.util.Pair; import org.apache.kylin.common.util.RandomUtil; +import org.apache.kylin.common.util.StringUtil; import org.apache.kylin.cube.CubeInstance; import org.apache.kylin.cube.CubeManager; import org.apache.kylin.cube.CubeSegment; @@ -55,11 +57,13 @@ import org.apache.kylin.engine.mr.common.CuboidStatsReaderUtil; import org.apache.kylin.job.JobInstance; import org.apache.kylin.job.JoinedFlatTable; import org.apache.kylin.job.exception.JobException; +import org.apache.kylin.metadata.model.DataModelDesc; import org.apache.kylin.metadata.model.IJoinedFlatTableDesc; import org.apache.kylin.metadata.model.ISourceAware; import org.apache.kylin.metadata.model.MeasureDesc; import org.apache.kylin.metadata.model.SegmentRange; import org.apache.kylin.metadata.model.SegmentRange.TSRange; +import org.apache.kylin.metadata.model.TableRef; import org.apache.kylin.metadata.project.ProjectInstance; import org.apache.kylin.metadata.realization.RealizationStatusEnum; import org.apache.kylin.rest.exception.BadRequestException; @@ -70,6 +74,7 @@ import org.apache.kylin.rest.exception.TooManyRequestException; import org.apache.kylin.rest.msg.Message; import org.apache.kylin.rest.msg.MsgPicker; import org.apache.kylin.rest.request.CubeRequest; +import org.apache.kylin.rest.request.CubeWithAllRequest; import org.apache.kylin.rest.request.JobBuildRequest; import org.apache.kylin.rest.request.JobBuildRequest2; import org.apache.kylin.rest.request.JobOptimizeRequest; @@ -84,8 +89,10 @@ import org.apache.kylin.rest.response.HBaseResponse; import org.apache.kylin.rest.response.ResponseCode; import org.apache.kylin.rest.service.CubeService; import org.apache.kylin.rest.service.JobService; +import org.apache.kylin.rest.service.ModelService; import org.apache.kylin.rest.service.ProjectService; import org.apache.kylin.rest.service.QueryService; +import org.apache.kylin.rest.service.TableService; import org.apache.kylin.rest.util.AclEvaluate; import org.apache.kylin.rest.util.ValidateUtil; import org.apache.kylin.shaded.com.google.common.collect.Lists; @@ -135,6 +142,14 @@ public class CubeController extends BasicController { private QueryService queryService; @Autowired + @Qualifier("modelMgmtService") + private ModelService modelService; + + @Autowired + @Qualifier("tableService") + private TableService tableService; + + @Autowired private AclEvaluate aclEvaluate; @RequestMapping(value = "{cubeName}/validate", method = RequestMethod.GET, produces = { "application/json" }) @@ -593,7 +608,7 @@ public class CubeController extends BasicController { //drop Cube try { - cubeService.deleteCube(cube); + cubeService.deleteCube(cube, true); } catch (Exception e) { logger.error(e.getLocalizedMessage(), e); throw new InternalErrorException("Failed to delete cube. " + " Caused by: " + e.getMessage(), e); @@ -601,6 +616,131 @@ public class CubeController extends BasicController { } + @RequestMapping(value = "/{cubeName}/withAll", method = { RequestMethod.DELETE }, produces = { "application/json" }) + @ResponseBody + public GeneralResponse deleteCubeWithAll(@PathVariable String cubeName) { + CubeInstance cube = getCube(cubeName); + + DataModelDesc model = cube.getModel(); + ProjectInstance project = cubeService.getProjectManager().getProjectOfModel(model.getName()); + Set<TableRef> tables = model.getAllTables(); + + //drop Cube + try { + cubeService.deleteCube(cube, false); + } catch (Exception e) { + logger.error(e.getLocalizedMessage(), e); + throw new InternalErrorException("Failed to delete cube caused by: " + e.getMessage(), e); + } + + GeneralResponse response = new GeneralResponse(); + StringBuilder warningMsg = new StringBuilder(); + + //drop Model + try { + modelService.dropModel(model); + } catch (Exception e) { + logger.warn("", e); + warningMsg.append(e); + + response.setProperty("Warning message:", warningMsg.toString()); + return response; + } + + //unload tables + Set<String> unLoadFail = com.google.common.collect.Sets.newHashSet(); + for (TableRef table : tables) { + try { + boolean ifUnloaded = tableService.unloadHiveTable(table.getTableIdentity(), project.getName()); + if (!ifUnloaded) { + unLoadFail.add(table.getTableIdentity()); + } + } catch (Exception e) { + logger.error("Failed to unload Hive Table", e); + throw new InternalErrorException(e.getLocalizedMessage()); + } + } + if (!unLoadFail.isEmpty()) { + warningMsg.append("Fail to unload tables: ").append(unLoadFail); + } + + response.setProperty("Warning message:", warningMsg.toString()); + return response; + } + + /** + * Save cubeDesc with loading tables, creating model + * @param cubeWithAllRequest + * @return + * @throws JsonProcessingException + */ + @RequestMapping(value = "/withAll", method = { RequestMethod.POST }, produces = { "application/json" }) + @ResponseBody + public CubeWithAllRequest saveCubeDescWithAll(@RequestBody CubeWithAllRequest cubeWithAllRequest) { + if (cubeWithAllRequest == null) { + logger.info("CubeWithAllRequest should not be null."); + throw new BadRequestException("CubeWithAllRequest should not be null."); + } + + Message msg = MsgPicker.getMsg(); + + String projectName = (null == cubeWithAllRequest.getProject()) ? ProjectInstance.DEFAULT_PROJECT_NAME + : cubeWithAllRequest.getProject(); + ProjectInstance project = cubeService.getProjectManager().getProject(projectName); + if (project == null) { + throw new BadRequestException(msg.getPROJECT_NOT_FOUND()); + } + + DataModelDesc model; + try { + model = JsonUtil.readValue(cubeWithAllRequest.getModelDescData(), DataModelDesc.class); + model.setUuid(UUID.randomUUID().toString()); + } catch (Exception e) { + throw new BadRequestException("Fail to deserialize modelDescData"); + } + + CubeDesc cubeDesc; + try { + cubeDesc = JsonUtil.readValue(cubeWithAllRequest.getCubeDescData(), CubeDesc.class); + cubeDesc.setUuid(UUID.randomUUID().toString()); + } catch (Exception e) { + throw new BadRequestException("Fail to deserialize cubeDescData"); + } + + String[] tableNames = StringUtil.splitAndTrim(cubeWithAllRequest.getTables(), ","); + try { + tableService.loadHiveTablesToProject(tableNames, project.getName()); + } catch (Exception e) { + throw new BadRequestException(String.format(Locale.ROOT, "Fail to load tables %s due to %s", + cubeWithAllRequest.getTables(), e.getLocalizedMessage())); + } + + try { + DataModelDesc existingModel = modelService.getDataModelManager().getDataModelDesc(model.getName()); + if (existingModel != null) { + cubeWithAllRequest + .appendMessage("model " + model.getName() + " already exist and will using existing one"); + model = existingModel; + } else { + modelService.createModelDesc(projectName, model); + } + } catch (Exception e) { + throw new BadRequestException(String.format(Locale.ROOT, "Fail to create model %s due to %s", + model.getName(), e.getLocalizedMessage())); + } + + try { + cubeService.createCubeAndDesc(project, cubeDesc); + } catch (Exception e) { + throw new BadRequestException(String.format(Locale.ROOT, "Fail to create cube %s due to %s", + cubeDesc.getName(), e.getLocalizedMessage())); + } + + cubeWithAllRequest.setModelUuid(model.getUuid()); + cubeWithAllRequest.setCubeUuid(cubeDesc.getUuid()); + return cubeWithAllRequest; + } + /** * save cubeDesc * diff --git a/server-base/src/main/java/org/apache/kylin/rest/request/CubeWithAllRequest.java b/server-base/src/main/java/org/apache/kylin/rest/request/CubeWithAllRequest.java new file mode 100644 index 0000000..6f4bbb6 --- /dev/null +++ b/server-base/src/main/java/org/apache/kylin/rest/request/CubeWithAllRequest.java @@ -0,0 +1,97 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kylin.rest.request; + +public class CubeWithAllRequest { + private String project; + + private String tables; + + private String modelUuid; + private String modelDescData; + + private String cubeUuid; + private String cubeDescData; + + private String message; + + public String getProject() { + return project; + } + + public void setProject(String project) { + this.project = project; + } + + public String getTables() { + return tables; + } + + public void setTables(String tables) { + this.tables = tables; + } + + public String getModelUuid() { + return modelUuid; + } + + public void setModelUuid(String modelUuid) { + this.modelUuid = modelUuid; + } + + public String getModelDescData() { + return modelDescData; + } + + public void setModelDescData(String modelDescData) { + this.modelDescData = modelDescData; + } + + public String getCubeUuid() { + return cubeUuid; + } + + public void setCubeUuid(String cubeUuid) { + this.cubeUuid = cubeUuid; + } + + public String getCubeDescData() { + return cubeDescData; + } + + public void setCubeDescData(String cubeDescData) { + this.cubeDescData = cubeDescData; + } + + public String getMessage() { + return message; + } + + public void setMessage(String message) { + this.message = message; + } + + public void appendMessage(String message) { + if (this.message == null) { + this.message = message; + } else { + this.message += message; + } + } +} diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java b/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java index 99be837..4793510 100644 --- a/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java +++ b/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java @@ -316,14 +316,16 @@ public class CubeService extends BasicService implements InitializingBean { return updatedCubeDesc; } - public void deleteCube(CubeInstance cube) throws IOException { + public void deleteCube(CubeInstance cube, boolean ifCheckJobs) throws IOException { aclEvaluate.checkProjectWritePermission(cube); Message msg = MsgPicker.getMsg(); - final List<CubingJob> cubingJobs = jobService.listJobsByRealizationName(cube.getName(), null, - EnumSet.of(ExecutableState.READY, ExecutableState.RUNNING, ExecutableState.ERROR)); - if (!cubingJobs.isEmpty()) { - throw new BadRequestException(String.format(Locale.ROOT, msg.getDISCARD_JOB_FIRST(), cube.getName())); + if (ifCheckJobs) { + final List<CubingJob> cubingJobs = jobService.listJobsByRealizationName(cube.getName(), null, + EnumSet.of(ExecutableState.READY, ExecutableState.RUNNING, ExecutableState.ERROR)); + if (!cubingJobs.isEmpty()) { + throw new BadRequestException(String.format(Locale.ROOT, msg.getDISCARD_JOB_FIRST(), cube.getName())); + } } try {