This is an automated email from the ASF dual-hosted git repository. xxyu pushed a commit to branch kylin-on-parquet-v2 in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 082df34f2d304d1e6237c09047cf2c008c5e7df0 Author: yangjiang <yangji...@ebay.com> AuthorDate: Mon Apr 12 09:52:22 2021 +0800 [KYLIN-4948] Provide an API to allow users to adjust cuboids manually --- .../kylin/rest/controller/CubeController.java | 59 ++++++++++++++++++++++ .../kylin/rest/request/JobOptimizeRequest2.java | 40 +++++++++++++++ 2 files changed, 99 insertions(+) diff --git a/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java b/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java index c514bd4..7df1527 100644 --- a/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java +++ b/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java @@ -70,6 +70,7 @@ import org.apache.kylin.rest.request.CubeRequest; import org.apache.kylin.rest.request.JobBuildRequest; import org.apache.kylin.rest.request.JobBuildRequest2; import org.apache.kylin.rest.request.JobOptimizeRequest; +import org.apache.kylin.rest.request.JobOptimizeRequest2; import org.apache.kylin.rest.response.CubeInstanceResponse; import org.apache.kylin.rest.response.CuboidTreeResponse; import org.apache.kylin.rest.response.EnvelopeResponse; @@ -440,6 +441,64 @@ public class CubeController extends BasicController { } /** + * Send a optimize cube job for delete or add cuboid + * + * @param cubeName Cube ID + * @param jobOptimizeRequest method (add or delete), cuboidsRecommend + * @return JobInstance of CheckpointExecutable + */ + @RequestMapping(value = "/{cubeName}/optimize2", method = {RequestMethod.PUT}) + @ResponseBody + public JobInstance optimize(@PathVariable String cubeName, @RequestBody JobOptimizeRequest2 jobOptimizeRequest) { + try { + String submitter = SecurityContextHolder.getContext().getAuthentication().getName(); + + checkCubeExists(cubeName); + CubeInstance cube = jobService.getCubeManager().getCube(cubeName); + + Set<Long> cuboidIds = cube.getCuboidScheduler().getAllCuboidIds(); + Set<Long> cuboidsAdd = jobOptimizeRequest.getCuboidsAdd(); + Set<Long> cuboidsDelete = jobOptimizeRequest.getCuboidsDelete(); + Set<Long> result = new HashSet<>(cuboidIds); + + if (cuboidsAdd == null && cuboidsDelete == null) { + throw new BadRequestException("must use cuboidsAdd or cuboidsDelete in request body."); + } + + if (cuboidsAdd != null && cuboidsAdd.size() != 0) { + result.addAll(cuboidsAdd); + logger.info( + "Add cuboid cubeName: " + cubeName + " contained cuboids: " + Sets.intersection(cuboidIds, cuboidsAdd)); + cuboidsAdd.removeAll(cuboidIds); + logger.info("Add cuboid cubeName: " + cubeName + " add cuboids: " + cuboidsAdd); + } else { + logger.info(cubeName + " no cuboids to add."); + } + + if (cuboidsDelete != null && cuboidsDelete.size() != 0) { + result.removeAll(cuboidsDelete); + logger.info("Remove cuboid cubeName: " + cubeName + " remove cuboids: " + + Sets.intersection(cuboidIds, cuboidsDelete)); + cuboidsDelete.removeAll(cuboidIds); + logger.info("Remove cuboid cubeName: " + cubeName + " missing cuboids: " + cuboidsDelete); + } else { + logger.info(cubeName + " no cuboids to delete."); + } + + return jobService.submitOptimizeJob(cube, result, submitter).getFirst(); + } catch (BadRequestException e) { + logger.error(e.getLocalizedMessage(), e); + throw e; + } catch (JobException e) { + logger.error(e.getLocalizedMessage(), e); + throw new BadRequestException(e.getLocalizedMessage()); + } catch (Exception e) { + logger.error(e.getLocalizedMessage(), e); + throw new InternalErrorException(e.getLocalizedMessage(), e); + } + } + + /** * Send a optimize cube segment job * * @param cubeName Cube ID diff --git a/server-base/src/main/java/org/apache/kylin/rest/request/JobOptimizeRequest2.java b/server-base/src/main/java/org/apache/kylin/rest/request/JobOptimizeRequest2.java new file mode 100644 index 0000000..75204fb --- /dev/null +++ b/server-base/src/main/java/org/apache/kylin/rest/request/JobOptimizeRequest2.java @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kylin.rest.request; + +import java.util.Set; +import java.util.stream.Collectors; + +public class JobOptimizeRequest2 { + private Set<String> cuboidsAdd; + private Set<String> cuboidsDelete; + + public Set<Long> getCuboidsAdd() { + if (cuboidsAdd == null) + return null; + return cuboidsAdd.stream().map(s -> Long.parseLong(s)).collect(Collectors.toSet()); + } + + public Set<Long> getCuboidsDelete() { + if (cuboidsDelete == null) + return null; + return cuboidsDelete.stream().map(s -> Long.parseLong(s)).collect(Collectors.toSet()); + } + +}