http://git-wip-us.apache.org/repos/asf/kylin/blob/73a78dbe/server-base/src/main/java/org/apache/kylin/rest/service/CubeServiceV2.java ---------------------------------------------------------------------- diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/CubeServiceV2.java b/server-base/src/main/java/org/apache/kylin/rest/service/CubeServiceV2.java new file mode 100644 index 0000000..2fae789 --- /dev/null +++ b/server-base/src/main/java/org/apache/kylin/rest/service/CubeServiceV2.java @@ -0,0 +1,497 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +package org.apache.kylin.rest.service; + +import static org.apache.kylin.cube.model.CubeDesc.STATUS_DRAFT; +import static org.apache.kylin.rest.controller2.CubeControllerV2.VALID_CUBENAME; + +import java.io.IOException; +import java.util.EnumSet; +import java.util.List; +import java.util.UUID; + +import org.apache.commons.lang.StringUtils; +import org.apache.kylin.cube.CubeInstance; +import org.apache.kylin.cube.CubeManager; +import org.apache.kylin.cube.CubeSegment; +import org.apache.kylin.cube.CubeUpdate; +import org.apache.kylin.cube.cuboid.CuboidCLI; +import org.apache.kylin.cube.model.CubeDesc; +import org.apache.kylin.engine.mr.CubingJob; +import org.apache.kylin.job.exception.JobException; +import org.apache.kylin.job.execution.ExecutableState; +import org.apache.kylin.metadata.model.DataModelDesc; +import org.apache.kylin.metadata.model.SegmentStatusEnum; +import org.apache.kylin.metadata.project.ProjectInstance; +import org.apache.kylin.metadata.project.ProjectManager; +import org.apache.kylin.metadata.realization.RealizationStatusEnum; +import org.apache.kylin.metadata.realization.RealizationType; +import org.apache.kylin.rest.constant.Constant; +import org.apache.kylin.rest.exception.BadRequestException; +import org.apache.kylin.rest.exception.ForbiddenException; +import org.apache.kylin.rest.msg.Message; +import org.apache.kylin.rest.msg.MsgPicker; +import org.apache.kylin.rest.security.AclPermission; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.security.access.AccessDeniedException; +import org.springframework.security.access.prepost.PreAuthorize; +import org.springframework.security.core.context.SecurityContextHolder; +import org.springframework.stereotype.Component; + +/** + * Created by luwei on 17-4-17. + */ + +@Component("cubeMgmtServiceV2") +public class CubeServiceV2 extends CubeService { + + private static final Logger logger = LoggerFactory.getLogger(CubeServiceV2.class); + + @Autowired + @Qualifier("accessService") + private AccessService accessService; + + @Autowired + @Qualifier("jobService") + private JobService jobService; + + @Autowired + @Qualifier("modelMgmtServiceV2") + private ModelServiceV2 modelServiceV2; + + + @PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN + " or hasPermission(#cube, 'ADMINISTRATION') or hasPermission(#cube, 'OPERATION') or hasPermission(#cube, 'MANAGEMENT')") + public CubeInstance deleteSegment(CubeInstance cube, String segmentName) throws IOException { + Message msg = MsgPicker.getMsg(); + + if (!segmentName.equals(cube.getSegments().get(0).getName()) && !segmentName.equals(cube.getSegments().get(cube.getSegments().size() - 1).getName())) { + throw new BadRequestException(String.format(msg.getDELETE_NOT_FIRST_LAST_SEG(), segmentName)); + } + CubeSegment toDelete = null; + for (CubeSegment seg : cube.getSegments()) { + if (seg.getName().equals(segmentName)) { + toDelete = seg; + } + } + + if (toDelete == null) { + throw new BadRequestException(String.format(msg.getSEG_NOT_FOUND(), segmentName)); + } + + if (toDelete.getStatus() != SegmentStatusEnum.READY) { + throw new BadRequestException(String.format(msg.getDELETE_NOT_READY_SEG(), segmentName)); + } + + CubeUpdate update = new CubeUpdate(cube); + update.setToRemoveSegs(new CubeSegment[] { toDelete }); + return CubeManager.getInstance(getConfig()).updateCube(update); + } + + /** + * Update a cube status from ready to disabled. + * + * @return + * @throws IOException + */ + @PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN + " or hasPermission(#cube, 'ADMINISTRATION') or hasPermission(#cube, 'OPERATION') or hasPermission(#cube, 'MANAGEMENT')") + public CubeInstance disableCube(CubeInstance cube) throws IOException { + Message msg = MsgPicker.getMsg(); + + String cubeName = cube.getName(); + + RealizationStatusEnum ostatus = cube.getStatus(); + if (null != ostatus && !RealizationStatusEnum.READY.equals(ostatus)) { + throw new BadRequestException(String.format(msg.getDISABLE_NOT_READY_CUBE(), cubeName, ostatus)); + } + + cube.setStatus(RealizationStatusEnum.DISABLED); + + try { + CubeUpdate cubeBuilder = new CubeUpdate(cube); + cubeBuilder.setStatus(RealizationStatusEnum.DISABLED); + return getCubeManager().updateCube(cubeBuilder); + } catch (IOException e) { + cube.setStatus(ostatus); + throw e; + } + } + + /** + * Stop all jobs belonging to this cube and clean out all segments + * + * @param cube + * @return + * @throws IOException + */ + @PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN + " or hasPermission(#cube, 'ADMINISTRATION') or hasPermission(#cube, 'OPERATION') or hasPermission(#cube, 'MANAGEMENT')") + public CubeInstance purgeCube(CubeInstance cube) throws IOException { + Message msg = MsgPicker.getMsg(); + + String cubeName = cube.getName(); + RealizationStatusEnum ostatus = cube.getStatus(); + if (null != ostatus && !RealizationStatusEnum.DISABLED.equals(ostatus)) { + throw new BadRequestException(String.format(msg.getPURGE_NOT_DISABLED_CUBE(), cubeName, ostatus)); + } + + this.releaseAllSegments(cube); + return cube; + + } + + /** + * purge the cube + * + * @throws IOException + * @throws JobException + */ + protected void releaseAllSegments(CubeInstance cube) throws IOException { + releaseAllJobs(cube); + + CubeUpdate update = new CubeUpdate(cube); + update.setToRemoveSegs(cube.getSegments().toArray(new CubeSegment[cube.getSegments().size()])); + CubeManager.getInstance(getConfig()).updateCube(update); + } + + protected void releaseAllJobs(CubeInstance cube) { + final List<CubingJob> cubingJobs = jobService.listAllCubingJobs(cube.getName(), null); + for (CubingJob cubingJob : cubingJobs) { + final ExecutableState status = cubingJob.getStatus(); + if (status != ExecutableState.SUCCEED && status != ExecutableState.DISCARDED) { + getExecutableManager().discardJob(cubingJob.getId()); + } + } + } + + @PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN + " or " + Constant.ACCESS_HAS_ROLE_MODELER) + public CubeInstance createCubeAndDesc(String cubeName, String projectName, CubeDesc desc) throws IOException { + Message msg = MsgPicker.getMsg(); + + if (getCubeManager().getCube(cubeName) != null) { + throw new BadRequestException(String.format(msg.getCUBE_ALREADY_EXIST(), cubeName)); + } + + if (getCubeDescManager().getCubeDesc(desc.getName()) != null) { + throw new BadRequestException(String.format(msg.getCUBE_DESC_ALREADY_EXIST(), desc.getName())); + } + + String owner = SecurityContextHolder.getContext().getAuthentication().getName(); + CubeDesc createdDesc; + CubeInstance createdCube; + + createdDesc = getCubeDescManager().createCubeDesc(desc); + + if (!createdDesc.getError().isEmpty()) { + getCubeDescManager().removeCubeDesc(createdDesc); + throw new BadRequestException(String.format(msg.getBROKEN_CUBE_DESC(), cubeName)); + } + + if (desc.getStatus() == null) { + try { + int cuboidCount = CuboidCLI.simulateCuboidGeneration(createdDesc, false); + logger.info("New cube " + cubeName + " has " + cuboidCount + " cuboids"); + } catch (Exception e) { + getCubeDescManager().removeCubeDesc(createdDesc); + throw e; + } + } + + createdCube = getCubeManager().createCube(cubeName, projectName, createdDesc, owner); + accessService.init(createdCube, AclPermission.ADMINISTRATION); + + ProjectInstance project = getProjectManager().getProject(projectName); + accessService.inherit(createdCube, project); + + return createdCube; + } + + /** + * Update a cube status from disable to ready. + * + * @return + * @throws IOException + */ + @PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN + " or hasPermission(#cube, 'ADMINISTRATION') or hasPermission(#cube, 'OPERATION') or hasPermission(#cube, 'MANAGEMENT')") + public CubeInstance enableCube(CubeInstance cube) throws IOException { + Message msg = MsgPicker.getMsg(); + + String cubeName = cube.getName(); + + RealizationStatusEnum ostatus = cube.getStatus(); + if (!cube.getStatus().equals(RealizationStatusEnum.DISABLED)) { + throw new BadRequestException(String.format(msg.getENABLE_NOT_DISABLED_CUBE(), cubeName, ostatus)); + } + + if (cube.getSegments(SegmentStatusEnum.READY).size() == 0) { + throw new BadRequestException(String.format(msg.getNO_READY_SEGMENT(), cubeName)); + } + + final List<CubingJob> cubingJobs = jobService.listAllCubingJobs(cube.getName(), null, EnumSet.of(ExecutableState.READY, ExecutableState.RUNNING)); + if (!cubingJobs.isEmpty()) { + throw new BadRequestException(msg.getENABLE_WITH_RUNNING_JOB()); + } + if (!cube.getDescriptor().checkSignature()) { + throw new BadRequestException(String.format(msg.getINCONSISTENT_CUBE_DESC_SIGNATURE(), cube.getDescriptor())); + } + + try { + CubeUpdate cubeBuilder = new CubeUpdate(cube); + cubeBuilder.setStatus(RealizationStatusEnum.READY); + return getCubeManager().updateCube(cubeBuilder); + } catch (IOException e) { + cube.setStatus(ostatus); + throw e; + } + } + + @PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN + " or hasPermission(#cube, 'ADMINISTRATION') or hasPermission(#cube, 'MANAGEMENT')") + public void deleteCube(CubeInstance cube) throws IOException { + Message msg = MsgPicker.getMsg(); + + final List<CubingJob> cubingJobs = jobService.listAllCubingJobs(cube.getName(), null, EnumSet.of(ExecutableState.READY, ExecutableState.RUNNING, ExecutableState.ERROR)); + if (!cubingJobs.isEmpty()) { + throw new BadRequestException(String.format(msg.getDISCARD_JOB_FIRST(), cube.getName())); + } + + try { + this.releaseAllJobs(cube); + } catch (Exception e) { + logger.error("error when releasing all jobs", e); + //ignore the exception + } + + int cubeNum = getCubeManager().getCubesByDesc(cube.getDescriptor().getName()).size(); + getCubeManager().dropCube(cube.getName(), cubeNum == 1);//only delete cube desc when no other cube is using it + accessService.clean(cube, true); + } + + @PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN + " or hasPermission(#cube, 'ADMINISTRATION') or hasPermission(#cube, 'MANAGEMENT')") + public CubeDesc updateCubeAndDesc(CubeInstance cube, CubeDesc desc, String newProjectName, boolean forceUpdate) throws IOException { + Message msg = MsgPicker.getMsg(); + + final List<CubingJob> cubingJobs = jobService.listAllCubingJobs(cube.getName(), null, EnumSet.of(ExecutableState.READY, ExecutableState.RUNNING)); + if (!cubingJobs.isEmpty()) { + throw new BadRequestException(String.format(msg.getDISCARD_JOB_FIRST(), cube.getName())); + } + + //double check again + if (!forceUpdate && !cube.getDescriptor().consistentWith(desc)) { + throw new BadRequestException(String.format(msg.getINCONSISTENT_CUBE_DESC(), desc.getName())); + } + + CubeDesc updatedCubeDesc = getCubeDescManager().updateCubeDesc(desc); + if (desc.getStatus() == null) { + int cuboidCount = CuboidCLI.simulateCuboidGeneration(updatedCubeDesc, false); + logger.info("Updated cube " + cube.getName() + " has " + cuboidCount + " cuboids"); + } + + ProjectManager projectManager = getProjectManager(); + if (!isCubeInProject(newProjectName, cube)) { + String owner = SecurityContextHolder.getContext().getAuthentication().getName(); + ProjectInstance newProject = projectManager.moveRealizationToProject(RealizationType.CUBE, cube.getName(), newProjectName, owner); + accessService.inherit(cube, newProject); + } + + return updatedCubeDesc; + } + + public void validateCubeDesc(CubeDesc desc, boolean isDraft) { + Message msg = MsgPicker.getMsg(); + + if (desc == null) { + throw new BadRequestException(msg.getINVALID_CUBE_DEFINITION()); + } + + String cubeName = desc.getName(); + if (StringUtils.isEmpty(cubeName)) { + logger.info("Cube name should not be empty."); + throw new BadRequestException(msg.getEMPTY_CUBE_NAME()); + } + if (!StringUtils.containsOnly(cubeName, VALID_CUBENAME)) { + logger.info("Invalid Cube name {}, only letters, numbers and underline supported.", cubeName); + throw new BadRequestException(String.format(msg.getINVALID_CUBE_NAME(), cubeName)); + } + + if (!isDraft) { + DataModelDesc modelDesc = modelServiceV2.getMetadataManager().getDataModelDesc(desc.getModelName()); + if (modelDesc == null) { + throw new BadRequestException(String.format(msg.getMODEL_NOT_FOUND(), desc.getModelName())); + } + + String modelDescStatus = modelDesc.getStatus(); + if (modelDescStatus != null && modelDescStatus.equals(DataModelDesc.STATUS_DRAFT)) { + logger.info("Cannot use draft model."); + throw new BadRequestException(String.format(msg.getUSE_DRAFT_MODEL(), desc.getModelName())); + } + } + } + + public boolean unifyCubeDesc(CubeDesc desc, boolean isDraft) throws IOException { + Message msg = MsgPicker.getMsg(); + + boolean createNew = false; + String cubeName = desc.getName(); + String originName = null; // for draft rename check + if (desc.getUuid() != null) { + originName = getNameByUuid(desc.getUuid()); + } + + if (!isDraft) { // save as official cube + if (desc.getStatus() != null && desc.getStatus().equals(STATUS_DRAFT)) { // from draft + if (originName == null) { + throw new BadRequestException(msg.getORIGIN_CUBE_NOT_FOUND()); + } + originName = originName.substring(0, originName.lastIndexOf("_draft")); + if (!originName.equals(cubeName)) { // if rename draft + CubeDesc parentDesc = getCubeDescManager().getCubeDesc(originName); + if (parentDesc == null) { // only allow rename when official cube has not been saved + createNew = true; + deleteCubeByUuid(desc.getUuid()); + desc.setStatus(null); + desc.setLastModified(0); + desc.setUuid(UUID.randomUUID().toString()); + } else { + throw new BadRequestException(msg.getCUBE_RENAME()); + } + } else { // without rename draft + desc.setStatus(null); + CubeDesc parentDesc = getCubeDescManager().getCubeDesc(cubeName); + if (parentDesc == null) { // official cube doesn't exist, create new one + createNew = true; + desc.setLastModified(0); + desc.setUuid(UUID.randomUUID().toString()); + } else { // update existing + desc.setLastModified(parentDesc.getLastModified()); + desc.setUuid(parentDesc.getUuid()); + } + } + } else { // from official + if (originName == null) { // official cube doesn't exist, create new one + createNew = true; + desc.setLastModified(0); + desc.setUuid(UUID.randomUUID().toString()); + } else { + if (!originName.equals(cubeName)) { // do not allow official cube rename + throw new BadRequestException(msg.getCUBE_RENAME()); + } + } + } + } else { // save as draft model + if (desc.getStatus() == null) { // from official + cubeName += "_draft"; + desc.setName(cubeName); + desc.setStatus(STATUS_DRAFT); + CubeDesc draftDesc = getCubeDescManager().getCubeDesc(cubeName); + if (draftDesc == null) { + createNew = true; + desc.setLastModified(0); + desc.setUuid(UUID.randomUUID().toString()); + } else if (draftDesc.getStatus() != null && draftDesc.getStatus().equals(STATUS_DRAFT)) { // update existing + desc.setLastModified(draftDesc.getLastModified()); + desc.setUuid(draftDesc.getUuid()); + } else { // already exist an official draft with name ends with '_draft' + throw new BadRequestException(String.format(msg.getNON_DRAFT_CUBE_ALREADY_EXIST(), cubeName)); + } + } else if (desc.getStatus().equals(STATUS_DRAFT)) { // from draft + if (originName == null) { + throw new BadRequestException(msg.getORIGIN_CUBE_NOT_FOUND()); + } + originName = originName.substring(0, originName.lastIndexOf("_draft")); + if (!originName.equals(cubeName)) { // if rename draft + CubeDesc parentDesc = getCubeDescManager().getCubeDesc(originName); + if (parentDesc == null) { // only allow rename when official cube has not been saved + createNew = true; + deleteCubeByUuid(desc.getUuid()); + cubeName += "_draft"; + desc.setName(cubeName); + desc.setStatus(STATUS_DRAFT); + desc.setLastModified(0); + desc.setUuid(UUID.randomUUID().toString()); + } else { + throw new BadRequestException(msg.getMODEL_RENAME()); + } + } else { // without rename draft + cubeName += "_draft"; + desc.setName(cubeName); + } + } + } + return createNew; + } + + public String getNameByUuid(String uuid) { + List<CubeInstance> cubes = getCubeManager().listAllCubes(); + for (CubeInstance cube : cubes) { + if (cube.getDescriptor().getUuid().equals(uuid)) { + return cube.getName(); + } + } + return null; + } + + public void deleteCubeByUuid(String uuid) throws IOException { + List<CubeInstance> cubes = getCubeManager().listAllCubes(); + for (CubeInstance cube : cubes) { + if (cube.getDescriptor().getUuid().equals(uuid)) { + deleteCube(cube); + } + } + } + + public CubeDesc updateCubeToResourceStore(CubeDesc desc, String projectName, boolean createNew, boolean isDraft) throws IOException { + Message msg = MsgPicker.getMsg(); + + String cubeName = desc.getName(); + if (createNew) { + createCubeAndDesc(cubeName, projectName, desc); + } else { + try { + CubeInstance cube = getCubeManager().getCube(desc.getName()); + + if (cube == null) { + throw new BadRequestException(String.format(msg.getCUBE_NOT_FOUND(), desc.getName())); + } + + if (cube.getSegments().size() != 0 && !cube.getDescriptor().consistentWith(desc)) { + throw new BadRequestException(String.format(msg.getINCONSISTENT_CUBE_DESC(), desc.getName())); + } + + desc = updateCubeAndDesc(cube, desc, projectName, true); + } catch (AccessDeniedException accessDeniedException) { + throw new ForbiddenException(msg.getUPDATE_CUBE_NO_RIGHT()); + } + + if (!desc.getError().isEmpty()) { + throw new BadRequestException(String.format(msg.getBROKEN_CUBE_DESC(), cubeName)); + } + } + + if (!isDraft) { + String draftName = desc.getName() + "_draft"; + CubeInstance draftCube = getCubeManager().getCube(draftName); + if (null != draftCube && draftCube.getDescriptor().getStatus() != null && draftCube.getDescriptor().getStatus().equals(STATUS_DRAFT)) { + //drop draft Cube + deleteCube(draftCube); + } + } + return desc; + } +}
http://git-wip-us.apache.org/repos/asf/kylin/blob/73a78dbe/server-base/src/main/java/org/apache/kylin/rest/service/DiagnosisService.java ---------------------------------------------------------------------- diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/DiagnosisService.java b/server-base/src/main/java/org/apache/kylin/rest/service/DiagnosisService.java index b473071..35b018c 100644 --- a/server-base/src/main/java/org/apache/kylin/rest/service/DiagnosisService.java +++ b/server-base/src/main/java/org/apache/kylin/rest/service/DiagnosisService.java @@ -28,6 +28,9 @@ import org.apache.kylin.common.util.CliCommandExecutor; import org.apache.kylin.common.util.Pair; import org.apache.kylin.metadata.badquery.BadQueryHistory; import org.apache.kylin.rest.constant.Constant; +import org.apache.kylin.rest.exception.BadRequestException; +import org.apache.kylin.rest.msg.Message; +import org.apache.kylin.rest.msg.MsgPicker; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.security.access.prepost.PreAuthorize; @@ -40,14 +43,16 @@ public class DiagnosisService extends BasicService { private static final Logger logger = LoggerFactory.getLogger(DiagnosisService.class); - private File getDumpDir() { + protected File getDumpDir() { return Files.createTempDir(); } private String getDiagnosisPackageName(File destDir) { + Message msg = MsgPicker.getMsg(); + File[] files = destDir.listFiles(); if (files == null) { - throw new RuntimeException("Diagnosis package is not available in directory: " + destDir.getAbsolutePath()); + throw new BadRequestException(String.format(msg.getDIAG_PACKAGE_NOT_AVAILABLE(), destDir.getAbsolutePath())); } for (File subDir : files) { if (subDir.isDirectory()) { @@ -58,7 +63,7 @@ public class DiagnosisService extends BasicService { } } } - throw new RuntimeException("Diagnosis package not found in directory: " + destDir.getAbsolutePath()); + throw new BadRequestException(String.format(msg.getDIAG_PACKAGE_NOT_FOUND(), destDir.getAbsolutePath())); } @PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN) @@ -83,13 +88,15 @@ public class DiagnosisService extends BasicService { } private void runDiagnosisCLI(String[] args) throws IOException { + Message msg = MsgPicker.getMsg(); + File cwd = new File(""); logger.debug("Current path: " + cwd.getAbsolutePath()); logger.debug("DiagnosisInfoCLI args: " + Arrays.toString(args)); File script = new File(KylinConfig.getKylinHome() + File.separator + "bin", "diag.sh"); if (!script.exists()) { - throw new RuntimeException("diag.sh not found at " + script.getAbsolutePath()); + throw new BadRequestException(String.format(msg.getDIAG_NOT_FOUND(), script.getAbsolutePath())); } String diagCmd = script.getAbsolutePath() + " " + StringUtils.join(args, " "); @@ -97,7 +104,8 @@ public class DiagnosisService extends BasicService { Pair<Integer, String> cmdOutput = executor.execute(diagCmd); if (cmdOutput.getKey() != 0) { - throw new RuntimeException("Failed to generate diagnosis package."); + throw new BadRequestException(msg.getGENERATE_DIAG_PACKAGE_FAIL()); } } + } http://git-wip-us.apache.org/repos/asf/kylin/blob/73a78dbe/server-base/src/main/java/org/apache/kylin/rest/service/EncodingService.java ---------------------------------------------------------------------- diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/EncodingService.java b/server-base/src/main/java/org/apache/kylin/rest/service/EncodingService.java index 60f9974..f3742de 100644 --- a/server-base/src/main/java/org/apache/kylin/rest/service/EncodingService.java +++ b/server-base/src/main/java/org/apache/kylin/rest/service/EncodingService.java @@ -28,6 +28,9 @@ import org.apache.kylin.dimension.FixedLenHexDimEnc; import org.apache.kylin.dimension.IntegerDimEnc; import org.apache.kylin.dimension.TimeDimEnc; import org.apache.kylin.metadata.datatype.DataType; +import org.apache.kylin.rest.exception.BadRequestException; +import org.apache.kylin.rest.msg.Message; +import org.apache.kylin.rest.msg.MsgPicker; import org.springframework.stereotype.Component; import com.google.common.collect.Lists; @@ -36,6 +39,8 @@ import com.google.common.collect.Lists; public class EncodingService extends BasicService { public List<String> getValidEncodings(DataType dataType) { + Message msg = MsgPicker.getMsg(); + if (dataType.isIntegerFamily()) { return Lists.newArrayList(BooleanDimEnc.ENCODING_NAME, DateDimEnc.ENCODING_NAME, TimeDimEnc.ENCODING_NAME, DictionaryDimEnc.ENCODING_NAME, IntegerDimEnc.ENCODING_NAME); } else if (dataType.isNumberFamily()) { //numbers include integers @@ -46,7 +51,7 @@ public class EncodingService extends BasicService { return Lists.newArrayList(BooleanDimEnc.ENCODING_NAME, DictionaryDimEnc.ENCODING_NAME, FixedLenDimEnc.ENCODING_NAME, // FixedLenHexDimEnc.ENCODING_NAME, IntegerDimEnc.ENCODING_NAME); } else { - throw new IllegalArgumentException("can't provide valid encodings for datatype:" + dataType); + throw new BadRequestException(String.format(msg.getVALID_ENCODING_NOT_AVAILABLE(), dataType)); } } http://git-wip-us.apache.org/repos/asf/kylin/blob/73a78dbe/server-base/src/main/java/org/apache/kylin/rest/service/ExtFilterService.java ---------------------------------------------------------------------- diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/ExtFilterService.java b/server-base/src/main/java/org/apache/kylin/rest/service/ExtFilterService.java index 0a8439d..90dac27 100644 --- a/server-base/src/main/java/org/apache/kylin/rest/service/ExtFilterService.java +++ b/server-base/src/main/java/org/apache/kylin/rest/service/ExtFilterService.java @@ -23,7 +23,9 @@ import java.util.List; import org.apache.kylin.metadata.model.ExternalFilterDesc; import org.apache.kylin.rest.constant.Constant; -import org.apache.kylin.rest.exception.InternalErrorException; +import org.apache.kylin.rest.exception.BadRequestException; +import org.apache.kylin.rest.msg.Message; +import org.apache.kylin.rest.msg.MsgPicker; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.security.access.prepost.PreAuthorize; @@ -36,16 +38,20 @@ public class ExtFilterService extends BasicService { @PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN) public void saveExternalFilter(ExternalFilterDesc desc) throws IOException { + Message msg = MsgPicker.getMsg(); + if (getMetadataManager().getExtFilterDesc(desc.getName()) != null) { - throw new InternalErrorException("The filter named " + desc.getName() + " already exists"); + throw new BadRequestException(String.format(msg.getFILTER_ALREADY_EXIST(), desc.getName())); } getMetadataManager().saveExternalFilter(desc); } @PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN) public void updateExternalFilter(ExternalFilterDesc desc) throws IOException { + Message msg = MsgPicker.getMsg(); + if (getMetadataManager().getExtFilterDesc(desc.getName()) == null) { - throw new InternalErrorException("The filter named " + desc.getName() + " does not exists"); + throw new BadRequestException(String.format(msg.getFILTER_NOT_FOUND(), desc.getName())); } getMetadataManager().saveExternalFilter(desc); } http://git-wip-us.apache.org/repos/asf/kylin/blob/73a78dbe/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java ---------------------------------------------------------------------- diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java b/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java index 56a148d..4e5f80f 100644 --- a/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java +++ b/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -45,7 +45,6 @@ import org.apache.kylin.job.SchedulerFactory; import org.apache.kylin.job.constant.JobStatusEnum; import org.apache.kylin.job.constant.JobTimeFilterEnum; import org.apache.kylin.job.engine.JobEngineConfig; -import org.apache.kylin.job.exception.JobException; import org.apache.kylin.job.exception.SchedulerException; import org.apache.kylin.job.execution.AbstractExecutable; import org.apache.kylin.job.execution.DefaultChainedExecutable; @@ -56,6 +55,8 @@ import org.apache.kylin.metadata.model.SegmentStatusEnum; import org.apache.kylin.metadata.realization.RealizationStatusEnum; import org.apache.kylin.rest.constant.Constant; import org.apache.kylin.rest.exception.BadRequestException; +import org.apache.kylin.rest.msg.Message; +import org.apache.kylin.rest.msg.MsgPicker; import org.apache.kylin.source.ISource; import org.apache.kylin.source.SourceFactory; import org.apache.kylin.source.SourcePartition; @@ -63,12 +64,12 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.InitializingBean; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.context.annotation.EnableAspectJAutoProxy; import org.springframework.security.access.prepost.PreAuthorize; import org.springframework.stereotype.Component; import com.google.common.base.Function; -import com.google.common.base.Preconditions; import com.google.common.base.Predicate; import com.google.common.base.Predicates; import com.google.common.collect.FluentIterable; @@ -88,6 +89,7 @@ public class JobService extends BasicService implements InitializingBean { private JobLock jobLock; @Autowired + @Qualifier("accessService") private AccessService accessService; /* @@ -148,54 +150,59 @@ public class JobService extends BasicService implements InitializingBean { return states; } - private long getTimeStartInMillis(Calendar calendar, JobTimeFilterEnum timeFilter) { - switch (timeFilter) { - case LAST_ONE_DAY: - calendar.add(Calendar.DAY_OF_MONTH, -1); - return calendar.getTimeInMillis(); - case LAST_ONE_WEEK: - calendar.add(Calendar.WEEK_OF_MONTH, -1); - return calendar.getTimeInMillis(); - case LAST_ONE_MONTH: - calendar.add(Calendar.MONTH, -1); - return calendar.getTimeInMillis(); - case LAST_ONE_YEAR: - calendar.add(Calendar.YEAR, -1); - return calendar.getTimeInMillis(); - case ALL: - return 0; - default: - throw new RuntimeException("illegal timeFilter for job history:" + timeFilter); + private ExecutableState parseToExecutableState(JobStatusEnum status) { + Message msg = MsgPicker.getMsg(); + + switch (status) { + case DISCARDED: + return ExecutableState.DISCARDED; + case ERROR: + return ExecutableState.ERROR; + case FINISHED: + return ExecutableState.SUCCEED; + case NEW: + return ExecutableState.READY; + case PENDING: + return ExecutableState.READY; + case RUNNING: + return ExecutableState.RUNNING; + case STOPPED: + return ExecutableState.STOPPED; + default: + throw new BadRequestException(String.format(msg.getILLEGAL_EXECUTABLE_STATE(), status)); } } - private ExecutableState parseToExecutableState(JobStatusEnum status) { - switch (status) { - case DISCARDED: - return ExecutableState.DISCARDED; - case ERROR: - return ExecutableState.ERROR; - case FINISHED: - return ExecutableState.SUCCEED; - case NEW: - return ExecutableState.READY; - case PENDING: - return ExecutableState.READY; - case RUNNING: - return ExecutableState.RUNNING; - case STOPPED: - return ExecutableState.STOPPED; - default: - throw new RuntimeException("illegal status:" + status); + private long getTimeStartInMillis(Calendar calendar, JobTimeFilterEnum timeFilter) { + Message msg = MsgPicker.getMsg(); + + switch (timeFilter) { + case LAST_ONE_DAY: + calendar.add(Calendar.DAY_OF_MONTH, -1); + return calendar.getTimeInMillis(); + case LAST_ONE_WEEK: + calendar.add(Calendar.WEEK_OF_MONTH, -1); + return calendar.getTimeInMillis(); + case LAST_ONE_MONTH: + calendar.add(Calendar.MONTH, -1); + return calendar.getTimeInMillis(); + case LAST_ONE_YEAR: + calendar.add(Calendar.YEAR, -1); + return calendar.getTimeInMillis(); + case ALL: + return 0; + default: + throw new BadRequestException(String.format(msg.getILLEGAL_TIME_FILTER(), timeFilter)); } } @PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN + " or hasPermission(#cube, 'ADMINISTRATION') or hasPermission(#cube, 'OPERATION') or hasPermission(#cube, 'MANAGEMENT')") public JobInstance submitJob(CubeInstance cube, long startDate, long endDate, long startOffset, long endOffset, // - Map<Integer, Long> sourcePartitionOffsetStart, Map<Integer, Long> sourcePartitionOffsetEnd, CubeBuildTypeEnum buildType, boolean force, String submitter) throws IOException, JobException { + Map<Integer, Long> sourcePartitionOffsetStart, Map<Integer, Long> sourcePartitionOffsetEnd, CubeBuildTypeEnum buildType, boolean force, String submitter) throws IOException { + Message msg = MsgPicker.getMsg(); if (cube.getStatus() == RealizationStatusEnum.DESCBROKEN) { - throw new BadRequestException("Broken cube " + cube.getName() + " can't be built"); + throw new BadRequestException(String.format(msg.getBUILD_BROKEN_CUBE(), cube.getName())); } checkCubeDescSignature(cube); @@ -216,7 +223,7 @@ public class JobService extends BasicService implements InitializingBean { newSeg = getCubeManager().refreshSegment(cube, startDate, endDate, startOffset, endOffset); job = EngineFactory.createBatchCubingJob(newSeg, submitter); } else { - throw new JobException("invalid build type:" + buildType); + throw new BadRequestException(String.format(msg.getINVALID_BUILD_TYPE(), buildType)); } getExecutableManager().addJob(job); @@ -246,11 +253,13 @@ public class JobService extends BasicService implements InitializingBean { } private void checkCubeDescSignature(CubeInstance cube) { + Message msg = MsgPicker.getMsg(); + if (!cube.getDescriptor().checkSignature()) - throw new IllegalStateException("Inconsistent cube desc signature for " + cube.getDescriptor() + ", if it's right after a upgrade, please try 'Edit CubeDesc' to delete the 'signature' field. Or use 'bin/metastore.sh refresh-cube-signature' to batch refresh all cubes' signatures, then reload metadata to take effect"); + throw new BadRequestException(String.format(msg.getINCONSISTENT_CUBE_DESC_SIGNATURE(), cube.getDescriptor())); } - public JobInstance getJobInstance(String uuid) throws IOException, JobException { + public JobInstance getJobInstance(String uuid) { return getSingleJobInstance(getExecutableManager().getJob(uuid)); } @@ -258,11 +267,16 @@ public class JobService extends BasicService implements InitializingBean { return getExecutableManager().getOutput(id); } - private JobInstance getSingleJobInstance(AbstractExecutable job) { + protected JobInstance getSingleJobInstance(AbstractExecutable job) { + Message msg = MsgPicker.getMsg(); + if (job == null) { return null; } - Preconditions.checkState(job instanceof CubingJob, "illegal job type, id:" + job.getId()); + if (!(job instanceof CubingJob)) { + throw new BadRequestException(String.format(msg.getILLEGAL_JOB_TYPE(), job.getId())); + } + CubingJob cubeJob = (CubingJob) job; final JobInstance result = new JobInstance(); result.setName(job.getName()); @@ -283,17 +297,17 @@ public class JobService extends BasicService implements InitializingBean { } @PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN + " or hasPermission(#job, 'ADMINISTRATION') or hasPermission(#job, 'OPERATION') or hasPermission(#job, 'MANAGEMENT')") - public void resumeJob(JobInstance job) throws IOException, JobException { + public void resumeJob(JobInstance job) { getExecutableManager().resumeJob(job.getId()); } @PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN + " or hasPermission(#job, 'ADMINISTRATION') or hasPermission(#job, 'OPERATION') or hasPermission(#job, 'MANAGEMENT')") - public void rollbackJob(JobInstance job, String stepId) throws IOException, JobException { + public void rollbackJob(JobInstance job, String stepId) { getExecutableManager().rollbackJob(job.getId(), stepId); } @PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN + " or hasPermission(#job, 'ADMINISTRATION') or hasPermission(#job, 'OPERATION') or hasPermission(#job, 'MANAGEMENT')") - public JobInstance cancelJob(JobInstance job) throws IOException, JobException { + public JobInstance cancelJob(JobInstance job) throws IOException { if (null == job.getRelatedCube() || null == getCubeManager().getCube(job.getRelatedCube())) { getExecutableManager().discardJob(job.getId()); return job; @@ -316,13 +330,13 @@ public class JobService extends BasicService implements InitializingBean { } @PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN + " or hasPermission(#job, 'ADMINISTRATION') or hasPermission(#job, 'OPERATION') or hasPermission(#job, 'MANAGEMENT')") - public JobInstance pauseJob(JobInstance job) throws IOException, JobException { + public JobInstance pauseJob(JobInstance job) { getExecutableManager().pauseJob(job.getId()); return job; } @PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN + " or hasPermission(#job, 'ADMINISTRATION') or hasPermission(#job, 'OPERATION') or hasPermission(#job, 'MANAGEMENT')") - public void dropJob(JobInstance job) throws IOException, JobException { + public void dropJob(JobInstance job) throws IOException { cancelJob(job); getExecutableManager().deleteJob(job.getId()); } @@ -331,7 +345,7 @@ public class JobService extends BasicService implements InitializingBean { * currently only support substring match * @return */ - public List<JobInstance> searchJobs(final String cubeNameSubstring, final String projectName, final List<JobStatusEnum> statusList, final Integer limitValue, final Integer offsetValue, final JobTimeFilterEnum timeFilter) throws IOException, JobException { + public List<JobInstance> searchJobs(final String cubeNameSubstring, final String projectName, final List<JobStatusEnum> statusList, final Integer limitValue, final Integer offsetValue, final JobTimeFilterEnum timeFilter) { Integer limit = (null == limitValue) ? 30 : limitValue; Integer offset = (null == offsetValue) ? 0 : offsetValue; List<JobInstance> jobs = searchJobs(cubeNameSubstring, projectName, statusList, timeFilter); @@ -348,7 +362,7 @@ public class JobService extends BasicService implements InitializingBean { return jobs.subList(offset, offset + limit); } - private List<JobInstance> searchJobs(final String cubeNameSubstring, final String projectName, final List<JobStatusEnum> statusList, final JobTimeFilterEnum timeFilter) { + public List<JobInstance> searchJobs(final String cubeNameSubstring, final String projectName, final List<JobStatusEnum> statusList, final JobTimeFilterEnum timeFilter) { Calendar calendar = Calendar.getInstance(); calendar.setTime(new Date()); long timeStartInMillis = getTimeStartInMillis(calendar, timeFilter); http://git-wip-us.apache.org/repos/asf/kylin/blob/73a78dbe/server-base/src/main/java/org/apache/kylin/rest/service/KafkaConfigService.java ---------------------------------------------------------------------- diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/KafkaConfigService.java b/server-base/src/main/java/org/apache/kylin/rest/service/KafkaConfigService.java index f7fc49b..0dbe6f2 100644 --- a/server-base/src/main/java/org/apache/kylin/rest/service/KafkaConfigService.java +++ b/server-base/src/main/java/org/apache/kylin/rest/service/KafkaConfigService.java @@ -23,7 +23,9 @@ import java.util.ArrayList; import java.util.List; import org.apache.kylin.rest.constant.Constant; -import org.apache.kylin.rest.exception.InternalErrorException; +import org.apache.kylin.rest.exception.BadRequestException; +import org.apache.kylin.rest.msg.Message; +import org.apache.kylin.rest.msg.MsgPicker; import org.apache.kylin.source.kafka.config.KafkaConfig; import org.springframework.security.access.prepost.PostFilter; import org.springframework.stereotype.Component; @@ -66,8 +68,10 @@ public class KafkaConfigService extends BasicService { } public KafkaConfig createKafkaConfig(KafkaConfig config) throws IOException { + Message msg = MsgPicker.getMsg(); + if (getKafkaManager().getKafkaConfig(config.getName()) != null) { - throw new InternalErrorException("The kafkaConfig named " + config.getName() + " already exists"); + throw new BadRequestException(String.format(msg.getKAFKA_CONFIG_ALREADY_EXIST(), config.getName())); } getKafkaManager().createKafkaConfig(config.getName(), config); return config; http://git-wip-us.apache.org/repos/asf/kylin/blob/73a78dbe/server-base/src/main/java/org/apache/kylin/rest/service/ModelService.java ---------------------------------------------------------------------- diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/ModelService.java b/server-base/src/main/java/org/apache/kylin/rest/service/ModelService.java index 614f0c9..9401624 100644 --- a/server-base/src/main/java/org/apache/kylin/rest/service/ModelService.java +++ b/server-base/src/main/java/org/apache/kylin/rest/service/ModelService.java @@ -18,10 +18,6 @@ package org.apache.kylin.rest.service; -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; - import org.apache.kylin.common.util.HadoopUtil; import org.apache.kylin.cube.model.CubeDesc; import org.apache.kylin.metadata.model.DataModelDesc; @@ -30,11 +26,16 @@ import org.apache.kylin.rest.constant.Constant; import org.apache.kylin.rest.exception.InternalErrorException; import org.apache.kylin.rest.security.AclPermission; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.security.access.prepost.PostFilter; import org.springframework.security.access.prepost.PreAuthorize; import org.springframework.security.core.context.SecurityContextHolder; import org.springframework.stereotype.Component; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + /** * @author jiazhong */ @@ -42,8 +43,13 @@ import org.springframework.stereotype.Component; public class ModelService extends BasicService { @Autowired + @Qualifier("accessService") private AccessService accessService; + @Autowired + @Qualifier("cubeMgmtService") + private CubeService cubeService; + @PostFilter(Constant.ACCESS_POST_FILTER_READ) public List<DataModelDesc> listAllModels(final String modelName, final String projectName) throws IOException { List<DataModelDesc> models; @@ -139,4 +145,10 @@ public class ModelService extends BasicService { tableName = dbTableName[0] + "." + dbTableName[1]; return getMetadataManager().getModelsUsingTable(tableName, projectName); } + + public boolean checkNameAvailability(String modelName) throws IOException { + List<DataModelDesc> models = listAllModels(modelName, null); + + return models.isEmpty(); + } } http://git-wip-us.apache.org/repos/asf/kylin/blob/73a78dbe/server-base/src/main/java/org/apache/kylin/rest/service/ModelServiceV2.java ---------------------------------------------------------------------- diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/ModelServiceV2.java b/server-base/src/main/java/org/apache/kylin/rest/service/ModelServiceV2.java new file mode 100644 index 0000000..1628964 --- /dev/null +++ b/server-base/src/main/java/org/apache/kylin/rest/service/ModelServiceV2.java @@ -0,0 +1,351 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +package org.apache.kylin.rest.service; + +import static org.apache.kylin.metadata.model.DataModelDesc.STATUS_DRAFT; +import static org.apache.kylin.rest.controller2.ModelControllerV2.VALID_MODELNAME; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.UUID; + +import org.apache.commons.lang.StringUtils; +import org.apache.kylin.cube.CubeInstance; +import org.apache.kylin.cube.model.CubeDesc; +import org.apache.kylin.metadata.model.DataModelDesc; +import org.apache.kylin.metadata.model.JoinsTree; +import org.apache.kylin.metadata.model.ModelDimensionDesc; +import org.apache.kylin.metadata.model.TblColRef; +import org.apache.kylin.metadata.project.ProjectInstance; +import org.apache.kylin.rest.constant.Constant; +import org.apache.kylin.rest.exception.BadRequestException; +import org.apache.kylin.rest.exception.ForbiddenException; +import org.apache.kylin.rest.msg.Message; +import org.apache.kylin.rest.msg.MsgPicker; +import org.apache.kylin.rest.security.AclPermission; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.security.access.AccessDeniedException; +import org.springframework.security.access.prepost.PreAuthorize; +import org.springframework.security.core.context.SecurityContextHolder; +import org.springframework.stereotype.Component; + +import com.google.common.collect.Maps; +import com.google.common.collect.Sets; + +/** + * Created by luwei on 17-4-19. + */ +@Component("modelMgmtServiceV2") +public class ModelServiceV2 extends ModelService { + + private static final Logger logger = LoggerFactory.getLogger(ModelServiceV2.class); + + @Autowired + @Qualifier("accessService") + private AccessService accessService; + + @Autowired + @Qualifier("cubeMgmtServiceV2") + private CubeServiceV2 cubeServiceV2; + + public DataModelDesc createModelDesc(String projectName, DataModelDesc desc) throws IOException { + Message msg = MsgPicker.getMsg(); + + if (getMetadataManager().getDataModelDesc(desc.getName()) != null) { + throw new BadRequestException(String.format(msg.getDUPLICATE_MODEL_NAME(), desc.getName())); + } + DataModelDesc createdDesc = null; + String owner = SecurityContextHolder.getContext().getAuthentication().getName(); + createdDesc = getMetadataManager().createDataModelDesc(desc, projectName, owner); + + accessService.init(createdDesc, AclPermission.ADMINISTRATION); + ProjectInstance project = getProjectManager().getProject(projectName); + accessService.inherit(createdDesc, project); + return createdDesc; + } + + @PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN + " or hasPermission(#desc, 'ADMINISTRATION') or hasPermission(#desc, 'MANAGEMENT')") + public void dropModel(DataModelDesc desc) throws IOException { + Message msg = MsgPicker.getMsg(); + + //check cube desc exist + List<CubeDesc> cubeDescs = getCubeDescManager().listAllDesc(); + for (CubeDesc cubeDesc : cubeDescs) { + if (cubeDesc.getModelName().equals(desc.getName())) { + throw new BadRequestException(String.format(msg.getDROP_REFERENCED_MODEL(), cubeDesc.getName())); + } + } + + getMetadataManager().dropModel(desc); + + accessService.clean(desc, true); + } + + public Map<TblColRef, Set<CubeInstance>> getUsedDimCols(String modelName) { + Map<TblColRef, Set<CubeInstance>> ret = Maps.newHashMap(); + List<CubeInstance> cubeInstances = cubeServiceV2.listAllCubes(null, null, modelName); + for (CubeInstance cubeInstance : cubeInstances) { + CubeDesc cubeDesc = cubeInstance.getDescriptor(); + for (TblColRef tblColRef : cubeDesc.listDimensionColumnsIncludingDerived()) { + if (ret.containsKey(tblColRef)) { + ret.get(tblColRef).add(cubeInstance); + } else { + Set<CubeInstance> set = Sets.newHashSet(cubeInstance); + ret.put(tblColRef, set); + } + } + } + return ret; + } + + public Map<TblColRef, Set<CubeInstance>> getUsedNonDimCols(String modelName) { + Map<TblColRef, Set<CubeInstance>> ret = Maps.newHashMap(); + List<CubeInstance> cubeInstances = cubeServiceV2.listAllCubes(null, null, modelName); + for (CubeInstance cubeInstance : cubeInstances) { + CubeDesc cubeDesc = cubeInstance.getDescriptor(); + Set<TblColRef> tblColRefs = Sets.newHashSet(cubeDesc.listAllColumns());//make a copy + tblColRefs.removeAll(cubeDesc.listDimensionColumnsIncludingDerived()); + for (TblColRef tblColRef : tblColRefs) { + if (ret.containsKey(tblColRef)) { + ret.get(tblColRef).add(cubeInstance); + } else { + Set<CubeInstance> set = Sets.newHashSet(cubeInstance); + ret.put(tblColRef, set); + } + } + } + return ret; + } + + public boolean validate(DataModelDesc dataModelDesc) throws IOException { + Message msg = MsgPicker.getMsg(); + + dataModelDesc.init(getConfig(), getMetadataManager().getAllTablesMap(), getMetadataManager().getCcInfoMap()); + + List<String> dimCols = new ArrayList<String>(); + List<String> dimAndMCols = new ArrayList<String>(); + + List<ModelDimensionDesc> dimensions = dataModelDesc.getDimensions(); + String[] measures = dataModelDesc.getMetrics(); + + for (ModelDimensionDesc dim : dimensions) { + String table = dim.getTable(); + for (String c : dim.getColumns()) { + dimCols.add(table + "." + c); + } + } + + dimAndMCols.addAll(dimCols); + + for (String measure : measures) { + dimAndMCols.add(measure); + } + + String modelName = dataModelDesc.getName(); + Set<TblColRef> usedDimCols = getUsedDimCols(modelName).keySet(); + Set<TblColRef> usedNonDimCols = getUsedNonDimCols(modelName).keySet(); + + for (TblColRef tblColRef : usedDimCols) { + if (!dimCols.contains(tblColRef.getTableAlias() + "." + tblColRef.getName())) + return false; + } + + for (TblColRef tblColRef : usedNonDimCols) { + if (!dimAndMCols.contains(tblColRef.getTableAlias() + "." + tblColRef.getName())) + return false; + } + + DataModelDesc originDataModelDesc = listAllModels(modelName, null).get(0); + + if (!dataModelDesc.getRootFactTable().equals(originDataModelDesc.getRootFactTable())) + return false; + + JoinsTree joinsTree = dataModelDesc.getJoinsTree(), originJoinsTree = originDataModelDesc.getJoinsTree(); + if (joinsTree.matchNum(originJoinsTree) != originDataModelDesc.getJoinTables().length + 1) + return false; + + return true; + } + + public void validateModelDesc(DataModelDesc modelDesc) { + Message msg = MsgPicker.getMsg(); + + if (modelDesc == null) { + throw new BadRequestException(msg.getINVALID_MODEL_DEFINITION()); + } + + String modelName = modelDesc.getName(); + + if (StringUtils.isEmpty(modelName)) { + logger.info("Model name should not be empty."); + throw new BadRequestException(msg.getEMPTY_MODEL_NAME()); + } + if (!StringUtils.containsOnly(modelName, VALID_MODELNAME)) { + logger.info("Invalid Model name {}, only letters, numbers and underline supported.", modelDesc.getName()); + throw new BadRequestException(String.format(msg.getINVALID_MODEL_NAME(), modelName)); + } + } + + public boolean unifyModelDesc(DataModelDesc modelDesc, boolean isDraft) throws IOException { + Message msg = MsgPicker.getMsg(); + + boolean createNew = false; + String modelName = modelDesc.getName(); + String originName = null; // for draft rename check + if (modelDesc.getUuid() != null) { + originName = getNameByUuid(modelDesc.getUuid()); + } + + if (!isDraft) { // save as official model + if (modelDesc.getStatus() != null && modelDesc.getStatus().equals(STATUS_DRAFT)) { // from draft + if (originName == null) { + throw new BadRequestException(msg.getORIGIN_MODEL_NOT_FOUND()); + } + originName = originName.substring(0, originName.lastIndexOf("_draft")); + if (!originName.equals(modelName)) { // if rename draft + DataModelDesc parentDesc = getMetadataManager().getDataModelDesc(originName); + if (parentDesc == null) { // only allow rename when official model has not been saved + createNew = true; + dropModelByUuid(modelDesc.getUuid()); + modelDesc.setStatus(null); + modelDesc.setLastModified(0); + modelDesc.setUuid(UUID.randomUUID().toString()); + } else { + throw new BadRequestException(msg.getMODEL_RENAME()); + } + } else { // without rename draft + modelDesc.setStatus(null); + DataModelDesc parentDesc = getMetadataManager().getDataModelDesc(modelName); + if (parentDesc == null) { // official model doesn't exist, create new one + createNew = true; + modelDesc.setLastModified(0); + modelDesc.setUuid(UUID.randomUUID().toString()); + } else { // update existing + modelDesc.setLastModified(parentDesc.getLastModified()); + modelDesc.setUuid(parentDesc.getUuid()); + } + } + } else { // from official + if (originName == null) { // official model doesn't exist, create new one + createNew = true; + modelDesc.setLastModified(0); + modelDesc.setUuid(UUID.randomUUID().toString()); + } else { + if (!originName.equals(modelName)) { // do not allow official model rename + throw new BadRequestException(msg.getMODEL_RENAME()); + } + } + } + } else { // save as draft model + if (modelDesc.getStatus() == null) { // from official + modelName += "_draft"; + modelDesc.setName(modelName); + modelDesc.setStatus(STATUS_DRAFT); + DataModelDesc draftDesc = getMetadataManager().getDataModelDesc(modelName); + if (draftDesc == null) { // draft model doesn't exist, create new one + createNew = true; + modelDesc.setLastModified(0); + modelDesc.setUuid(UUID.randomUUID().toString()); + } else if (draftDesc.getStatus() != null && draftDesc.getStatus().equals(STATUS_DRAFT)) { // update existing + modelDesc.setLastModified(draftDesc.getLastModified()); + modelDesc.setUuid(draftDesc.getUuid()); + } else { // already exist an official draft with name ends with '_draft' + throw new BadRequestException(String.format(msg.getNON_DRAFT_MODEL_ALREADY_EXIST(), modelName)); + } + } else if (modelDesc.getStatus().equals(STATUS_DRAFT)) { // from draft + if (originName == null) { + throw new BadRequestException(msg.getORIGIN_MODEL_NOT_FOUND()); + } + originName = originName.substring(0, originName.lastIndexOf("_draft")); + if (!originName.equals(modelName)) { // if rename draft + DataModelDesc parentDesc = getMetadataManager().getDataModelDesc(originName); + if (parentDesc == null) { // only allow rename when official model has not been saved + createNew = true; + dropModelByUuid(modelDesc.getUuid()); + modelName += "_draft"; + modelDesc.setName(modelName); + modelDesc.setStatus(STATUS_DRAFT); + modelDesc.setLastModified(0); + modelDesc.setUuid(UUID.randomUUID().toString()); + } else { + throw new BadRequestException(msg.getMODEL_RENAME()); + } + } else { // without rename draft + modelName += "_draft"; + modelDesc.setName(modelName); + } + } + } + return createNew; + } + + public String getNameByUuid(String uuid) { + List<DataModelDesc> models = getMetadataManager().getModels(); + for (DataModelDesc model : models) { + if (model.getUuid().equals(uuid)) { + return model.getName(); + } + } + return null; + } + + public void dropModelByUuid(String uuid) throws IOException { + List<DataModelDesc> models = getMetadataManager().getModels(); + for (DataModelDesc model : models) { + if (model.getUuid().equals(uuid)) { + dropModel(model); + } + } + } + + public DataModelDesc updateModelToResourceStore(DataModelDesc modelDesc, String projectName, boolean createNew, boolean isDraft) throws IOException { + Message msg = MsgPicker.getMsg(); + + if (createNew) { + createModelDesc(projectName, modelDesc); + } else { + try { + if (!isDraft && !validate(modelDesc)) { + throw new BadRequestException(msg.getUPDATE_MODEL_KEY_FIELD()); + } + modelDesc = updateModelAndDesc(modelDesc); + } catch (AccessDeniedException accessDeniedException) { + throw new ForbiddenException(msg.getUPDATE_MODEL_NO_RIGHT()); + } + + if (!modelDesc.getError().isEmpty()) { + throw new BadRequestException(String.format(msg.getBROKEN_MODEL_DESC(), modelDesc.getName())); + } + } + + if (!isDraft) { + DataModelDesc draftDesc = getMetadataManager().getDataModelDesc(modelDesc.getName() + "_draft"); + if (null != draftDesc && draftDesc.getStatus() != null && draftDesc.getStatus().equals(STATUS_DRAFT)) { + dropModel(draftDesc); + } + } + return modelDesc; + } +} http://git-wip-us.apache.org/repos/asf/kylin/blob/73a78dbe/server-base/src/main/java/org/apache/kylin/rest/service/ProjectService.java ---------------------------------------------------------------------- diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/ProjectService.java b/server-base/src/main/java/org/apache/kylin/rest/service/ProjectService.java index 8c0cf7a..cf86d3e 100644 --- a/server-base/src/main/java/org/apache/kylin/rest/service/ProjectService.java +++ b/server-base/src/main/java/org/apache/kylin/rest/service/ProjectService.java @@ -31,6 +31,7 @@ import org.apache.kylin.rest.security.AclPermission; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.security.access.prepost.PostFilter; import org.springframework.security.access.prepost.PreAuthorize; import org.springframework.security.core.context.SecurityContextHolder; @@ -46,6 +47,7 @@ public class ProjectService extends BasicService { private static final Logger logger = LoggerFactory.getLogger(ProjectService.class); @Autowired + @Qualifier("accessService") private AccessService accessService; @PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN) http://git-wip-us.apache.org/repos/asf/kylin/blob/73a78dbe/server-base/src/main/java/org/apache/kylin/rest/service/ProjectServiceV2.java ---------------------------------------------------------------------- diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/ProjectServiceV2.java b/server-base/src/main/java/org/apache/kylin/rest/service/ProjectServiceV2.java new file mode 100644 index 0000000..fb1ac94 --- /dev/null +++ b/server-base/src/main/java/org/apache/kylin/rest/service/ProjectServiceV2.java @@ -0,0 +1,89 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +package org.apache.kylin.rest.service; + +import java.io.IOException; +import java.util.LinkedHashMap; +import java.util.List; + +import org.apache.kylin.metadata.project.ProjectInstance; +import org.apache.kylin.rest.constant.Constant; +import org.apache.kylin.rest.exception.BadRequestException; +import org.apache.kylin.rest.msg.Message; +import org.apache.kylin.rest.msg.MsgPicker; +import org.apache.kylin.rest.security.AclPermission; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.security.access.prepost.PreAuthorize; +import org.springframework.security.core.context.SecurityContextHolder; +import org.springframework.stereotype.Component; + +/** + * Created by luwei on 17-4-20. + */ +@Component("projectServiceV2") +public class ProjectServiceV2 extends ProjectService { + + private static final Logger logger = LoggerFactory.getLogger(ProjectServiceV2.class); + + @Autowired + @Qualifier("accessService") + private AccessService accessService; + + @PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN) + public ProjectInstance createProject(ProjectInstance newProject) throws IOException { + Message msg = MsgPicker.getMsg(); + + String projectName = newProject.getName(); + String description = newProject.getDescription(); + LinkedHashMap<String, String> overrideProps = newProject.getOverrideKylinProps(); + + ProjectInstance currentProject = getProjectManager().getProject(projectName); + + if (currentProject != null) { + throw new BadRequestException(String.format(msg.getPROJECT_ALREADY_EXIST(), projectName)); + } + String owner = SecurityContextHolder.getContext().getAuthentication().getName(); + ProjectInstance createdProject = getProjectManager().createProject(projectName, owner, description, overrideProps); + accessService.init(createdProject, AclPermission.ADMINISTRATION); + logger.debug("New project created."); + + return createdProject; + } + + @PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN + " or hasPermission(#project, 'ADMINISTRATION') or hasPermission(#project, 'MANAGEMENT')") + public void deleteProject(String projectName, ProjectInstance project) throws IOException { + getProjectManager().dropProject(projectName); + + accessService.clean(project, true); + } + + public String getProjectOfModel(String modelName) { + List<ProjectInstance> projectInstances = listProjects(null, null); + for (ProjectInstance projectInstance : projectInstances) { + if (projectInstance.containsModel(modelName)) + return projectInstance.getName(); + } + return null; + } + + +} http://git-wip-us.apache.org/repos/asf/kylin/blob/73a78dbe/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java ---------------------------------------------------------------------- diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java index 9eab5a3..c4b5c0d 100644 --- a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java +++ b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java @@ -86,6 +86,7 @@ import org.apache.kylin.storage.hybrid.HybridInstance; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.security.access.AccessDeniedException; import org.springframework.security.core.GrantedAuthority; import org.springframework.security.core.context.SecurityContextHolder; @@ -114,15 +115,16 @@ public class QueryService extends BasicService { public static final String EXCEPTION_QUERY_CACHE = "ExceptionQueryCache"; private final Serializer<Query[]> querySerializer = new Serializer<Query[]>(Query[].class); - private final BadQueryDetector badQueryDetector = new BadQueryDetector(); + protected final BadQueryDetector badQueryDetector = new BadQueryDetector(); private final StorageURL hbaseUrl; private final String userTableName; @Autowired - private CacheManager cacheManager; + protected CacheManager cacheManager; @Autowired + @Qualifier("cacheService") private CacheService cacheService; @Autowired @@ -524,7 +526,7 @@ public class QueryService extends BasicService { return tableMetas; } - private void processStatementAttr(Statement s, SQLRequest sqlRequest) throws SQLException { + protected void processStatementAttr(Statement s, SQLRequest sqlRequest) throws SQLException { Integer statementMaxRows = BackdoorToggles.getStatementMaxRows(); if (statementMaxRows != null) { logger.info("Setting current statement's max rows to {}", statementMaxRows); @@ -672,7 +674,7 @@ public class QueryService extends BasicService { } } - private int getInt(String content) { + protected int getInt(String content) { try { return Integer.parseInt(content); } catch (Exception e) { @@ -680,7 +682,7 @@ public class QueryService extends BasicService { } } - private short getShort(String content) { + protected short getShort(String content) { try { return Short.parseShort(content); } catch (Exception e) { @@ -688,7 +690,7 @@ public class QueryService extends BasicService { } } - private static void close(ResultSet resultSet, Statement stat, Connection conn) { + protected static void close(ResultSet resultSet, Statement stat, Connection conn) { OLAPContext.clearParameter(); DBUtils.closeQuietly(resultSet); DBUtils.closeQuietly(stat);
