Repository: kylin Updated Branches: refs/heads/master 4cd733a14 -> db5f89bae
KYLIN-1854 Allow deleting cube instance when its underlying cubedesc went wrong Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/edfb37d0 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/edfb37d0 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/edfb37d0 Branch: refs/heads/master Commit: edfb37d08c94294801307c396b94df103b8706c8 Parents: 4cd733a Author: Hongbin Ma <mahong...@apache.org> Authored: Thu Jul 7 15:31:56 2016 +0800 Committer: Hongbin Ma <mahong...@apache.org> Committed: Thu Jul 7 15:31:56 2016 +0800 ---------------------------------------------------------------------- .../java/org/apache/kylin/job/DeployUtil.java | 6 ++-- .../java/org/apache/kylin/cube/CubeManager.java | 15 ++++++--- .../apache/kylin/rest/service/CubeService.java | 32 ++++++++++++++------ 3 files changed, 35 insertions(+), 18 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/edfb37d0/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java ---------------------------------------------------------------------- diff --git a/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java b/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java index 22fe48a..da97df3 100644 --- a/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java +++ b/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java @@ -32,9 +32,9 @@ import org.apache.kylin.common.KylinConfig; import org.apache.kylin.common.persistence.ResourceStore; import org.apache.kylin.common.persistence.ResourceTool; import org.apache.kylin.common.util.LocalFileMetadataTestCase; +import org.apache.kylin.cube.CubeDescManager; import org.apache.kylin.cube.CubeInstance; import org.apache.kylin.cube.CubeManager; -import org.apache.kylin.cube.CubeUpdate; import org.apache.kylin.job.dataGen.FactTableGenerator; import org.apache.kylin.job.streaming.KafkaDataLoader; import org.apache.kylin.job.streaming.StreamingTableDataGenerator; @@ -71,9 +71,7 @@ public class DeployUtil { // update cube desc signature. for (CubeInstance cube : CubeManager.getInstance(config()).listAllCubes()) { - cube.getDescriptor().setSignature(cube.getDescriptor().calculateSignature()); - CubeUpdate cubeBuilder = new CubeUpdate(cube); - CubeManager.getInstance(config()).updateCube(cubeBuilder); + CubeDescManager.getInstance(config()).updateCubeDesc(cube.getDescriptor());//enforce signature updating } } http://git-wip-us.apache.org/repos/asf/kylin/blob/edfb37d0/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java index a200f5d..0941d56 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java @@ -810,13 +810,20 @@ public class CubeManager implements IRealizationProvider { ResourceStore store = getStore(); List<String> paths = store.collectResourceRecursively(ResourceStore.CUBE_RESOURCE_ROOT, ".json"); - logger.debug("Loading Cube from folder " + store.getReadableResourcePath(ResourceStore.CUBE_RESOURCE_ROOT)); + logger.info("Loading Cube from folder " + store.getReadableResourcePath(ResourceStore.CUBE_RESOURCE_ROOT)); + int succeed = 0; + int fail = 0; for (String path : paths) { - reloadCubeLocalAt(path); + CubeInstance cube = reloadCubeLocalAt(path); + if (cube == null) { + fail++; + } else { + succeed++; + } } - logger.debug("Loaded " + paths.size() + " Cube(s)"); + logger.info("Loaded " + succeed + " cubes, fail on " + fail + " cubes"); } private synchronized CubeInstance reloadCubeLocalAt(String path) { @@ -855,7 +862,7 @@ public class CubeManager implements IRealizationProvider { return cubeInstance; } catch (Exception e) { - logger.error("Error during load cube instance " + path, e); + logger.error("Error during load cube instance, skipping : " + path, e); return null; } } http://git-wip-us.apache.org/repos/asf/kylin/blob/edfb37d0/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java ---------------------------------------------------------------------- diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java b/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java index a9d4bfc..72942e8 100644 --- a/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java +++ b/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java @@ -282,8 +282,15 @@ public class CubeService extends BasicService { throw new JobException("The cube " + cube.getName() + " has running job, please discard it and try again."); } - this.releaseAllSegments(cube); - getCubeManager().dropCube(cube.getName(), true); + try { + this.releaseAllJobs(cube); + } catch (Exception e) { + logger.error("error when releasing all jobs", e); + //ignore the exception + } + + int cubeNum = getCubeManager().getCubesByDesc(cube.getDescriptor().getName()).size(); + getCubeManager().dropCube(cube.getName(), cubeNum == 1);//only delete cube desc when no other cube is using it accessService.clean(cube, true); } @@ -550,13 +557,7 @@ public class CubeService extends BasicService { return CubeManager.getInstance(getConfig()).updateCube(update); } - /** - * purge the cube - * - * @throws IOException - * @throws JobException - */ - private CubeInstance releaseAllSegments(CubeInstance cube) throws IOException, JobException { + private void releaseAllJobs(CubeInstance cube) { final List<CubingJob> cubingJobs = listAllCubingJobs(cube.getName(), null); for (CubingJob cubingJob : cubingJobs) { final ExecutableState status = cubingJob.getStatus(); @@ -564,9 +565,20 @@ public class CubeService extends BasicService { getExecutableManager().discardJob(cubingJob.getId()); } } + } + + /** + * purge the cube + * + * @throws IOException + * @throws JobException + */ + private void releaseAllSegments(CubeInstance cube) throws IOException, JobException { + releaseAllJobs(cube); + CubeUpdate update = new CubeUpdate(cube); update.setToRemoveSegs(cube.getSegments().toArray(new CubeSegment[cube.getSegments().size()])); - return CubeManager.getInstance(getConfig()).updateCube(update); + CubeManager.getInstance(getConfig()).updateCube(update); } @PreAuthorize(Constant.ACCESS_HAS_ROLE_MODELER + " or " + Constant.ACCESS_HAS_ROLE_ADMIN)