Repository: kylin Updated Branches: refs/heads/master 425d56bb5 -> 2e03c9c38
KYLIN-1540 REST API for deleting segment Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/2e03c9c3 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/2e03c9c3 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/2e03c9c3 Branch: refs/heads/master Commit: 2e03c9c38a02635ce245ca8e6fe5157e7f1db618 Parents: 425d56b Author: shaofengshi <[email protected]> Authored: Mon Mar 28 17:53:19 2016 +0800 Committer: shaofengshi <[email protected]> Committed: Mon Mar 28 17:57:54 2016 +0800 ---------------------------------------------------------------------- build/conf/kylin.properties | 3 + .../org/apache/kylin/cube/CubeInstance.java | 2 +- ...st_kylin_cube_with_slr_ready_3_segments.json | 88 ++++++++++++++++++++ .../test_case_data/sandbox/kylin.properties | 2 +- .../kylin/rest/controller/CubeController.java | 27 ++++++ .../apache/kylin/rest/service/CubeService.java | 32 +++++-- .../rest/controller/CubeControllerTest.java | 47 ++++++++++- 7 files changed, 190 insertions(+), 11 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/2e03c9c3/build/conf/kylin.properties ---------------------------------------------------------------------- diff --git a/build/conf/kylin.properties b/build/conf/kylin.properties index 289afd1..5fb4e36 100644 --- a/build/conf/kylin.properties +++ b/build/conf/kylin.properties @@ -15,6 +15,9 @@ # limitations under the License. # +# kylin server's mode +kylin.server.mode=all + # optional information for the owner of kylin platform, it can be your team's email # currently it will be attached to each kylin's htable attribute [email protected] http://git-wip-us.apache.org/repos/asf/kylin/blob/2e03c9c3/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java index d89e736..aa063a6 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java @@ -296,7 +296,7 @@ public class CubeInstance extends RootPersistentEntity implements IRealization, public CubeSegment getSegment(String name, SegmentStatusEnum status) { for (CubeSegment segment : segments) { - if ((null != segment.getName() && segment.getName().equals(name)) && segment.getStatus() == status) { + if ((null != segment.getName() && segment.getName().equals(name)) && (status == null || segment.getStatus() == status)) { return segment; } } http://git-wip-us.apache.org/repos/asf/kylin/blob/2e03c9c3/examples/test_case_data/localmeta/cube/test_kylin_cube_with_slr_ready_3_segments.json ---------------------------------------------------------------------- diff --git a/examples/test_case_data/localmeta/cube/test_kylin_cube_with_slr_ready_3_segments.json b/examples/test_case_data/localmeta/cube/test_kylin_cube_with_slr_ready_3_segments.json new file mode 100644 index 0000000..e39de87 --- /dev/null +++ b/examples/test_case_data/localmeta/cube/test_kylin_cube_with_slr_ready_3_segments.json @@ -0,0 +1,88 @@ +{ + "uuid" : "3eaca32a-a33e-4b69-83dd-0bb8b1f8c53c", + "last_modified" : 1404098141020, + "name" : "test_kylin_cube_with_slr_ready_3_segments", + "owner" : null, + "descriptor" : "test_kylin_cube_with_slr_desc", + "cost" : 50, + "segments" : [ { + "uuid" : "1aeca32a-a33e-4b69-83dd-xxe8b1f8dddd", + "name" : "19691231160000_20131112000000", + "storage_location_identifier" : "KYLIN-CUBE-TEST_KYLIN_CUBE_WITH_SLR_READY-F24668F6-DCFF-4CB6-A89B-77F1119DF8FA", + "date_range_start" : 1384240200000, + "date_range_end" : 1384243200000, + "status" : "READY", + "size_kb" : 7801, + "source_records" : 10000, + "source_records_size" : 608012, + "last_build_time" : 1404098140902, + "last_build_job_id" : "f24668f6-dcff-4cb6-a89b-77f1119df8fa", + "binary_signature" : null, + "dictionaries" : { + "DEFAULT.TEST_CATEGORY_GROUPINGS/CATEG_LVL2_NAME" : "/dict/TEST_CATEGORY_GROUPINGS/CATEG_LVL2_NAME/16d8185c-ee6b-4f8c-a919-756d9809f937.dict", + "DEFAULT.TEST_KYLIN_FACT/SLR_SEGMENT_CD" : "/dict/TEST_SELLER_TYPE_DIM/SELLER_TYPE_CD/c6bf9b51-6e90-4337-8082-4e2fdf78307f.dict", + "DEFAULT.TEST_KYLIN_FACT/LSTG_SITE_ID" : "/dict/TEST_SITES/SITE_ID/652bd393-678a-4f16-a504-fd8ce1229355.dict", + "EDW.TEST_SELLER_TYPE_DIM/SELLER_TYPE_CD" : "/dict/TEST_SELLER_TYPE_DIM/SELLER_TYPE_CD/2a44ff38-f64b-42e7-9fcf-66afccac8047.dict", + "DEFAULT.TEST_KYLIN_FACT/CAL_DT" : "/dict/PREDEFINED/date(yyyy-mm-dd)/64ac4f82-f2af-476e-85b9-f0805001014e.dict", + "DEFAULT.TEST_CATEGORY_GROUPINGS/CATEG_LVL3_NAME" : "/dict/TEST_CATEGORY_GROUPINGS/CATEG_LVL3_NAME/ad09f2d5-054a-4e1b-a776-7cc07399a6c1.dict", + "EDW.TEST_CAL_DT/CAL_DT" : "/dict/TEST_CAL_DT/CAL_DT/ed0c3451-593c-494c-9019-64f63fcb0b8e.dict", + "DEFAULT.TEST_KYLIN_FACT/LEAF_CATEG_ID" : "/dict/TEST_CATEGORY_GROUPINGS/LEAF_CATEG_ID/4243889f-bc81-4807-a975-7041bbbf35e7.dict", + "EDW.TEST_SITES/SITE_ID" : "/dict/TEST_SITES/SITE_ID/ff7e8943-ac0f-4e66-b9ed-510f6a0b875d.dict", + "DEFAULT.TEST_CATEGORY_GROUPINGS/META_CATEG_NAME" : "/dict/TEST_CATEGORY_GROUPINGS/META_CATEG_NAME/aceae914-4246-4251-a0c2-692fe7a300df.dict" + }, + "snapshots" : { + "EDW.TEST_SELLER_TYPE_DIM" : "/table_snapshot/TEST_SELLER_TYPE_DIM.csv/4fe75ccd-9b24-4cdf-ac9d-b4038e947f89.snapshot", + "EDW.TEST_CAL_DT" : "/table_snapshot/TEST_CAL_DT.csv/8ff1339e-f804-47f3-b42c-1d4fa4ff0cf7.snapshot", + "DEFAULT.TEST_CATEGORY_GROUPINGS" : "/table_snapshot/TEST_CATEGORY_GROUPINGS.csv/e172b442-ae10-447e-9071-c7dbb2bb38cc.snapshot", + "EDW.TEST_SITES" : "/table_snapshot/TEST_SITES.csv/28130338-fcf4-429e-91b0-cd8dfd397280.snapshot" + } + }, { + "uuid" : "2aeca32a-a33e-4b69-83dd-xxe8b1f8dddd", + "name" : "20131112000000_20131212000000", + "storage_location_identifier" : "KYLIN-CUBE-TEST_KYLIN_CUBE_WITH_SLR_READY-F24668F6-DCFF-4CB6-A89B-77F1119DF8FB", + "date_range_start" : 1384243200000, + "date_range_end" : 1386835200000, + "status" : "READY", + "size_kb" : 7801, + "source_records" : 10000, + "source_records_size" : 608012, + "last_build_time" : 1404098140902, + "last_build_job_id" : "f24668f6-dcff-4cb6-a89b-77f1119df8fb", + "binary_signature" : null, + "dictionaries" : { + "DEFAULT.TEST_CATEGORY_GROUPINGS/CATEG_LVL2_NAME" : "/dict/TEST_CATEGORY_GROUPINGS/CATEG_LVL2_NAME/16d8185c-ee6b-4f8c-a919-756d9809f937.dict", + "DEFAULT.TEST_KYLIN_FACT/SLR_SEGMENT_CD" : "/dict/TEST_SELLER_TYPE_DIM/SELLER_TYPE_CD/c6bf9b51-6e90-4337-8082-4e2fdf78307f.dict", + "DEFAULT.TEST_KYLIN_FACT/LSTG_SITE_ID" : "/dict/TEST_SITES/SITE_ID/652bd393-678a-4f16-a504-fd8ce1229355.dict", + "EDW.TEST_SELLER_TYPE_DIM/SELLER_TYPE_CD" : "/dict/TEST_SELLER_TYPE_DIM/SELLER_TYPE_CD/2a44ff38-f64b-42e7-9fcf-66afccac8047.dict", + "DEFAULT.TEST_KYLIN_FACT/CAL_DT" : "/dict/PREDEFINED/date(yyyy-mm-dd)/64ac4f82-f2af-476e-85b9-f0805001014e.dict", + "DEFAULT.TEST_CATEGORY_GROUPINGS/CATEG_LVL3_NAME" : "/dict/TEST_CATEGORY_GROUPINGS/CATEG_LVL3_NAME/ad09f2d5-054a-4e1b-a776-7cc07399a6c1.dict", + "EDW.TEST_CAL_DT/CAL_DT" : "/dict/TEST_CAL_DT/CAL_DT/ed0c3451-593c-494c-9019-64f63fcb0b8e.dict", + "DEFAULT.TEST_KYLIN_FACT/LEAF_CATEG_ID" : "/dict/TEST_CATEGORY_GROUPINGS/LEAF_CATEG_ID/4243889f-bc81-4807-a975-7041bbbf35e7.dict", + "EDW.TEST_SITES/SITE_ID" : "/dict/TEST_SITES/SITE_ID/ff7e8943-ac0f-4e66-b9ed-510f6a0b875d.dict", + "DEFAULT.TEST_CATEGORY_GROUPINGS/META_CATEG_NAME" : "/dict/TEST_CATEGORY_GROUPINGS/META_CATEG_NAME/aceae914-4246-4251-a0c2-692fe7a300df.dict" + }, + "snapshots" : { + "EDW.TEST_SELLER_TYPE_DIM" : "/table_snapshot/TEST_SELLER_TYPE_DIM.csv/4fe75ccd-9b24-4cdf-ac9d-b4038e947f89.snapshot", + "EDW.TEST_CAL_DT" : "/table_snapshot/TEST_CAL_DT.csv/8ff1339e-f804-47f3-b42c-1d4fa4ff0cf7.snapshot", + "DEFAULT.TEST_CATEGORY_GROUPINGS" : "/table_snapshot/TEST_CATEGORY_GROUPINGS.csv/e172b442-ae10-447e-9071-c7dbb2bb38cc.snapshot", + "EDW.TEST_SITES" : "/table_snapshot/TEST_SITES.csv/28130338-fcf4-429e-91b0-cd8dfd397280.snapshot" + } + }, { + "uuid" : "3aeca32a-a33e-4b69-83dd-xxe8b1f8dddd", + "name" : "20131212000000_20140112000000", + "storage_location_identifier" : "KYLIN-CUBE-TEST_KYLIN_CUBE_WITH_SLR_READY-F24668F6-DCFF-4CB6-A89B-77F1119DF8FB", + "date_range_start" : 1386835200000, + "date_range_end" : 1389484800000, + "status" : "NEW", + "size_kb" : 0, + "source_records" : 0, + "source_records_size" : 0, + "last_build_time" : 1404098140902, + "last_build_job_id" : "", + "binary_signature" : null + } ], + "status" : "READY", + "create_time" : null, + "notify_list" : null, + "auto_merge_time_ranges" : [2595000000] +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/kylin/blob/2e03c9c3/examples/test_case_data/sandbox/kylin.properties ---------------------------------------------------------------------- diff --git a/examples/test_case_data/sandbox/kylin.properties b/examples/test_case_data/sandbox/kylin.properties index f1ecd64..2fa647d 100644 --- a/examples/test_case_data/sandbox/kylin.properties +++ b/examples/test_case_data/sandbox/kylin.properties @@ -63,7 +63,7 @@ kylin.job.concurrent.max.limit=10 kylin.job.mapreduce.max.reducer.number=5 #the percentage of the sampling, default 100% -kylin.job.cubing.inMem.sampling.percent=100 +kylin.job.cubing.inmem.sampling.percent=100 # The cut size for hbase region, in GB. # E.g, for cube whose capacity be marked as "SMALL", split region per 10GB by default http://git-wip-us.apache.org/repos/asf/kylin/blob/2e03c9c3/server/src/main/java/org/apache/kylin/rest/controller/CubeController.java ---------------------------------------------------------------------- diff --git a/server/src/main/java/org/apache/kylin/rest/controller/CubeController.java b/server/src/main/java/org/apache/kylin/rest/controller/CubeController.java index b17378c..6c62b52 100644 --- a/server/src/main/java/org/apache/kylin/rest/controller/CubeController.java +++ b/server/src/main/java/org/apache/kylin/rest/controller/CubeController.java @@ -209,6 +209,33 @@ public class CubeController extends BasicController { } /** + * Delete a cube segment + * + * @throws IOException + */ + @RequestMapping(value = "/{cubeName}/segs/{segmentName}", method = { RequestMethod.DELETE }) + @ResponseBody + public CubeInstance deleteSegment(@PathVariable String cubeName, @PathVariable String segmentName) { + CubeInstance cube = cubeService.getCubeManager().getCube(cubeName); + + if (cube == null) { + throw new InternalErrorException("Cannot find cube " + cubeName); + } + + CubeSegment segment = cube.getSegment(segmentName, null); + if (segment == null) { + throw new InternalErrorException("Cannot find segment '" + segmentName + "'"); + } + + try { + return cubeService.deleteSegment(cube, segmentName); + } catch (Exception e) { + logger.error(e.getLocalizedMessage(), e); + throw new InternalErrorException(e.getLocalizedMessage()); + } + } + + /** * Send a rebuild cube job * * @param cubeName Cube ID http://git-wip-us.apache.org/repos/asf/kylin/blob/2e03c9c3/server/src/main/java/org/apache/kylin/rest/service/CubeService.java ---------------------------------------------------------------------- diff --git a/server/src/main/java/org/apache/kylin/rest/service/CubeService.java b/server/src/main/java/org/apache/kylin/rest/service/CubeService.java index eb15beb..8ed0802 100644 --- a/server/src/main/java/org/apache/kylin/rest/service/CubeService.java +++ b/server/src/main/java/org/apache/kylin/rest/service/CubeService.java @@ -19,17 +19,11 @@ package org.apache.kylin.rest.service; import java.io.IOException; -import java.util.ArrayList; -import java.util.Collections; -import java.util.Date; -import java.util.EnumSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.WeakHashMap; +import java.util.*; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.client.HTable; +import org.apache.hadoop.mapred.Merger; import org.apache.kylin.common.KylinConfig; import org.apache.kylin.cube.CubeInstance; import org.apache.kylin.cube.CubeManager; @@ -527,6 +521,28 @@ public class CubeService extends BasicService { return cube; } + @PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN + " or hasPermission(#cube, 'ADMINISTRATION') or hasPermission(#cube, 'OPERATION') or hasPermission(#cube, 'MANAGEMENT')") + public CubeInstance deleteSegment(CubeInstance cube, String segmentName) throws IOException { + + if (!segmentName.equals(cube.getSegments().get(0).getName()) && !segmentName.equals(cube.getSegments().get(cube.getSegments().size() - 1).getName())) { + throw new IllegalArgumentException("Cannot delete segment '" + segmentName + "' as it is neither the first nor the last segment."); + } + CubeSegment toDelete = null; + for (CubeSegment seg : cube.getSegments()) { + if (seg.getName().equals(segmentName)) { + toDelete = seg; + } + } + + if (toDelete.getStatus() != SegmentStatusEnum.READY) { + throw new IllegalArgumentException("Cannot delete segment '" + segmentName + "' as its status is not READY. Discard the on-going job for it."); + } + + CubeUpdate update = new CubeUpdate(cube); + update.setToRemoveSegs(new CubeSegment[]{toDelete}); + return CubeManager.getInstance(getConfig()).updateCube(update); + } + /** * purge the cube * http://git-wip-us.apache.org/repos/asf/kylin/blob/2e03c9c3/server/src/test/java/org/apache/kylin/rest/controller/CubeControllerTest.java ---------------------------------------------------------------------- diff --git a/server/src/test/java/org/apache/kylin/rest/controller/CubeControllerTest.java b/server/src/test/java/org/apache/kylin/rest/controller/CubeControllerTest.java index 1230886..20ddd58 100644 --- a/server/src/test/java/org/apache/kylin/rest/controller/CubeControllerTest.java +++ b/server/src/test/java/org/apache/kylin/rest/controller/CubeControllerTest.java @@ -24,6 +24,7 @@ import java.util.List; import org.apache.kylin.cube.CubeInstance; import org.apache.kylin.cube.model.CubeDesc; +import org.apache.kylin.rest.exception.InternalErrorException; import org.apache.kylin.rest.request.CubeRequest; import org.apache.kylin.rest.service.CubeService; import org.apache.kylin.rest.service.JobService; @@ -68,7 +69,7 @@ public class CubeControllerTest extends ServiceTestBase { @Test public void testBasics() throws IOException { - CubeDesc[] cubes = (CubeDesc[]) cubeDescController.getCube("test_kylin_cube_with_slr_ready"); + CubeDesc[] cubes = cubeDescController.getCube("test_kylin_cube_with_slr_ready"); Assert.assertNotNull(cubes); Assert.assertNotNull(cubeController.getSql("test_kylin_cube_with_slr_ready", "20130331080000_20131212080000")); Assert.assertNotNull(cubeController.getCubes(null, null, null, 0, 5)); @@ -121,4 +122,48 @@ public class CubeControllerTest extends ServiceTestBase { cubeController.deleteCube(newCubeName); } + @Test (expected=InternalErrorException.class) + public void testDeleteSegmentNew() throws IOException { + String cubeName = "test_kylin_cube_with_slr_ready_3_segments"; + CubeDesc[] cubes = cubeDescController.getCube(cubeName); + Assert.assertNotNull(cubes); + + cubeController.deleteSegment(cubeName, "20131212000000_20140112000000"); + } + + @Test (expected=InternalErrorException.class) + public void testDeleteSegmentNotExist() throws IOException { + String cubeName = "test_kylin_cube_with_slr_ready_3_segments"; + CubeDesc[] cubes = cubeDescController.getCube(cubeName); + Assert.assertNotNull(cubes); + + cubeController.deleteSegment(cubeName, "not_exist_segment"); + } + + + @Test (expected=InternalErrorException.class) + public void testDeleteSegmentInMiddle() throws IOException { + String cubeName = "test_kylin_cube_with_slr_ready_3_segments"; + CubeDesc[] cubes = cubeDescController.getCube(cubeName); + Assert.assertNotNull(cubes); + + cubeController.deleteSegment(cubeName, "20131112000000_20131212000000"); + } + + @Test + public void testDeleteSegmentFromHead() throws IOException { + String cubeName = "test_kylin_cube_with_slr_ready_3_segments"; + CubeDesc[] cubes = cubeDescController.getCube(cubeName); + Assert.assertNotNull(cubes); + + int segNumber = cubeService.getCubeManager().getCube(cubeName).getSegments().size(); + + cubeController.deleteSegment(cubeName, "19691231160000_20131112000000"); + + int newSegNumber = cubeService.getCubeManager().getCube(cubeName).getSegments().size(); + + Assert.assertTrue(segNumber == newSegNumber + 1); + } + + }
