KYLIN-2317 Hybrid Cube CLI Tools
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/d7196d20 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/d7196d20 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/d7196d20 Branch: refs/heads/sparkcubing-rebase Commit: d7196d20ed7a70d11192d6f36eb802ee21f5539a Parents: e7e29f1 Author: Billy Liu <billy...@apache.org> Authored: Tue Jan 3 12:53:01 2017 +0800 Committer: Billy Liu <billy...@apache.org> Committed: Tue Jan 3 12:53:01 2017 +0800 ---------------------------------------------------------------------- .../kylin/metadata/cachesync/Broadcaster.java | 6 +- .../kylin/storage/hybrid/HybridManager.java | 3 +- .../localmeta/cube/ssb_cube1.json | 14 ++ .../localmeta/cube/ssb_cube2.json | 14 ++ .../localmeta/cube/ssb_cube3.json | 14 ++ .../localmeta/cube_desc/ssb_cube1.json | 155 ++++++++++++++ .../localmeta/cube_desc/ssb_cube2.json | 155 ++++++++++++++ .../localmeta/cube_desc/ssb_cube3.json | 143 +++++++++++++ .../apache/kylin/rest/service/ModelService.java | 4 +- tool/pom.xml | 2 + .../org/apache/kylin/tool/HybridCubeCLI.java | 206 +++++++++++++++++++ .../apache/kylin/tool/MrJobInfoExtractor.java | 25 +-- .../apache/kylin/tool/HybridCubeCLITest.java | 93 +++++++++ 13 files changed, 816 insertions(+), 18 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/d7196d20/core-metadata/src/main/java/org/apache/kylin/metadata/cachesync/Broadcaster.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/cachesync/Broadcaster.java b/core-metadata/src/main/java/org/apache/kylin/metadata/cachesync/Broadcaster.java index 6bc5771..d00c490 100644 --- a/core-metadata/src/main/java/org/apache/kylin/metadata/cachesync/Broadcaster.java +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/cachesync/Broadcaster.java @@ -115,7 +115,7 @@ public class Broadcaster { for (String node : config.getRestServers()) { restClients.add(new RestClient(node)); } - final ExecutorService wipingCachePool = Executors.newFixedThreadPool(restClients.size()); + final ExecutorService wipingCachePool = Executors.newFixedThreadPool(restClients.size(), new DaemonThreadFactory()); while (true) { try { final BroadcastEvent broadcastEvent = broadcastEvents.takeFirst(); @@ -127,7 +127,7 @@ public class Broadcaster { try { restClient.wipeCache(broadcastEvent.getEntity(), broadcastEvent.getEvent(), broadcastEvent.getCacheKey()); } catch (IOException e) { - logger.warn("Thread failed during wipe cache at " + broadcastEvent); + logger.warn("Thread failed during wipe cache at " + broadcastEvent, e); } } }); @@ -228,7 +228,7 @@ public class Broadcaster { try { counter.incrementAndGet(); - broadcastEvents.putFirst(new BroadcastEvent(entity, event, key)); + broadcastEvents.putLast(new BroadcastEvent(entity, event, key)); } catch (Exception e) { counter.decrementAndGet(); logger.error("error putting BroadcastEvent", e); http://git-wip-us.apache.org/repos/asf/kylin/blob/d7196d20/core-storage/src/main/java/org/apache/kylin/storage/hybrid/HybridManager.java ---------------------------------------------------------------------- diff --git a/core-storage/src/main/java/org/apache/kylin/storage/hybrid/HybridManager.java b/core-storage/src/main/java/org/apache/kylin/storage/hybrid/HybridManager.java index 4f81b09..748e873 100644 --- a/core-storage/src/main/java/org/apache/kylin/storage/hybrid/HybridManager.java +++ b/core-storage/src/main/java/org/apache/kylin/storage/hybrid/HybridManager.java @@ -124,10 +124,11 @@ public class HybridManager implements IRealizationProvider { } } - private void reloadAllHybridInstance() throws IOException { + public void reloadAllHybridInstance() throws IOException { ResourceStore store = getStore(); List<String> paths = store.collectResourceRecursively(ResourceStore.HYBRID_RESOURCE_ROOT, ".json"); + hybridMap.clear(); logger.debug("Loading Hybrid from folder " + store.getReadableResourcePath(ResourceStore.HYBRID_RESOURCE_ROOT)); for (String path : paths) { http://git-wip-us.apache.org/repos/asf/kylin/blob/d7196d20/examples/test_case_data/localmeta/cube/ssb_cube1.json ---------------------------------------------------------------------- diff --git a/examples/test_case_data/localmeta/cube/ssb_cube1.json b/examples/test_case_data/localmeta/cube/ssb_cube1.json new file mode 100644 index 0000000..b1eb2a5 --- /dev/null +++ b/examples/test_case_data/localmeta/cube/ssb_cube1.json @@ -0,0 +1,14 @@ +{ + "uuid" : "70a9f288-3c01-4745-a04b-5641e82d6c70", + "name" : "ssb_cube1", + "owner" : "ADMIN", + "cost" : 50, + "status" : "DISABLED", + "segments" : [ ], + "last_modified" : 1457534216410, + "descriptor" : "ssb_cube1", + "create_time_utc" : 1457444500888, + "size_kb" : 0, + "input_records_count" : 0, + "input_records_size" : 0 +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/kylin/blob/d7196d20/examples/test_case_data/localmeta/cube/ssb_cube2.json ---------------------------------------------------------------------- diff --git a/examples/test_case_data/localmeta/cube/ssb_cube2.json b/examples/test_case_data/localmeta/cube/ssb_cube2.json new file mode 100644 index 0000000..58f935d --- /dev/null +++ b/examples/test_case_data/localmeta/cube/ssb_cube2.json @@ -0,0 +1,14 @@ +{ + "uuid" : "70a9f288-3c01-4745-a04b-5641e82d6c71", + "name" : "ssb_cube2", + "owner" : "ADMIN", + "cost" : 50, + "status" : "DISABLED", + "segments" : [ ], + "last_modified" : 1457534216410, + "descriptor" : "ssb_cube2", + "create_time_utc" : 1457444500888, + "size_kb" : 0, + "input_records_count" : 0, + "input_records_size" : 0 +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/kylin/blob/d7196d20/examples/test_case_data/localmeta/cube/ssb_cube3.json ---------------------------------------------------------------------- diff --git a/examples/test_case_data/localmeta/cube/ssb_cube3.json b/examples/test_case_data/localmeta/cube/ssb_cube3.json new file mode 100644 index 0000000..fc09006 --- /dev/null +++ b/examples/test_case_data/localmeta/cube/ssb_cube3.json @@ -0,0 +1,14 @@ +{ + "uuid" : "70a9f288-3c01-4745-a04b-5641e82d6c72", + "name" : "ssb_cube3", + "owner" : "ADMIN", + "cost" : 50, + "status" : "DISABLED", + "segments" : [ ], + "last_modified" : 1457534216410, + "descriptor" : "ssb_cube3", + "create_time_utc" : 1457444500888, + "size_kb" : 0, + "input_records_count" : 0, + "input_records_size" : 0 +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/kylin/blob/d7196d20/examples/test_case_data/localmeta/cube_desc/ssb_cube1.json ---------------------------------------------------------------------- diff --git a/examples/test_case_data/localmeta/cube_desc/ssb_cube1.json b/examples/test_case_data/localmeta/cube_desc/ssb_cube1.json new file mode 100644 index 0000000..5986057 --- /dev/null +++ b/examples/test_case_data/localmeta/cube_desc/ssb_cube1.json @@ -0,0 +1,155 @@ +{ + "uuid" : "5c44df30-daec-486e-af90-927bf7851058", + "name" : "ssb_cube1", + "description" : "", + "dimensions" : [ { + "name" : "SSB.PART_DERIVED", + "table" : "SSB.PART", + "column" : null, + "derived" : [ "P_MFGR", "P_CATEGORY", "P_BRAND" ] + }, { + "name" : "C_CITY", + "table" : "SSB.CUSTOMER", + "column" : "C_CITY", + "derived" : null + }, { + "name" : "C_REGION", + "table" : "SSB.CUSTOMER", + "column" : "C_REGION", + "derived" : null + }, { + "name" : "C_NATION", + "table" : "SSB.CUSTOMER", + "column" : "C_NATION", + "derived" : null + }, { + "name" : "D_YEAR", + "table" : "SSB.DATES", + "column" : "D_YEAR", + "derived" : null + }, { + "name" : "D_YEARMONTH", + "table" : "SSB.DATES", + "column" : "D_YEARMONTH", + "derived" : null + }, { + "name" : "D_YEARMONTHNUM", + "table" : "SSB.DATES", + "column" : "D_YEARMONTHNUM", + "derived" : null + }, { + "name" : "D_WEEKNUMINYEAR", + "table" : "SSB.DATES", + "column" : "D_WEEKNUMINYEAR", + "derived" : null + } ], + "measures" : [ { + "name" : "_COUNT_", + "function" : { + "expression" : "COUNT", + "parameter" : { + "type" : "constant", + "value" : "1", + "next_parameter" : null + }, + "returntype" : "bigint" + }, + "dependent_measure_ref" : null + }, { + "name" : "TOTAL_REVENUE", + "function" : { + "expression" : "SUM", + "parameter" : { + "type" : "column", + "value" : "LO_REVENUE", + "next_parameter" : null + }, + "returntype" : "bigint" + }, + "dependent_measure_ref" : null + }, { + "name" : "TOTAL_SUPPLYCOST", + "function" : { + "expression" : "SUM", + "parameter" : { + "type" : "column", + "value" : "LO_SUPPLYCOST", + "next_parameter" : null + }, + "returntype" : "bigint" + }, + "dependent_measure_ref" : null + }, { + "name" : "TOTAL_V_REVENUE", + "function" : { + "expression" : "SUM", + "parameter" : { + "type" : "column", + "value" : "V_REVENUE", + "next_parameter" : null + }, + "returntype" : "bigint" + }, + "dependent_measure_ref" : null + } ], + "rowkey" : { + "rowkey_columns" : [ { + "column" : "LO_PARTKEY", + "encoding" : "dict" + }, { + "column" : "C_CITY", + "encoding" : "dict" + }, { + "column" : "C_REGION", + "encoding" : "dict" + }, { + "column" : "C_NATION", + "encoding" : "dict" + }, { + "column" : "D_YEAR", + "encoding" : "dict" + }, { + "column" : "D_YEARMONTH", + "encoding" : "dict" + }, { + "column" : "D_YEARMONTHNUM", + "encoding" : "dict" + }, { + "column" : "D_WEEKNUMINYEAR", + "encoding" : "dict" + } ] + }, + "signature" : "", + "last_modified" : 1457503036686, + "model_name" : "ssb", + "null_string" : null, + "hbase_mapping" : { + "column_family" : [ { + "name" : "F1", + "columns" : [ { + "qualifier" : "M", + "measure_refs" : [ "_COUNT_", "TOTAL_REVENUE", "TOTAL_SUPPLYCOST", "TOTAL_V_REVENUE" ] + } ] + } ] + }, + "aggregation_groups" : [ { + "includes" : [ "LO_PARTKEY", "C_CITY", "C_REGION", "C_NATION", "D_YEAR", "D_YEARMONTH", "D_YEARMONTHNUM", "D_WEEKNUMINYEAR" ], + "select_rule" : { + "hierarchy_dims" : [ [ "C_REGION", "C_NATION", "C_CITY" ], [ "D_YEARMONTH", "D_YEARMONTHNUM", "D_WEEKNUMINYEAR" ] ], + "mandatory_dims" : [ "D_YEAR" ], + "joint_dims" : [ ] + } + } ], + "notify_list" : [ ], + "status_need_notify" : [ ], + "partition_date_start" : 694224000000, + "partition_date_end" : 3153000000000, + "auto_merge_time_ranges" : [ 604800000, 2419200000 ], + "retention_range" : 0, + "engine_type" : 2, + "storage_type" : 2, + "override_kylin_properties" : { + "kylin.hbase.default.compression.codec" : "lz4", + "kylin.cube.aggrgroup.isMandatoryOnlyValid" : "true" + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/kylin/blob/d7196d20/examples/test_case_data/localmeta/cube_desc/ssb_cube2.json ---------------------------------------------------------------------- diff --git a/examples/test_case_data/localmeta/cube_desc/ssb_cube2.json b/examples/test_case_data/localmeta/cube_desc/ssb_cube2.json new file mode 100644 index 0000000..fc75ea8 --- /dev/null +++ b/examples/test_case_data/localmeta/cube_desc/ssb_cube2.json @@ -0,0 +1,155 @@ +{ + "uuid" : "5c44df30-daec-486e-af90-927bf7851059", + "name" : "ssb_cube2", + "description" : "", + "dimensions" : [ { + "name" : "SSB.PART_DERIVED", + "table" : "SSB.PART", + "column" : null, + "derived" : [ "P_MFGR", "P_CATEGORY", "P_BRAND" ] + }, { + "name" : "S_CITY", + "table" : "SSB.SUPPLIER", + "column" : "S_CITY", + "derived" : null + }, { + "name" : "S_REGION", + "table" : "SSB.SUPPLIER", + "column" : "S_REGION", + "derived" : null + }, { + "name" : "S_NATION", + "table" : "SSB.SUPPLIER", + "column" : "S_NATION", + "derived" : null + }, { + "name" : "D_YEAR", + "table" : "SSB.DATES", + "column" : "D_YEAR", + "derived" : null + }, { + "name" : "D_YEARMONTH", + "table" : "SSB.DATES", + "column" : "D_YEARMONTH", + "derived" : null + }, { + "name" : "D_YEARMONTHNUM", + "table" : "SSB.DATES", + "column" : "D_YEARMONTHNUM", + "derived" : null + }, { + "name" : "D_WEEKNUMINYEAR", + "table" : "SSB.DATES", + "column" : "D_WEEKNUMINYEAR", + "derived" : null + } ], + "measures" : [ { + "name" : "_COUNT_", + "function" : { + "expression" : "COUNT", + "parameter" : { + "type" : "constant", + "value" : "1", + "next_parameter" : null + }, + "returntype" : "bigint" + }, + "dependent_measure_ref" : null + }, { + "name" : "TOTAL_REVENUE", + "function" : { + "expression" : "SUM", + "parameter" : { + "type" : "column", + "value" : "LO_REVENUE", + "next_parameter" : null + }, + "returntype" : "bigint" + }, + "dependent_measure_ref" : null + }, { + "name" : "TOTAL_SUPPLYCOST", + "function" : { + "expression" : "SUM", + "parameter" : { + "type" : "column", + "value" : "LO_SUPPLYCOST", + "next_parameter" : null + }, + "returntype" : "bigint" + }, + "dependent_measure_ref" : null + }, { + "name" : "TOTAL_V_REVENUE", + "function" : { + "expression" : "SUM", + "parameter" : { + "type" : "column", + "value" : "V_REVENUE", + "next_parameter" : null + }, + "returntype" : "bigint" + }, + "dependent_measure_ref" : null + } ], + "rowkey" : { + "rowkey_columns" : [ { + "column" : "LO_PARTKEY", + "encoding" : "dict" + }, { + "column" : "S_CITY", + "encoding" : "dict" + }, { + "column" : "S_REGION", + "encoding" : "dict" + }, { + "column" : "S_NATION", + "encoding" : "dict" + }, { + "column" : "D_YEAR", + "encoding" : "dict" + }, { + "column" : "D_YEARMONTH", + "encoding" : "dict" + }, { + "column" : "D_YEARMONTHNUM", + "encoding" : "dict" + }, { + "column" : "D_WEEKNUMINYEAR", + "encoding" : "dict" + } ] + }, + "signature" : "", + "last_modified" : 1457503036686, + "model_name" : "ssb", + "null_string" : null, + "hbase_mapping" : { + "column_family" : [ { + "name" : "F1", + "columns" : [ { + "qualifier" : "M", + "measure_refs" : [ "_COUNT_", "TOTAL_REVENUE", "TOTAL_SUPPLYCOST", "TOTAL_V_REVENUE" ] + } ] + } ] + }, + "aggregation_groups" : [ { + "includes" : [ "LO_PARTKEY", "S_CITY", "S_REGION", "S_NATION", "D_YEAR", "D_YEARMONTH", "D_YEARMONTHNUM", "D_WEEKNUMINYEAR" ], + "select_rule" : { + "hierarchy_dims" : [ [ "S_REGION", "S_NATION", "S_CITY" ], [ "D_YEARMONTH", "D_YEARMONTHNUM", "D_WEEKNUMINYEAR" ] ], + "mandatory_dims" : [ "D_YEAR" ], + "joint_dims" : [ ] + } + } ], + "notify_list" : [ ], + "status_need_notify" : [ ], + "partition_date_start" : 3153000000000, + "partition_date_end" : 3153600000000, + "auto_merge_time_ranges" : [ 604800000, 2419200000 ], + "retention_range" : 0, + "engine_type" : 2, + "storage_type" : 2, + "override_kylin_properties" : { + "kylin.hbase.default.compression.codec" : "lz4", + "kylin.cube.aggrgroup.isMandatoryOnlyValid" : "true" + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/kylin/blob/d7196d20/examples/test_case_data/localmeta/cube_desc/ssb_cube3.json ---------------------------------------------------------------------- diff --git a/examples/test_case_data/localmeta/cube_desc/ssb_cube3.json b/examples/test_case_data/localmeta/cube_desc/ssb_cube3.json new file mode 100644 index 0000000..4509116 --- /dev/null +++ b/examples/test_case_data/localmeta/cube_desc/ssb_cube3.json @@ -0,0 +1,143 @@ +{ + "uuid" : "5c44df30-daec-486e-af90-927bf7851060", + "name" : "ssb_cube3", + "description" : "", + "dimensions" : [ { + "name" : "SSB.PART_DERIVED", + "table" : "SSB.PART", + "column" : null, + "derived" : [ "P_MFGR", "P_CATEGORY", "P_BRAND" ] + }, { + "name" : "S_CITY", + "table" : "SSB.SUPPLIER", + "column" : "S_CITY", + "derived" : null + }, { + "name" : "S_REGION", + "table" : "SSB.SUPPLIER", + "column" : "S_REGION", + "derived" : null + }, { + "name" : "S_NATION", + "table" : "SSB.SUPPLIER", + "column" : "S_NATION", + "derived" : null + }, { + "name" : "D_YEAR", + "table" : "SSB.DATES", + "column" : "D_YEAR", + "derived" : null + }, { + "name" : "D_YEARMONTH", + "table" : "SSB.DATES", + "column" : "D_YEARMONTH", + "derived" : null + }, { + "name" : "D_YEARMONTHNUM", + "table" : "SSB.DATES", + "column" : "D_YEARMONTHNUM", + "derived" : null + }, { + "name" : "D_WEEKNUMINYEAR", + "table" : "SSB.DATES", + "column" : "D_WEEKNUMINYEAR", + "derived" : null + } ], + "measures" : [ { + "name" : "_COUNT_", + "function" : { + "expression" : "COUNT", + "parameter" : { + "type" : "constant", + "value" : "1", + "next_parameter" : null + }, + "returntype" : "bigint" + }, + "dependent_measure_ref" : null + }, { + "name" : "TOTAL_REVENUE", + "function" : { + "expression" : "SUM", + "parameter" : { + "type" : "column", + "value" : "LO_REVENUE", + "next_parameter" : null + }, + "returntype" : "bigint" + }, + "dependent_measure_ref" : null + }, { + "name" : "TOTAL_SUPPLYCOST", + "function" : { + "expression" : "SUM", + "parameter" : { + "type" : "column", + "value" : "LO_SUPPLYCOST", + "next_parameter" : null + }, + "returntype" : "bigint" + }, + "dependent_measure_ref" : null + } ], + "rowkey" : { + "rowkey_columns" : [ { + "column" : "LO_PARTKEY", + "encoding" : "dict" + }, { + "column" : "S_CITY", + "encoding" : "dict" + }, { + "column" : "S_REGION", + "encoding" : "dict" + }, { + "column" : "S_NATION", + "encoding" : "dict" + }, { + "column" : "D_YEAR", + "encoding" : "dict" + }, { + "column" : "D_YEARMONTH", + "encoding" : "dict" + }, { + "column" : "D_YEARMONTHNUM", + "encoding" : "dict" + }, { + "column" : "D_WEEKNUMINYEAR", + "encoding" : "dict" + } ] + }, + "signature" : "", + "last_modified" : 1457503036686, + "model_name" : "ssb", + "null_string" : null, + "hbase_mapping" : { + "column_family" : [ { + "name" : "F1", + "columns" : [ { + "qualifier" : "M", + "measure_refs" : [ "_COUNT_", "TOTAL_REVENUE", "TOTAL_SUPPLYCOST"] + } ] + } ] + }, + "aggregation_groups" : [ { + "includes" : [ "LO_PARTKEY", "S_CITY", "S_REGION", "S_NATION", "D_YEAR", "D_YEARMONTH", "D_YEARMONTHNUM", "D_WEEKNUMINYEAR" ], + "select_rule" : { + "hierarchy_dims" : [ [ "S_REGION", "S_NATION", "S_CITY" ], [ "D_YEARMONTH", "D_YEARMONTHNUM", "D_WEEKNUMINYEAR" ] ], + "mandatory_dims" : [ "D_YEAR" ], + "joint_dims" : [ ] + } + } ], + "notify_list" : [ ], + "status_need_notify" : [ ], + "partition_date_start" : 3153000000000, + "partition_date_end" : 3153600000000, + "auto_merge_time_ranges" : [ 604800000, 2419200000 ], + "retention_range" : 0, + "engine_type" : 2, + "storage_type" : 2, + "override_kylin_properties" : { + "kylin.hbase.default.compression.codec" : "lz4", + "kylin.cube.aggrgroup.isMandatoryOnlyValid" : "true" + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/kylin/blob/d7196d20/server-base/src/main/java/org/apache/kylin/rest/service/ModelService.java ---------------------------------------------------------------------- diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/ModelService.java b/server-base/src/main/java/org/apache/kylin/rest/service/ModelService.java index 2f8667c..7dd7b0c 100644 --- a/server-base/src/main/java/org/apache/kylin/rest/service/ModelService.java +++ b/server-base/src/main/java/org/apache/kylin/rest/service/ModelService.java @@ -84,7 +84,7 @@ public class ModelService extends BasicService { public DataModelDesc createModelDesc(String projectName, DataModelDesc desc) throws IOException { if (getMetadataManager().getDataModelDesc(desc.getName()) != null) { - throw new InternalErrorException("The model named " + desc.getName() + " already exists"); + throw new InternalErrorException("Model name " + desc.getName() + "is duplicated, could not create"); } DataModelDesc createdDesc = null; String owner = SecurityContextHolder.getContext().getAuthentication().getName(); @@ -110,7 +110,7 @@ public class ModelService extends BasicService { List<CubeDesc> cubeDescs = getCubeDescManager().listAllDesc(); for (CubeDesc cubeDesc : cubeDescs) { if (cubeDesc.getModelName().equals(desc.getName())) { - throw new InternalErrorException("Model referenced by cube,drop cubes under model and try again."); + throw new InternalErrorException("Model is referenced by Cube: " + cubeDesc.getName() + " , could not drop"); } } http://git-wip-us.apache.org/repos/asf/kylin/blob/d7196d20/tool/pom.xml ---------------------------------------------------------------------- diff --git a/tool/pom.xml b/tool/pom.xml index 213f05c..8dba3f7 100644 --- a/tool/pom.xml +++ b/tool/pom.xml @@ -81,6 +81,8 @@ <shadedClassifierName>assembly</shadedClassifierName> <artifactSet> <includes> + <!-- shade the httpcore to avoid the lower version conflict with HBase one --> + <include>org.apache.httpcomponents:httpcore</include> <include>org.apache.kylin:kylin-tool</include> </includes> </artifactSet> http://git-wip-us.apache.org/repos/asf/kylin/blob/d7196d20/tool/src/main/java/org/apache/kylin/tool/HybridCubeCLI.java ---------------------------------------------------------------------- diff --git a/tool/src/main/java/org/apache/kylin/tool/HybridCubeCLI.java b/tool/src/main/java/org/apache/kylin/tool/HybridCubeCLI.java new file mode 100644 index 0000000..8320213 --- /dev/null +++ b/tool/src/main/java/org/apache/kylin/tool/HybridCubeCLI.java @@ -0,0 +1,206 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kylin.tool; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import org.apache.commons.cli.Option; +import org.apache.commons.cli.OptionBuilder; +import org.apache.commons.cli.Options; +import org.apache.commons.lang3.StringUtils; +import org.apache.kylin.common.KylinConfig; +import org.apache.kylin.common.persistence.ResourceStore; +import org.apache.kylin.common.util.AbstractApplication; +import org.apache.kylin.common.util.OptionsHelper; +import org.apache.kylin.cube.CubeInstance; +import org.apache.kylin.cube.CubeManager; +import org.apache.kylin.cube.CubeSegment; +import org.apache.kylin.metadata.MetadataManager; +import org.apache.kylin.metadata.model.DataModelDesc; +import org.apache.kylin.metadata.project.ProjectManager; +import org.apache.kylin.metadata.project.RealizationEntry; +import org.apache.kylin.metadata.realization.RealizationType; +import org.apache.kylin.storage.hybrid.HybridInstance; +import org.apache.kylin.storage.hybrid.HybridManager; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * 1. Create new HybridCube + * bin/kylin.sh org.apache.kylin.tool.HybridCubeCLI -action create -name hybrid_name -project project_name -model model_name -cubes cube1,cube2 + * 2. Update existing HybridCube + * bin/kylin.sh org.apache.kylin.tool.HybridCubeCLI -action update -name hybrid_name -project project_name -model model_name -cubes cube1,cube2,cube3 + * 3. Delete the HybridCube + * bin/kylin.sh org.apache.kylin.tool.HybridCubeCLI -action delete -name hybrid_name -project project_name -model model_name + */ +public class HybridCubeCLI extends AbstractApplication { + + private static final Logger logger = LoggerFactory.getLogger(HybridCubeCLI.class); + + private static final Option OPTION_ACTION = OptionBuilder.withArgName("action").hasArg().isRequired(true).withDescription("create/update/delete").create("action"); + + private static final Option OPTION_HYBRID_NAME = OptionBuilder.withArgName("name").hasArg().isRequired(true).withDescription("HybridCube name").create("name"); + + private static final Option OPTION_PROJECT = OptionBuilder.withArgName("project").hasArg().isRequired(true).withDescription("the target project for the hybrid cube").create("project"); + + private static final Option OPTION_MODEL = OptionBuilder.withArgName("model").hasArg().isRequired(true).withDescription("the target model for the hybrid cube").create("model"); + + private static final Option OPTION_CUBES = OptionBuilder.withArgName("cubes").hasArg().isRequired(false).withDescription("the cubes used in HybridCube, seperated by comma, empty if to delete HybridCube").create("cubes"); + + private final Options options; + + private KylinConfig kylinConfig; + private CubeManager cubeManager; + private HybridManager hybridManager; + private MetadataManager metadataManager; + private ResourceStore store; + + public HybridCubeCLI() { + options = new Options(); + options.addOption(OPTION_ACTION); + options.addOption(OPTION_HYBRID_NAME); + options.addOption(OPTION_PROJECT); + options.addOption(OPTION_MODEL); + options.addOption(OPTION_CUBES); + + this.kylinConfig = KylinConfig.getInstanceFromEnv(); + this.store = ResourceStore.getStore(kylinConfig); + this.cubeManager = CubeManager.getInstance(kylinConfig); + this.hybridManager = HybridManager.getInstance(kylinConfig); + this.metadataManager = MetadataManager.getInstance(kylinConfig); + } + + public static void main(String[] args) { + HybridCubeCLI cli = new HybridCubeCLI(); + cli.execute(args); + } + + @Override + protected Options getOptions() { + return options; + } + + @Override + protected void execute(OptionsHelper optionsHelper) throws Exception { + String action = optionsHelper.getOptionValue(OPTION_ACTION); + String hybridName = optionsHelper.getOptionValue(OPTION_HYBRID_NAME); + String projectName = optionsHelper.getOptionValue(OPTION_PROJECT); + String modelName = optionsHelper.getOptionValue(OPTION_MODEL); + String cubeNamesStr = optionsHelper.getOptionValue(OPTION_CUBES); + String[] cubeNames = new String[] {}; + if (cubeNamesStr != null) + cubeNames = cubeNamesStr.split(","); + String owner = null; + + DataModelDesc modelDesc = metadataManager.getDataModelDesc(modelName); + if (modelDesc == null) { + throw new RuntimeException("Could not find model: " + modelName); + } + + List<RealizationEntry> realizationEntries = new ArrayList<RealizationEntry>(); + for (String cubeName : cubeNames) { + if (StringUtils.isEmpty(cubeName)) + continue; + CubeInstance cube = cubeManager.getCube(cubeName); + if (cube == null) { + throw new RuntimeException("Could not find cube: " + cubeName); + } + if (owner == null) { + owner = cube.getOwner(); + } + realizationEntries.add(RealizationEntry.create(RealizationType.CUBE, cube.getName())); + } + + HybridInstance hybridInstance = hybridManager.getHybridInstance(hybridName); + if ("create".equals(action)) { + if (hybridInstance != null) { + throw new RuntimeException("The Hybrid Cube does exist, could not create: " + hybridName); + } + //Create new Hybrid + create(hybridName, realizationEntries, projectName, owner); + } else if ("update".equals(action)) { + if (hybridInstance == null) { + throw new RuntimeException("The Hybrid Cube doesn't exist, could not update: " + hybridName); + } + // Update the Hybrid + update(hybridInstance, realizationEntries, projectName, owner); + } else if ("delete".equals(action)) { + if (hybridInstance == null) { + throw new RuntimeException("The Hybrid Cube doesn't exist, could not delete: " + hybridName); + } + // Delete the Hybrid + delete(hybridInstance); + } + + } + + private HybridInstance create(String hybridName, List<RealizationEntry> realizationEntries, String projectName, String owner) throws IOException { + checkSegmentOffset(realizationEntries); + HybridInstance hybridInstance = HybridInstance.create(kylinConfig, hybridName, realizationEntries); + store.putResource(hybridInstance.getResourcePath(), hybridInstance, HybridManager.HYBRID_SERIALIZER); + ProjectManager.getInstance(kylinConfig).moveRealizationToProject(RealizationType.HYBRID, hybridInstance.getName(), projectName, owner); + hybridManager.reloadHybridInstance(hybridName); + logger.info("HybridInstance was created at: " + hybridInstance.getResourcePath()); + return hybridInstance; + } + + private void update(HybridInstance hybridInstance, List<RealizationEntry> realizationEntries, String projectName, String owner) throws IOException { + checkSegmentOffset(realizationEntries); + hybridInstance.setRealizationEntries(realizationEntries); + store.putResource(hybridInstance.getResourcePath(), hybridInstance, HybridManager.HYBRID_SERIALIZER); + ProjectManager.getInstance(kylinConfig).moveRealizationToProject(RealizationType.HYBRID, hybridInstance.getName(), projectName, owner); + hybridManager.reloadHybridInstance(hybridInstance.getName()); + logger.info("HybridInstance was updated at: " + hybridInstance.getResourcePath()); + } + + private void delete(HybridInstance hybridInstance) throws IOException { + ProjectManager.getInstance(kylinConfig).removeRealizationsFromProjects(RealizationType.HYBRID, hybridInstance.getName()); + store.deleteResource(hybridInstance.getResourcePath()); + hybridManager.reloadAllHybridInstance(); + logger.info("HybridInstance was deleted at: " + hybridInstance.getResourcePath()); + } + + private void checkSegmentOffset(List<RealizationEntry> realizationEntries) { + if (realizationEntries == null || realizationEntries.size() == 0) + throw new RuntimeException("No realization found"); + if (realizationEntries.size() == 1) + throw new RuntimeException("Hybrid needs at least 2 cubes"); + long lastOffset = -1; + for (RealizationEntry entry : realizationEntries) { + if (entry.getType() != RealizationType.CUBE) { + throw new RuntimeException("Wrong realization type: " + entry.getType() + ", only cube supported. "); + } + + CubeInstance cubeInstance = cubeManager.getCube(entry.getRealization()); + CubeSegment segment = cubeInstance.getLastSegment(); + if (segment == null) + continue; + if (lastOffset == -1) { + lastOffset = segment.getSourceOffsetEnd(); + } else { + if (lastOffset > segment.getSourceOffsetStart()) { + throw new RuntimeException("Segments has overlap, could not hybrid. Last Segment End: " + lastOffset + ", Next Segment Start: " + segment.getSourceOffsetStart()); + } + lastOffset = segment.getSourceOffsetEnd(); + } + } + } +} http://git-wip-us.apache.org/repos/asf/kylin/blob/d7196d20/tool/src/main/java/org/apache/kylin/tool/MrJobInfoExtractor.java ---------------------------------------------------------------------- diff --git a/tool/src/main/java/org/apache/kylin/tool/MrJobInfoExtractor.java b/tool/src/main/java/org/apache/kylin/tool/MrJobInfoExtractor.java index 9dd4c81..1050bbe 100644 --- a/tool/src/main/java/org/apache/kylin/tool/MrJobInfoExtractor.java +++ b/tool/src/main/java/org/apache/kylin/tool/MrJobInfoExtractor.java @@ -26,15 +26,16 @@ import java.util.regex.Pattern; import org.apache.commons.cli.Option; import org.apache.commons.cli.OptionBuilder; -import org.apache.commons.httpclient.HttpClient; -import org.apache.commons.httpclient.HttpMethod; -import org.apache.commons.httpclient.methods.GetMethod; import org.apache.commons.io.FileUtils; import org.apache.commons.lang.StringUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.yarn.conf.HAUtil; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.util.RMHAUtils; +import org.apache.http.HttpResponse; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.impl.client.DefaultHttpClient; +import org.apache.http.util.EntityUtils; import org.apache.kylin.common.KylinConfig; import org.apache.kylin.common.util.OptionsHelper; import org.apache.kylin.engine.mr.HadoopUtil; @@ -94,24 +95,24 @@ public class MrJobInfoExtractor extends AbstractInfoExtractor { } private String getHttpResponse(String url) { - HttpClient client = new HttpClient(); - String response = null; + DefaultHttpClient client = new DefaultHttpClient(); + String msg = null; int retry_times = 0; - while (response == null && retry_times < HTTP_RETRY) { + while (msg == null && retry_times < HTTP_RETRY) { retry_times++; - HttpMethod get = new GetMethod(url); + HttpGet request = new HttpGet(url); try { - get.addRequestHeader("accept", "application/json"); - client.executeMethod(get); - response = get.getResponseBodyAsString(); + request.addHeader("accept", "application/json"); + HttpResponse response = client.execute(request); + msg = EntityUtils.toString(response.getEntity()); } catch (Exception e) { logger.warn("Failed to fetch http response. Retry={}", retry_times, e); } finally { - get.releaseConnection(); + request.releaseConnection(); } } - return response; + return msg; } private void extractTaskCounter(String taskId, File exportDir, String taskUrl) throws IOException { http://git-wip-us.apache.org/repos/asf/kylin/blob/d7196d20/tool/src/test/java/org/apache/kylin/tool/HybridCubeCLITest.java ---------------------------------------------------------------------- diff --git a/tool/src/test/java/org/apache/kylin/tool/HybridCubeCLITest.java b/tool/src/test/java/org/apache/kylin/tool/HybridCubeCLITest.java new file mode 100644 index 0000000..5734d0c --- /dev/null +++ b/tool/src/test/java/org/apache/kylin/tool/HybridCubeCLITest.java @@ -0,0 +1,93 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kylin.tool; + +import java.io.IOException; + +import org.apache.kylin.common.KylinConfig; +import org.apache.kylin.common.util.LocalFileMetadataTestCase; +import org.apache.kylin.metadata.cachesync.Broadcaster; +import org.apache.kylin.metadata.project.ProjectManager; +import org.apache.kylin.metadata.realization.RealizationType; +import org.apache.kylin.storage.hybrid.HybridInstance; +import org.apache.kylin.storage.hybrid.HybridManager; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.FixMethodOrder; +import org.junit.Test; +import org.junit.runners.MethodSorters; + +@FixMethodOrder(MethodSorters.NAME_ASCENDING) +public class HybridCubeCLITest extends LocalFileMetadataTestCase { + + @Before + public void setUp() throws Exception { + this.createTestMetadata(); + } + + @After + public void after() throws Exception { + this.cleanupTestMetadata(); + } + + @Test + public void test1Create() throws IOException { + HybridManager hybridManager = HybridManager.getInstance(KylinConfig.getInstanceFromEnv()); + Assert.assertNull(hybridManager.getHybridInstance("ssb_hybrid")); + HybridCubeCLI.main(new String[] { "-name", "ssb_hybrid", "-project", "default", "-model", "ssb", "-cubes", "ssb_cube1,ssb_cube2", "-action", "create" }); + + HybridInstance hybridInstance = hybridManager.getHybridInstance("ssb_hybrid"); + Assert.assertNotNull(hybridInstance); + Assert.assertEquals("ssb_hybrid", hybridInstance.getName()); + Assert.assertEquals(2, hybridInstance.getRealizationEntries().size()); + } + + @Test + public void test2Update() throws IOException { + HybridManager hybridManager = HybridManager.getInstance(KylinConfig.getInstanceFromEnv()); + Assert.assertNull(hybridManager.getHybridInstance("ssb_hybrid")); + HybridCubeCLI.main(new String[] { "-name", "ssb_hybrid", "-project", "default", "-model", "ssb", "-cubes", "ssb_cube1,ssb_cube2", "-action", "create" }); + + HybridInstance hybridInstance = hybridManager.getHybridInstance("ssb_hybrid"); + Assert.assertNotNull(hybridManager.getHybridInstance("ssb_hybrid")); + Assert.assertEquals("ssb_hybrid", hybridInstance.getName()); + Assert.assertEquals(2, hybridInstance.getRealizationEntries().size()); + HybridCubeCLI.main(new String[] { "-name", "ssb_hybrid", "-project", "default", "-model", "ssb", "-cubes", "ssb_cube1,ssb_cube2,ssb_cube3", "-action", "update" }); + + hybridInstance = hybridManager.getHybridInstance("ssb_hybrid"); + Assert.assertNotNull(hybridInstance); + Assert.assertEquals("ssb_hybrid", hybridInstance.getName()); + Assert.assertEquals(3, hybridInstance.getRealizationEntries().size()); + } + + @Test + public void test3Delete() throws IOException { + HybridManager hybridManager = HybridManager.getInstance(KylinConfig.getInstanceFromEnv()); + Assert.assertNull(hybridManager.getHybridInstance("ssb_hybrid")); + HybridCubeCLI.main(new String[] { "-name", "ssb_hybrid", "-project", "default", "-model", "ssb", "-cubes", "ssb_cube1,ssb_cube2", "-action", "create" }); + Assert.assertNotNull(hybridManager.getHybridInstance("ssb_hybrid")); + HybridCubeCLI.main(new String[] { "-name", "ssb_hybrid", "-project", "default", "-model", "ssb", "-action", "delete" }); + + HybridInstance hybridInstance = hybridManager.getHybridInstance("ssb_hybrid"); + Assert.assertNull(hybridInstance); + Assert.assertEquals(0, ProjectManager.getInstance(KylinConfig.getInstanceFromEnv()).findProjects(RealizationType.HYBRID, "ssb_hybrid").size()); + } + +}