kylin git commit: KYLIN-1591 enabling different block size setting for small column families
Repository: kylin Updated Branches: refs/heads/master 41014d6a5 -> 22e6969cb KYLIN-1591 enabling different block size setting for small column families Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/22e6969c Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/22e6969c Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/22e6969c Branch: refs/heads/master Commit: 22e6969cb30f2146b23a9c81701b8a39dba15fd4 Parents: 41014d6 Author: Hongbin Ma Authored: Mon Apr 18 15:40:05 2016 +0800 Committer: Hongbin Ma Committed: Mon Apr 18 15:40:54 2016 +0800 -- .../java/org/apache/kylin/common/KylinConfigBase.java | 10 +++--- .../apache/kylin/cube/model/HBaseColumnFamilyDesc.java | 13 + .../java/org/apache/kylin/gridtable/GTScanRequest.java | 3 ++- .../kylin/storage/hbase/steps/CubeHTableUtil.java | 9 ++--- 4 files changed, 28 insertions(+), 7 deletions(-) -- http://git-wip-us.apache.org/repos/asf/kylin/blob/22e6969c/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java -- diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java index 51aa8aa..4c7995b 100644 --- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java +++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java @@ -489,7 +489,7 @@ abstract public class KylinConfigBase implements Serializable { } public int getQueryScanFuzzyKeyMax() { -return Integer.parseInt(this.getOptional("kylin.query.scan.fuzzykey.max","200")); +return Integer.parseInt(this.getOptional("kylin.query.scan.fuzzykey.max", "200")); } public int getQueryStorageVisitScanRangeMax() { @@ -558,8 +558,12 @@ abstract public class KylinConfigBase implements Serializable { return getOptional("kylin.hbase.default.encoding", "FAST_DIFF"); } -public String getHbaseDefaultBlockSize() { -return getOptional("kylin.hbase.default.block.size", "4194304"); +public int getHbaseDefaultBlockSize() { +return Integer.valueOf(getOptional("kylin.hbase.default.block.size", "1048576")); +} + +public int getHbaseSmallFamilyBlockSize() { +return Integer.valueOf(getOptional("kylin.hbase.small.family.block.size", "65536")); } public boolean isHiveKeepFlatTable() { http://git-wip-us.apache.org/repos/asf/kylin/blob/22e6969c/core-cube/src/main/java/org/apache/kylin/cube/model/HBaseColumnFamilyDesc.java -- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/HBaseColumnFamilyDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/HBaseColumnFamilyDesc.java index 5db2a9b..c5b2e19 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/model/HBaseColumnFamilyDesc.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/model/HBaseColumnFamilyDesc.java @@ -20,6 +20,8 @@ package org.apache.kylin.cube.model; import java.util.Arrays; +import org.apache.kylin.metadata.model.MeasureDesc; + import com.fasterxml.jackson.annotation.JsonAutoDetect; import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility; import com.fasterxml.jackson.annotation.JsonProperty; @@ -55,4 +57,15 @@ public class HBaseColumnFamilyDesc { return "HBaseColumnFamilyDesc [name=" + name + ", columns=" + Arrays.toString(columns) + "]"; } +public boolean isMemoryHungry() { +for (HBaseColumnDesc hBaseColumnDesc : columns) { +for (MeasureDesc measureDesc : hBaseColumnDesc.getMeasures()) { +if (measureDesc.getFunction().getMeasureType().isMemoryHungry()) { +return true; +} +} +} +return false; +} + } http://git-wip-us.apache.org/repos/asf/kylin/blob/22e6969c/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java -- diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java index 5681057..bc478c9 100644 --- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java +++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java @@ -161,7 +161,8 @@ public class GTScanRequest { } if (!doAggr) {//Skip reading this section if you're not profiling! -int scanned = lookAndForget(result); +int scanned = result.getScannedRowCount(); +lookAndForget(result); return new Em
[3/3] kylin git commit: KYLIN-1591 enabling different block size setting for small column families
KYLIN-1591 enabling different block size setting for small column families Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/3aad1fc6 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/3aad1fc6 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/3aad1fc6 Branch: refs/heads/yang-m1 Commit: 3aad1fc60cc0278833d4b5e0e5f2db02788cd3ba Parents: ad09341 Author: Hongbin Ma Authored: Mon Apr 18 15:40:05 2016 +0800 Committer: Hongbin Ma Committed: Mon Apr 18 15:51:31 2016 +0800 -- .../java/org/apache/kylin/common/KylinConfigBase.java | 10 +++--- .../apache/kylin/cube/model/HBaseColumnFamilyDesc.java | 13 + .../java/org/apache/kylin/gridtable/GTScanRequest.java | 3 ++- .../kylin/storage/hbase/steps/CubeHTableUtil.java | 9 ++--- 4 files changed, 28 insertions(+), 7 deletions(-) -- http://git-wip-us.apache.org/repos/asf/kylin/blob/3aad1fc6/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java -- diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java index 94d8f62..23f0fa7 100644 --- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java +++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java @@ -481,7 +481,7 @@ abstract public class KylinConfigBase implements Serializable { } public int getQueryScanFuzzyKeyMax() { -return Integer.parseInt(this.getOptional("kylin.query.scan.fuzzykey.max","200")); +return Integer.parseInt(this.getOptional("kylin.query.scan.fuzzykey.max", "200")); } public int getHBaseKeyValueSize() { @@ -546,8 +546,12 @@ abstract public class KylinConfigBase implements Serializable { return getOptional("kylin.hbase.default.encoding", "FAST_DIFF"); } -public String getHbaseDefaultBlockSize() { -return getOptional("kylin.hbase.default.block.size", "4194304"); +public int getHbaseDefaultBlockSize() { +return Integer.valueOf(getOptional("kylin.hbase.default.block.size", "1048576")); +} + +public int getHbaseSmallFamilyBlockSize() { +return Integer.valueOf(getOptional("kylin.hbase.small.family.block.size", "65536")); } public boolean isHiveKeepFlatTable() { http://git-wip-us.apache.org/repos/asf/kylin/blob/3aad1fc6/core-cube/src/main/java/org/apache/kylin/cube/model/HBaseColumnFamilyDesc.java -- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/HBaseColumnFamilyDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/HBaseColumnFamilyDesc.java index 5db2a9b..c5b2e19 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/model/HBaseColumnFamilyDesc.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/model/HBaseColumnFamilyDesc.java @@ -20,6 +20,8 @@ package org.apache.kylin.cube.model; import java.util.Arrays; +import org.apache.kylin.metadata.model.MeasureDesc; + import com.fasterxml.jackson.annotation.JsonAutoDetect; import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility; import com.fasterxml.jackson.annotation.JsonProperty; @@ -55,4 +57,15 @@ public class HBaseColumnFamilyDesc { return "HBaseColumnFamilyDesc [name=" + name + ", columns=" + Arrays.toString(columns) + "]"; } +public boolean isMemoryHungry() { +for (HBaseColumnDesc hBaseColumnDesc : columns) { +for (MeasureDesc measureDesc : hBaseColumnDesc.getMeasures()) { +if (measureDesc.getFunction().getMeasureType().isMemoryHungry()) { +return true; +} +} +} +return false; +} + } http://git-wip-us.apache.org/repos/asf/kylin/blob/3aad1fc6/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java -- diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java index 97900c1..befa866 100644 --- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java +++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java @@ -153,7 +153,8 @@ public class GTScanRequest { } if (!doAggr) {//Skip reading this section if you're not profiling! -int scanned = lookAndForget(result); +int scanned = result.getScannedRowCount(); +lookAndForget(result); return new EmptyGTScanner(scanned); } http://git-wip-us.apache.org/repos/asf/kylin/blob/3
[1/3] kylin git commit: stop testing storage type 0 as it is no longer used
Repository: kylin Updated Branches: refs/heads/yang-m1 2f8df5160 -> 3aad1fc60 stop testing storage type 0 as it is no longer used Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/372d4335 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/372d4335 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/372d4335 Branch: refs/heads/yang-m1 Commit: 372d4335a0ef6cf913e68beaf599528f43eeb37c Parents: 2f8df51 Author: Hongbin Ma Authored: Mon Apr 18 11:27:45 2016 +0800 Committer: Hongbin Ma Committed: Mon Apr 18 15:49:04 2016 +0800 -- .../cube_desc/test_kylin_cube_with_slr_left_join_desc.json | 2 +- .../localmeta/cube_desc/test_kylin_cube_without_slr_desc.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) -- http://git-wip-us.apache.org/repos/asf/kylin/blob/372d4335/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_slr_left_join_desc.json -- diff --git a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_slr_left_join_desc.json b/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_slr_left_join_desc.json index bb59a05..9d82ac3 100644 --- a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_slr_left_join_desc.json +++ b/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_slr_left_join_desc.json @@ -166,6 +166,6 @@ "auto_merge_time_ranges" : null, "retention_range" : 0, "engine_type" : 0, - "storage_type" : 0, + "storage_type" : 2, "partition_date_start" : 0 } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/kylin/blob/372d4335/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_desc.json -- diff --git a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_desc.json b/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_desc.json index e1d740a..3741b22 100644 --- a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_desc.json +++ b/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_desc.json @@ -330,6 +330,6 @@ "auto_merge_time_ranges": null, "retention_range": 0, "engine_type": 2, - "storage_type": 0, + "storage_type": 2, "partition_date_start": 0 }
[2/3] kylin git commit: stop testing storage type 0 as it is no longer used
stop testing storage type 0 as it is no longer used Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/ad09341c Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/ad09341c Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/ad09341c Branch: refs/heads/yang-m1 Commit: ad09341ca038c61351517d894920c64632bd5f5b Parents: 372d433 Author: Hongbin Ma Authored: Mon Apr 18 13:56:35 2016 +0800 Committer: Hongbin Ma Committed: Mon Apr 18 15:51:14 2016 +0800 -- .../java/org/apache/kylin/cube/common/RowKeySplitterTest.java | 6 +++--- .../test/java/org/apache/kylin/cube/kv/RowKeyDecoderTest.java | 2 +- .../test/java/org/apache/kylin/cube/kv/RowKeyEncoderTest.java | 3 +-- .../cube_desc/test_kylin_cube_with_slr_left_join_desc.json | 2 +- 4 files changed, 6 insertions(+), 7 deletions(-) -- http://git-wip-us.apache.org/repos/asf/kylin/blob/ad09341c/core-cube/src/test/java/org/apache/kylin/cube/common/RowKeySplitterTest.java -- diff --git a/core-cube/src/test/java/org/apache/kylin/cube/common/RowKeySplitterTest.java b/core-cube/src/test/java/org/apache/kylin/cube/common/RowKeySplitterTest.java index 1ddc745..b41f23b 100644 --- a/core-cube/src/test/java/org/apache/kylin/cube/common/RowKeySplitterTest.java +++ b/core-cube/src/test/java/org/apache/kylin/cube/common/RowKeySplitterTest.java @@ -48,7 +48,7 @@ public class RowKeySplitterTest extends LocalFileMetadataTestCase { RowKeySplitter rowKeySplitter = new RowKeySplitter(cube.getFirstSegment(), 11, 20); // base cuboid rowkey -byte[] input = { 0, 0, 0, 0, 0, 0, 0, 0, 1, -1, 0,-104,-106,-128, 11, 54, -105, 55,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9, 13, 71, 114, 65, 66, 73, 78, 9, 9, 9, 9, 9, 9, 9, 9, 0, 10, 0 }; +byte[] input = { 0, 0, 0, 0, 0, 0, 0, 0, 1, -1, 0, -104, -106, -128, 11, 54, -105, 55, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 13, 71, 114, 65, 66, 73, 78, 9, 9, 9, 9, 9, 9, 9, 9, 0, 10, 0 }; rowKeySplitter.split(input); assertEquals(11, rowKeySplitter.getBufferSize()); @@ -61,9 +61,9 @@ public class RowKeySplitterTest extends LocalFileMetadataTestCase { RowKeySplitter rowKeySplitter = new RowKeySplitter(cube.getFirstSegment(), 11, 20); // base cuboid rowkey -byte[] input = { 0, 0, 0, 0, 0, 0, 0, -1, 11, 55, -13, 13, 22, 34, 121, 70, 80, 45, 71, 84, 67, 9, 9, 9, 9, 9, 9, 0, 10, 5 }; +byte[] input = { 0, 0, 0, 0, 0, 0, 0, 0, 0, -1, 11, 55, -13, 13, 22, 34, 121, 70, 80, 45, 71, 84, 67, 9, 9, 9, 9, 9, 9, 0, 10, 5 }; rowKeySplitter.split(input); -assertEquals(9, rowKeySplitter.getBufferSize()); +assertEquals(10, rowKeySplitter.getBufferSize()); } } http://git-wip-us.apache.org/repos/asf/kylin/blob/ad09341c/core-cube/src/test/java/org/apache/kylin/cube/kv/RowKeyDecoderTest.java -- diff --git a/core-cube/src/test/java/org/apache/kylin/cube/kv/RowKeyDecoderTest.java b/core-cube/src/test/java/org/apache/kylin/cube/kv/RowKeyDecoderTest.java index 0fe37f7..43498d3 100644 --- a/core-cube/src/test/java/org/apache/kylin/cube/kv/RowKeyDecoderTest.java +++ b/core-cube/src/test/java/org/apache/kylin/cube/kv/RowKeyDecoderTest.java @@ -53,7 +53,7 @@ public class RowKeyDecoderTest extends LocalFileMetadataTestCase { RowKeyDecoder rowKeyDecoder = new RowKeyDecoder(cube.getFirstSegment()); -byte[] key = { 0, 0, 0, 0, 0, 0, 0, -1, 11, 55, -13, 13, 22, 34, 121, 70, 80, 45, 71, 84, 67, 9, 9, 9, 9, 9, 9, 0, 10, 5 }; +byte[] key = { 0, 0, 0, 0, 0, 0, 0, 0, 0, -1, 11, 55, -13, 13, 22, 34, 121, 70, 80, 45, 71, 84, 67, 9, 9, 9, 9, 9, 9, 0, 10, 5 }; rowKeyDecoder.decode(key); List values = rowKeyDecoder.getValues(); http://git-wip-us.apache.org/repos/asf/kylin/blob/ad09341c/core-cube/src/test/java/org/apache/kylin/cube/kv/RowKeyEncoderTest.java -- diff --git a/core-cube/src/test/java/org/apache/kylin/cube/kv/RowKeyEncoderTest.java b/core-cube/src/test/java/org/apache/kylin/cube/kv/RowKeyEncoderTest.java index 1a6b4dd..ba71387 100644 --- a/core-cube/src/test/java/org/apache/kylin/cube/kv/RowKeyEncoderTest.java +++ b/core-cube/src/test/java/org/apache/kylin/cube/kv/RowKeyEncoderTest.java @@ -70,13 +70,12 @@ public class RowKeyEncoderTest extends LocalFileMetadataTestCase { byte[] encodedKey = rowKeyEncoder.encode(data); assertEquals(22 + rowKeyEncoder.getHeaderLength(), encodedKey.length); -byte[] cuboidId = Arrays.copyOfRange(encodedKey, 0, rowKeyEncoder.getHeaderLength()); +byte[] cuboidId = Arrays.copyOfRange(encodedKey, RowCon
kylin git commit: KYLIN-1458: run CubeMigrationCheckCLI under the condition with parameter 'realExecute' set to true
Repository: kylin Updated Branches: refs/heads/master 22e6969cb -> f0473fcaf KYLIN-1458: run CubeMigrationCheckCLI under the condition with parameter 'realExecute' set to true Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/f0473fca Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/f0473fca Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/f0473fca Branch: refs/heads/master Commit: f0473fcafe37ecf5caaf977d105c5bb7c1654c56 Parents: 22e6969 Author: Zhong Authored: Mon Apr 18 17:11:11 2016 +0800 Committer: Zhong Committed: Mon Apr 18 17:11:11 2016 +0800 -- .../org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) -- http://git-wip-us.apache.org/repos/asf/kylin/blob/f0473fca/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java -- diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java index 09aab48..2de7f01 100644 --- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java +++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java @@ -138,11 +138,10 @@ public class CubeMigrationCLI { if (realExecute.equalsIgnoreCase("true")) { doOpts(); +checkMigrationSuccess(dstConfig, cubeName, true); } else { showOpts(); } - -checkMigrationSuccess(dstConfig, cubeName, true); } public static void moveCube(String srcCfgUri, String dstCfgUri, String cubeName, String projectName, String copyAcl, String purgeAndDisable, String overwriteIfExists, String realExecute) throws IOException, InterruptedException {
kylin git commit: Add Env and Project Info in job email notification
Repository: kylin Updated Branches: refs/heads/1.4-rc 08b2051f8 -> 85113c11f Add Env and Project Info in job email notification Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/85113c11 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/85113c11 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/85113c11 Branch: refs/heads/1.4-rc Commit: 85113c11f443c8ee702c5a8d201b926349e80306 Parents: 08b2051 Author: Zhong Authored: Mon Apr 18 17:39:57 2016 +0800 Committer: Zhong Committed: Mon Apr 18 17:39:57 2016 +0800 -- .../apache/kylin/common/KylinConfigBase.java| 11 ++-- .../kylin/job/constant/ExecutableConstants.java | 2 +- .../org/apache/kylin/engine/mr/CubingJob.java | 56 3 files changed, 52 insertions(+), 17 deletions(-) -- http://git-wip-us.apache.org/repos/asf/kylin/blob/85113c11/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java -- diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java index 9bbcbb4..6f1fccc 100644 --- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java +++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java @@ -173,10 +173,7 @@ public class KylinConfigBase implements Serializable { if (!root.endsWith("/")) { root += "/"; } -return new StringBuffer(root) -.append(StringUtils.replaceChars(getMetadataUrlPrefix(), ':', '-')) -.append("/") -.toString(); +return new StringBuffer(root).append(StringUtils.replaceChars(getMetadataUrlPrefix(), ':', '-')).append("/").toString(); } public CliCommandExecutor getCliCommandExecutor() throws IOException { @@ -485,7 +482,7 @@ public class KylinConfigBase implements Serializable { public boolean isGetJobStatusWithKerberos() { return Boolean.valueOf(this.getOptional("kylin.job.status.with.kerberos", "false")); } - + public boolean isTransformPathToMasterNN() { return Boolean.valueOf(this.getOptional("kylin.transform.hdfs.path.enable", "false")); } @@ -526,6 +523,10 @@ public class KylinConfigBase implements Serializable { return getOptional("mail.sender", ""); } +public String getDeployEnv() { +return getOptional("deploy.env", "DEV"); +} + public String toString() { return getMetadataUrl(); } http://git-wip-us.apache.org/repos/asf/kylin/blob/85113c11/core-job/src/main/java/org/apache/kylin/job/constant/ExecutableConstants.java -- diff --git a/core-job/src/main/java/org/apache/kylin/job/constant/ExecutableConstants.java b/core-job/src/main/java/org/apache/kylin/job/constant/ExecutableConstants.java index d370b0d..ab2fed7 100644 --- a/core-job/src/main/java/org/apache/kylin/job/constant/ExecutableConstants.java +++ b/core-job/src/main/java/org/apache/kylin/job/constant/ExecutableConstants.java @@ -75,5 +75,5 @@ public final class ExecutableConstants { public static final String PROP_JOB_KILLED = "jobKilled"; public static final String PROP_JOB_RUNTIME_FLOWS = "jobFlows"; -public static final String NOTIFY_EMAIL_TEMPLATE = "Build Result of Job ${job_name}" + "Build Result: ${result}" + "Job Engine: ${job_engine}" + "Cube Name: ${cube_name}" + "Source Records Count: ${source_records_count}" + "Start Time: ${start_time}" + "Duration: ${duration}" + "MR Waiting: ${mr_waiting}" + "Last Update Time: ${last_update_time}" + "Submitter: ${submitter}" + "Error Log: ${error_log}" + ""; +public static final String NOTIFY_EMAIL_TEMPLATE = "Build Result of Job ${job_name}" + "Build Result: ${result}" + "Job Engine: ${job_engine}" + "Env: ${env_name}" + "Project: ${project_name}" + "Cube Name: ${cube_name}" + "Source Records Count: ${source_records_count}" + "Start Time: ${start_time}" + "Duration: ${duration}" + "MR Waiting: ${mr_waiting}" + "Last Update Time: ${last_update_time}" + "Submitter: ${submitter}" + "Error Log: ${error_log}" + ""; } http://git-wip-us.apache.org/repos/asf/kylin/blob/85113c11/engine-mr/src/main/java/org/apache/kylin/engine/mr/CubingJob.java -- diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/CubingJob.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/CubingJob.java index 979ff75..1353fe8 100644 --- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/CubingJob.java +++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/CubingJob.java @@ -30,6 +30,7 @@ import java.
kylin git commit: KYLIN-1595 columns in metric could also be in filter/groupby
Repository: kylin Updated Branches: refs/heads/master f0473fcaf -> 78ff89470 KYLIN-1595 columns in metric could also be in filter/groupby Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/78ff8947 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/78ff8947 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/78ff8947 Branch: refs/heads/master Commit: 78ff89470a9f774af3a6d9d189393a67f823d483 Parents: f0473fc Author: Hongbin Ma Authored: Mon Apr 18 18:27:05 2016 +0800 Committer: Hongbin Ma Committed: Mon Apr 18 18:27:09 2016 +0800 -- .../localmeta/cube_desc/test_kylin_cube_with_slr_desc.json | 2 +- .../org/apache/kylin/storage/hbase/cube/v2/CubeStorageQuery.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) -- http://git-wip-us.apache.org/repos/asf/kylin/blob/78ff8947/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_slr_desc.json -- diff --git a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_slr_desc.json b/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_slr_desc.json index 417eb5c..7db40ec 100644 --- a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_slr_desc.json +++ b/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_slr_desc.json @@ -158,7 +158,7 @@ "includes" : [ "cal_dt", "categ_lvl2_name", "categ_lvl3_name", "leaf_categ_id", "lstg_format_name", "lstg_site_id", "meta_categ_name", "seller_id", "slr_segment_cd" ], "select_rule" : { "hierarchy_dims" : [ [ "META_CATEG_NAME", "CATEG_LVL2_NAME", "CATEG_LVL3_NAME" ] ], - "mandatory_dims" : [ "seller_id" ], + "mandatory_dims" : [], "joint_dims" : [ [ "lstg_format_name", "lstg_site_id", "slr_segment_cd" ] ] } } ], http://git-wip-us.apache.org/repos/asf/kylin/blob/78ff8947/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeStorageQuery.java -- diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeStorageQuery.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeStorageQuery.java index e0e6d83..b765798 100644 --- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeStorageQuery.java +++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeStorageQuery.java @@ -136,7 +136,7 @@ public class CubeStorageQuery implements IStorageQuery { for (TblColRef column : sqlDigest.allColumns) { // skip measure columns -if (sqlDigest.metricColumns.contains(column)) { +if (sqlDigest.metricColumns.contains(column) && !(sqlDigest.groupbyColumns.contains(column) || sqlDigest.filterColumns.contains(column))) { continue; }
kylin git commit: KYLIN-1595 columns in metric could also be in filter/groupby
Repository: kylin Updated Branches: refs/heads/yang-m1 3aad1fc60 -> ae51cf5cf KYLIN-1595 columns in metric could also be in filter/groupby Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/ae51cf5c Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/ae51cf5c Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/ae51cf5c Branch: refs/heads/yang-m1 Commit: ae51cf5cf924e1d7c374c96b224eea8e3bd371e6 Parents: 3aad1fc Author: Hongbin Ma Authored: Mon Apr 18 18:27:05 2016 +0800 Committer: Hongbin Ma Committed: Mon Apr 18 18:28:35 2016 +0800 -- .../localmeta/cube_desc/test_kylin_cube_with_slr_desc.json | 2 +- .../org/apache/kylin/storage/hbase/cube/v2/CubeStorageQuery.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) -- http://git-wip-us.apache.org/repos/asf/kylin/blob/ae51cf5c/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_slr_desc.json -- diff --git a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_slr_desc.json b/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_slr_desc.json index 417eb5c..7db40ec 100644 --- a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_slr_desc.json +++ b/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_slr_desc.json @@ -158,7 +158,7 @@ "includes" : [ "cal_dt", "categ_lvl2_name", "categ_lvl3_name", "leaf_categ_id", "lstg_format_name", "lstg_site_id", "meta_categ_name", "seller_id", "slr_segment_cd" ], "select_rule" : { "hierarchy_dims" : [ [ "META_CATEG_NAME", "CATEG_LVL2_NAME", "CATEG_LVL3_NAME" ] ], - "mandatory_dims" : [ "seller_id" ], + "mandatory_dims" : [], "joint_dims" : [ [ "lstg_format_name", "lstg_site_id", "slr_segment_cd" ] ] } } ], http://git-wip-us.apache.org/repos/asf/kylin/blob/ae51cf5c/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeStorageQuery.java -- diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeStorageQuery.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeStorageQuery.java index 863dd67..577b2e6 100644 --- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeStorageQuery.java +++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeStorageQuery.java @@ -136,7 +136,7 @@ public class CubeStorageQuery implements IStorageQuery { for (TblColRef column : sqlDigest.allColumns) { // skip measure columns -if (sqlDigest.metricColumns.contains(column)) { +if (sqlDigest.metricColumns.contains(column) && !(sqlDigest.groupbyColumns.contains(column) || sqlDigest.filterColumns.contains(column))) { continue; }
kylin git commit: KYLIN-1595 refine test case
Repository: kylin Updated Branches: refs/heads/master 78ff89470 -> 96167fc3e KYLIN-1595 refine test case Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/96167fc3 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/96167fc3 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/96167fc3 Branch: refs/heads/master Commit: 96167fc3ed249500aa1c6bad9f464a8464dd07a1 Parents: 78ff894 Author: Hongbin Ma Authored: Mon Apr 18 18:56:26 2016 +0800 Committer: Hongbin Ma Committed: Mon Apr 18 18:56:26 2016 +0800 -- .../org/apache/kylin/gridtable/AggregationCacheSpillTest.java| 4 ++-- .../apache/kylin/job/impl/threadpool/DefaultSchedulerTest.java | 2 ++ .../localmeta/cube_desc/test_kylin_cube_with_slr_desc.json | 2 +- .../cube_desc/test_kylin_cube_with_slr_left_join_desc.json | 2 +- 4 files changed, 6 insertions(+), 4 deletions(-) -- http://git-wip-us.apache.org/repos/asf/kylin/blob/96167fc3/core-cube/src/test/java/org/apache/kylin/gridtable/AggregationCacheSpillTest.java -- diff --git a/core-cube/src/test/java/org/apache/kylin/gridtable/AggregationCacheSpillTest.java b/core-cube/src/test/java/org/apache/kylin/gridtable/AggregationCacheSpillTest.java index 0e399dd..73e4908 100644 --- a/core-cube/src/test/java/org/apache/kylin/gridtable/AggregationCacheSpillTest.java +++ b/core-cube/src/test/java/org/apache/kylin/gridtable/AggregationCacheSpillTest.java @@ -88,7 +88,7 @@ public class AggregationCacheSpillTest { assertEquals(21, ((BigDecimal) returnRecord[4]).longValue()); count++; -System.out.println(record); +//System.out.println(record); } assertEquals(DATA_CARDINALITY, count); scanner.close(); @@ -130,7 +130,7 @@ public class AggregationCacheSpillTest { assertEquals(84000, ((BigDecimal) returnRecord[4]).longValue()); count++; -System.out.println(record); +//System.out.println(record); } assertEquals(10, count); scanner.close(); http://git-wip-us.apache.org/repos/asf/kylin/blob/96167fc3/core-job/src/test/java/org/apache/kylin/job/impl/threadpool/DefaultSchedulerTest.java -- diff --git a/core-job/src/test/java/org/apache/kylin/job/impl/threadpool/DefaultSchedulerTest.java b/core-job/src/test/java/org/apache/kylin/job/impl/threadpool/DefaultSchedulerTest.java index df521f9..6acbbe5 100644 --- a/core-job/src/test/java/org/apache/kylin/job/impl/threadpool/DefaultSchedulerTest.java +++ b/core-job/src/test/java/org/apache/kylin/job/impl/threadpool/DefaultSchedulerTest.java @@ -37,10 +37,12 @@ import org.apache.kylin.job.SucceedTestExecutable; import org.apache.kylin.job.execution.DefaultChainedExecutable; import org.apache.kylin.job.execution.ExecutableState; import org.junit.Assert; +import org.junit.Ignore; import org.junit.Test; /** */ +@Ignore public class DefaultSchedulerTest extends BaseSchedulerTest { @Test http://git-wip-us.apache.org/repos/asf/kylin/blob/96167fc3/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_slr_desc.json -- diff --git a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_slr_desc.json b/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_slr_desc.json index 7db40ec..4064fcb 100644 --- a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_slr_desc.json +++ b/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_slr_desc.json @@ -158,7 +158,7 @@ "includes" : [ "cal_dt", "categ_lvl2_name", "categ_lvl3_name", "leaf_categ_id", "lstg_format_name", "lstg_site_id", "meta_categ_name", "seller_id", "slr_segment_cd" ], "select_rule" : { "hierarchy_dims" : [ [ "META_CATEG_NAME", "CATEG_LVL2_NAME", "CATEG_LVL3_NAME" ] ], - "mandatory_dims" : [], + "mandatory_dims" : ["seller_id"], "joint_dims" : [ [ "lstg_format_name", "lstg_site_id", "slr_segment_cd" ] ] } } ], http://git-wip-us.apache.org/repos/asf/kylin/blob/96167fc3/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_slr_left_join_desc.json -- diff --git a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_slr_left_join_desc.json b/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_slr_left_join_desc.json index 33a7080..f0fe9c0 100644 --- a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_slr_left_join_desc.json +++ b/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_slr_l
kylin git commit: minor, update documentation link on README.md
Repository: kylin Updated Branches: refs/heads/master 96167fc3e -> 59477501c minor, update documentation link on README.md Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/59477501 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/59477501 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/59477501 Branch: refs/heads/master Commit: 59477501cf6e82806309d50017ed121219e29b76 Parents: 96167fc Author: lidongsjtu Authored: Mon Apr 18 21:47:23 2016 +0800 Committer: lidongsjtu Committed: Mon Apr 18 21:47:23 2016 +0800 -- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/kylin/blob/59477501/README.md -- diff --git a/README.md b/README.md index 657212f..0c136ef 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@ For more details, see the website [http://kylin.apache.org](http://kylin.apache. Documentation = -Please refer to [http://kylin.apache.org/docs/](http://kylin.apache.org/docs/). +Please refer to [http://kylin.apache.org/docs15/](http://kylin.apache.org/docs15/). Get Help
kylin git commit: KYLIN-1595 refine test case
Repository: kylin Updated Branches: refs/heads/yang-m1 ae51cf5cf -> 41fb735de KYLIN-1595 refine test case KYLIN-1595 add mass distinct count query Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/41fb735d Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/41fb735d Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/41fb735d Branch: refs/heads/yang-m1 Commit: 41fb735de8521864659e384e5a0117c06da09f80 Parents: ae51cf5 Author: Hongbin Ma Authored: Mon Apr 18 18:56:26 2016 +0800 Committer: Hongbin Ma Committed: Mon Apr 18 22:28:10 2016 +0800 -- .../gridtable/AggregationCacheSpillTest.java| 4 +-- .../test_kylin_cube_with_slr_desc.json | 2 +- ...test_kylin_cube_with_slr_left_join_desc.json | 2 +- .../apache/kylin/query/ITMassInQueryTest.java | 27 - .../query/sql_massin_distinct/query01.sql | 31 5 files changed, 61 insertions(+), 5 deletions(-) -- http://git-wip-us.apache.org/repos/asf/kylin/blob/41fb735d/core-cube/src/test/java/org/apache/kylin/gridtable/AggregationCacheSpillTest.java -- diff --git a/core-cube/src/test/java/org/apache/kylin/gridtable/AggregationCacheSpillTest.java b/core-cube/src/test/java/org/apache/kylin/gridtable/AggregationCacheSpillTest.java index 0e399dd..73e4908 100644 --- a/core-cube/src/test/java/org/apache/kylin/gridtable/AggregationCacheSpillTest.java +++ b/core-cube/src/test/java/org/apache/kylin/gridtable/AggregationCacheSpillTest.java @@ -88,7 +88,7 @@ public class AggregationCacheSpillTest { assertEquals(21, ((BigDecimal) returnRecord[4]).longValue()); count++; -System.out.println(record); +//System.out.println(record); } assertEquals(DATA_CARDINALITY, count); scanner.close(); @@ -130,7 +130,7 @@ public class AggregationCacheSpillTest { assertEquals(84000, ((BigDecimal) returnRecord[4]).longValue()); count++; -System.out.println(record); +//System.out.println(record); } assertEquals(10, count); scanner.close(); http://git-wip-us.apache.org/repos/asf/kylin/blob/41fb735d/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_slr_desc.json -- diff --git a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_slr_desc.json b/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_slr_desc.json index 7db40ec..4064fcb 100644 --- a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_slr_desc.json +++ b/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_slr_desc.json @@ -158,7 +158,7 @@ "includes" : [ "cal_dt", "categ_lvl2_name", "categ_lvl3_name", "leaf_categ_id", "lstg_format_name", "lstg_site_id", "meta_categ_name", "seller_id", "slr_segment_cd" ], "select_rule" : { "hierarchy_dims" : [ [ "META_CATEG_NAME", "CATEG_LVL2_NAME", "CATEG_LVL3_NAME" ] ], - "mandatory_dims" : [], + "mandatory_dims" : ["seller_id"], "joint_dims" : [ [ "lstg_format_name", "lstg_site_id", "slr_segment_cd" ] ] } } ], http://git-wip-us.apache.org/repos/asf/kylin/blob/41fb735d/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_slr_left_join_desc.json -- diff --git a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_slr_left_join_desc.json b/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_slr_left_join_desc.json index 33a7080..f0fe9c0 100644 --- a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_slr_left_join_desc.json +++ b/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_slr_left_join_desc.json @@ -157,7 +157,7 @@ "includes" : [ "cal_dt", "categ_lvl2_name", "categ_lvl3_name", "leaf_categ_id", "lstg_format_name", "lstg_site_id", "meta_categ_name", "seller_id", "slr_segment_cd" ], "select_rule" : { "hierarchy_dims" : [ [ "META_CATEG_NAME", "CATEG_LVL2_NAME", "CATEG_LVL3_NAME" ] ], - "mandatory_dims" : [ "seller_id" ], + "mandatory_dims" : [ ], "joint_dims" : [ [ "lstg_format_name", "lstg_site_id", "slr_segment_cd" ] ] } } ], http://git-wip-us.apache.org/repos/asf/kylin/blob/41fb735d/kylin-it/src/test/java/org/apache/kylin/query/ITMassInQueryTest.java -- diff --git a/kylin-it/src/test/java/org/apache/kylin/query/ITMassInQueryTest.java b/kylin-it/src/test/java/org/apache/kylin/query/ITMassInQueryTest.java index a92e298..5f2db43 100644 --- a/kylin-it/src/test/java/or
kylin git commit: KYLIN-1595 add mass distinct count query
Repository: kylin Updated Branches: refs/heads/master 59477501c -> 205471998 KYLIN-1595 add mass distinct count query Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/20547199 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/20547199 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/20547199 Branch: refs/heads/master Commit: 205471998d551bf5bc1988e0da73f5490f0a622c Parents: 5947750 Author: Hongbin Ma Authored: Mon Apr 18 22:24:44 2016 +0800 Committer: Hongbin Ma Committed: Mon Apr 18 22:24:52 2016 +0800 -- .../impl/threadpool/DefaultSchedulerTest.java | 2 -- .../apache/kylin/query/ITMassInQueryTest.java | 27 - .../query/sql_massin_distinct/query01.sql | 31 3 files changed, 57 insertions(+), 3 deletions(-) -- http://git-wip-us.apache.org/repos/asf/kylin/blob/20547199/core-job/src/test/java/org/apache/kylin/job/impl/threadpool/DefaultSchedulerTest.java -- diff --git a/core-job/src/test/java/org/apache/kylin/job/impl/threadpool/DefaultSchedulerTest.java b/core-job/src/test/java/org/apache/kylin/job/impl/threadpool/DefaultSchedulerTest.java index 6acbbe5..df521f9 100644 --- a/core-job/src/test/java/org/apache/kylin/job/impl/threadpool/DefaultSchedulerTest.java +++ b/core-job/src/test/java/org/apache/kylin/job/impl/threadpool/DefaultSchedulerTest.java @@ -37,12 +37,10 @@ import org.apache.kylin.job.SucceedTestExecutable; import org.apache.kylin.job.execution.DefaultChainedExecutable; import org.apache.kylin.job.execution.ExecutableState; import org.junit.Assert; -import org.junit.Ignore; import org.junit.Test; /** */ -@Ignore public class DefaultSchedulerTest extends BaseSchedulerTest { @Test http://git-wip-us.apache.org/repos/asf/kylin/blob/20547199/kylin-it/src/test/java/org/apache/kylin/query/ITMassInQueryTest.java -- diff --git a/kylin-it/src/test/java/org/apache/kylin/query/ITMassInQueryTest.java b/kylin-it/src/test/java/org/apache/kylin/query/ITMassInQueryTest.java index a92e298..5f2db43 100644 --- a/kylin-it/src/test/java/org/apache/kylin/query/ITMassInQueryTest.java +++ b/kylin-it/src/test/java/org/apache/kylin/query/ITMassInQueryTest.java @@ -63,7 +63,7 @@ public class ITMassInQueryTest extends KylinTestBase { public void setup() throws Exception { ITKylinQueryTest.clean(); -ITKylinQueryTest.joinType = "inner"; +ITKylinQueryTest.joinType = "left"; ITKylinQueryTest.setupAll(); Configuration hconf = HadoopUtil.getCurrentConfiguration(); @@ -96,6 +96,31 @@ public class ITMassInQueryTest extends KylinTestBase { compare("src/test/resources/query/sql_massin", null, true); } +@Test +public void testMassInWithDistinctCount() throws Exception { +run("src/test/resources/query/sql_massin_distinct", null, true); +} + +protected void run(String queryFolder, String[] exclusiveQuerys, boolean needSort) throws Exception { +printInfo("-- test folder: " + queryFolder); +Set exclusiveSet = buildExclusiveSet(exclusiveQuerys); + +List sqlFiles = getFilesFromFolder(new File(queryFolder), ".sql"); +for (File sqlFile : sqlFiles) { +String queryName = StringUtils.split(sqlFile.getName(), '.')[0]; +if (exclusiveSet.contains(queryName)) { +continue; +} +String sql = getTextFromFile(sqlFile); + +// execute Kylin +printInfo("Query Result from Kylin - " + queryName + " (" + queryFolder + ")"); +IDatabaseConnection kylinConn = new DatabaseConnection(cubeConnection); +ITable kylinTable = executeQuery(kylinConn, queryName, sql, needSort); + +} +} + protected void compare(String queryFolder, String[] exclusiveQuerys, boolean needSort) throws Exception { printInfo("-- test folder: " + queryFolder); Set exclusiveSet = buildExclusiveSet(exclusiveQuerys); http://git-wip-us.apache.org/repos/asf/kylin/blob/20547199/kylin-it/src/test/resources/query/sql_massin_distinct/query01.sql -- diff --git a/kylin-it/src/test/resources/query/sql_massin_distinct/query01.sql b/kylin-it/src/test/resources/query/sql_massin_distinct/query01.sql new file mode 100644 index 000..70afa87 --- /dev/null +++ b/kylin-it/src/test/resources/query/sql_massin_distinct/query01.sql @@ -0,0 +1,31 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional inf
[1/2] kylin git commit: KYLIN-1587 show cube level configuration overwrites properties in CubeDesigner
Repository: kylin Updated Branches: refs/heads/yang-m1 41fb735de -> aef842442 KYLIN-1587 show cube level configuration overwrites properties in CubeDesigner Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/aef84244 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/aef84244 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/aef84244 Branch: refs/heads/yang-m1 Commit: aef842442a7d7e2ee1ad931a376b36dd36157ac5 Parents: 209aa66 Author: Jason Authored: Mon Apr 18 22:50:32 2016 +0800 Committer: Jason Committed: Mon Apr 18 22:50:51 2016 +0800 -- webapp/app/index.html | 1 + webapp/app/js/controllers/cubeOverwriteProp.js | 71 webapp/app/js/controllers/cubeSchema.js | 49 +++--- .../cubeDesigner/cubeOverwriteProp.html | 70 +++ 4 files changed, 166 insertions(+), 25 deletions(-) -- http://git-wip-us.apache.org/repos/asf/kylin/blob/aef84244/webapp/app/index.html -- diff --git a/webapp/app/index.html b/webapp/app/index.html index 82cef34..a25f812 100644 --- a/webapp/app/index.html +++ b/webapp/app/index.html @@ -179,6 +179,7 @@ + http://git-wip-us.apache.org/repos/asf/kylin/blob/aef84244/webapp/app/js/controllers/cubeOverwriteProp.js -- diff --git a/webapp/app/js/controllers/cubeOverwriteProp.js b/webapp/app/js/controllers/cubeOverwriteProp.js new file mode 100644 index 000..25e99d5 --- /dev/null +++ b/webapp/app/js/controllers/cubeOverwriteProp.js @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +'use strict'; + +KylinApp.controller('CubeOverWriteCtrl', function ($scope, $modal,cubeConfig,MetaModel,cubesManager,CubeDescModel) { + $scope.cubesManager = cubesManager; + + + //rowkey + $scope.convertedProperties = []; + + for(var key in $scope.cubeMetaFrame.override_kylin_properties){ +$scope.convertedProperties.push({ + name:key, + value:$scope.cubeMetaFrame.override_kylin_properties[key] +}); + } + + + $scope.addNewProperty = function () { +if($scope.cubeMetaFrame.override_kylin_properties.hasOwnProperty('')){ + return; +} +$scope.cubeMetaFrame.override_kylin_properties['']=''; +$scope.convertedProperties.push({ + name:'', + value:'' +}); + + }; + + $scope.refreshPropertiesObj = function(){ +$scope.cubeMetaFrame.override_kylin_properties = {}; +angular.forEach($scope.convertedProperties,function(item,index){ + $scope.cubeMetaFrame.override_kylin_properties[item.name] = item.value; +}) + } + + + $scope.refreshProperty = function(list,index,item){ +$scope.convertedProperties[index] = item; +$scope.refreshPropertiesObj(); + } + + + $scope.removeProperty= function(arr,index,item){ +if (index > -1) { + arr.splice(index, 1); +} +delete $scope.cubeMetaFrame.override_kylin_properties[item.name]; + } + + + +}); http://git-wip-us.apache.org/repos/asf/kylin/blob/aef84244/webapp/app/js/controllers/cubeSchema.js -- diff --git a/webapp/app/js/controllers/cubeSchema.js b/webapp/app/js/controllers/cubeSchema.js index 618e14e..f9d9dc1 100755 --- a/webapp/app/js/controllers/cubeSchema.js +++ b/webapp/app/js/controllers/cubeSchema.js @@ -35,6 +35,7 @@ KylinApp.controller('CubeSchemaCtrl', function ($scope, QueryService, UserServic $scope.wizardSteps.push({title: 'Advanced Setting', src: 'partials/cubeDesigner/advanced_settings.html', isComplete: false,form:'cube_setting_form'}); } //$scope.wizardSteps.push({title: 'Streaming', src: 'partials/cubeDesigner/streamingConfig.html', isComplete: false,form:'cube_streaming_form'}); +$scope.wizardSteps.push({title: 'Configuration Overwrites ', src: 'partials/cubeDesigner/cubeOverwriteProp.html', isComplete: false,form:'cube_overwrite_prop_form'})
kylin git commit: KYLIN-1587 show cube level configuration overwrites properties in CubeDesigner
Repository: kylin Updated Branches: refs/heads/master 205471998 -> 0320d1f78 KYLIN-1587 show cube level configuration overwrites properties in CubeDesigner Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/0320d1f7 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/0320d1f7 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/0320d1f7 Branch: refs/heads/master Commit: 0320d1f78f834bd941c7d531009a9434007254a6 Parents: 2054719 Author: Jason Authored: Mon Apr 18 22:53:15 2016 +0800 Committer: Jason Committed: Mon Apr 18 22:53:38 2016 +0800 -- webapp/app/index.html | 1 + webapp/app/js/controllers/cubeOverwriteProp.js | 71 webapp/app/js/controllers/cubeSchema.js | 49 +++--- .../cubeDesigner/cubeOverwriteProp.html | 70 +++ 4 files changed, 166 insertions(+), 25 deletions(-) -- http://git-wip-us.apache.org/repos/asf/kylin/blob/0320d1f7/webapp/app/index.html -- diff --git a/webapp/app/index.html b/webapp/app/index.html index 82cef34..a25f812 100644 --- a/webapp/app/index.html +++ b/webapp/app/index.html @@ -179,6 +179,7 @@ + http://git-wip-us.apache.org/repos/asf/kylin/blob/0320d1f7/webapp/app/js/controllers/cubeOverwriteProp.js -- diff --git a/webapp/app/js/controllers/cubeOverwriteProp.js b/webapp/app/js/controllers/cubeOverwriteProp.js new file mode 100644 index 000..25e99d5 --- /dev/null +++ b/webapp/app/js/controllers/cubeOverwriteProp.js @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +'use strict'; + +KylinApp.controller('CubeOverWriteCtrl', function ($scope, $modal,cubeConfig,MetaModel,cubesManager,CubeDescModel) { + $scope.cubesManager = cubesManager; + + + //rowkey + $scope.convertedProperties = []; + + for(var key in $scope.cubeMetaFrame.override_kylin_properties){ +$scope.convertedProperties.push({ + name:key, + value:$scope.cubeMetaFrame.override_kylin_properties[key] +}); + } + + + $scope.addNewProperty = function () { +if($scope.cubeMetaFrame.override_kylin_properties.hasOwnProperty('')){ + return; +} +$scope.cubeMetaFrame.override_kylin_properties['']=''; +$scope.convertedProperties.push({ + name:'', + value:'' +}); + + }; + + $scope.refreshPropertiesObj = function(){ +$scope.cubeMetaFrame.override_kylin_properties = {}; +angular.forEach($scope.convertedProperties,function(item,index){ + $scope.cubeMetaFrame.override_kylin_properties[item.name] = item.value; +}) + } + + + $scope.refreshProperty = function(list,index,item){ +$scope.convertedProperties[index] = item; +$scope.refreshPropertiesObj(); + } + + + $scope.removeProperty= function(arr,index,item){ +if (index > -1) { + arr.splice(index, 1); +} +delete $scope.cubeMetaFrame.override_kylin_properties[item.name]; + } + + + +}); http://git-wip-us.apache.org/repos/asf/kylin/blob/0320d1f7/webapp/app/js/controllers/cubeSchema.js -- diff --git a/webapp/app/js/controllers/cubeSchema.js b/webapp/app/js/controllers/cubeSchema.js index 618e14e..f9d9dc1 100755 --- a/webapp/app/js/controllers/cubeSchema.js +++ b/webapp/app/js/controllers/cubeSchema.js @@ -35,6 +35,7 @@ KylinApp.controller('CubeSchemaCtrl', function ($scope, QueryService, UserServic $scope.wizardSteps.push({title: 'Advanced Setting', src: 'partials/cubeDesigner/advanced_settings.html', isComplete: false,form:'cube_setting_form'}); } //$scope.wizardSteps.push({title: 'Streaming', src: 'partials/cubeDesigner/streamingConfig.html', isComplete: false,form:'cube_streaming_form'}); +$scope.wizardSteps.push({title: 'Configuration Overwrites ', src: 'partials/cubeDesigner/cubeOverwriteProp.html', isComplete: false,form:'cube_overwrite_prop_form'});
[2/2] kylin git commit: KYLIN-1569 Select any column when adding a custom aggregation in GUI
KYLIN-1569 Select any column when adding a custom aggregation in GUI Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/209aa665 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/209aa665 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/209aa665 Branch: refs/heads/yang-m1 Commit: 209aa6657f595020e1201b2d6e79bd549fe91284 Parents: 41fb735 Author: Jason Authored: Mon Apr 18 12:17:02 2016 +0800 Committer: Jason Committed: Mon Apr 18 22:50:51 2016 +0800 -- webapp/app/js/controllers/cubeEdit.js | 25 ++--- webapp/app/js/controllers/cubeMeasures.js | 4 ++-- webapp/app/partials/cubeDesigner/measures.html | 16 - 3 files changed, 18 insertions(+), 27 deletions(-) -- http://git-wip-us.apache.org/repos/asf/kylin/blob/209aa665/webapp/app/js/controllers/cubeEdit.js -- diff --git a/webapp/app/js/controllers/cubeEdit.js b/webapp/app/js/controllers/cubeEdit.js index b34393a..2cae518 100755 --- a/webapp/app/js/controllers/cubeEdit.js +++ b/webapp/app/js/controllers/cubeEdit.js @@ -88,8 +88,9 @@ KylinApp.controller('CubeEditCtrl', function ($scope, $q, $routeParams, $locatio return avaColObject; }; - //SUM,COUNT,MAX,MIN - $scope.getCommonMetricColumns = function () { + $scope.getCommonMetricColumns = function (measure) { +var nonCustomMeasures = ['SUM','MIN','MAX','COUNT']; +var expression = measure.function.expression; //metric from model var me_columns = []; @@ -98,18 +99,9 @@ KylinApp.controller('CubeEditCtrl', function ($scope, $q, $routeParams, $locatio me_columns.push(metric); }) } -return me_columns; - }; - //COUNT_DISTINCT,TopN - $scope.getMetricColumns = function () { - -//metric from model -var me_columns = []; -if($scope.metaModel.model.metrics){ - angular.forEach($scope.metaModel.model.metrics,function(metric,index){ -me_columns.push(metric); - }) +if(nonCustomMeasures.indexOf(expression)!==-1){ + return me_columns; } //add cube dimension column for specific measure @@ -117,12 +109,19 @@ KylinApp.controller('CubeEditCtrl', function ($scope, $q, $routeParams, $locatio if(dimension.column && dimension.derived == null){ me_columns.push(dimension.column); } + + if(dimension.derived&&dimension.derived.length>=1){ +me_columns = me_columns.concat(dimension.derived); + } + }); return me_columns; + }; + $scope.getColumnType = function (_column, table) { var columns = $scope.getColumnsByTable(table); var type; http://git-wip-us.apache.org/repos/asf/kylin/blob/209aa665/webapp/app/js/controllers/cubeMeasures.js -- diff --git a/webapp/app/js/controllers/cubeMeasures.js b/webapp/app/js/controllers/cubeMeasures.js index 11cb1eb..8693bd1 100644 --- a/webapp/app/js/controllers/cubeMeasures.js +++ b/webapp/app/js/controllers/cubeMeasures.js @@ -179,8 +179,8 @@ var NextParameterModalCtrl = function ($scope, scope,para,$modalInstance,cubeCon $modalInstance.dismiss('cancel'); }; - $scope. getMetricColumns = function(){ -return scope.getMetricColumns(); + $scope.getCommonMetricColumns = function(measure){ +return scope.getCommonMetricColumns(measure); } $scope.nextPara = { http://git-wip-us.apache.org/repos/asf/kylin/blob/209aa665/webapp/app/partials/cubeDesigner/measures.html -- diff --git a/webapp/app/partials/cubeDesigner/measures.html b/webapp/app/partials/cubeDesigner/measures.html index 6014a78..f0ae86f 100755 --- a/webapp/app/partials/cubeDesigner/measures.html +++ b/webapp/app/partials/cubeDesigner/measures.html @@ -146,18 +146,10 @@ ng-init="newMeasure.function.parameter.value = 1"> 1 - -- Select a Column -- - - - +ng-options="column as column for column in getCommonMetricColumns(newMeasure)" > -- Select a Column -- @@ -313,7 +305,7 @@ +ng-options="column as column for column in getCommonMetricColumns(newmea.measure)" > +ng-options="column as column for column in getCommonMetricColumns(newmea.measure)" >
kylin git commit: minor, remove outdated information from pom
Repository: kylin Updated Branches: refs/heads/master 0320d1f78 -> ef36d6cb4 minor, remove outdated information from pom Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/ef36d6cb Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/ef36d6cb Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/ef36d6cb Branch: refs/heads/master Commit: ef36d6cb49dc4d970f53af123419ea302c11c4f3 Parents: 0320d1f Author: lidongsjtu Authored: Mon Apr 18 21:56:41 2016 +0800 Committer: lidongsjtu Committed: Mon Apr 18 23:21:29 2016 +0800 -- pom.xml | 1 - 1 file changed, 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/kylin/blob/ef36d6cb/pom.xml -- diff --git a/pom.xml b/pom.xml index c223b12..3b6c184 100644 --- a/pom.xml +++ b/pom.xml @@ -169,7 +169,6 @@ scm:git:https://git-wip-us.apache.org/repos/asf/kylin.git scm:git:https://git-wip-us.apache.org/repos/asf/kylin.git https://git-wip-us.apache.org/repos/asf/kylin.git -apache-kylin-2.1
[2/2] kylin git commit: KYLIN-1592: fix the bug of missing cube statistics while migrating cube
KYLIN-1592: fix the bug of missing cube statistics while migrating cube Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/01d63946 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/01d63946 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/01d63946 Branch: refs/heads/1.4-rc Commit: 01d6394686bb5a2cdc62d536e3ffb5b387248d19 Parents: 5adc113 Author: Zhong Authored: Tue Apr 19 01:24:50 2016 +0800 Committer: Zhong Committed: Tue Apr 19 01:24:50 2016 +0800 -- .../java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java | 1 + 1 file changed, 1 insertion(+) -- http://git-wip-us.apache.org/repos/asf/kylin/blob/01d63946/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java -- diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java index ce2b135..a84 100644 --- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java +++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java @@ -243,6 +243,7 @@ public class CubeMigrationCLI { } for (CubeSegment segment : cube.getSegments()) { +metaResource.add(segment.getStatisticsResourcePath()); dictAndSnapshot.addAll(segment.getSnapshotPaths()); dictAndSnapshot.addAll(segment.getDictionaryPaths()); }
[1/2] kylin git commit: KYLIN-1592: copy all of the cube segment statistics after an old version of migration
Repository: kylin Updated Branches: refs/heads/1.4-rc 85113c11f -> 01d639468 KYLIN-1592: copy all of the cube segment statistics after an old version of migration Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/5adc1139 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/5adc1139 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/5adc1139 Branch: refs/heads/1.4-rc Commit: 5adc1139b390d333f81cde57d53abd403027802b Parents: 85113c1 Author: Zhong Authored: Tue Apr 19 01:16:08 2016 +0800 Committer: Zhong Committed: Tue Apr 19 01:16:08 2016 +0800 -- .../hbase/util/CubeMigrationSupplementCLI.java | 141 +++ 1 file changed, 141 insertions(+) -- http://git-wip-us.apache.org/repos/asf/kylin/blob/5adc1139/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationSupplementCLI.java -- diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationSupplementCLI.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationSupplementCLI.java new file mode 100644 index 000..e16fe58 --- /dev/null +++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationSupplementCLI.java @@ -0,0 +1,141 @@ +package org.apache.kylin.storage.hbase.util; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import org.apache.commons.cli.Option; +import org.apache.commons.cli.OptionBuilder; +import org.apache.commons.cli.Options; +import org.apache.commons.cli.ParseException; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HBaseConfiguration; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.HBaseAdmin; +import org.apache.kylin.common.KylinConfig; +import org.apache.kylin.common.persistence.RawResource; +import org.apache.kylin.common.persistence.ResourceStore; +import org.apache.kylin.common.util.OptionsHelper; +import org.apache.kylin.cube.CubeInstance; +import org.apache.kylin.cube.CubeManager; +import org.apache.kylin.cube.CubeSegment; +import org.apache.kylin.cube.model.CubeDesc; +import org.apache.kylin.metadata.model.DataModelDesc; +import org.apache.kylin.metadata.model.SegmentStatusEnum; +import org.apache.kylin.metadata.model.TableDesc; +import org.apache.kylin.metadata.realization.IRealizationConstants; +import org.apache.kylin.metadata.realization.RealizationStatusEnum; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.common.collect.Lists; + +/** + * + * This tool serves for the purpose of + * checking the "KYLIN_HOST" property to be consistent with the dst's MetadataUrlPrefix + * for all of cube segments' corresponding HTables after migrating a cube + * + */ +public class CubeMigrationSupplementCLI { + +private static final Logger logger = LoggerFactory.getLogger(CubeMigrationSupplementCLI.class); + +private static KylinConfig srcConfig; +private static KylinConfig dstConfig; +private static ResourceStore srcStore; +private static ResourceStore dstStore; + +public static void main(String[] args) throws InterruptedException, IOException { + +if (args.length != 5) { +usage(); +System.exit(1); +} + +moveCubeSupplement(args[0], args[1], args[2], args[3], args[4]); +} + +private static void usage() { +System.out.println("Usage: CubeMigrationSupplementCLI srcKylinConfigUri dstKylinConfigUri cubeName projectName realExecute"); +System.out.println(" srcKylinConfigUri: The KylinConfig of the cubeâs source \n" + "dstKylinConfigUri: The KylinConfig of the cubeâs new home \n" + "cubeName: the name of cube to be migrated. \n" + "projectName: The target project in the target environment.(Make sure it exist) \n" + "realExecute: if false, just print the operations to take, if true, do the real migration. \n"); + +} + +public static void moveCubeSupplement(String srcCfgUri, String dstCfgUri, String cubeName, String projectName, String realExecute) throws IOException, InterruptedException { + +moveCubeSupplement(KylinConfig.createInstanceFromUri(srcCfgUri), KylinConfig.createInstanceFromUri(dstCfgUri), cubeName, projectName, realExecute); +} + +public static void moveCubeSupplement(KylinConfig srcCfg, KylinConfig dstCfg, String cubeName, String projectName, String realExecute) throws IOException, InterruptedException { + +srcConfig = srcCfg; +srcStore = ResourceStore.getStore(srcConfig); +dstConfig = dstCfg; +dstStore = ResourceStore.getStore(dstConfig); + +CubeMa
kylin git commit: KYLIN-1592: modify the comments
Repository: kylin Updated Branches: refs/heads/1.4-rc 01d639468 -> b25ed7b53 KYLIN-1592: modify the comments Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/b25ed7b5 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/b25ed7b5 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/b25ed7b5 Branch: refs/heads/1.4-rc Commit: b25ed7b5394e6c68bbaed70f659e3b2748d07563 Parents: 01d6394 Author: kyotoYaho Authored: Tue Apr 19 08:00:39 2016 +0800 Committer: kyotoYaho Committed: Tue Apr 19 08:00:39 2016 +0800 -- .../kylin/storage/hbase/util/CubeMigrationSupplementCLI.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) -- http://git-wip-us.apache.org/repos/asf/kylin/blob/b25ed7b5/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationSupplementCLI.java -- diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationSupplementCLI.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationSupplementCLI.java index e16fe58..9451001 100644 --- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationSupplementCLI.java +++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationSupplementCLI.java @@ -35,8 +35,8 @@ import com.google.common.collect.Lists; /** * * This tool serves for the purpose of - * checking the "KYLIN_HOST" property to be consistent with the dst's MetadataUrlPrefix - * for all of cube segments' corresponding HTables after migrating a cube + * copying all of the cube segment statistics after an old version of migration + * without copying the cube segment statistics * */ public class CubeMigrationSupplementCLI {
kylin git commit: KYLIN-1596 UT fail, due to String encoding CharsetEncoder mismatch
Repository: kylin Updated Branches: refs/heads/yang-m1 aef842442 -> 6ff4b940e KYLIN-1596 UT fail, due to String encoding CharsetEncoder mismatch Signed-off-by: wangxianbin1987 Signed-off-by: Hongbin Ma Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/6ff4b940 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/6ff4b940 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/6ff4b940 Branch: refs/heads/yang-m1 Commit: 6ff4b940ef7c723cfb7f9c2508e8759992fdea2c Parents: aef8424 Author: wangxianbin1987 Authored: Tue Apr 19 10:04:02 2016 +0800 Committer: Hongbin Ma Committed: Tue Apr 19 10:23:46 2016 +0800 -- .../apache/kylin/invertedindex/InvertedIndexLocalTest.java| 7 ++- 1 file changed, 6 insertions(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/kylin/blob/6ff4b940/invertedindex/src/test/java/org/apache/kylin/invertedindex/InvertedIndexLocalTest.java -- diff --git a/invertedindex/src/test/java/org/apache/kylin/invertedindex/InvertedIndexLocalTest.java b/invertedindex/src/test/java/org/apache/kylin/invertedindex/InvertedIndexLocalTest.java index 4b74fdd..5aff1b0 100644 --- a/invertedindex/src/test/java/org/apache/kylin/invertedindex/InvertedIndexLocalTest.java +++ b/invertedindex/src/test/java/org/apache/kylin/invertedindex/InvertedIndexLocalTest.java @@ -23,6 +23,7 @@ import static org.junit.Assert.assertEquals; import java.io.File; import java.io.FileInputStream; import java.io.IOException; +import java.io.UnsupportedEncodingException; import java.util.Collections; import java.util.Comparator; import java.util.HashSet; @@ -171,7 +172,11 @@ public class InvertedIndexLocalTest extends LocalFileMetadataTestCase { @Nullable @Override public byte[] apply(String input) { -return input.getBytes(); +try { +return input.getBytes("UTF-8"); +} catch (UnsupportedEncodingException e) { +throw new RuntimeException(e); +} } }))); }
kylin git commit: KYLIN-1596 UT fail, due to String encoding CharsetEncoder mismatch
Repository: kylin Updated Branches: refs/heads/master ef36d6cb4 -> 8ee67a9cd KYLIN-1596 UT fail, due to String encoding CharsetEncoder mismatch Signed-off-by: wangxianbin1987 Signed-off-by: Hongbin Ma Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/8ee67a9c Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/8ee67a9c Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/8ee67a9c Branch: refs/heads/master Commit: 8ee67a9cde465f806a8520009b50b73074564440 Parents: ef36d6c Author: wangxianbin1987 Authored: Tue Apr 19 10:04:02 2016 +0800 Committer: Hongbin Ma Committed: Tue Apr 19 10:24:18 2016 +0800 -- .../apache/kylin/invertedindex/InvertedIndexLocalTest.java| 7 ++- 1 file changed, 6 insertions(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/kylin/blob/8ee67a9c/invertedindex/src/test/java/org/apache/kylin/invertedindex/InvertedIndexLocalTest.java -- diff --git a/invertedindex/src/test/java/org/apache/kylin/invertedindex/InvertedIndexLocalTest.java b/invertedindex/src/test/java/org/apache/kylin/invertedindex/InvertedIndexLocalTest.java index 4b74fdd..5aff1b0 100644 --- a/invertedindex/src/test/java/org/apache/kylin/invertedindex/InvertedIndexLocalTest.java +++ b/invertedindex/src/test/java/org/apache/kylin/invertedindex/InvertedIndexLocalTest.java @@ -23,6 +23,7 @@ import static org.junit.Assert.assertEquals; import java.io.File; import java.io.FileInputStream; import java.io.IOException; +import java.io.UnsupportedEncodingException; import java.util.Collections; import java.util.Comparator; import java.util.HashSet; @@ -171,7 +172,11 @@ public class InvertedIndexLocalTest extends LocalFileMetadataTestCase { @Nullable @Override public byte[] apply(String input) { -return input.getBytes(); +try { +return input.getBytes("UTF-8"); +} catch (UnsupportedEncodingException e) { +throw new RuntimeException(e); +} } }))); }
kylin git commit: KYLIN-1595 add hll to with slr left cube
Repository: kylin Updated Branches: refs/heads/master 8ee67a9cd -> 01ae37d5d KYLIN-1595 add hll to with slr left cube Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/01ae37d5 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/01ae37d5 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/01ae37d5 Branch: refs/heads/master Commit: 01ae37d5dab5dcc837a1286344f7f7d85350d7c5 Parents: 8ee67a9 Author: Hongbin Ma Authored: Tue Apr 19 11:59:39 2016 +0800 Committer: Hongbin Ma Committed: Tue Apr 19 11:59:39 2016 +0800 -- ...test_kylin_cube_with_slr_left_join_desc.json | 407 --- .../apache/kylin/query/ITMassInQueryTest.java | 4 +- .../query/sql_massin_distinct/query01.sql | 2 +- 3 files changed, 255 insertions(+), 158 deletions(-) -- http://git-wip-us.apache.org/repos/asf/kylin/blob/01ae37d5/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_slr_left_join_desc.json -- diff --git a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_slr_left_join_desc.json b/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_slr_left_join_desc.json index f0fe9c0..608b9de 100644 --- a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_slr_left_join_desc.json +++ b/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_slr_left_join_desc.json @@ -1,171 +1,266 @@ { - "uuid" : "a905-1fc6-4f67-985c-38fa5aeafd92", - - "name" : "test_kylin_cube_with_slr_left_join_desc", - "description" : null, - "dimensions" : [ { -"name" : "CAL_DT", -"table" : "EDW.TEST_CAL_DT", -"column" : "{FK}", -"derived" : [ "WEEK_BEG_DT" ] - }, { -"name" : "CATEGORY", -"table" : "DEFAULT.TEST_CATEGORY_GROUPINGS", -"column" : "META_CATEG_NAME", -"derived" : null - }, { -"name" : "CATEGORY", -"table" : "DEFAULT.TEST_CATEGORY_GROUPINGS", -"column" : "CATEG_LVL2_NAME", -"derived" : null - }, { -"name" : "CATEGORY", -"table" : "DEFAULT.TEST_CATEGORY_GROUPINGS", -"column" : "CATEG_LVL3_NAME", -"derived" : null - }, { -"name" : "CATEGORY_DERIVED", -"table" : "DEFAULT.TEST_CATEGORY_GROUPINGS", -"column" : "{FK}", -"derived" : [ "USER_DEFINED_FIELD1", "USER_DEFINED_FIELD3", "UPD_DATE", "UPD_USER" ] - }, { -"name" : "LSTG_FORMAT_NAME", -"table" : "DEFAULT.TEST_KYLIN_FACT", -"column" : "LSTG_FORMAT_NAME", -"derived" : null - }, { -"name" : "SITE_ID", -"table" : "EDW.TEST_SITES", -"column" : "{FK}", -"derived" : [ "SITE_NAME", "CRE_USER" ] - }, { -"name" : "SELLER_TYPE_CD", -"table" : "EDW.TEST_SELLER_TYPE_DIM", -"column" : "{FK}", -"derived" : [ "SELLER_TYPE_DESC" ] - }, { -"name" : "SELLER_ID", -"table" : "DEFAULT.TEST_KYLIN_FACT", -"column" : "SELLER_ID", -"derived" : null - } ], - "measures" : [ { -"name" : "GMV_SUM", -"function" : { - "expression" : "SUM", - "parameter" : { -"type" : "column", -"value" : "PRICE", -"next_parameter" : null + "uuid": "a905-1fc6-4f67-985c-38fa5aeafd92", + "name": "test_kylin_cube_with_slr_left_join_desc", + "description": null, + "dimensions": [ +{ + "name": "CAL_DT", + "table": "EDW.TEST_CAL_DT", + "column": "{FK}", + "derived": [ +"WEEK_BEG_DT" + ] +}, +{ + "name": "CATEGORY", + "table": "DEFAULT.TEST_CATEGORY_GROUPINGS", + "column": "META_CATEG_NAME", + "derived": null +}, +{ + "name": "CATEGORY", + "table": "DEFAULT.TEST_CATEGORY_GROUPINGS", + "column": "CATEG_LVL2_NAME", + "derived": null +}, +{ + "name": "CATEGORY", + "table": "DEFAULT.TEST_CATEGORY_GROUPINGS", + "column": "CATEG_LVL3_NAME", + "derived": null +}, +{ + "name": "CATEGORY_DERIVED", + "table": "DEFAULT.TEST_CATEGORY_GROUPINGS", + "column": "{FK}", + "derived": [ +"USER_DEFINED_FIELD1", +"USER_DEFINED_FIELD3", +"UPD_DATE", +"UPD_USER" + ] +}, +{ + "name": "LSTG_FORMAT_NAME", + "table": "DEFAULT.TEST_KYLIN_FACT", + "column": "LSTG_FORMAT_NAME", + "derived": null +}, +{ + "name": "SITE_ID", + "table": "EDW.TEST_SITES", + "column": "{FK}", + "derived": [ +"SITE_NAME", +"CRE_USER" + ] +}, +{ + "name": "SELLER_TYPE_CD", + "table": "EDW.TEST_SELLER_TYPE_DIM", + "column": "{FK}", + "derived": [ +"SELLER_TYPE_DESC" + ] +}, +{ + "name": "SELLER_ID", + "table": "DEFAULT.TEST_KYLIN_FACT", + "column": "SELLER_ID", + "derived": null +} + ], + "measures": [ +
kylin git commit: KYLIN-1595 add hll to with slr left cube
Repository: kylin Updated Branches: refs/heads/yang-m1 6ff4b940e -> 74ba02972 KYLIN-1595 add hll to with slr left cube Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/74ba0297 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/74ba0297 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/74ba0297 Branch: refs/heads/yang-m1 Commit: 74ba0297271efa37b0b04f46f95611e8a01ea3db Parents: 6ff4b94 Author: Hongbin Ma Authored: Tue Apr 19 11:59:39 2016 +0800 Committer: Hongbin Ma Committed: Tue Apr 19 12:04:24 2016 +0800 -- ...test_kylin_cube_with_slr_left_join_desc.json | 407 --- .../apache/kylin/query/ITMassInQueryTest.java | 4 +- .../query/sql_massin_distinct/query01.sql | 2 +- 3 files changed, 255 insertions(+), 158 deletions(-) -- http://git-wip-us.apache.org/repos/asf/kylin/blob/74ba0297/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_slr_left_join_desc.json -- diff --git a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_slr_left_join_desc.json b/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_slr_left_join_desc.json index f0fe9c0..608b9de 100644 --- a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_slr_left_join_desc.json +++ b/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_slr_left_join_desc.json @@ -1,171 +1,266 @@ { - "uuid" : "a905-1fc6-4f67-985c-38fa5aeafd92", - - "name" : "test_kylin_cube_with_slr_left_join_desc", - "description" : null, - "dimensions" : [ { -"name" : "CAL_DT", -"table" : "EDW.TEST_CAL_DT", -"column" : "{FK}", -"derived" : [ "WEEK_BEG_DT" ] - }, { -"name" : "CATEGORY", -"table" : "DEFAULT.TEST_CATEGORY_GROUPINGS", -"column" : "META_CATEG_NAME", -"derived" : null - }, { -"name" : "CATEGORY", -"table" : "DEFAULT.TEST_CATEGORY_GROUPINGS", -"column" : "CATEG_LVL2_NAME", -"derived" : null - }, { -"name" : "CATEGORY", -"table" : "DEFAULT.TEST_CATEGORY_GROUPINGS", -"column" : "CATEG_LVL3_NAME", -"derived" : null - }, { -"name" : "CATEGORY_DERIVED", -"table" : "DEFAULT.TEST_CATEGORY_GROUPINGS", -"column" : "{FK}", -"derived" : [ "USER_DEFINED_FIELD1", "USER_DEFINED_FIELD3", "UPD_DATE", "UPD_USER" ] - }, { -"name" : "LSTG_FORMAT_NAME", -"table" : "DEFAULT.TEST_KYLIN_FACT", -"column" : "LSTG_FORMAT_NAME", -"derived" : null - }, { -"name" : "SITE_ID", -"table" : "EDW.TEST_SITES", -"column" : "{FK}", -"derived" : [ "SITE_NAME", "CRE_USER" ] - }, { -"name" : "SELLER_TYPE_CD", -"table" : "EDW.TEST_SELLER_TYPE_DIM", -"column" : "{FK}", -"derived" : [ "SELLER_TYPE_DESC" ] - }, { -"name" : "SELLER_ID", -"table" : "DEFAULT.TEST_KYLIN_FACT", -"column" : "SELLER_ID", -"derived" : null - } ], - "measures" : [ { -"name" : "GMV_SUM", -"function" : { - "expression" : "SUM", - "parameter" : { -"type" : "column", -"value" : "PRICE", -"next_parameter" : null + "uuid": "a905-1fc6-4f67-985c-38fa5aeafd92", + "name": "test_kylin_cube_with_slr_left_join_desc", + "description": null, + "dimensions": [ +{ + "name": "CAL_DT", + "table": "EDW.TEST_CAL_DT", + "column": "{FK}", + "derived": [ +"WEEK_BEG_DT" + ] +}, +{ + "name": "CATEGORY", + "table": "DEFAULT.TEST_CATEGORY_GROUPINGS", + "column": "META_CATEG_NAME", + "derived": null +}, +{ + "name": "CATEGORY", + "table": "DEFAULT.TEST_CATEGORY_GROUPINGS", + "column": "CATEG_LVL2_NAME", + "derived": null +}, +{ + "name": "CATEGORY", + "table": "DEFAULT.TEST_CATEGORY_GROUPINGS", + "column": "CATEG_LVL3_NAME", + "derived": null +}, +{ + "name": "CATEGORY_DERIVED", + "table": "DEFAULT.TEST_CATEGORY_GROUPINGS", + "column": "{FK}", + "derived": [ +"USER_DEFINED_FIELD1", +"USER_DEFINED_FIELD3", +"UPD_DATE", +"UPD_USER" + ] +}, +{ + "name": "LSTG_FORMAT_NAME", + "table": "DEFAULT.TEST_KYLIN_FACT", + "column": "LSTG_FORMAT_NAME", + "derived": null +}, +{ + "name": "SITE_ID", + "table": "EDW.TEST_SITES", + "column": "{FK}", + "derived": [ +"SITE_NAME", +"CRE_USER" + ] +}, +{ + "name": "SELLER_TYPE_CD", + "table": "EDW.TEST_SELLER_TYPE_DIM", + "column": "{FK}", + "derived": [ +"SELLER_TYPE_DESC" + ] +}, +{ + "name": "SELLER_ID", + "table": "DEFAULT.TEST_KYLIN_FACT", + "column": "SELLER_ID", + "derived": null +} + ], + "measures": [ +