Repository: kylin Updated Branches: refs/heads/KYLIN-2215 [created] a6e3ccd7b
minor, various code review Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/d4a18c11 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/d4a18c11 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/d4a18c11 Branch: refs/heads/KYLIN-2215 Commit: d4a18c113131dc316d78ad98e1eaf1fb18af95c7 Parents: c8d4136 Author: Yang Li <liy...@apache.org> Authored: Sat Nov 19 15:04:10 2016 +0800 Committer: Yang Li <liy...@apache.org> Committed: Sat Nov 19 15:04:10 2016 +0800 ---------------------------------------------------------------------- .../org/apache/kylin/cube/kv/RowKeyEncoder.java | 14 +++---- .../storage/gtrecord/CubeScanRangePlanner.java | 9 +++-- .../mr/steps/NumberDictionaryForestTest.java | 39 ++++++++++++++++++++ .../kylin/provision/BuildCubeWithEngine.java | 5 +-- 4 files changed, 52 insertions(+), 15 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/d4a18c11/core-cube/src/main/java/org/apache/kylin/cube/kv/RowKeyEncoder.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/kv/RowKeyEncoder.java b/core-cube/src/main/java/org/apache/kylin/cube/kv/RowKeyEncoder.java index ff37752..672e1bc 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/kv/RowKeyEncoder.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/kv/RowKeyEncoder.java @@ -42,8 +42,8 @@ public class RowKeyEncoder extends AbstractRowKeyEncoder { private RowKeyColumnIO colIO; protected boolean enableSharding; - private int UHCOffset = -1;//it's a offset to the beginning of body - private int UHCLength = -1; + private int uhcOffset = -1;//it's a offset to the beginning of body + private int uhcLength = -1; public RowKeyEncoder(CubeSegment cubeSeg, Cuboid cuboid) { super(cubeSeg, cuboid); @@ -55,8 +55,8 @@ public class RowKeyEncoder extends AbstractRowKeyEncoder { colIO = new RowKeyColumnIO(cubeSeg.getDimensionEncodingMap()); for (TblColRef column : cuboid.getColumns()) { if (shardByColumns.contains(column)) { - UHCOffset = bodyLength; - UHCLength = colIO.getColumnLength(column); + uhcOffset = bodyLength; + uhcLength = colIO.getColumnLength(column); } bodyLength += colIO.getColumnLength(column); } @@ -72,8 +72,8 @@ public class RowKeyEncoder extends AbstractRowKeyEncoder { protected short calculateShard(byte[] key) { if (enableSharding) { - int shardSeedOffset = UHCOffset == -1 ? 0 : UHCOffset; - int shardSeedLength = UHCLength == -1 ? bodyLength : UHCLength; + int shardSeedOffset = uhcOffset == -1 ? 0 : uhcOffset; + int shardSeedLength = uhcLength == -1 ? bodyLength : uhcLength; short cuboidShardNum = cubeSeg.getCuboidShardNum(cuboid.getId()); short shardOffset = ShardingHash.getShard(key, RowConstants.ROWKEY_SHARD_AND_CUBOID_LEN + shardSeedOffset, shardSeedLength, cuboidShardNum); return ShardingHash.normalize(cubeSeg.getCuboidBaseShard(cuboid.getId()), shardOffset, cubeSeg.getTotalShards(cuboid.getId())); @@ -102,7 +102,7 @@ public class RowKeyEncoder extends AbstractRowKeyEncoder { } //ByteArray representing dimension does not have extra header - public void encodeDims(GTRecord record, ImmutableBitSet selectedCols, ByteArray buf, byte defaultValue) { + private void encodeDims(GTRecord record, ImmutableBitSet selectedCols, ByteArray buf, byte defaultValue) { int pos = 0; for (int i = 0; i < selectedCols.trueBitCount(); i++) { int c = selectedCols.trueBitAt(i); http://git-wip-us.apache.org/repos/asf/kylin/blob/d4a18c11/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeScanRangePlanner.java ---------------------------------------------------------------------- diff --git a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeScanRangePlanner.java b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeScanRangePlanner.java index b011f40..8d5a3d4 100644 --- a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeScanRangePlanner.java +++ b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeScanRangePlanner.java @@ -71,7 +71,7 @@ public class CubeScanRangePlanner extends ScanRangePlannerBase { protected StorageContext context; - public CubeScanRangePlanner(CubeSegment cubeSegment, Cuboid cuboid, TupleFilter filter, Set<TblColRef> dimensions, Set<TblColRef> groupbyDims, // + public CubeScanRangePlanner(CubeSegment cubeSegment, Cuboid cuboid, TupleFilter filter, Set<TblColRef> dimensions, Set<TblColRef> groupByDims, // Collection<FunctionDesc> metrics, StorageContext context) { this.context = context; @@ -96,11 +96,12 @@ public class CubeScanRangePlanner extends ScanRangePlannerBase { //start key GTRecord compare to stop key GTRecord this.rangeStartEndComparator = RecordComparators.getRangeStartEndComparator(comp); - //replace the constant values in filter to dictionary codes - this.gtFilter = GTUtil.convertFilterColumnsAndConstants(filter, gtInfo, mapping.getCuboidDimensionsInGTOrder(), groupbyDims); + //replace the constant values in filter to dictionary codes + Set<TblColRef> groupByPushDown = Sets.newHashSet(groupByDims); + this.gtFilter = GTUtil.convertFilterColumnsAndConstants(filter, gtInfo, mapping.getCuboidDimensionsInGTOrder(), groupByPushDown); this.gtDimensions = mapping.makeGridTableColumns(dimensions); - this.gtAggrGroups = mapping.makeGridTableColumns(replaceDerivedColumns(groupbyDims, cubeSegment.getCubeDesc())); + this.gtAggrGroups = mapping.makeGridTableColumns(replaceDerivedColumns(groupByPushDown, cubeSegment.getCubeDesc())); this.gtAggrMetrics = mapping.makeGridTableColumns(metrics); this.gtAggrFuncs = mapping.makeAggrFuncs(metrics); http://git-wip-us.apache.org/repos/asf/kylin/blob/d4a18c11/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/NumberDictionaryForestTest.java ---------------------------------------------------------------------- diff --git a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/NumberDictionaryForestTest.java b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/NumberDictionaryForestTest.java index 33dca01..47b10e8 100644 --- a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/NumberDictionaryForestTest.java +++ b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/NumberDictionaryForestTest.java @@ -111,6 +111,45 @@ public class NumberDictionaryForestTest { dict2.dump(System.out); } + + @Test + public void testMerge() { + // mimic the logic as in MergeCuboidMapper + NumberDictionaryForestBuilder b1 = new NumberDictionaryForestBuilder(); + b1.addValue("0"); + b1.addValue("3"); + b1.addValue("23"); + TrieDictionaryForest<String> dict1 = b1.build(); + + NumberDictionaryForestBuilder b2 = new NumberDictionaryForestBuilder(); + b2.addValue("0"); + b2.addValue("2"); + b2.addValue("3"); + b2.addValue("15"); + b2.addValue("23"); + TrieDictionaryForest<String> dict2 = b2.build(); + + assertTrue(dict1.getSizeOfId() == dict2.getSizeOfId()); + assertTrue(dict1.getSizeOfValue() == dict2.getSizeOfValue()); + + byte[] buf = new byte[dict1.getSizeOfValue()]; + + { + int len = dict1.getValueBytesFromId(0, buf, 0); + int newId = dict2.getIdFromValueBytes(buf, 0, len); + assertTrue(newId == 0); + } + { + int len = dict1.getValueBytesFromId(1, buf, 0); + int newId = dict2.getIdFromValueBytes(buf, 0, len); + assertTrue(newId == 2); + } + { + int len = dict1.getValueBytesFromId(2, buf, 0); + int newId = dict2.getIdFromValueBytes(buf, 0, len); + assertTrue(newId == 4); + } + } private static TrieDictionaryForest<String> testSerialize(TrieDictionaryForest<String> dict) { try { http://git-wip-us.apache.org/repos/asf/kylin/blob/d4a18c11/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java ---------------------------------------------------------------------- diff --git a/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java b/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java index c3902a2..3b88dab 100644 --- a/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java +++ b/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java @@ -247,10 +247,6 @@ public class BuildCubeWithEngine { } } - private void assertJobSuccess() { - - } - @SuppressWarnings("unused") // called by reflection private boolean testInnerJoinCubeWithSlr() throws Exception { @@ -433,6 +429,7 @@ public class BuildCubeWithEngine { return 0; } + @SuppressWarnings("unused") private void checkHFilesInHBase(CubeSegment segment) throws IOException { Configuration conf = HBaseConfiguration.create(HadoopUtil.getCurrentConfiguration()); String tableName = segment.getStorageLocationIdentifier();
