refactor: minor changes
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/b361d8aa Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/b361d8aa Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/b361d8aa Branch: refs/heads/stream_m1 Commit: b361d8aaa2b1b5d0d86107bedb9f34397be000a5 Parents: ea24450 Author: Hongbin Ma <mahong...@apache.org> Authored: Wed Jun 22 13:59:58 2016 +0800 Committer: Hongbin Ma <mahong...@apache.org> Committed: Wed Jun 22 14:00:15 2016 +0800 ---------------------------------------------------------------------- .../org/apache/kylin/cube/kv/RowKeyEncoder.java | 21 ++++++- .../org/apache/kylin/gridtable/GTRecord.java | 17 +----- .../gridtable/memstore/GTSimpleMemStore.java | 20 +++++- .../kylin/gridtable/DictGridTableTest.java | 64 ++++++++++---------- .../coprocessor/endpoint/CubeVisitService.java | 2 +- 5 files changed, 70 insertions(+), 54 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/b361d8aa/core-cube/src/main/java/org/apache/kylin/cube/kv/RowKeyEncoder.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/kv/RowKeyEncoder.java b/core-cube/src/main/java/org/apache/kylin/cube/kv/RowKeyEncoder.java index db704dc..ebcbadd 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/kv/RowKeyEncoder.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/kv/RowKeyEncoder.java @@ -94,12 +94,31 @@ public class RowKeyEncoder extends AbstractRowKeyEncoder { @Override public void encode(GTRecord record, ImmutableBitSet keyColumns, byte[] buf) { ByteArray byteArray = new ByteArray(buf, getHeaderLength(), 0); - record.exportColumns(keyColumns, byteArray, defaultValue()); + + encodeDims(record, keyColumns, byteArray, defaultValue()); //fill shard and cuboid fillHeader(buf); } + //ByteArray representing dimension does not have extra header + public void encodeDims(GTRecord record, ImmutableBitSet selectedCols, ByteArray buf, byte defaultValue) { + int pos = 0; + for (int i = 0; i < selectedCols.trueBitCount(); i++) { + int c = selectedCols.trueBitAt(i); + ByteArray columnC = record.get(c); + if (columnC.array() != null) { + System.arraycopy(record.get(c).array(), columnC.offset(), buf.array(), buf.offset() + pos, columnC.length()); + pos += columnC.length(); + } else { + int maxLength = record.getInfo().getCodeSystem().maxCodeLength(c); + Arrays.fill(buf.array(), buf.offset() + pos, buf.offset() + pos + maxLength, defaultValue); + pos += maxLength; + } + } + buf.setLength(pos); + } + @Override public void encode(ByteArray bodyBytes, ByteArray outputBuf) { Preconditions.checkState(bodyBytes.length() == bodyLength); http://git-wip-us.apache.org/repos/asf/kylin/blob/b361d8aa/core-cube/src/main/java/org/apache/kylin/gridtable/GTRecord.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/GTRecord.java b/core-cube/src/main/java/org/apache/kylin/gridtable/GTRecord.java index b676693..276f66e 100644 --- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTRecord.java +++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTRecord.java @@ -235,22 +235,7 @@ public class GTRecord implements Comparable<GTRecord> { buf.setLength(pos); } - /** write data to given buffer, like serialize, use defaultValue when required column is not set*/ - public void exportColumns(ImmutableBitSet selectedCols, ByteArray buf, byte defaultValue) { - int pos = 0; - for (int i = 0; i < selectedCols.trueBitCount(); i++) { - int c = selectedCols.trueBitAt(i); - if (cols[c].array() != null) { - System.arraycopy(cols[c].array(), cols[c].offset(), buf.array(), buf.offset() + pos, cols[c].length()); - pos += cols[c].length(); - } else { - int maxLength = info.codeSystem.maxCodeLength(c); - Arrays.fill(buf.array(), buf.offset() + pos, buf.offset() + pos + maxLength, defaultValue); - pos += maxLength; - } - } - buf.setLength(pos); - } + /** write data to given buffer, like serialize */ public void exportColumns(ImmutableBitSet selectedCols, ByteBuffer buf) { http://git-wip-us.apache.org/repos/asf/kylin/blob/b361d8aa/core-cube/src/main/java/org/apache/kylin/gridtable/memstore/GTSimpleMemStore.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/memstore/GTSimpleMemStore.java b/core-cube/src/main/java/org/apache/kylin/gridtable/memstore/GTSimpleMemStore.java index a1e81a8..ec5d22d 100644 --- a/core-cube/src/main/java/org/apache/kylin/gridtable/memstore/GTSimpleMemStore.java +++ b/core-cube/src/main/java/org/apache/kylin/gridtable/memstore/GTSimpleMemStore.java @@ -25,6 +25,7 @@ import java.util.Iterator; import java.util.List; import org.apache.kylin.common.util.ByteArray; +import org.apache.kylin.common.util.ImmutableBitSet; import org.apache.kylin.gridtable.GTInfo; import org.apache.kylin.gridtable.GTRecord; import org.apache.kylin.gridtable.GTScanRequest; @@ -34,14 +35,23 @@ import org.apache.kylin.gridtable.IGTWriter; public class GTSimpleMemStore implements IGTStore { - final GTInfo info; - final List<byte[]> rowList; + final protected GTInfo info; + final protected List<byte[]> rowList; + + protected GTSimpleMemStore(GTInfo info, List<byte[]> rowList) { + this.info = info; + this.rowList = rowList; + } public GTSimpleMemStore(GTInfo info) { this.info = info; this.rowList = new ArrayList<byte[]>(); } + public List<byte[]> getRowList() { + return rowList; + } + @Override public GTInfo getInfo() { return info; @@ -80,6 +90,10 @@ public class GTSimpleMemStore implements IGTStore { } } + protected ImmutableBitSet getColumns() { + return info.getAllColumns(); + } + @Override public IGTScanner scan(GTScanRequest scanRequest) { @@ -115,7 +129,7 @@ public class GTSimpleMemStore implements IGTStore { @Override public GTRecord next() { byte[] bytes = it.next(); - oneRecord.loadColumns(info.getAllColumns(), ByteBuffer.wrap(bytes)); + oneRecord.loadColumns(getColumns(), ByteBuffer.wrap(bytes)); count++; return oneRecord; } http://git-wip-us.apache.org/repos/asf/kylin/blob/b361d8aa/core-cube/src/test/java/org/apache/kylin/gridtable/DictGridTableTest.java ---------------------------------------------------------------------- diff --git a/core-cube/src/test/java/org/apache/kylin/gridtable/DictGridTableTest.java b/core-cube/src/test/java/org/apache/kylin/gridtable/DictGridTableTest.java index dd2b98e..1687a4f 100644 --- a/core-cube/src/test/java/org/apache/kylin/gridtable/DictGridTableTest.java +++ b/core-cube/src/test/java/org/apache/kylin/gridtable/DictGridTableTest.java @@ -17,7 +17,7 @@ package org.apache.kylin.gridtable; -import static org.junit.Assert.*; +import static org.junit.Assert.assertEquals; import java.io.IOException; import java.math.BigDecimal; @@ -60,7 +60,7 @@ import org.junit.Test; import com.google.common.collect.Lists; -public class DictGridTableTest extends LocalFileMetadataTestCase { +public class DictGridTableTest extends LocalFileMetadataTestCase { private GridTable table; private GTInfo info; @@ -78,15 +78,15 @@ public class DictGridTableTest extends LocalFileMetadataTestCase { @After public void after() throws Exception { - + this.cleanupTestMetadata(); } @Before public void setup() throws IOException { - + this.createTestMetadata(); - + table = newTestTable(); info = table.getInfo(); @@ -112,10 +112,9 @@ public class DictGridTableTest extends LocalFileMetadataTestCase { ByteArray segmentEnd = enc(info, 0, "2015-01-15"); assertEquals(segmentStart, segmentStartX); - { LogicalTupleFilter filter = and(timeComp0, ageComp1); - GTScanRangePlanner planner = new GTScanRangePlanner(info, Pair.newPair(segmentStart, segmentEnd), info.colRef(0),filter); + GTScanRangePlanner planner = new GTScanRangePlanner(info, Pair.newPair(segmentStart, segmentEnd), info.colRef(0), filter); List<GTScanRange> r = planner.planScanRanges(); assertEquals(1, r.size());//scan range are [close,close] assertEquals("[null, 10]-[1421193600000, 10]", r.get(0).toString()); @@ -124,25 +123,25 @@ public class DictGridTableTest extends LocalFileMetadataTestCase { } { LogicalTupleFilter filter = and(timeComp2, ageComp1); - GTScanRangePlanner planner = new GTScanRangePlanner(info, Pair.newPair(segmentStart, segmentEnd), info.colRef(0),filter); + GTScanRangePlanner planner = new GTScanRangePlanner(info, Pair.newPair(segmentStart, segmentEnd), info.colRef(0), filter); List<GTScanRange> r = planner.planScanRanges(); assertEquals(0, r.size()); } { LogicalTupleFilter filter = and(timeComp4, ageComp1); - GTScanRangePlanner planner = new GTScanRangePlanner(info, Pair.newPair(segmentStart, segmentEnd), info.colRef(0),filter); + GTScanRangePlanner planner = new GTScanRangePlanner(info, Pair.newPair(segmentStart, segmentEnd), info.colRef(0), filter); List<GTScanRange> r = planner.planScanRanges(); assertEquals(1, r.size()); } { LogicalTupleFilter filter = and(timeComp5, ageComp1); - GTScanRangePlanner planner = new GTScanRangePlanner(info, Pair.newPair(segmentStart, segmentEnd), info.colRef(0),filter); + GTScanRangePlanner planner = new GTScanRangePlanner(info, Pair.newPair(segmentStart, segmentEnd), info.colRef(0), filter); List<GTScanRange> r = planner.planScanRanges(); assertEquals(0, r.size()); } { LogicalTupleFilter filter = or(and(timeComp2, ageComp1), and(timeComp1, ageComp1), and(timeComp6, ageComp1)); - GTScanRangePlanner planner = new GTScanRangePlanner(info, Pair.newPair(segmentStart, segmentEnd), info.colRef(0),filter); + GTScanRangePlanner planner = new GTScanRangePlanner(info, Pair.newPair(segmentStart, segmentEnd), info.colRef(0), filter); List<GTScanRange> r = planner.planScanRanges(); assertEquals(1, r.size()); assertEquals("[1421193600000, 10]-[null, 10]", r.get(0).toString()); @@ -150,7 +149,7 @@ public class DictGridTableTest extends LocalFileMetadataTestCase { } { LogicalTupleFilter filter = or(timeComp2, timeComp1, timeComp6); - GTScanRangePlanner planner = new GTScanRangePlanner(info, Pair.newPair(segmentStart, segmentEnd), info.colRef(0),filter); + GTScanRangePlanner planner = new GTScanRangePlanner(info, Pair.newPair(segmentStart, segmentEnd), info.colRef(0), filter); List<GTScanRange> r = planner.planScanRanges(); assertEquals(1, r.size()); assertEquals("[1421193600000, null]-[null, null]", r.get(0).toString()); @@ -159,14 +158,14 @@ public class DictGridTableTest extends LocalFileMetadataTestCase { { //skip FALSE filter LogicalTupleFilter filter = and(ageComp1, ConstantTupleFilter.FALSE); - GTScanRangePlanner planner = new GTScanRangePlanner(info, Pair.newPair(segmentStart, segmentEnd), info.colRef(0),filter); + GTScanRangePlanner planner = new GTScanRangePlanner(info, Pair.newPair(segmentStart, segmentEnd), info.colRef(0), filter); List<GTScanRange> r = planner.planScanRanges(); assertEquals(0, r.size()); } { //TRUE or FALSE filter LogicalTupleFilter filter = or(ConstantTupleFilter.TRUE, ConstantTupleFilter.FALSE); - GTScanRangePlanner planner = new GTScanRangePlanner(info, Pair.newPair(segmentStart, segmentEnd), info.colRef(0),filter); + GTScanRangePlanner planner = new GTScanRangePlanner(info, Pair.newPair(segmentStart, segmentEnd), info.colRef(0), filter); List<GTScanRange> r = planner.planScanRanges(); assertEquals(1, r.size()); assertEquals("[null, null]-[null, null]", r.get(0).toString()); @@ -174,7 +173,7 @@ public class DictGridTableTest extends LocalFileMetadataTestCase { { //TRUE or other filter LogicalTupleFilter filter = or(ageComp1, ConstantTupleFilter.TRUE); - GTScanRangePlanner planner = new GTScanRangePlanner(info, Pair.newPair(segmentStart, segmentEnd), info.colRef(0),filter); + GTScanRangePlanner planner = new GTScanRangePlanner(info, Pair.newPair(segmentStart, segmentEnd), info.colRef(0), filter); List<GTScanRange> r = planner.planScanRanges(); assertEquals(1, r.size()); assertEquals("[null, null]-[null, null]", r.get(0).toString()); @@ -187,7 +186,7 @@ public class DictGridTableTest extends LocalFileMetadataTestCase { { LogicalTupleFilter filter = and(timeComp0, ageComp1); - GTScanRangePlanner planner = new GTScanRangePlanner(info, Pair.newPair(new ByteArray(), segmentEnd), info.colRef(0),filter); + GTScanRangePlanner planner = new GTScanRangePlanner(info, Pair.newPair(new ByteArray(), segmentEnd), info.colRef(0), filter); List<GTScanRange> r = planner.planScanRanges(); assertEquals(1, r.size());//scan range are [close,close] assertEquals("[null, 10]-[1421193600000, 10]", r.get(0).toString()); @@ -197,7 +196,7 @@ public class DictGridTableTest extends LocalFileMetadataTestCase { { LogicalTupleFilter filter = and(timeComp5, ageComp1); - GTScanRangePlanner planner = new GTScanRangePlanner(info, Pair.newPair(new ByteArray(), segmentEnd), info.colRef(0),filter); + GTScanRangePlanner planner = new GTScanRangePlanner(info, Pair.newPair(new ByteArray(), segmentEnd), info.colRef(0), filter); List<GTScanRange> r = planner.planScanRanges(); assertEquals(0, r.size());//scan range are [close,close] } @@ -206,11 +205,10 @@ public class DictGridTableTest extends LocalFileMetadataTestCase { @Test public void verifyScanRangePlanner() { - // flatten or-and & hbase fuzzy value { LogicalTupleFilter filter = and(timeComp1, or(ageComp1, ageComp2)); - GTScanRangePlanner planner = new GTScanRangePlanner(info, null, null,filter); + GTScanRangePlanner planner = new GTScanRangePlanner(info, null, null, filter); List<GTScanRange> r = planner.planScanRanges(); assertEquals(1, r.size()); assertEquals("[1421193600000, 10]-[null, 20]", r.get(0).toString()); @@ -220,7 +218,7 @@ public class DictGridTableTest extends LocalFileMetadataTestCase { // pre-evaluate ever false { LogicalTupleFilter filter = and(timeComp1, timeComp2); - GTScanRangePlanner planner = new GTScanRangePlanner(info, null, null,filter); + GTScanRangePlanner planner = new GTScanRangePlanner(info, null, null, filter); List<GTScanRange> r = planner.planScanRanges(); assertEquals(0, r.size()); } @@ -228,7 +226,7 @@ public class DictGridTableTest extends LocalFileMetadataTestCase { // pre-evaluate ever true { LogicalTupleFilter filter = or(timeComp1, ageComp4); - GTScanRangePlanner planner = new GTScanRangePlanner(info, null, null,filter); + GTScanRangePlanner planner = new GTScanRangePlanner(info, null, null, filter); List<GTScanRange> r = planner.planScanRanges(); assertEquals("[[null, null]-[null, null]]", r.toString()); } @@ -236,7 +234,7 @@ public class DictGridTableTest extends LocalFileMetadataTestCase { // merge overlap range { LogicalTupleFilter filter = or(timeComp1, timeComp3); - GTScanRangePlanner planner = new GTScanRangePlanner(info, null, null,filter); + GTScanRangePlanner planner = new GTScanRangePlanner(info, null, null, filter); List<GTScanRange> r = planner.planScanRanges(); assertEquals("[[null, null]-[null, null]]", r.toString()); } @@ -244,7 +242,7 @@ public class DictGridTableTest extends LocalFileMetadataTestCase { // merge too many ranges { LogicalTupleFilter filter = or(and(timeComp4, ageComp1), and(timeComp4, ageComp2), and(timeComp4, ageComp3)); - GTScanRangePlanner planner = new GTScanRangePlanner(info, null, null,filter); + GTScanRangePlanner planner = new GTScanRangePlanner(info, null, null, filter); List<GTScanRange> r = planner.planScanRanges(); assertEquals(3, r.size()); assertEquals("[1421280000000, 10]-[1421280000000, 10]", r.get(0).toString()); @@ -271,7 +269,7 @@ public class DictGridTableTest extends LocalFileMetadataTestCase { } //for testing GTScanRequest serialization and deserialization - private GTScanRequest useDeserializedGTScanRequest(GTScanRequest origin) { + public static GTScanRequest useDeserializedGTScanRequest(GTScanRequest origin) { ByteBuffer buffer = ByteBuffer.allocate(BytesSerializer.SERIALIZE_BUFFER_SIZE); GTScanRequest.serializer.serialize(origin, buffer); buffer.flip(); @@ -445,38 +443,38 @@ public class DictGridTableTest extends LocalFileMetadataTestCase { scanner.close(); } - private ByteArray enc(GTInfo info, int col, String value) { + public static ByteArray enc(GTInfo info, int col, String value) { ByteBuffer buf = ByteBuffer.allocate(info.getMaxColumnLength()); info.codeSystem.encodeColumnValue(col, value, buf); return ByteArray.copyOf(buf.array(), buf.arrayOffset(), buf.position()); } - private ExtractTupleFilter unevaluatable(TblColRef col) { + public static ExtractTupleFilter unevaluatable(TblColRef col) { ExtractTupleFilter r = new ExtractTupleFilter(FilterOperatorEnum.EXTRACT); r.addChild(new ColumnTupleFilter(col)); return r; } - private CompareTupleFilter compare(TblColRef col, FilterOperatorEnum op, Object... value) { + public static CompareTupleFilter compare(TblColRef col, FilterOperatorEnum op, Object... value) { CompareTupleFilter result = new CompareTupleFilter(op); result.addChild(new ColumnTupleFilter(col)); result.addChild(new ConstantTupleFilter(Arrays.asList(value))); return result; } - private LogicalTupleFilter and(TupleFilter... children) { + public static LogicalTupleFilter and(TupleFilter... children) { return logic(FilterOperatorEnum.AND, children); } - private LogicalTupleFilter or(TupleFilter... children) { + public static LogicalTupleFilter or(TupleFilter... children) { return logic(FilterOperatorEnum.OR, children); } - private LogicalTupleFilter not(TupleFilter child) { + public static LogicalTupleFilter not(TupleFilter child) { return logic(FilterOperatorEnum.NOT, child); } - private LogicalTupleFilter logic(FilterOperatorEnum op, TupleFilter... children) { + public static LogicalTupleFilter logic(FilterOperatorEnum op, TupleFilter... children) { LogicalTupleFilter result = new LogicalTupleFilter(op); for (TupleFilter c : children) { result.addChild(c); @@ -484,7 +482,7 @@ public class DictGridTableTest extends LocalFileMetadataTestCase { return result; } - static GridTable newTestTable() throws IOException { + public static GridTable newTestTable() throws IOException { GTInfo info = newInfo(); GTSimpleMemStore store = new GTSimpleMemStore(info); GridTable table = new GridTable(info, store); @@ -609,7 +607,7 @@ public class DictGridTableTest extends LocalFileMetadataTestCase { return builder.build(0); } - private static ImmutableBitSet setOf(int... values) { + public static ImmutableBitSet setOf(int... values) { BitSet set = new BitSet(); for (int i : values) set.set(i); http://git-wip-us.apache.org/repos/asf/kylin/blob/b361d8aa/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java ---------------------------------------------------------------------- diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java index c3d2a0d..3827aa9 100644 --- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java +++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java @@ -312,7 +312,7 @@ public class CubeVisitService extends CubeVisitProtos.CubeVisitService implement try { oneRecord.exportColumns(scanReq.getColumns(), buffer); } catch (BufferOverflowException boe) { - buffer = ByteBuffer.allocate((int) (oneRecord.sizeOf(scanReq.getColumns()) * 1.5)); + buffer = ByteBuffer.allocate(oneRecord.sizeOf(scanReq.getColumns()) * 2); oneRecord.exportColumns(scanReq.getColumns(), buffer); }