KYLIN-2821 Fix bug in GTScanRequest and add checking for duplicated columns in HBase mapping
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/309c872a Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/309c872a Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/309c872a Branch: refs/heads/ranger Commit: 309c872abf2344f820504fef574bbd41278d2d0c Parents: 01f1756 Author: auphyroc99 <454530...@qq.com> Authored: Tue Sep 5 19:31:34 2017 +0800 Committer: Hongbin Ma <m...@kyligence.io> Committed: Wed Sep 6 10:55:54 2017 +0800 ---------------------------------------------------------------------- .../apache/kylin/common/KylinConfigBase.java | 4 +++ .../org/apache/kylin/cube/model/CubeDesc.java | 29 ++++++++++++++++++-- .../kylin/cube/model/HBaseMappingDesc.java | 18 ++++++++++++ .../org/apache/kylin/gridtable/GTRecord.java | 20 ++++++++++++-- .../apache/kylin/gridtable/GTScanRequest.java | 9 +++--- .../localmeta/cube_desc/ci_inner_join_cube.json | 6 ++-- .../localmeta/cube_desc/ci_left_join_cube.json | 4 +-- 7 files changed, 76 insertions(+), 14 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/309c872a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java ---------------------------------------------------------------------- diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java index 9db2f0b..647b953 100644 --- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java +++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java @@ -286,6 +286,10 @@ abstract public class KylinConfigBase implements Serializable { "org.apache.kylin.storage.hbase.util.ZookeeperDistributedLock$Factory"); return (DistributedLockFactory) ClassUtil.newInstance(clsName); } + + public String getHBaseMappingAdapter() { + return getOptional("kylin.metadata.hbasemapping-adapter"); + } // ============================================================================ // DICTIONARY & SNAPSHOT http://git-wip-us.apache.org/repos/asf/kylin/blob/309c872a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java index 3cbba50..92695a8 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java @@ -22,6 +22,7 @@ import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkNotNull; import static com.google.common.base.Preconditions.checkState; +import java.lang.reflect.Method; import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; import java.util.ArrayList; @@ -602,8 +603,21 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware { validateAggregationGroups(); // check if aggregation group is valid validateAggregationGroupsCombination(); - if (hbaseMapping != null) { - hbaseMapping.init(this); + String hbaseMappingAdapterName = config.getHBaseMappingAdapter(); + + if (hbaseMappingAdapterName != null) { + try { + Class<?> hbaseMappingAdapterClass = Class.forName(hbaseMappingAdapterName); + Method initMethod = hbaseMappingAdapterClass.getMethod("initHBaseMapping", CubeDesc.class); + initMethod.invoke(null, this); + } catch (Exception e) { + logger.error("Wrong configuration for kylin.metadata.hbasemapping-adapter: class " + + hbaseMappingAdapterName + " not found. "); + } + } else { + if (hbaseMapping != null) { + hbaseMapping.init(this); + } } initMeasureReferenceToColumnFamily(); @@ -999,16 +1013,24 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware { measureIndexLookup.put(measures.get(i).getName(), i); BitSet checkEachMeasureExist = new BitSet(); + Set<String> measureSet = Sets.newHashSet(); for (HBaseColumnFamilyDesc cf : getHbaseMapping().getColumnFamily()) { for (HBaseColumnDesc c : cf.getColumns()) { String[] colMeasureRefs = c.getMeasureRefs(); MeasureDesc[] measureDescs = new MeasureDesc[colMeasureRefs.length]; int[] measureIndex = new int[colMeasureRefs.length]; + int lastMeasureIndex = -1; for (int i = 0; i < colMeasureRefs.length; i++) { measureDescs[i] = measureLookup.get(colMeasureRefs[i]); checkState(measureDescs[i] != null, "measure desc at (%s) is null", i); measureIndex[i] = measureIndexLookup.get(colMeasureRefs[i]); checkState(measureIndex[i] >= 0, "measure index at (%s) not positive", i); + + checkState(measureIndex[i] > lastMeasureIndex, "measure (%s) is not in order", colMeasureRefs[i]); + lastMeasureIndex = measureIndex[i]; + + checkState(!measureSet.contains(colMeasureRefs[i]), "column (%s) duplicates", colMeasureRefs[i]); + measureSet.add(colMeasureRefs[i]); checkEachMeasureExist.set(measureIndex[i]); } @@ -1020,8 +1042,9 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware { for (int i = 0; i < measures.size(); i++) { checkState(checkEachMeasureExist.get(i), - "measure (%s) does not exist in column familyï¼or measure duplicates", measures.get(i)); + "measure (%s) does not exist in column family, or measure duplicates", measures.get(i)); } + } private void initDictionaryDesc() { http://git-wip-us.apache.org/repos/asf/kylin/blob/309c872a/core-cube/src/main/java/org/apache/kylin/cube/model/HBaseMappingDesc.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/HBaseMappingDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/HBaseMappingDesc.java index d1e5829..77b4b14 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/model/HBaseMappingDesc.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/model/HBaseMappingDesc.java @@ -87,6 +87,24 @@ public class HBaseMappingDesc implements java.io.Serializable { } } + public void initAsSeparatedColumns(CubeDesc cubeDesc) { + cubeRef = cubeDesc; + + int cfNum = cubeDesc.getMeasures().size(); + columnFamily = new HBaseColumnFamilyDesc[cfNum]; + + for (int i = 0; i < cfNum; i++) { + HBaseColumnFamilyDesc cf = new HBaseColumnFamilyDesc(); + HBaseColumnDesc col = new HBaseColumnDesc(); + String measureRef = cubeDesc.getMeasures().get(i).getName(); + col.setMeasureRefs(new String[] { measureRef }); + col.setQualifier("M"); + cf.setColumns(new HBaseColumnDesc[] { col }); + cf.setName("F" + (i + 1)); + columnFamily[i] = cf; + } + } + @Override public String toString() { return "HBaseMappingDesc [columnFamily=" + Arrays.toString(columnFamily) + "]"; http://git-wip-us.apache.org/repos/asf/kylin/blob/309c872a/core-cube/src/main/java/org/apache/kylin/gridtable/GTRecord.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/GTRecord.java b/core-cube/src/main/java/org/apache/kylin/gridtable/GTRecord.java index f65e4b5..115fd91 100644 --- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTRecord.java +++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTRecord.java @@ -45,7 +45,7 @@ public class GTRecord implements Comparable<GTRecord>, Cloneable { } this.info = info; } - + @Override public GTRecord clone() { // deep copy ByteArray[] cols = new ByteArray[this.cols.length]; @@ -114,7 +114,6 @@ public class GTRecord implements Comparable<GTRecord>, Cloneable { return result; } - /** decode and return the values of this record */ public Object[] getValues(int[] selectedColumns, Object[] result) { assert selectedColumns.length <= result.length; @@ -296,6 +295,7 @@ public class GTRecord implements Comparable<GTRecord>, Cloneable { } } + /** change pointers to point to data in given buffer, this * method allows to defined specific column to load */ public void loadColumns(int selectedCol, ByteBuffer buf) { @@ -304,4 +304,20 @@ public class GTRecord implements Comparable<GTRecord>, Cloneable { cols[selectedCol].set(buf.array(), buf.arrayOffset() + pos, len); } + public void loadColumnsFromColumnBlocks(ImmutableBitSet[] selectedColumnBlocks, ImmutableBitSet selectedCols, + ByteBuffer buf) { + int pos = buf.position(); + for (ImmutableBitSet selectedColBlock : selectedColumnBlocks) { + for (int i = 0; i < selectedColBlock.trueBitCount(); i++) { + int c = selectedColBlock.trueBitAt(i); + int len = info.codeSystem.codeLength(c, buf); + if (selectedCols.get(c)) { + cols[c].set(buf.array(), buf.arrayOffset() + pos, len); + } + pos += len; + buf.position(pos); + } + } + } + } http://git-wip-us.apache.org/repos/asf/kylin/blob/309c872a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java index 523814b..412ff7f 100644 --- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java +++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java @@ -128,12 +128,13 @@ public class GTScanRequest { if (columns == null) columns = info.colAll; - - this.selectedColBlocks = info.selectColumnBlocks(columns); - + if (hasFilterPushDown()) { validateFilterPushDown(info); } + + this.selectedColBlocks = info.selectColumnBlocks(columns); + } public void setTimeout(long timeout) { @@ -248,7 +249,7 @@ public class GTScanRequest { } } } - System.out.println("Meaningless byte is " + meaninglessByte); + logger.info("Meaningless byte is " + meaninglessByte); IOUtils.closeQuietly(scanner); return scanned; } http://git-wip-us.apache.org/repos/asf/kylin/blob/309c872a/examples/test_case_data/localmeta/cube_desc/ci_inner_join_cube.json ---------------------------------------------------------------------- diff --git a/examples/test_case_data/localmeta/cube_desc/ci_inner_join_cube.json b/examples/test_case_data/localmeta/cube_desc/ci_inner_join_cube.json index efed0f9..cfd8cd1 100644 --- a/examples/test_case_data/localmeta/cube_desc/ci_inner_join_cube.json +++ b/examples/test_case_data/localmeta/cube_desc/ci_inner_join_cube.json @@ -512,11 +512,11 @@ "qualifier": "m", "measure_refs": [ "TEST_EXTENDED_COLUMN", + "BUYER_CONTACT", + "SELLER_CONTACT", "TRANS_ID_RAW", "PRICE_RAW", - "CAL_DT_RAW", - "BUYER_CONTACT", - "SELLER_CONTACT" + "CAL_DT_RAW" ] } ] http://git-wip-us.apache.org/repos/asf/kylin/blob/309c872a/examples/test_case_data/localmeta/cube_desc/ci_left_join_cube.json ---------------------------------------------------------------------- diff --git a/examples/test_case_data/localmeta/cube_desc/ci_left_join_cube.json b/examples/test_case_data/localmeta/cube_desc/ci_left_join_cube.json index 6cb0722..597f95f 100644 --- a/examples/test_case_data/localmeta/cube_desc/ci_left_join_cube.json +++ b/examples/test_case_data/localmeta/cube_desc/ci_left_join_cube.json @@ -523,11 +523,11 @@ "qualifier": "m", "measure_refs": [ "TEST_EXTENDED_COLUMN", + "BUYER_CONTACT", + "SELLER_CONTACT", "TRANS_ID_RAW", "PRICE_RAW", "CAL_DT_RAW", - "BUYER_CONTACT", - "SELLER_CONTACT", "GVM_PERCENTILE" ] }
