Repository: kylin Updated Branches: refs/heads/2.x-staging 5db430706 -> 829f5d312
KYLIN-1306 Null strings are not applied during fast cubing Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/829f5d31 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/829f5d31 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/829f5d31 Branch: refs/heads/2.x-staging Commit: 829f5d3129d0521d22d42bd43ba07a83bc2945be Parents: 5db4307 Author: shaofengshi <shaofeng...@apache.org> Authored: Tue Jan 12 16:23:04 2016 +0800 Committer: shaofengshi <shaofeng...@apache.org> Committed: Tue Jan 12 17:28:34 2016 +0800 ---------------------------------------------------------------------- .../InMemCubeBuilderInputConverter.java | 30 ++++++++++++++++++++ 1 file changed, 30 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/829f5d31/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilderInputConverter.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilderInputConverter.java b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilderInputConverter.java index 3e9a67d..a1eb6da 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilderInputConverter.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilderInputConverter.java @@ -17,10 +17,13 @@ */ package org.apache.kylin.cube.inmemcubing; +import org.apache.kylin.common.util.Bytes; + import java.util.List; import java.util.Map; import org.apache.kylin.common.util.Dictionary; +import com.google.common.collect.Lists; import org.apache.kylin.cube.model.CubeDesc; import org.apache.kylin.cube.model.CubeJoinedFlatTableDesc; import org.apache.kylin.gridtable.GTInfo; @@ -35,12 +38,15 @@ import org.apache.kylin.metadata.model.TblColRef; */ public class InMemCubeBuilderInputConverter { + public static final byte[] HIVE_NULL = Bytes.toBytes("\\N"); + private final CubeJoinedFlatTableDesc intermediateTableDesc; private final MeasureDesc[] measureDescs; private final MeasureIngester<?>[] measureIngesters; private final int measureCount; private final Map<TblColRef, Dictionary<String>> dictionaryMap; private final GTInfo gtInfo; + protected List<byte[]> nullBytes; public InMemCubeBuilderInputConverter(CubeDesc cubeDesc, Map<TblColRef, Dictionary<String>> dictionaryMap, GTInfo gtInfo) { @@ -50,6 +56,7 @@ public class InMemCubeBuilderInputConverter { this.measureDescs = cubeDesc.getMeasures().toArray(new MeasureDesc[measureCount]); this.measureIngesters = MeasureIngester.create(cubeDesc.getMeasures()); this.dictionaryMap = dictionaryMap; + initNullBytes(cubeDesc); } public final GTRecord convert(List<String> row) { @@ -73,6 +80,9 @@ public class InMemCubeBuilderInputConverter { for (int i = 0; i < keySize; i++) { key[i] = row.get(intermediateTableDesc.getRowKeyColumnIndexes()[i]); + if (key[i] != null && isNull(Bytes.toBytes((String)key[i]))) { + key[i] = null; + } } return key; @@ -112,4 +122,24 @@ public class InMemCubeBuilderInputConverter { return measureIngesters[idxOfMeasure].valueOf(inputToMeasure, measure, dictionaryMap); } + private void initNullBytes(CubeDesc cubeDesc) { + nullBytes = Lists.newArrayList(); + nullBytes.add(HIVE_NULL); + String[] nullStrings = cubeDesc.getNullStrings(); + if (nullStrings != null) { + for (String s : nullStrings) { + nullBytes.add(Bytes.toBytes(s)); + } + } + } + + private boolean isNull(byte[] v) { + for (byte[] nullByte : nullBytes) { + if (Bytes.equals(v, nullByte)) + return true; + } + return false; + } + + }
