http://git-wip-us.apache.org/repos/asf/kylin/blob/07537a9a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java ---------------------------------------------------------------------- diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java index 981e19a..2a1304a 100644 --- a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java +++ b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java @@ -223,11 +223,11 @@ public class DictionaryManager { } } - public DictionaryInfo buildDictionary(DataModelDesc model, String dict, TblColRef col, DistinctColumnValuesProvider factTableValueProvider) throws IOException { + public DictionaryInfo buildDictionary(DataModelDesc model, boolean usingDict, TblColRef col, DistinctColumnValuesProvider factTableValueProvider) throws IOException { logger.info("building dictionary for " + col); - TblColRef srcCol = decideSourceData(model, dict, col); + TblColRef srcCol = decideSourceData(model, usingDict, col); String srcTable = srcCol.getTable(); String srcColName = srcCol.getName(); int srcColIdx = srcCol.getColumnDesc().getZeroBasedIndex(); @@ -259,9 +259,9 @@ public class DictionaryManager { /** * Decide a dictionary's source data, leverage PK-FK relationship. */ - public TblColRef decideSourceData(DataModelDesc model, String dict, TblColRef col) throws IOException { + public TblColRef decideSourceData(DataModelDesc model, boolean usingDict, TblColRef col) throws IOException { // Note FK on fact table is supported by scan the related PK on lookup table - if ("true".equals(dict) || "string".equals(dict) || "number".equals(dict) || "any".equals(dict)) { + if (usingDict) { // FK on fact table and join type is inner, use PK from lookup instead if (model.isFactTable(col.getTable())) { TblColRef pkCol = model.findPKByFK(col, "inner"); @@ -270,7 +270,7 @@ public class DictionaryManager { } return col; } else - throw new IllegalArgumentException("Unknown dictionary value: " + dict); + throw new IllegalArgumentException("Not using Dictionary "); } private String checkDupByInfo(DictionaryInfo dictInfo) throws IOException {
http://git-wip-us.apache.org/repos/asf/kylin/blob/07537a9a/core-job/src/main/java/org/apache/kylin/job/manager/ExecutableManager.java ---------------------------------------------------------------------- diff --git a/core-job/src/main/java/org/apache/kylin/job/manager/ExecutableManager.java b/core-job/src/main/java/org/apache/kylin/job/manager/ExecutableManager.java index ff92a50..d73fd26 100644 --- a/core-job/src/main/java/org/apache/kylin/job/manager/ExecutableManager.java +++ b/core-job/src/main/java/org/apache/kylin/job/manager/ExecutableManager.java @@ -299,6 +299,7 @@ public class ExecutableManager { private static AbstractExecutable parseTo(ExecutablePO executablePO) { if (executablePO == null) { + logger.warn("executablePO is null"); return null; } String type = executablePO.getType(); @@ -318,8 +319,7 @@ public class ExecutableManager { } return result; } catch (ReflectiveOperationException e) { - logger.error("cannot parse this job:" + executablePO.getId(), e); - return null; + throw new IllegalStateException("cannot parse this job:" + executablePO.getId(), e); } } http://git-wip-us.apache.org/repos/asf/kylin/blob/07537a9a/core-metadata/src/main/java/org/apache/kylin/metadata/model/MeasureDesc.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/MeasureDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/MeasureDesc.java index 618d25a..5d65771 100644 --- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/MeasureDesc.java +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/MeasureDesc.java @@ -31,8 +31,7 @@ import com.fasterxml.jackson.annotation.JsonProperty; @JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE) public class MeasureDesc { - @JsonProperty("id") - private int id; + @JsonProperty("name") private String name; @JsonProperty("function") @@ -49,13 +48,7 @@ public class MeasureDesc { } } - public int getId() { - return id; - } - - public void setId(int id) { - this.id = id; - } + public String getName() { return name; @@ -103,11 +96,6 @@ public class MeasureDesc { } @Override - public int hashCode() { - return id; - } - - @Override public String toString() { return "MeasureDesc [name=" + name + ", function=" + function + "]"; } http://git-wip-us.apache.org/repos/asf/kylin/blob/07537a9a/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchCubingJobBuilder.java ---------------------------------------------------------------------- diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchCubingJobBuilder.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchCubingJobBuilder.java index 12cac94..3c10c09 100644 --- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchCubingJobBuilder.java +++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchCubingJobBuilder.java @@ -63,7 +63,7 @@ public class BatchCubingJobBuilder extends JobBuilderSupport { // Phase 3: Build Cube RowKeyDesc rowKeyDesc = ((CubeSegment) seg).getCubeDesc().getRowkey(); - final int groupRowkeyColumnsCount = rowKeyDesc.getNCuboidBuildLevels(); + final int groupRowkeyColumnsCount = ((CubeSegment) seg).getCubeDesc().getBuildLevel(); final int totalRowkeyColumnsCount = rowKeyDesc.getRowKeyColumns().length; final String[] cuboidOutputTempPath = getCuboidOutputPaths(cuboidRootPath, totalRowkeyColumnsCount, groupRowkeyColumnsCount); // base cuboid step http://git-wip-us.apache.org/repos/asf/kylin/blob/07537a9a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsMapperBase.java ---------------------------------------------------------------------- diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsMapperBase.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsMapperBase.java index 2279b8b..5e83add 100644 --- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsMapperBase.java +++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsMapperBase.java @@ -61,7 +61,7 @@ public class FactDistinctColumnsMapperBase<KEYIN, VALUEIN> extends KylinMapper<K for (int i = 0; i < dictionaryColumns.size(); i++) { TblColRef col = dictionaryColumns.get(i); - String scanTable = dictMgr.decideSourceData(cubeDesc.getModel(), "true", col).getTable(); + String scanTable = dictMgr.decideSourceData(cubeDesc.getModel(), true, col).getTable(); if (cubeDesc.getModel().isFactTable(scanTable)) { factDictCols.add(i); } http://git-wip-us.apache.org/repos/asf/kylin/blob/07537a9a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctHiveColumnsMapper.java ---------------------------------------------------------------------- diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctHiveColumnsMapper.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctHiveColumnsMapper.java index eb67ad9..a4fd59b 100644 --- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctHiveColumnsMapper.java +++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctHiveColumnsMapper.java @@ -91,8 +91,7 @@ public class FactDistinctHiveColumnsMapper<KEYIN> extends FactDistinctColumnsMap private void addCuboidBitSet(long cuboidId, List<Integer[]> allCuboidsBitSet, List<Long> allCuboids) { allCuboids.add(cuboidId); - BitSet bitSet = BitSet.valueOf(new long[] { cuboidId }); - Integer[] indice = new Integer[bitSet.cardinality()]; + Integer[] indice = new Integer[Long.bitCount(cuboidId)]; long mask = Long.highestOneBit(baseCuboidId); int position = 0; http://git-wip-us.apache.org/repos/asf/kylin/blob/07537a9a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MapContextGTRecordWriter.java ---------------------------------------------------------------------- diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MapContextGTRecordWriter.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MapContextGTRecordWriter.java index 6098381..8416d95 100644 --- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MapContextGTRecordWriter.java +++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MapContextGTRecordWriter.java @@ -88,7 +88,7 @@ public class MapContextGTRecordWriter implements ICuboidWriter { rowKeyEncoder = AbstractRowKeyEncoder.createInstance(cubeSegment, Cuboid.findById(cubeDesc, cuboidId)); keyBuf = rowKeyEncoder.createBuf(); - dimensions = BitSet.valueOf(new long[] { cuboidId }).cardinality(); + dimensions = Long.bitCount(cuboidId); measureColumnsIndex = new int[measureCount]; for (int i = 0; i < measureCount; i++) { measureColumnsIndex[i] = dimensions + i; http://git-wip-us.apache.org/repos/asf/kylin/blob/07537a9a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidFromStorageMapper.java ---------------------------------------------------------------------- diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidFromStorageMapper.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidFromStorageMapper.java index fc616fa..22324ba 100644 --- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidFromStorageMapper.java +++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidFromStorageMapper.java @@ -209,7 +209,7 @@ public class MergeCuboidFromStorageMapper extends KylinMapper<Object, Object, By else { ret = cubeDesc.getRowkey().isUseDictionary(col); if (ret) { - String dictTable = DictionaryManager.getInstance(config).decideSourceData(cubeDesc.getModel(), cubeDesc.getRowkey().getDictionary(col), col).getTable(); + String dictTable = DictionaryManager.getInstance(config).decideSourceData(cubeDesc.getModel(), cubeDesc.getRowkey().isUseDictionary(col), col).getTable(); ret = cubeDesc.getFactTable().equalsIgnoreCase(dictTable); } dimensionsNeedDict.put(col, ret); http://git-wip-us.apache.org/repos/asf/kylin/blob/07537a9a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidMapper.java ---------------------------------------------------------------------- diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidMapper.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidMapper.java index 6c2679e..b5a63cb 100644 --- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidMapper.java +++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidMapper.java @@ -233,7 +233,7 @@ public class MergeCuboidMapper extends KylinMapper<Text, Text, Text, Text> { else { ret = cubeDesc.getRowkey().isUseDictionary(col); if (ret) { - String dictTable = DictionaryManager.getInstance(config).decideSourceData(cubeDesc.getModel(), cubeDesc.getRowkey().getDictionary(col), col).getTable(); + String dictTable = DictionaryManager.getInstance(config).decideSourceData(cubeDesc.getModel(), cubeDesc.getRowkey().isUseDictionary(col), col).getTable(); ret = cubeDesc.getFactTable().equalsIgnoreCase(dictTable); } dimensionsNeedDict.put(col, ret); http://git-wip-us.apache.org/repos/asf/kylin/blob/07537a9a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeDictionaryStep.java ---------------------------------------------------------------------- diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeDictionaryStep.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeDictionaryStep.java index b73fda4..8fced61 100644 --- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeDictionaryStep.java +++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeDictionaryStep.java @@ -112,7 +112,7 @@ public class MergeDictionaryStep extends AbstractExecutable { CubeDesc cubeDesc = cube.getDescriptor(); for (TblColRef col : cubeDesc.getAllColumnsNeedDictionary()) { - String dictTable = dictMgr.decideSourceData(cubeDesc.getModel(), "true", col).getTable(); + String dictTable = dictMgr.decideSourceData(cubeDesc.getModel(),true, col).getTable(); if (cubeDesc.getFactTable().equalsIgnoreCase(dictTable)) { colsNeedMeringDict.add(col); } else { http://git-wip-us.apache.org/repos/asf/kylin/blob/07537a9a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/CubeSamplingTest.java ---------------------------------------------------------------------- diff --git a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/CubeSamplingTest.java b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/CubeSamplingTest.java index fc016fc..410cec7 100644 --- a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/CubeSamplingTest.java +++ b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/CubeSamplingTest.java @@ -77,8 +77,7 @@ public class CubeSamplingTest { } private void addCuboidBitSet(long cuboidId, List<Integer[]> allCuboidsBitSet) { - BitSet bitSet = BitSet.valueOf(new long[] { cuboidId }); - Integer[] indice = new Integer[bitSet.cardinality()]; + Integer[] indice = new Integer[Long.bitCount(cuboidId)]; long mask = Long.highestOneBit(baseCuboidId); int position = 0; http://git-wip-us.apache.org/repos/asf/kylin/blob/07537a9a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubing.java ---------------------------------------------------------------------- diff --git a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubing.java b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubing.java index 01d97fd..32d4d2a 100644 --- a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubing.java +++ b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubing.java @@ -229,8 +229,7 @@ public class SparkCubing extends AbstractApplication { final ByteArray[] row_hashcodes = new ByteArray[nRowKey]; for (Long cuboidId : allCuboidIds) { - BitSet bitSet = BitSet.valueOf(new long[]{cuboidId}); - Integer[] cuboidBitSet = new Integer[bitSet.cardinality()]; + Integer[] cuboidBitSet = new Integer[Long.bitCount(cuboidId)]; long mask = Long.highestOneBit(baseCuboidId); int position = 0; @@ -480,7 +479,7 @@ public class SparkCubing extends AbstractApplication { final Configuration hbaseConf = HBaseConnection.getCurrentHBaseConfiguration(); FileSystem fs = FileSystem.get(hbaseConf); FsPermission permission = new FsPermission((short) 0777); - for (HBaseColumnFamilyDesc cf : cubeDesc.getHBaseMapping().getColumnFamily()) { + for (HBaseColumnFamilyDesc cf : cubeDesc.getHbaseMapping().getColumnFamily()) { String cfName = cf.getName(); Path columnFamilyPath = new Path(hfileLocation, cfName); http://git-wip-us.apache.org/repos/asf/kylin/blob/07537a9a/engine-spark/src/main/java/org/apache/kylin/engine/spark/cube/DefaultTupleConverter.java ---------------------------------------------------------------------- diff --git a/engine-spark/src/main/java/org/apache/kylin/engine/spark/cube/DefaultTupleConverter.java b/engine-spark/src/main/java/org/apache/kylin/engine/spark/cube/DefaultTupleConverter.java index 6b3a82c..9c1d681 100644 --- a/engine-spark/src/main/java/org/apache/kylin/engine/spark/cube/DefaultTupleConverter.java +++ b/engine-spark/src/main/java/org/apache/kylin/engine/spark/cube/DefaultTupleConverter.java @@ -18,7 +18,6 @@ package org.apache.kylin.engine.spark.cube; import java.nio.ByteBuffer; -import java.util.BitSet; import java.util.Map; import org.apache.kylin.common.util.ByteArray; @@ -70,7 +69,7 @@ public final class DefaultTupleConverter implements TupleConverter { Cuboid cuboid = Cuboid.findById(segment.getCubeDesc(), cuboidId); RowKeyEncoder rowkeyEncoder = rowKeyEncoderProvider.getRowkeyEncoder(cuboid); - final int dimensions = BitSet.valueOf(new long[] { cuboidId }).cardinality(); + final int dimensions = Long.bitCount(cuboidId); int[] measureColumnsIndex = getMeasureColumnsIndex(); for (int i = 0; i < measureCount; i++) { measureColumnsIndex[i] = dimensions + i; http://git-wip-us.apache.org/repos/asf/kylin/blob/07537a9a/server/src/main/java/org/apache/kylin/rest/service/CubeService.java ---------------------------------------------------------------------- diff --git a/server/src/main/java/org/apache/kylin/rest/service/CubeService.java b/server/src/main/java/org/apache/kylin/rest/service/CubeService.java index 5e1a3d9..5767f15 100644 --- a/server/src/main/java/org/apache/kylin/rest/service/CubeService.java +++ b/server/src/main/java/org/apache/kylin/rest/service/CubeService.java @@ -194,7 +194,7 @@ public class CubeService extends BasicService { } try { - int cuboidCount = CuboidCLI.simulateCuboidGeneration(createdDesc); + int cuboidCount = CuboidCLI.simulateCuboidGeneration(createdDesc,false); logger.info("New cube " + cubeName + " has " + cuboidCount + " cuboids"); } catch (Exception e) { getCubeDescManager().removeCubeDesc(createdDesc); @@ -260,7 +260,7 @@ public class CubeService extends BasicService { } CubeDesc updatedCubeDesc = getCubeDescManager().updateCubeDesc(desc); - int cuboidCount = CuboidCLI.simulateCuboidGeneration(updatedCubeDesc); + int cuboidCount = CuboidCLI.simulateCuboidGeneration(updatedCubeDesc,false); logger.info("Updated cube " + cube.getName() + " has " + cuboidCount + " cuboids"); ProjectManager projectManager = getProjectManager(); http://git-wip-us.apache.org/repos/asf/kylin/blob/07537a9a/server/src/test/java/org/apache/kylin/rest/controller/CubeControllerTest.java ---------------------------------------------------------------------- diff --git a/server/src/test/java/org/apache/kylin/rest/controller/CubeControllerTest.java b/server/src/test/java/org/apache/kylin/rest/controller/CubeControllerTest.java index c8496a0..1230886 100644 --- a/server/src/test/java/org/apache/kylin/rest/controller/CubeControllerTest.java +++ b/server/src/test/java/org/apache/kylin/rest/controller/CubeControllerTest.java @@ -60,11 +60,9 @@ public class CubeControllerTest extends ServiceTestBase { cubeController.setCubeService(cubeService); cubeController.setJobService(jobService); cubeController.setStreamingService(streamingService); - - + cubeDescController = new CubeDescController(); cubeDescController.setCubeService(cubeService); - } @@ -89,10 +87,11 @@ public class CubeControllerTest extends ServiceTestBase { newCube.setModelName(cube.getModelName()); newCube.setModel(cube.getModel()); newCube.setDimensions(cube.getDimensions()); - newCube.setHBaseMapping(cube.getHBaseMapping()); + newCube.setHbaseMapping(cube.getHbaseMapping()); newCube.setMeasures(cube.getMeasures()); newCube.setConfig(cube.getConfig()); newCube.setRowkey(cube.getRowkey()); + newCube.setAggregationGroups(cube.getAggregationGroups()); newCube.getModel().setLastModified(0); http://git-wip-us.apache.org/repos/asf/kylin/blob/07537a9a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeStorageQuery.java ---------------------------------------------------------------------- diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeStorageQuery.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeStorageQuery.java index 2fa0490..652d558 100644 --- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeStorageQuery.java +++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeStorageQuery.java @@ -141,7 +141,7 @@ public class CubeStorageQuery implements ICachableStorageQuery { List<HBaseKeyRange> scans = buildScanRanges(flattenToOrAndFilter(filterD), dimensionsD); // check involved measures, build value decoder for each each family:column - List<RowValueDecoder> valueDecoders = translateAggregation(cubeDesc.getHBaseMapping(), metrics, context); + List<RowValueDecoder> valueDecoders = translateAggregation(cubeDesc.getHbaseMapping(), metrics, context); // memory hungry distinct count are pushed down to coprocessor, no need to set threshold any more // setThreshold(dimensionsD, valueDecoders, context); // set cautious threshold to prevent out of memory http://git-wip-us.apache.org/repos/asf/kylin/blob/07537a9a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeTupleConverter.java ---------------------------------------------------------------------- diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeTupleConverter.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeTupleConverter.java index 8813901..6a15bc3 100644 --- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeTupleConverter.java +++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeTupleConverter.java @@ -12,7 +12,6 @@ import org.apache.kylin.common.topn.Counter; import org.apache.kylin.common.topn.TopNCounter; import org.apache.kylin.common.util.Array; import org.apache.kylin.common.util.ByteArray; -import org.apache.kylin.common.util.Bytes; import org.apache.kylin.common.util.BytesUtil; import org.apache.kylin.cube.CubeManager; import org.apache.kylin.cube.CubeSegment; @@ -21,12 +20,9 @@ import org.apache.kylin.cube.kv.RowKeyDecoder; import org.apache.kylin.cube.model.CubeDesc.DeriveInfo; import org.apache.kylin.dict.Dictionary; import org.apache.kylin.dict.lookup.LookupStringTable; -import org.apache.kylin.metadata.model.DataType; import org.apache.kylin.metadata.model.FunctionDesc; import org.apache.kylin.metadata.model.MeasureDesc; import org.apache.kylin.metadata.model.TblColRef; -import org.apache.kylin.metadata.tuple.ITuple; -import org.apache.kylin.storage.StorageContext; import org.apache.kylin.storage.hbase.steps.RowValueDecoder; import org.apache.kylin.storage.tuple.Tuple; import org.apache.kylin.storage.tuple.TupleInfo; @@ -104,7 +100,7 @@ public class CubeTupleConverter { } // prepare derived columns and filler - Map<Array<TblColRef>, List<DeriveInfo>> hostToDerivedInfo = cuboid.getCube().getHostToDerivedInfo(dimCols, null); + Map<Array<TblColRef>, List<DeriveInfo>> hostToDerivedInfo = cuboid.getCubeDesc().getHostToDerivedInfo(dimCols, null); for (Entry<Array<TblColRef>, List<DeriveInfo>> entry : hostToDerivedInfo.entrySet()) { TblColRef[] hostCols = entry.getKey().data; for (DeriveInfo deriveInfo : entry.getValue()) { http://git-wip-us.apache.org/repos/asf/kylin/blob/07537a9a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeStorageQuery.java ---------------------------------------------------------------------- diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeStorageQuery.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeStorageQuery.java index 59ed99b..ca4c8aa 100644 --- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeStorageQuery.java +++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeStorageQuery.java @@ -101,7 +101,7 @@ public class CubeStorageQuery implements ICachableStorageQuery { try { scanner = new CubeSegmentScanner(cubeSeg, cuboid, dimensionsD, groupsD, metrics, filterD, !isExactAggregation); } catch (NotEnoughGTInfoException e) { - logger.info("Cannot construct Segment {}'s GTInfo, this may due to empty segment or broken metadata"); + logger.info("Cannot construct Segment {}'s GTInfo, this may due to empty segment or broken metadata", cubeSeg); continue; } scanners.add(scanner); http://git-wip-us.apache.org/repos/asf/kylin/blob/07537a9a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeTupleConverter.java ---------------------------------------------------------------------- diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeTupleConverter.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeTupleConverter.java index c89cce2..f09ce8a 100644 --- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeTupleConverter.java +++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeTupleConverter.java @@ -24,7 +24,6 @@ import java.util.Map; import java.util.Map.Entry; import java.util.Set; -import org.apache.hadoop.hbase.client.Result; import org.apache.kylin.common.topn.Counter; import org.apache.kylin.common.topn.TopNCounter; import org.apache.kylin.common.util.Array; @@ -114,7 +113,7 @@ public class CubeTupleConverter { } // prepare derived columns and filler - Map<Array<TblColRef>, List<DeriveInfo>> hostToDerivedInfo = cuboid.getCube().getHostToDerivedInfo(cuboidDims, null); + Map<Array<TblColRef>, List<DeriveInfo>> hostToDerivedInfo = cuboid.getCubeDesc().getHostToDerivedInfo(cuboidDims, null); for (Entry<Array<TblColRef>, List<DeriveInfo>> entry : hostToDerivedInfo.entrySet()) { TblColRef[] hostCols = entry.getKey().data; for (DeriveInfo deriveInfo : entry.getValue()) { http://git-wip-us.apache.org/repos/asf/kylin/blob/07537a9a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/BulkLoadJob.java ---------------------------------------------------------------------- diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/BulkLoadJob.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/BulkLoadJob.java index b31a078..d2a18c9 100644 --- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/BulkLoadJob.java +++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/BulkLoadJob.java @@ -69,7 +69,7 @@ public class BulkLoadJob extends AbstractHadoopJob { CubeInstance cube = cubeMgr.getCube(cubeName); CubeDesc cubeDesc = cube.getDescriptor(); FsPermission permission = new FsPermission((short) 0777); - for (HBaseColumnFamilyDesc cf : cubeDesc.getHBaseMapping().getColumnFamily()) { + for (HBaseColumnFamilyDesc cf : cubeDesc.getHbaseMapping().getColumnFamily()) { String cfName = cf.getName(); Path columnFamilyPath = new Path(input, cfName); http://git-wip-us.apache.org/repos/asf/kylin/blob/07537a9a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHFileMapper.java ---------------------------------------------------------------------- diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHFileMapper.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHFileMapper.java index 9f97b0e..509df42 100644 --- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHFileMapper.java +++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHFileMapper.java @@ -66,7 +66,7 @@ public class CubeHFileMapper extends KylinMapper<Text, Text, ImmutableBytesWrita inputMeasures = new Object[cubeDesc.getMeasures().size()]; keyValueCreators = Lists.newArrayList(); - for (HBaseColumnFamilyDesc cfDesc : cubeDesc.getHBaseMapping().getColumnFamily()) { + for (HBaseColumnFamilyDesc cfDesc : cubeDesc.getHbaseMapping().getColumnFamily()) { for (HBaseColumnDesc colDesc : cfDesc.getColumns()) { keyValueCreators.add(new KeyValueCreator(cubeDesc, colDesc)); } http://git-wip-us.apache.org/repos/asf/kylin/blob/07537a9a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHTableUtil.java ---------------------------------------------------------------------- diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHTableUtil.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHTableUtil.java index cdc259b..864765e 100644 --- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHTableUtil.java +++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHTableUtil.java @@ -47,7 +47,7 @@ public class CubeHTableUtil { tableDesc.addCoprocessor("org.apache.hadoop.hbase.security.access.SecureBulkLoadEndpoint"); } - for (HBaseColumnFamilyDesc cfDesc : cubeDesc.getHBaseMapping().getColumnFamily()) { + for (HBaseColumnFamilyDesc cfDesc : cubeDesc.getHbaseMapping().getColumnFamily()) { HColumnDescriptor cf = new HColumnDescriptor(cfDesc.getName()); cf.setMaxVersions(1); http://git-wip-us.apache.org/repos/asf/kylin/blob/07537a9a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseCuboidWriter.java ---------------------------------------------------------------------- diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseCuboidWriter.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseCuboidWriter.java index 31cce7b..c4dc0b5 100644 --- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseCuboidWriter.java +++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseCuboidWriter.java @@ -76,7 +76,7 @@ public final class HBaseCuboidWriter implements ICuboidWriter { this.keyValueCreators = Lists.newArrayList(); this.cubeSegment = segment; this.cubeDesc = cubeSegment.getCubeDesc(); - for (HBaseColumnFamilyDesc cfDesc : cubeDesc.getHBaseMapping().getColumnFamily()) { + for (HBaseColumnFamilyDesc cfDesc : cubeDesc.getHbaseMapping().getColumnFamily()) { for (HBaseColumnDesc colDesc : cfDesc.getColumns()) { keyValueCreators.add(new KeyValueCreator(cubeDesc, colDesc)); } http://git-wip-us.apache.org/repos/asf/kylin/blob/07537a9a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseMROutput2.java ---------------------------------------------------------------------- diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseMROutput2.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseMROutput2.java index f99ddb5..205c0db 100644 --- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseMROutput2.java +++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseMROutput2.java @@ -140,7 +140,7 @@ public class HBaseMROutput2 implements IMROutput2 { List<RowValueDecoder> valueDecoderList = Lists.newArrayList(); List<MeasureDesc> measuresDescs = Lists.newArrayList(); - for (HBaseColumnFamilyDesc cfDesc : seg.getCubeDesc().getHBaseMapping().getColumnFamily()) { + for (HBaseColumnFamilyDesc cfDesc : seg.getCubeDesc().getHbaseMapping().getColumnFamily()) { for (HBaseColumnDesc colDesc : cfDesc.getColumns()) { valueDecoderList.add(new RowValueDecoder(colDesc)); for (MeasureDesc measure : colDesc.getMeasures()) { @@ -238,7 +238,7 @@ public class HBaseMROutput2 implements IMROutput2 { @Override public void doReducerOutput(ByteArrayWritable key, Object[] value, Reducer.Context context) throws IOException, InterruptedException { if (keyValueCreators.size() == 0) { - for (HBaseColumnFamilyDesc cfDesc : seg.getCubeDesc().getHBaseMapping().getColumnFamily()) { + for (HBaseColumnFamilyDesc cfDesc : seg.getCubeDesc().getHbaseMapping().getColumnFamily()) { for (HBaseColumnDesc colDesc : cfDesc.getColumns()) { keyValueCreators.add(new KeyValueCreator(seg.getCubeDesc(), colDesc)); } http://git-wip-us.apache.org/repos/asf/kylin/blob/07537a9a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseMROutput2Transition.java ---------------------------------------------------------------------- diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseMROutput2Transition.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseMROutput2Transition.java index e7c4cf5..8cb8f13 100644 --- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseMROutput2Transition.java +++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseMROutput2Transition.java @@ -274,7 +274,7 @@ public class HBaseMROutput2Transition implements IMROutput2 { if (rowValueDecoders == null) { List<RowValueDecoder> valueDecoderList = Lists.newArrayList(); List<MeasureDesc> measuresDescs = Lists.newArrayList(); - for (HBaseColumnFamilyDesc cfDesc : seg.getCubeDesc().getHBaseMapping().getColumnFamily()) { + for (HBaseColumnFamilyDesc cfDesc : seg.getCubeDesc().getHbaseMapping().getColumnFamily()) { for (HBaseColumnDesc colDesc : cfDesc.getColumns()) { valueDecoderList.add(new RowValueDecoder(colDesc)); for (MeasureDesc measure : colDesc.getMeasures()) { http://git-wip-us.apache.org/repos/asf/kylin/blob/07537a9a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/RowValueDecoderTest.java ---------------------------------------------------------------------- diff --git a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/RowValueDecoderTest.java b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/RowValueDecoderTest.java index ba79305..68a2e76 100644 --- a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/RowValueDecoderTest.java +++ b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/RowValueDecoderTest.java @@ -55,7 +55,7 @@ public class RowValueDecoderTest extends LocalFileMetadataTestCase { @Test public void testDecode() throws Exception { CubeDesc cubeDesc = CubeManager.getInstance(getTestConfig()).getCube("test_kylin_cube_with_slr_ready").getDescriptor(); - HBaseColumnDesc hbaseCol = cubeDesc.getHBaseMapping().getColumnFamily()[0].getColumns()[0]; + HBaseColumnDesc hbaseCol = cubeDesc.getHbaseMapping().getColumnFamily()[0].getColumns()[0]; MeasureCodec codec = new MeasureCodec(hbaseCol.getMeasures()); BigDecimal sum = new BigDecimal("333.1234567"); @@ -86,7 +86,7 @@ public class RowValueDecoderTest extends LocalFileMetadataTestCase { @Test(expected = IllegalArgumentException.class) public void testError() throws Exception { CubeDesc cubeDesc = CubeManager.getInstance(getTestConfig()).getCube("test_kylin_cube_with_slr_ready").getDescriptor(); - HBaseColumnDesc hbaseCol = cubeDesc.getHBaseMapping().getColumnFamily()[0].getColumns()[0]; + HBaseColumnDesc hbaseCol = cubeDesc.getHbaseMapping().getColumnFamily()[0].getColumns()[0]; MeasureCodec codec = new MeasureCodec(hbaseCol.getMeasures()); BigDecimal sum = new BigDecimal("11111111111111111111333.1234567");
