http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/DoggedCubeBuilder.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/DoggedCubeBuilder.java b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/DoggedCubeBuilder.java index 0a6314c..a9211da 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/DoggedCubeBuilder.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/DoggedCubeBuilder.java @@ -57,8 +57,7 @@ public class DoggedCubeBuilder extends AbstractInMemCubeBuilder { private int splitRowThreshold = Integer.MAX_VALUE; private int unitRows = 1000; - public DoggedCubeBuilder(CubeDesc cubeDesc, IJoinedFlatTableDesc flatDesc, - Map<TblColRef, Dictionary<String>> dictionaryMap) { + public DoggedCubeBuilder(CubeDesc cubeDesc, IJoinedFlatTableDesc flatDesc, Map<TblColRef, Dictionary<String>> dictionaryMap) { super(cubeDesc, flatDesc, dictionaryMap); // check memory more often if a single row is big @@ -199,10 +198,7 @@ public class DoggedCubeBuilder extends AbstractInMemCubeBuilder { } else { for (Throwable t : errors) logger.error("Exception during in-mem cube build", t); - throw new IOException( - errors.size() - + " exceptions during in-mem cube build, cause set to the first, check log for more", - errors.get(0)); + throw new IOException(errors.size() + " exceptions during in-mem cube build, cause set to the first, check log for more", errors.get(0)); } } @@ -255,8 +251,7 @@ public class DoggedCubeBuilder extends AbstractInMemCubeBuilder { int nSplit = splits.size(); long splitRowCount = nSplit == 0 ? 0 : splits.get(nSplit - 1).inputRowCount; - logger.info(splitRowCount + " records went into split #" + nSplit + "; " + systemAvailMB + " MB left, " - + reserveMemoryMB + " MB threshold"); + logger.info(splitRowCount + " records went into split #" + nSplit + "; " + systemAvailMB + " MB left, " + reserveMemoryMB + " MB threshold"); if (splitRowCount >= splitRowThreshold) { logger.info("Split cut due to hitting splitRowThreshold " + splitRowThreshold); @@ -264,8 +259,7 @@ public class DoggedCubeBuilder extends AbstractInMemCubeBuilder { } if (systemAvailMB <= reserveMemoryMB * 1.5) { - logger.info("Split cut due to hitting memory threshold, system avail " + systemAvailMB - + " MB <= reserve " + reserveMemoryMB + "*1.5 MB"); + logger.info("Split cut due to hitting memory threshold, system avail " + systemAvailMB + " MB <= reserve " + reserveMemoryMB + "*1.5 MB"); return true; } @@ -411,8 +405,7 @@ public class DoggedCubeBuilder extends AbstractInMemCubeBuilder { if (cuboidIterator.hasNext()) { CuboidResult cuboid = cuboidIterator.next(); currentCuboidId = cuboid.cuboidId; - scanner = cuboid.table.scan(new GTScanRequestBuilder().setInfo(cuboid.table.getInfo()) - .setRanges(null).setDimensions(null).setFilterPushDown(null).createGTScanRequest()); + scanner = cuboid.table.scan(new GTScanRequestBuilder().setInfo(cuboid.table.getInfo()).setRanges(null).setDimensions(null).setFilterPushDown(null).createGTScanRequest()); recordIterator = scanner.iterator(); } else { return false;
http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilder.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilder.java b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilder.java index 5233def..a26e948 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilder.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilder.java @@ -33,8 +33,8 @@ import java.util.concurrent.atomic.AtomicInteger; import org.apache.kylin.common.util.Dictionary; import org.apache.kylin.common.util.ImmutableBitSet; import org.apache.kylin.common.util.MemoryBudgetController; -import org.apache.kylin.common.util.MemoryBudgetController.MemoryWaterLevel; import org.apache.kylin.common.util.Pair; +import org.apache.kylin.common.util.MemoryBudgetController.MemoryWaterLevel; import org.apache.kylin.cube.cuboid.Cuboid; import org.apache.kylin.cube.cuboid.CuboidScheduler; import org.apache.kylin.cube.gridtable.CubeGridTable; @@ -90,8 +90,7 @@ public class InMemCubeBuilder extends AbstractInMemCubeBuilder { private Object[] totalSumForSanityCheck; private ICuboidCollector resultCollector; - public InMemCubeBuilder(CubeDesc cubeDesc, IJoinedFlatTableDesc flatDesc, - Map<TblColRef, Dictionary<String>> dictionaryMap) { + public InMemCubeBuilder(CubeDesc cubeDesc, IJoinedFlatTableDesc flatDesc, Map<TblColRef, Dictionary<String>> dictionaryMap) { super(cubeDesc, flatDesc, dictionaryMap); this.cuboidScheduler = new CuboidScheduler(cubeDesc); this.baseCuboidId = Cuboid.getBaseCuboidId(cubeDesc); @@ -110,8 +109,10 @@ public class InMemCubeBuilder extends AbstractInMemCubeBuilder { } private GridTable newGridTableByCuboidID(long cuboidID) throws IOException { - GTInfo info = CubeGridTable.newGTInfo(Cuboid.findById(cubeDesc, cuboidID), - new CubeDimEncMap(cubeDesc, dictionaryMap)); + GTInfo info = CubeGridTable.newGTInfo( + Cuboid.findById(cubeDesc, cuboidID), + new CubeDimEncMap(cubeDesc, dictionaryMap) + ); // Below several store implementation are very similar in performance. The ConcurrentDiskStore is the simplest. // MemDiskStore store = new MemDiskStore(info, memBudget == null ? MemoryBudgetController.ZERO_BUDGET : memBudget); @@ -231,9 +232,7 @@ public class InMemCubeBuilder extends AbstractInMemCubeBuilder { } else { for (Throwable t : errors) logger.error("Exception during in-mem cube build", t); - throw new IOException( - errors.size() + " exceptions during in-mem cube build, cause set to the first, check log for more", - errors.get(0)); + throw new IOException(errors.size() + " exceptions during in-mem cube build, cause set to the first, check log for more", errors.get(0)); } } @@ -323,9 +322,7 @@ public class InMemCubeBuilder extends AbstractInMemCubeBuilder { if (budget < baseResult.aggrCacheMB) { // make sure we have base aggr cache as minimal budget = baseResult.aggrCacheMB; - logger.warn("System avail memory (" + systemAvailMB + " MB) is less than base aggr cache (" - + baseResult.aggrCacheMB + " MB) + minimal reservation (" + reserve - + " MB), consider increase JVM heap -Xmx"); + logger.warn("System avail memory (" + systemAvailMB + " MB) is less than base aggr cache (" + baseResult.aggrCacheMB + " MB) + minimal reservation (" + reserve + " MB), consider increase JVM heap -Xmx"); } logger.debug("Memory Budget is " + budget + " MB"); @@ -340,11 +337,8 @@ public class InMemCubeBuilder extends AbstractInMemCubeBuilder { GTBuilder baseBuilder = baseCuboid.rebuild(); IGTScanner baseInput = new InputConverter(baseCuboid.getInfo(), input); - Pair<ImmutableBitSet, ImmutableBitSet> dimensionMetricsBitSet = InMemCubeBuilderUtils - .getDimensionAndMetricColumnBitSet(baseCuboidId, measureCount); - GTScanRequest req = new GTScanRequestBuilder().setInfo(baseCuboid.getInfo()).setRanges(null).setDimensions(null) - .setAggrGroupBy(dimensionMetricsBitSet.getFirst()).setAggrMetrics(dimensionMetricsBitSet.getSecond()) - .setAggrMetricsFuncs(metricsAggrFuncs).setFilterPushDown(null).createGTScanRequest(); + Pair<ImmutableBitSet, ImmutableBitSet> dimensionMetricsBitSet = InMemCubeBuilderUtils.getDimensionAndMetricColumnBitSet(baseCuboidId, measureCount); + GTScanRequest req = new GTScanRequestBuilder().setInfo(baseCuboid.getInfo()).setRanges(null).setDimensions(null).setAggrGroupBy(dimensionMetricsBitSet.getFirst()).setAggrMetrics(dimensionMetricsBitSet.getSecond()).setAggrMetricsFuncs(metricsAggrFuncs).setFilterPushDown(null).createGTScanRequest(); GTAggregateScanner aggregationScanner = new GTAggregateScanner(baseInput, req); aggregationScanner.trackMemoryLevel(baseCuboidMemTracker); @@ -362,15 +356,13 @@ public class InMemCubeBuilder extends AbstractInMemCubeBuilder { long timeSpent = System.currentTimeMillis() - startTime; logger.info("Cuboid " + baseCuboidId + " has " + count + " rows, build takes " + timeSpent + "ms"); - int mbEstimateBaseAggrCache = (int) (aggregationScanner.getEstimateSizeOfAggrCache() - / MemoryBudgetController.ONE_MB); + int mbEstimateBaseAggrCache = (int) (aggregationScanner.getEstimateSizeOfAggrCache() / MemoryBudgetController.ONE_MB); logger.info("Wild estimate of base aggr cache is " + mbEstimateBaseAggrCache + " MB"); return updateCuboidResult(baseCuboidId, baseCuboid, count, timeSpent, 0); } - private CuboidResult updateCuboidResult(long cuboidId, GridTable table, int nRows, long timeSpent, - int aggrCacheMB) { + private CuboidResult updateCuboidResult(long cuboidId, GridTable table, int nRows, long timeSpent, int aggrCacheMB) { if (aggrCacheMB <= 0 && baseResult != null) { aggrCacheMB = (int) Math.round(// (DERIVE_AGGR_CACHE_CONSTANT_FACTOR + DERIVE_AGGR_CACHE_VARIABLE_FACTOR * nRows / baseResult.nRows) // @@ -408,18 +400,13 @@ public class InMemCubeBuilder extends AbstractInMemCubeBuilder { } private CuboidResult aggregateCuboid(CuboidResult parent, long cuboidId) throws IOException { - final Pair<ImmutableBitSet, ImmutableBitSet> allNeededColumns = InMemCubeBuilderUtils - .getDimensionAndMetricColumnBitSet(parent.cuboidId, cuboidId, measureCount); - return scanAndAggregateGridTable(parent.table, parent.cuboidId, cuboidId, allNeededColumns.getFirst(), - allNeededColumns.getSecond()); + final Pair<ImmutableBitSet, ImmutableBitSet> allNeededColumns = InMemCubeBuilderUtils.getDimensionAndMetricColumnBitSet(parent.cuboidId, cuboidId, measureCount); + return scanAndAggregateGridTable(parent.table, parent.cuboidId, cuboidId, allNeededColumns.getFirst(), allNeededColumns.getSecond()); } - private GTAggregateScanner prepareGTAggregationScanner(GridTable gridTable, long parentId, long cuboidId, - ImmutableBitSet aggregationColumns, ImmutableBitSet measureColumns) throws IOException { + private GTAggregateScanner prepareGTAggregationScanner(GridTable gridTable, long parentId, long cuboidId, ImmutableBitSet aggregationColumns, ImmutableBitSet measureColumns) throws IOException { GTInfo info = gridTable.getInfo(); - GTScanRequest req = new GTScanRequestBuilder().setInfo(info).setRanges(null).setDimensions(null) - .setAggrGroupBy(aggregationColumns).setAggrMetrics(measureColumns).setAggrMetricsFuncs(metricsAggrFuncs) - .setFilterPushDown(null).createGTScanRequest(); + GTScanRequest req = new GTScanRequestBuilder().setInfo(info).setRanges(null).setDimensions(null).setAggrGroupBy(aggregationColumns).setAggrMetrics(measureColumns).setAggrMetricsFuncs(metricsAggrFuncs).setFilterPushDown(null).createGTScanRequest(); GTAggregateScanner scanner = (GTAggregateScanner) gridTable.scan(req); // for child cuboid, some measures don't need aggregation. @@ -438,13 +425,11 @@ public class InMemCubeBuilder extends AbstractInMemCubeBuilder { return scanner; } - private CuboidResult scanAndAggregateGridTable(GridTable gridTable, long parentId, long cuboidId, - ImmutableBitSet aggregationColumns, ImmutableBitSet measureColumns) throws IOException { + private CuboidResult scanAndAggregateGridTable(GridTable gridTable, long parentId, long cuboidId, ImmutableBitSet aggregationColumns, ImmutableBitSet measureColumns) throws IOException { long startTime = System.currentTimeMillis(); logger.info("Calculating cuboid " + cuboidId); - GTAggregateScanner scanner = prepareGTAggregationScanner(gridTable, parentId, cuboidId, aggregationColumns, - measureColumns); + GTAggregateScanner scanner = prepareGTAggregationScanner(gridTable, parentId, cuboidId, aggregationColumns, measureColumns); GridTable newGridTable = newGridTableByCuboidID(cuboidId); GTBuilder builder = newGridTable.rebuild(); @@ -539,8 +524,7 @@ public class InMemCubeBuilder extends AbstractInMemCubeBuilder { this.info = info; this.input = input; this.record = new GTRecord(info); - this.inMemCubeBuilderInputConverter = new InMemCubeBuilderInputConverter(cubeDesc, flatDesc, dictionaryMap, - info); + this.inMemCubeBuilderInputConverter = new InMemCubeBuilderInputConverter(cubeDesc, flatDesc, dictionaryMap, info); } @Override http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilderInputConverter.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilderInputConverter.java b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilderInputConverter.java index 6a2e40f..ab44f63 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilderInputConverter.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilderInputConverter.java @@ -49,8 +49,7 @@ public class InMemCubeBuilderInputConverter { private final GTInfo gtInfo; protected List<byte[]> nullBytes; - public InMemCubeBuilderInputConverter(CubeDesc cubeDesc, IJoinedFlatTableDesc flatDesc, - Map<TblColRef, Dictionary<String>> dictionaryMap, GTInfo gtInfo) { + public InMemCubeBuilderInputConverter(CubeDesc cubeDesc, IJoinedFlatTableDesc flatDesc, Map<TblColRef, Dictionary<String>> dictionaryMap, GTInfo gtInfo) { this.gtInfo = gtInfo; this.flatDesc = new CubeJoinedFlatTableEnrich(flatDesc, cubeDesc); this.measureCount = cubeDesc.getMeasures().size(); http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilderUtils.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilderUtils.java b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilderUtils.java index a0661f0..8557acf 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilderUtils.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilderUtils.java @@ -26,8 +26,7 @@ import org.apache.kylin.common.util.Pair; */ public final class InMemCubeBuilderUtils { - public static final Pair<ImmutableBitSet, ImmutableBitSet> getDimensionAndMetricColumnBitSet(final long cuboidId, - final int measureCount) { + public static final Pair<ImmutableBitSet, ImmutableBitSet> getDimensionAndMetricColumnBitSet(final long cuboidId, final int measureCount) { int cardinality = Long.bitCount(cuboidId); BitSet dimension = new BitSet(); dimension.set(0, cardinality); @@ -36,15 +35,13 @@ public final class InMemCubeBuilderUtils { return Pair.newPair(new ImmutableBitSet(dimension), new ImmutableBitSet(metrics)); } - public static final Pair<ImmutableBitSet, ImmutableBitSet> getDimensionAndMetricColumnBitSet( - final long baseCuboidId, final long childCuboidId, final int measureCount) { - final Pair<ImmutableBitSet, ImmutableBitSet> parentDimensionAndMetricColumnBitSet = getDimensionAndMetricColumnBitSet( - baseCuboidId, measureCount); + public static final Pair<ImmutableBitSet, ImmutableBitSet> getDimensionAndMetricColumnBitSet(final long baseCuboidId, final long childCuboidId, final int measureCount) { + final Pair<ImmutableBitSet, ImmutableBitSet> parentDimensionAndMetricColumnBitSet = getDimensionAndMetricColumnBitSet(baseCuboidId, measureCount); ImmutableBitSet parentDimensions = parentDimensionAndMetricColumnBitSet.getFirst(); ImmutableBitSet measureColumns = parentDimensionAndMetricColumnBitSet.getSecond(); ImmutableBitSet childDimensions = parentDimensions; long mask = Long.highestOneBit(baseCuboidId); - long parentCuboidIdActualLength = (long) Long.SIZE - Long.numberOfLeadingZeros(baseCuboidId); + long parentCuboidIdActualLength = (long)Long.SIZE - Long.numberOfLeadingZeros(baseCuboidId); int index = 0; for (int i = 0; i < parentCuboidIdActualLength; i++) { if ((mask & baseCuboidId) > 0) { http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/MemDiskStore.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/MemDiskStore.java b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/MemDiskStore.java index 9d4602e..a5471df 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/MemDiskStore.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/MemDiskStore.java @@ -71,8 +71,7 @@ public class MemDiskStore implements IGTStore, Closeable { this(info, budgetCtrl, diskFile, false); } - private MemDiskStore(GTInfo info, MemoryBudgetController budgetCtrl, File diskFile, boolean delOnClose) - throws IOException { + private MemDiskStore(GTInfo info, MemoryBudgetController budgetCtrl, File diskFile, boolean delOnClose) throws IOException { this.info = info; this.lock = this; this.memPart = new MemPart(budgetCtrl); @@ -168,8 +167,7 @@ public class MemDiskStore implements IGTStore, Closeable { if (available() <= 0) return -1; - if (memChunk == null && memPart.headOffset() <= readOffset - && readOffset < memPart.tailOffset()) { + if (memChunk == null && memPart.headOffset() <= readOffset && readOffset < memPart.tailOffset()) { memChunk = memPart.seekMemChunk(readOffset); } @@ -221,8 +219,7 @@ public class MemDiskStore implements IGTStore, Closeable { din.close(); diskPart.closeRead(); if (debug) - logger.debug(MemDiskStore.this + " read end @ " + readOffset + ", " + (memRead) + " from mem, " - + (diskRead) + " from disk, " + nReadCalls + " read() calls"); + logger.debug(MemDiskStore.this + " read end @ " + readOffset + ", " + (memRead) + " from mem, " + (diskRead) + " from disk, " + nReadCalls + " read() calls"); } } @@ -361,8 +358,7 @@ public class MemDiskStore implements IGTStore, Closeable { diskPart.closeWrite(); ongoingWriter = null; if (debug) - logger.debug(MemDiskStore.this + " write end @ " + writeOffset + ", " + (memWrite) + " to mem, " - + (diskWrite) + " to disk, " + nWriteCalls + " write() calls"); + logger.debug(MemDiskStore.this + " write end @ " + writeOffset + ", " + (memWrite) + " to mem, " + (diskWrite) + " to disk, " + nWriteCalls + " write() calls"); } else { // the asyncFlusher will call this close() again later } @@ -667,13 +663,11 @@ public class MemDiskStore implements IGTStore, Closeable { public void openWrite(boolean append) throws IOException { if (append) { - writeChannel = FileChannel.open(diskFile.toPath(), StandardOpenOption.CREATE, StandardOpenOption.APPEND, - StandardOpenOption.WRITE); + writeChannel = FileChannel.open(diskFile.toPath(), StandardOpenOption.CREATE, StandardOpenOption.APPEND, StandardOpenOption.WRITE); tailOffset = diskFile.length(); } else { diskFile.delete(); - writeChannel = FileChannel.open(diskFile.toPath(), StandardOpenOption.CREATE_NEW, - StandardOpenOption.WRITE); + writeChannel = FileChannel.open(diskFile.toPath(), StandardOpenOption.CREATE_NEW, StandardOpenOption.WRITE); tailOffset = 0; } } http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-cube/src/main/java/org/apache/kylin/cube/kv/AbstractRowKeyEncoder.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/kv/AbstractRowKeyEncoder.java b/core-cube/src/main/java/org/apache/kylin/cube/kv/AbstractRowKeyEncoder.java index 91a3bd4..2becde4 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/kv/AbstractRowKeyEncoder.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/kv/AbstractRowKeyEncoder.java @@ -18,8 +18,6 @@ package org.apache.kylin.cube.kv; -import java.util.Map; - import org.apache.kylin.common.util.ByteArray; import org.apache.kylin.common.util.ImmutableBitSet; import org.apache.kylin.cube.CubeSegment; @@ -30,6 +28,8 @@ import org.apache.kylin.metadata.model.TblColRef; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.util.Map; + /** * * @author xjiang http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-cube/src/main/java/org/apache/kylin/cube/kv/CubeDimEncMap.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/kv/CubeDimEncMap.java b/core-cube/src/main/java/org/apache/kylin/cube/kv/CubeDimEncMap.java index 2a17af4..bd9554a 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/kv/CubeDimEncMap.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/kv/CubeDimEncMap.java @@ -18,8 +18,7 @@ package org.apache.kylin.cube.kv; -import java.util.Map; - +import com.google.common.collect.Maps; import org.apache.kylin.common.util.Dictionary; import org.apache.kylin.cube.CubeSegment; import org.apache.kylin.cube.model.CubeDesc; @@ -33,7 +32,7 @@ import org.apache.kylin.metadata.model.TblColRef; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.google.common.collect.Maps; +import java.util.Map; public class CubeDimEncMap implements IDimensionEncodingMap, java.io.Serializable { @@ -72,8 +71,7 @@ public class CubeDimEncMap implements IDimensionEncodingMap, java.io.Serializabl } } else { // normal case - result = DimensionEncodingFactory.create(colDesc.getEncodingName(), colDesc.getEncodingArgs(), - colDesc.getEncodingVersion()); + result = DimensionEncodingFactory.create(colDesc.getEncodingName(), colDesc.getEncodingArgs(), colDesc.getEncodingVersion()); } encMap.put(col, result); } http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-cube/src/main/java/org/apache/kylin/cube/kv/FuzzyMaskEncoder.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/kv/FuzzyMaskEncoder.java b/core-cube/src/main/java/org/apache/kylin/cube/kv/FuzzyMaskEncoder.java index c2db871..0cbb7d2 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/kv/FuzzyMaskEncoder.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/kv/FuzzyMaskEncoder.java @@ -77,8 +77,7 @@ public class FuzzyMaskEncoder extends RowKeyEncoder { } @Override - protected void fillColumnValue(TblColRef column, int columnLen, String valueStr, byte[] outputValue, - int outputValueOffset) { + protected void fillColumnValue(TblColRef column, int columnLen, String valueStr, byte[] outputValue, int outputValueOffset) { if (valueStr == null) { Arrays.fill(outputValue, outputValueOffset, outputValueOffset + columnLen, RowConstants.BYTE_ONE); } else { http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-cube/src/main/java/org/apache/kylin/cube/kv/RowKeyColumnIO.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/kv/RowKeyColumnIO.java b/core-cube/src/main/java/org/apache/kylin/cube/kv/RowKeyColumnIO.java index 30c885c..65911a0 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/kv/RowKeyColumnIO.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/kv/RowKeyColumnIO.java @@ -47,8 +47,7 @@ public class RowKeyColumnIO implements java.io.Serializable { return dimEncMap.getDictionary(col); } - public void writeColumn(TblColRef col, String value, int roundingFlag, byte defaultValue, byte[] output, - int outputOffset) { + public void writeColumn(TblColRef col, String value, int roundingFlag, byte defaultValue, byte[] output, int outputOffset) { DimensionEncoding dimEnc = dimEncMap.get(col); if (dimEnc instanceof DictionaryDimEnc) dimEnc = ((DictionaryDimEnc) dimEnc).copy(roundingFlag, defaultValue); http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-cube/src/main/java/org/apache/kylin/cube/kv/RowKeyEncoder.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/kv/RowKeyEncoder.java b/core-cube/src/main/java/org/apache/kylin/cube/kv/RowKeyEncoder.java index d11dcb0..a669fb1 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/kv/RowKeyEncoder.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/kv/RowKeyEncoder.java @@ -18,11 +18,7 @@ package org.apache.kylin.cube.kv; -import java.util.Arrays; -import java.util.List; -import java.util.Map; -import java.util.Set; - +import com.google.common.base.Preconditions; import org.apache.kylin.common.util.ByteArray; import org.apache.kylin.common.util.BytesUtil; import org.apache.kylin.common.util.ImmutableBitSet; @@ -32,7 +28,10 @@ import org.apache.kylin.cube.cuboid.Cuboid; import org.apache.kylin.gridtable.GTRecord; import org.apache.kylin.metadata.model.TblColRef; -import com.google.common.base.Preconditions; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.Set; public class RowKeyEncoder extends AbstractRowKeyEncoder implements java.io.Serializable { @@ -75,10 +74,8 @@ public class RowKeyEncoder extends AbstractRowKeyEncoder implements java.io.Seri int shardSeedOffset = uhcOffset == -1 ? 0 : uhcOffset; int shardSeedLength = uhcLength == -1 ? bodyLength : uhcLength; short cuboidShardNum = cubeSeg.getCuboidShardNum(cuboid.getId()); - short shardOffset = ShardingHash.getShard(key, RowConstants.ROWKEY_SHARD_AND_CUBOID_LEN + shardSeedOffset, - shardSeedLength, cuboidShardNum); - return ShardingHash.normalize(cubeSeg.getCuboidBaseShard(cuboid.getId()), shardOffset, - cubeSeg.getTotalShards(cuboid.getId())); + short shardOffset = ShardingHash.getShard(key, RowConstants.ROWKEY_SHARD_AND_CUBOID_LEN + shardSeedOffset, shardSeedLength, cuboidShardNum); + return ShardingHash.normalize(cubeSeg.getCuboidBaseShard(cuboid.getId()), shardOffset, cubeSeg.getTotalShards(cuboid.getId())); } else { throw new RuntimeException("If enableSharding false, you should never calculate shard"); } @@ -110,8 +107,7 @@ public class RowKeyEncoder extends AbstractRowKeyEncoder implements java.io.Seri int c = selectedCols.trueBitAt(i); ByteArray columnC = record.get(c); if (columnC.array() != null) { - System.arraycopy(record.get(c).array(), columnC.offset(), buf.array(), buf.offset() + pos, - columnC.length()); + System.arraycopy(record.get(c).array(), columnC.offset(), buf.array(), buf.offset() + pos, columnC.length()); pos += columnC.length(); } else { int maxLength = record.getInfo().getCodeSystem().maxCodeLength(c); @@ -126,8 +122,7 @@ public class RowKeyEncoder extends AbstractRowKeyEncoder implements java.io.Seri public void encode(ByteArray bodyBytes, ByteArray outputBuf) { Preconditions.checkState(bodyBytes.length() == bodyLength); Preconditions.checkState(bodyBytes.length() + getHeaderLength() == outputBuf.length(), // - "bodybytes length: " + bodyBytes.length() + " outputBuf length: " + outputBuf.length() - + " header length: " + getHeaderLength()); + "bodybytes length: " + bodyBytes.length() + " outputBuf length: " + outputBuf.length() + " header length: " + getHeaderLength()); System.arraycopy(bodyBytes.array(), bodyBytes.offset(), outputBuf.array(), getHeaderLength(), bodyLength); //fill shard and cuboid @@ -176,8 +171,7 @@ public class RowKeyEncoder extends AbstractRowKeyEncoder implements java.io.Seri //return offset; } - protected void fillColumnValue(TblColRef column, int columnLen, String valueStr, byte[] outputValue, - int outputValueOffset) { + protected void fillColumnValue(TblColRef column, int columnLen, String valueStr, byte[] outputValue, int outputValueOffset) { // special null value case if (valueStr == null) { Arrays.fill(outputValue, outputValueOffset, outputValueOffset + columnLen, defaultValue()); http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-cube/src/main/java/org/apache/kylin/cube/model/AggregationGroup.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/AggregationGroup.java b/core-cube/src/main/java/org/apache/kylin/cube/model/AggregationGroup.java index 8e8faf1..5a32a92 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/model/AggregationGroup.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/model/AggregationGroup.java @@ -118,8 +118,7 @@ public class AggregationGroup implements Serializable { // check no dup Set<String> set = new HashSet<>(Arrays.asList(names)); if (set.size() < names.length) - throw new IllegalStateException( - "Columns in aggrgroup must not contain duplication: " + Arrays.asList(names)); + throw new IllegalStateException("Columns in aggrgroup must not contain duplication: " + Arrays.asList(names)); } private void buildPartialCubeFullMask(RowKeyDesc rowKeyDesc) { http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java index d362616..82d0b1b 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java @@ -114,8 +114,7 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware { @Override public String toString() { - return "DeriveInfo [type=" + type + ", join=" + join + ", columns=" + Arrays.toString(columns) - + ", isOneToOne=" + isOneToOne + "]"; + return "DeriveInfo [type=" + type + ", join=" + join + ", columns=" + Arrays.toString(columns) + ", isOneToOne=" + isOneToOne + "]"; } } @@ -289,8 +288,7 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware { throw new RuntimeException("Cannot get host info for " + derived); } - public Map<Array<TblColRef>, List<DeriveInfo>> getHostToDerivedInfo(List<TblColRef> rowCols, - Collection<TblColRef> wantedCols) { + public Map<Array<TblColRef>, List<DeriveInfo>> getHostToDerivedInfo(List<TblColRef> rowCols, Collection<TblColRef> wantedCols) { Map<Array<TblColRef>, List<DeriveInfo>> result = new HashMap<Array<TblColRef>, List<DeriveInfo>>(); for (Entry<Array<TblColRef>, List<DeriveInfo>> entry : hostToDerivedMap.entrySet()) { Array<TblColRef> hostCols = entry.getKey(); @@ -500,15 +498,12 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware { KylinVersion cubeVersion = new KylinVersion(getVersion()); KylinVersion kylinVersion = KylinVersion.getCurrentVersion(); if (!kylinVersion.isCompatibleWith(cubeVersion)) { - logger.info("checkSignature on {} is skipped as the its version {} is different from kylin version {}", - getName(), cubeVersion, kylinVersion); + logger.info("checkSignature on {} is skipped as the its version {} is different from kylin version {}", getName(), cubeVersion, kylinVersion); return true; } if (kylinVersion.isCompatibleWith(cubeVersion) && !kylinVersion.isSignatureCompatibleWith(cubeVersion)) { - logger.info( - "checkSignature on {} is skipped as the its version is {} (not signature compatible but compatible) ", - getName(), cubeVersion); + logger.info("checkSignature on {} is skipped as the its version is {} (not signature compatible but compatible) ", getName(), cubeVersion); return true; } @@ -616,9 +611,7 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware { // check all dimension columns are presented on rowkey List<TblColRef> dimCols = listDimensionColumnsExcludingDerived(true); - checkState(rowkey.getRowKeyColumns().length == dimCols.size(), - "RowKey columns count (%s) doesn't match dimensions columns count (%s)", - rowkey.getRowKeyColumns().length, dimCols.size()); + checkState(rowkey.getRowKeyColumns().length == dimCols.size(), "RowKey columns count (%s) doesn't match dimensions columns count (%s)", rowkey.getRowKeyColumns().length, dimCols.size()); initDictionaryDesc(); amendAllColumns(); @@ -672,8 +665,7 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware { combination = config.getCubeAggrGroupMaxCombination() + 1; } finally { if (combination > config.getCubeAggrGroupMaxCombination()) { - String msg = "Aggregation group " + index - + " has too many combinations, use 'mandatory'/'hierarchy'/'joint' to optimize; or update 'kylin.cube.aggrgroup.max-combination' to a bigger value."; + String msg = "Aggregation group " + index + " has too many combinations, use 'mandatory'/'hierarchy'/'joint' to optimize; or update 'kylin.cube.aggrgroup.max-combination' to a bigger value."; logger.error("Aggregation group " + index + " has " + combination + " combinations;"); logger.error(msg); throw new IllegalStateException(msg); @@ -712,64 +704,50 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware { Set<String> jointDims = new TreeSet<>(String.CASE_INSENSITIVE_ORDER); getDims(jointDimsList, jointDims, agg.getSelectRule().jointDims); - if (!includeDims.containsAll(mandatoryDims) || !includeDims.containsAll(hierarchyDims) - || !includeDims.containsAll(jointDims)) { + if (!includeDims.containsAll(mandatoryDims) || !includeDims.containsAll(hierarchyDims) || !includeDims.containsAll(jointDims)) { List<String> notIncluded = Lists.newArrayList(); - final Iterable<String> all = Iterables - .unmodifiableIterable(Iterables.concat(mandatoryDims, hierarchyDims, jointDims)); + final Iterable<String> all = Iterables.unmodifiableIterable(Iterables.concat(mandatoryDims, hierarchyDims, jointDims)); for (String dim : all) { if (includeDims.contains(dim) == false) { notIncluded.add(dim); } } Collections.sort(notIncluded); - logger.error( - "Aggregation group " + index + " Include dimensions not containing all the used dimensions"); - throw new IllegalStateException("Aggregation group " + index - + " 'includes' dimensions not include all the dimensions:" + notIncluded.toString()); + logger.error("Aggregation group " + index + " Include dimensions not containing all the used dimensions"); + throw new IllegalStateException("Aggregation group " + index + " 'includes' dimensions not include all the dimensions:" + notIncluded.toString()); } if (CollectionUtils.containsAny(mandatoryDims, hierarchyDims)) { - logger.warn("Aggregation group " + index + " mandatory dimensions overlap with hierarchy dimensions: " - + ensureOrder(CollectionUtils.intersection(mandatoryDims, hierarchyDims))); + logger.warn("Aggregation group " + index + " mandatory dimensions overlap with hierarchy dimensions: " + ensureOrder(CollectionUtils.intersection(mandatoryDims, hierarchyDims))); } if (CollectionUtils.containsAny(mandatoryDims, jointDims)) { - logger.warn("Aggregation group " + index + " mandatory dimensions overlap with joint dimensions: " - + ensureOrder(CollectionUtils.intersection(mandatoryDims, jointDims))); + logger.warn("Aggregation group " + index + " mandatory dimensions overlap with joint dimensions: " + ensureOrder(CollectionUtils.intersection(mandatoryDims, jointDims))); } if (CollectionUtils.containsAny(hierarchyDims, jointDims)) { logger.error("Aggregation group " + index + " hierarchy dimensions overlap with joint dimensions"); - throw new IllegalStateException( - "Aggregation group " + index + " hierarchy dimensions overlap with joint dimensions: " - + ensureOrder(CollectionUtils.intersection(hierarchyDims, jointDims))); + throw new IllegalStateException("Aggregation group " + index + " hierarchy dimensions overlap with joint dimensions: " + ensureOrder(CollectionUtils.intersection(hierarchyDims, jointDims))); } if (hasSingle(hierarchyDimsList)) { logger.error("Aggregation group " + index + " require at least 2 dimensions in a hierarchy"); - throw new IllegalStateException( - "Aggregation group " + index + " require at least 2 dimensions in a hierarchy."); + throw new IllegalStateException("Aggregation group " + index + " require at least 2 dimensions in a hierarchy."); } if (hasSingle(jointDimsList)) { logger.error("Aggregation group " + index + " require at least 2 dimensions in a joint"); - throw new IllegalStateException( - "Aggregation group " + index + " require at least 2 dimensions in a joint"); + throw new IllegalStateException("Aggregation group " + index + " require at least 2 dimensions in a joint"); } Pair<Boolean, Set<String>> overlap = hasOverlap(hierarchyDimsList, hierarchyDims); if (overlap.getFirst() == true) { - logger.error("Aggregation group " + index + " a dimension exist in more than one hierarchy: " - + ensureOrder(overlap.getSecond())); - throw new IllegalStateException("Aggregation group " + index - + " a dimension exist in more than one hierarchy: " + ensureOrder(overlap.getSecond())); + logger.error("Aggregation group " + index + " a dimension exist in more than one hierarchy: " + ensureOrder(overlap.getSecond())); + throw new IllegalStateException("Aggregation group " + index + " a dimension exist in more than one hierarchy: " + ensureOrder(overlap.getSecond())); } overlap = hasOverlap(jointDimsList, jointDims); if (overlap.getFirst() == true) { - logger.error("Aggregation group " + index + " a dimension exist in more than one joint: " - + ensureOrder(overlap.getSecond())); - throw new IllegalStateException("Aggregation group " + index - + " a dimension exist in more than one joint: " + ensureOrder(overlap.getSecond())); + logger.error("Aggregation group " + index + " a dimension exist in more than one joint: " + ensureOrder(overlap.getSecond())); + throw new IllegalStateException("Aggregation group " + index + " a dimension exist in more than one joint: " + ensureOrder(overlap.getSecond())); } index++; @@ -897,8 +875,7 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware { return new String[][] { cols, extra }; } - private void initDerivedMap(TblColRef[] hostCols, DeriveType type, JoinDesc join, TblColRef[] derivedCols, - String[] extra) { + private void initDerivedMap(TblColRef[] hostCols, DeriveType type, JoinDesc join, TblColRef[] derivedCols, String[] extra) { if (hostCols.length == 0 || derivedCols.length == 0) throw new IllegalStateException("host/derived columns must not be empty"); @@ -919,8 +896,7 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware { for (int i = 0; i < derivedCols.length; i++) { TblColRef derivedCol = derivedCols[i]; - boolean isOneToOne = type == DeriveType.PK_FK || ArrayUtils.contains(hostCols, derivedCol) - || (extra != null && extra[i].contains("1-1")); + boolean isOneToOne = type == DeriveType.PK_FK || ArrayUtils.contains(hostCols, derivedCol) || (extra != null && extra[i].contains("1-1")); derivedToHostMap.put(derivedCol, new DeriveInfo(type, join, hostCols, isOneToOne)); } @@ -952,8 +928,7 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware { whatsLeft.add(derCol); } if (whatsLeft.size() > 0) { - infoList.add(new DeriveInfo(type, join, (TblColRef[]) whatsLeft.toArray(new TblColRef[whatsLeft.size()]), - false)); + infoList.add(new DeriveInfo(type, join, (TblColRef[]) whatsLeft.toArray(new TblColRef[whatsLeft.size()]), false)); } } @@ -1044,8 +1019,7 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware { } for (int i = 0; i < measures.size(); i++) { - checkState(checkEachMeasureExist.get(i), - "measure (%s) does not exist in column familyï¼or measure duplicates", measures.get(i)); + checkState(checkEachMeasureExist.get(i), "measure (%s) does not exist in column familyï¼or measure duplicates", measures.get(i)); } } http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-cube/src/main/java/org/apache/kylin/cube/model/CubeJoinedFlatTableDesc.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeJoinedFlatTableDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeJoinedFlatTableDesc.java index 2f8eaa1..c49d37a 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeJoinedFlatTableDesc.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeJoinedFlatTableDesc.java @@ -65,8 +65,7 @@ public class CubeJoinedFlatTableDesc implements IJoinedFlatTableDesc, java.io.Se if (cubeSegment == null) { return "kylin_intermediate_" + cubeDesc.getName().toLowerCase(); } else { - return "kylin_intermediate_" + cubeDesc.getName().toLowerCase() + "_" - + cubeSegment.getUuid().replaceAll("-", "_"); + return "kylin_intermediate_" + cubeDesc.getName().toLowerCase() + "_" + cubeSegment.getUuid().replaceAll("-", "_"); } } @@ -116,8 +115,7 @@ public class CubeJoinedFlatTableDesc implements IJoinedFlatTableDesc, java.io.Se // sanity check the input record (in bytes) matches what's expected public void sanityCheck(BytesSplitter bytesSplitter) { if (columnCount != bytesSplitter.getBufferSize()) { - throw new IllegalArgumentException("Expect " + columnCount + " columns, but see " - + bytesSplitter.getBufferSize() + " -- " + bytesSplitter); + throw new IllegalArgumentException("Expect " + columnCount + " columns, but see " + bytesSplitter.getBufferSize() + " -- " + bytesSplitter); } // TODO: check data types here http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-cube/src/main/java/org/apache/kylin/cube/model/CubeJoinedFlatTableEnrich.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeJoinedFlatTableEnrich.java b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeJoinedFlatTableEnrich.java index e590d07..e829aeb 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeJoinedFlatTableEnrich.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeJoinedFlatTableEnrich.java @@ -18,8 +18,6 @@ package org.apache.kylin.cube.model; -import java.util.List; - import org.apache.kylin.cube.cuboid.Cuboid; import org.apache.kylin.metadata.model.DataModelDesc; import org.apache.kylin.metadata.model.FunctionDesc; @@ -28,6 +26,8 @@ import org.apache.kylin.metadata.model.ISegment; import org.apache.kylin.metadata.model.MeasureDesc; import org.apache.kylin.metadata.model.TblColRef; +import java.util.List; + /** * An enrich of IJoinedFlatTableDesc for cubes */ http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-cube/src/main/java/org/apache/kylin/cube/model/DimensionDesc.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/DimensionDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/DimensionDesc.java index 1762b26..578831f 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/model/DimensionDesc.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/model/DimensionDesc.java @@ -81,8 +81,7 @@ public class DimensionDesc implements java.io.Serializable { } } if (derived != null && join == null) { - throw new IllegalStateException( - "Derived can only be defined on lookup table, cube " + cubeDesc + ", " + this); + throw new IllegalStateException("Derived can only be defined on lookup table, cube " + cubeDesc + ", " + this); } } @@ -141,8 +140,7 @@ public class DimensionDesc implements java.io.Serializable { @Override public String toString() { - return Objects.toStringHelper(this).add("name", name).add("table", table).add("column", column) - .add("derived", Arrays.toString(derived)).add("join", join).toString(); + return Objects.toStringHelper(this).add("name", name).add("table", table).add("column", column).add("derived", Arrays.toString(derived)).add("join", join).toString(); } @Override http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-cube/src/main/java/org/apache/kylin/cube/model/HBaseColumnDesc.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/HBaseColumnDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/HBaseColumnDesc.java index 2a57a34..7007342 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/model/HBaseColumnDesc.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/model/HBaseColumnDesc.java @@ -18,14 +18,13 @@ package org.apache.kylin.cube.model; -import java.util.Arrays; - -import org.apache.kylin.metadata.model.FunctionDesc; -import org.apache.kylin.metadata.model.MeasureDesc; - import com.fasterxml.jackson.annotation.JsonAutoDetect; import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility; import com.fasterxml.jackson.annotation.JsonProperty; +import org.apache.kylin.metadata.model.FunctionDesc; +import org.apache.kylin.metadata.model.MeasureDesc; + +import java.util.Arrays; /** */ http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-cube/src/main/java/org/apache/kylin/cube/model/HBaseColumnFamilyDesc.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/HBaseColumnFamilyDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/HBaseColumnFamilyDesc.java index 1d88549..85c2c17 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/model/HBaseColumnFamilyDesc.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/model/HBaseColumnFamilyDesc.java @@ -18,13 +18,12 @@ package org.apache.kylin.cube.model; -import java.util.Arrays; - -import org.apache.kylin.metadata.model.MeasureDesc; - import com.fasterxml.jackson.annotation.JsonAutoDetect; import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility; import com.fasterxml.jackson.annotation.JsonProperty; +import org.apache.kylin.metadata.model.MeasureDesc; + +import java.util.Arrays; /** */ http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-cube/src/main/java/org/apache/kylin/cube/model/HBaseMappingDesc.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/HBaseMappingDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/HBaseMappingDesc.java index 7ae932a..d1e5829 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/model/HBaseMappingDesc.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/model/HBaseMappingDesc.java @@ -22,13 +22,12 @@ import java.util.Arrays; import java.util.Collection; import java.util.LinkedList; -import org.apache.kylin.common.util.StringUtil; -import org.apache.kylin.metadata.model.FunctionDesc; -import org.apache.kylin.metadata.model.MeasureDesc; - import com.fasterxml.jackson.annotation.JsonAutoDetect; import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility; import com.fasterxml.jackson.annotation.JsonProperty; +import org.apache.kylin.common.util.StringUtil; +import org.apache.kylin.metadata.model.FunctionDesc; +import org.apache.kylin.metadata.model.MeasureDesc; /** */ http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-cube/src/main/java/org/apache/kylin/cube/model/RowKeyColDesc.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/RowKeyColDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/RowKeyColDesc.java index c62055d..b6f0a27 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/model/RowKeyColDesc.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/model/RowKeyColDesc.java @@ -88,7 +88,7 @@ public class RowKeyColDesc implements java.io.Serializable { } encodingArgs = DateDimEnc.replaceEncodingArgs(encoding, encodingArgs, encodingName, type); - + if (encodingName.startsWith(FixedLenDimEnc.ENCODING_NAME) && (type.isIntegerFamily() || type.isNumberFamily())) throw new IllegalArgumentException(colRef + " type is " + type + " and cannot apply fixed_length encoding"); } http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-cube/src/main/java/org/apache/kylin/cube/model/v1_4_0/CubeDesc.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/v1_4_0/CubeDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/v1_4_0/CubeDesc.java index d2f8253..4019e4f 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/model/v1_4_0/CubeDesc.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/model/v1_4_0/CubeDesc.java @@ -85,8 +85,7 @@ public class CubeDesc extends RootPersistentEntity { @Override public String toString() { - return "DeriveInfo [type=" + type + ", dimension=" + dimension + ", columns=" + Arrays.toString(columns) - + ", isOneToOne=" + isOneToOne + "]"; + return "DeriveInfo [type=" + type + ", dimension=" + dimension + ", columns=" + Arrays.toString(columns) + ", isOneToOne=" + isOneToOne + "]"; } } @@ -243,8 +242,7 @@ public class CubeDesc extends RootPersistentEntity { return derivedToHostMap.get(derived); } - public Map<Array<TblColRef>, List<DeriveInfo>> getHostToDerivedInfo(List<TblColRef> rowCols, - Collection<TblColRef> wantedCols) { + public Map<Array<TblColRef>, List<DeriveInfo>> getHostToDerivedInfo(List<TblColRef> rowCols, Collection<TblColRef> wantedCols) { Map<Array<TblColRef>, List<DeriveInfo>> result = new HashMap<Array<TblColRef>, List<DeriveInfo>>(); for (Entry<Array<TblColRef>, List<DeriveInfo>> entry : hostToDerivedMap.entrySet()) { Array<TblColRef> hostCols = entry.getKey(); @@ -431,12 +429,7 @@ public class CubeDesc extends RootPersistentEntity { try { md = MessageDigest.getInstance("MD5"); StringBuilder sigString = new StringBuilder(); - sigString.append(this.name).append("|").append(this.getFactTable()).append("|") - .append(JsonUtil.writeValueAsString(this.model.getPartitionDesc())).append("|") - .append(JsonUtil.writeValueAsString(this.dimensions)).append("|") - .append(JsonUtil.writeValueAsString(this.measures)).append("|") - .append(JsonUtil.writeValueAsString(this.rowkey)).append("|") - .append(JsonUtil.writeValueAsString(this.hbaseMapping)); + sigString.append(this.name).append("|").append(this.getFactTable()).append("|").append(JsonUtil.writeValueAsString(this.model.getPartitionDesc())).append("|").append(JsonUtil.writeValueAsString(this.dimensions)).append("|").append(JsonUtil.writeValueAsString(this.measures)).append("|").append(JsonUtil.writeValueAsString(this.rowkey)).append("|").append(JsonUtil.writeValueAsString(this.hbaseMapping)); byte[] signature = md.digest(sigString.toString().getBytes()); return new String(Base64.encodeBase64(signature)); @@ -485,8 +478,7 @@ public class CubeDesc extends RootPersistentEntity { // check all dimension columns are presented on rowkey List<TblColRef> dimCols = listDimensionColumnsExcludingDerived(); if (rowkey.getRowKeyColumns().length != dimCols.size()) { - addError("RowKey columns count (" + rowkey.getRowKeyColumns().length - + ") does not match dimension columns count (" + dimCols.size() + "). "); + addError("RowKey columns count (" + rowkey.getRowKeyColumns().length + ") does not match dimension columns count (" + dimCols.size() + "). "); } } @@ -584,8 +576,7 @@ public class CubeDesc extends RootPersistentEntity { initDerivedMap(new TblColRef[] { hostCol }, type, dimension, new TblColRef[] { derivedCol }, null); } - private void initDerivedMap(TblColRef[] hostCols, DeriveType type, DimensionDesc dimension, TblColRef[] derivedCols, - String[] extra) { + private void initDerivedMap(TblColRef[] hostCols, DeriveType type, DimensionDesc dimension, TblColRef[] derivedCols, String[] extra) { if (hostCols.length == 0 || derivedCols.length == 0) throw new IllegalStateException("host/derived columns must not be empty"); @@ -609,8 +600,7 @@ public class CubeDesc extends RootPersistentEntity { for (int i = 0; i < derivedCols.length; i++) { TblColRef derivedCol = derivedCols[i]; - boolean isOneToOne = type == DeriveType.PK_FK || ArrayUtils.contains(hostCols, derivedCol) - || (extra != null && extra[i].contains("1-1")); + boolean isOneToOne = type == DeriveType.PK_FK || ArrayUtils.contains(hostCols, derivedCol) || (extra != null && extra[i].contains("1-1")); derivedToHostMap.put(derivedCol, new DeriveInfo(type, dimension, hostCols, isOneToOne)); } } http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-cube/src/main/java/org/apache/kylin/cube/model/v1_4_0/DimensionDesc.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/v1_4_0/DimensionDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/v1_4_0/DimensionDesc.java index e63a5b0..3903b2b 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/model/v1_4_0/DimensionDesc.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/model/v1_4_0/DimensionDesc.java @@ -108,8 +108,7 @@ public class DimensionDesc { } if (derived != null && join == null) { - throw new IllegalStateException( - "Derived can only be defined on lookup table, cube " + cubeDesc + ", " + this); + throw new IllegalStateException("Derived can only be defined on lookup table, cube " + cubeDesc + ", " + this); } } @@ -234,9 +233,7 @@ public class DimensionDesc { @Override public String toString() { - return "DimensionDesc [name=" + name + ", join=" + join + ", hierarchy=" + Arrays.toString(hierarchy) - + ", table=" + table + ", column=" + Arrays.toString(column) + ", derived=" + Arrays.toString(derived) - + "]"; + return "DimensionDesc [name=" + name + ", join=" + join + ", hierarchy=" + Arrays.toString(hierarchy) + ", table=" + table + ", column=" + Arrays.toString(column) + ", derived=" + Arrays.toString(derived) + "]"; } } http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-cube/src/main/java/org/apache/kylin/cube/model/v1_4_0/RowKeyColDesc.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/v1_4_0/RowKeyColDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/v1_4_0/RowKeyColDesc.java index 3151513..8ef3698 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/model/v1_4_0/RowKeyColDesc.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/model/v1_4_0/RowKeyColDesc.java @@ -86,8 +86,7 @@ public class RowKeyColDesc { @Override public String toString() { - return "RowKeyColDesc [column=" + column + ", length=" + length + ", dictionary=" + dictionary + ", mandatory=" - + mandatory + "]"; + return "RowKeyColDesc [column=" + column + ", length=" + length + ", dictionary=" + dictionary + ", mandatory=" + mandatory + "]"; } } http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-cube/src/main/java/org/apache/kylin/cube/model/v1_4_0/RowKeyDesc.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/v1_4_0/RowKeyDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/v1_4_0/RowKeyDesc.java index 2b77c6b..7d37a76 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/model/v1_4_0/RowKeyDesc.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/model/v1_4_0/RowKeyDesc.java @@ -188,8 +188,7 @@ public class RowKeyDesc { @Override public String toString() { - return "RowKeyDesc [rowkeyColumns=" + Arrays.toString(rowkeyColumns) + ", aggregationGroups=" - + Arrays.toString(aggregationGroups) + "]"; + return "RowKeyDesc [rowkeyColumns=" + Arrays.toString(rowkeyColumns) + ", aggregationGroups=" + Arrays.toString(aggregationGroups) + "]"; } private void buildRowKey(Map<String, TblColRef> colNameAbbr) { @@ -236,8 +235,7 @@ public class RowKeyDesc { for (int j = 0; j < aggGrp.length; j++) { TblColRef aggCol = colNameAbbr.get(aggGrp[j].toUpperCase()); if (aggCol == null) { - throw new IllegalArgumentException( - "Can't find aggregation column " + aggGrp[j] + " in cube " + this.cubeDesc.getName()); + throw new IllegalArgumentException("Can't find aggregation column " + aggGrp[j] + " in cube " + this.cubeDesc.getName()); } Integer index = getColumnBitIndex(aggCol); mask.groupMask |= 1L << index; http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-cube/src/main/java/org/apache/kylin/cube/model/validation/CubeMetadataValidator.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/validation/CubeMetadataValidator.java b/core-cube/src/main/java/org/apache/kylin/cube/model/validation/CubeMetadataValidator.java index 3c9da15..c2c5f89 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/model/validation/CubeMetadataValidator.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/model/validation/CubeMetadataValidator.java @@ -33,8 +33,7 @@ import org.apache.kylin.cube.model.validation.rule.StreamingCubeRule; */ public class CubeMetadataValidator { @SuppressWarnings("unchecked") - private IValidatorRule<CubeDesc>[] rules = new IValidatorRule[] { new FunctionRule(), new AggregationGroupRule(), - new RowKeyAttrRule(), new DictionaryRule(), new StreamingCubeRule() }; + private IValidatorRule<CubeDesc>[] rules = new IValidatorRule[] { new FunctionRule(), new AggregationGroupRule(), new RowKeyAttrRule(), new DictionaryRule(), new StreamingCubeRule() }; public ValidateContext validate(CubeDesc cube) { ValidateContext context = new ValidateContext(); http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/AggregationGroupRule.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/AggregationGroupRule.java b/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/AggregationGroupRule.java index e939b38..33fc390 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/AggregationGroupRule.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/AggregationGroupRule.java @@ -48,7 +48,6 @@ public class AggregationGroupRule implements IValidatorRule<CubeDesc> { public AggregationGroupRule() { } - @SuppressWarnings("checkstyle:methodlength") private void inner(CubeDesc cube, ValidateContext context) { if (cube.getAggregationGroups() == null || cube.getAggregationGroups().size() == 0) { @@ -100,33 +99,28 @@ public class AggregationGroupRule implements IValidatorRule<CubeDesc> { } } - if (!includeDims.containsAll(mandatoryDims) || !includeDims.containsAll(hierarchyDims) - || !includeDims.containsAll(jointDims)) { + if (!includeDims.containsAll(mandatoryDims) || !includeDims.containsAll(hierarchyDims) || !includeDims.containsAll(jointDims)) { List<String> notIncluded = Lists.newArrayList(); - final Iterable<String> all = Iterables - .unmodifiableIterable(Iterables.concat(mandatoryDims, hierarchyDims, jointDims)); + final Iterable<String> all = Iterables.unmodifiableIterable(Iterables.concat(mandatoryDims, hierarchyDims, jointDims)); for (String dim : all) { if (includeDims.contains(dim) == false) { notIncluded.add(dim); } } - context.addResult(ResultLevel.ERROR, "Aggregation group " + index - + " 'includes' dimensions not include all the dimensions:" + notIncluded.toString()); + context.addResult(ResultLevel.ERROR, "Aggregation group " + index + " 'includes' dimensions not include all the dimensions:" + notIncluded.toString()); continue; } if (CollectionUtils.containsAny(mandatoryDims, hierarchyDims)) { Set<String> intersection = new HashSet<>(mandatoryDims); intersection.retainAll(hierarchyDims); - context.addResult(ResultLevel.ERROR, "Aggregation group " + index - + " mandatory dimension has overlap with hierarchy dimension: " + intersection.toString()); + context.addResult(ResultLevel.ERROR, "Aggregation group " + index + " mandatory dimension has overlap with hierarchy dimension: " + intersection.toString()); continue; } if (CollectionUtils.containsAny(mandatoryDims, jointDims)) { Set<String> intersection = new HashSet<>(mandatoryDims); intersection.retainAll(jointDims); - context.addResult(ResultLevel.ERROR, "Aggregation group " + index - + " mandatory dimension has overlap with joint dimension: " + intersection.toString()); + context.addResult(ResultLevel.ERROR, "Aggregation group " + index + " mandatory dimension has overlap with joint dimension: " + intersection.toString()); continue; } @@ -140,8 +134,7 @@ public class AggregationGroupRule implements IValidatorRule<CubeDesc> { } if (oneJoint.size() < 2) { - context.addResult(ResultLevel.ERROR, "Aggregation group " + index - + " require at least 2 dimensions in a joint: " + oneJoint.toString()); + context.addResult(ResultLevel.ERROR, "Aggregation group " + index + " require at least 2 dimensions in a joint: " + oneJoint.toString()); continue; } jointDimNum += oneJoint.size(); @@ -156,17 +149,13 @@ public class AggregationGroupRule implements IValidatorRule<CubeDesc> { overlapHierarchies++; } if (share.size() > 1) { - context.addResult(ResultLevel.ERROR, - "Aggregation group " + index - + " joint dimensions has overlap with more than 1 dimensions in same hierarchy: " - + share.toString()); + context.addResult(ResultLevel.ERROR, "Aggregation group " + index + " joint dimensions has overlap with more than 1 dimensions in same hierarchy: " + share.toString()); continue; } } if (overlapHierarchies > 1) { - context.addResult(ResultLevel.ERROR, "Aggregation group " + index - + " joint dimensions has overlap with more than 1 hierarchies"); + context.addResult(ResultLevel.ERROR, "Aggregation group " + index + " joint dimensions has overlap with more than 1 hierarchies"); continue; } } @@ -186,8 +175,7 @@ public class AggregationGroupRule implements IValidatorRule<CubeDesc> { } existing.addAll(oneJoint); } - context.addResult(ResultLevel.ERROR, "Aggregation group " + index - + " a dimension exists in more than one joint: " + overlap.toString()); + context.addResult(ResultLevel.ERROR, "Aggregation group " + index + " a dimension exists in more than one joint: " + overlap.toString()); continue; } } @@ -198,9 +186,7 @@ public class AggregationGroupRule implements IValidatorRule<CubeDesc> { combination = getMaxCombinations(cube) + 1; } finally { if (combination > getMaxCombinations(cube)) { - String msg = "Aggregation group " + index + " has too many combinations, current combination is " - + combination + ", max allowed combination is " + getMaxCombinations(cube) - + "; use 'mandatory'/'hierarchy'/'joint' to optimize; or update 'kylin.cube.aggrgroup.max-combination' to a bigger value."; + String msg = "Aggregation group " + index + " has too many combinations, current combination is " + combination + ", max allowed combination is " + getMaxCombinations(cube) + "; use 'mandatory'/'hierarchy'/'joint' to optimize; or update 'kylin.cube.aggrgroup.max-combination' to a bigger value."; context.addResult(ResultLevel.ERROR, msg); continue; } http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/DictionaryRule.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/DictionaryRule.java b/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/DictionaryRule.java index 449cafc..8da3ca0 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/DictionaryRule.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/DictionaryRule.java @@ -82,9 +82,7 @@ public class DictionaryRule implements IValidatorRule<CubeDesc> { return; } - if (StringUtils.isNotEmpty(builderClass) - && builderClass.equalsIgnoreCase(GlobalDictionaryBuilder.class.getName()) - && dimensionColumns.contains(dictCol)) { + if (StringUtils.isNotEmpty(builderClass) && builderClass.equalsIgnoreCase(GlobalDictionaryBuilder.class.getName()) && dimensionColumns.contains(dictCol)) { context.addResult(ResultLevel.ERROR, ERROR_GLOBAL_DICTIONNARY_ONLY_MEASURE + dictCol); return; } http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/FunctionRule.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/FunctionRule.java b/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/FunctionRule.java index 4d295b0..36631ce 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/FunctionRule.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/FunctionRule.java @@ -72,26 +72,22 @@ public class FunctionRule implements IValidatorRule<CubeDesc> { FunctionDesc func = measure.getFunction(); ParameterDesc parameter = func.getParameter(); if (parameter == null) { - context.addResult(ResultLevel.ERROR, - "Must define parameter for function " + func.getExpression() + " in " + measure.getName()); + context.addResult(ResultLevel.ERROR, "Must define parameter for function " + func.getExpression() + " in " + measure.getName()); return; } String type = func.getParameter().getType(); String value = func.getParameter().getValue(); if (StringUtils.isEmpty(type)) { - context.addResult(ResultLevel.ERROR, - "Must define type for parameter type " + func.getExpression() + " in " + measure.getName()); + context.addResult(ResultLevel.ERROR, "Must define type for parameter type " + func.getExpression() + " in " + measure.getName()); return; } if (StringUtils.isEmpty(value)) { - context.addResult(ResultLevel.ERROR, - "Must define type for parameter value " + func.getExpression() + " in " + measure.getName()); + context.addResult(ResultLevel.ERROR, "Must define type for parameter value " + func.getExpression() + " in " + measure.getName()); return; } if (StringUtils.isEmpty(func.getReturnType())) { - context.addResult(ResultLevel.ERROR, - "Must define return type for function " + func.getExpression() + " in " + measure.getName()); + context.addResult(ResultLevel.ERROR, "Must define return type for function " + func.getExpression() + " in " + measure.getName()); return; } @@ -112,8 +108,7 @@ public class FunctionRule implements IValidatorRule<CubeDesc> { if (TopNMeasureType.FUNC_TOP_N.equalsIgnoreCase(func.getExpression())) { if (parameter.getNextParameter() == null) { - context.addResult(ResultLevel.ERROR, "Must define at least 2 parameters for function " - + func.getExpression() + " in " + measure.getName()); + context.addResult(ResultLevel.ERROR, "Must define at least 2 parameters for function " + func.getExpression() + " in " + measure.getName()); return; } @@ -122,8 +117,7 @@ public class FunctionRule implements IValidatorRule<CubeDesc> { while (groupByCol != null) { String embeded_groupby = groupByCol.getValue(); for (DimensionDesc dimensionDesc : cube.getDimensions()) { - if (dimensionDesc.getColumn() != null - && dimensionDesc.getColumn().equalsIgnoreCase(embeded_groupby)) { + if (dimensionDesc.getColumn() != null && dimensionDesc.getColumn().equalsIgnoreCase(embeded_groupby)) { duplicatedCol.add(embeded_groupby); } } @@ -134,8 +128,7 @@ public class FunctionRule implements IValidatorRule<CubeDesc> { } if (countFuncs.size() != 1) { - context.addResult(ResultLevel.ERROR, "Must define one and only one count(1) function, but there are " - + countFuncs.size() + " -- " + countFuncs); + context.addResult(ResultLevel.ERROR, "Must define one and only one count(1) function, but there are " + countFuncs.size() + " -- " + countFuncs); } } @@ -171,8 +164,8 @@ public class FunctionRule implements IValidatorRule<CubeDesc> { */ private boolean validateMeasureNamesDuplicated(List<MeasureDesc> measures, ValidateContext context) { Set<String> nameSet = new HashSet<>(); - for (MeasureDesc measure : measures) { - if (nameSet.contains(measure.getName())) { + for (MeasureDesc measure: measures){ + if (nameSet.contains(measure.getName())){ context.addResult(ResultLevel.ERROR, "There is duplicated measure's name: " + measure.getName()); return true; } else { http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/StreamingCubeRule.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/StreamingCubeRule.java b/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/StreamingCubeRule.java index 9e4a37a..4438706 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/StreamingCubeRule.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/StreamingCubeRule.java @@ -26,6 +26,7 @@ import org.apache.kylin.cube.model.validation.ValidateContext; import org.apache.kylin.metadata.model.DataModelDesc; import org.apache.kylin.metadata.model.IEngineAware; import org.apache.kylin.metadata.model.ISourceAware; + import org.apache.kylin.metadata.model.TblColRef; /** @@ -43,20 +44,18 @@ public class StreamingCubeRule implements IValidatorRule<CubeDesc> { @Override public void validate(CubeDesc cube, ValidateContext context) { DataModelDesc model = cube.getModel(); - + if (model.getRootFactTable().getTableDesc().getSourceType() != ISourceAware.ID_STREAMING) { return; } if (model.getLookupTables().size() > 0) { - context.addResult(ResultLevel.ERROR, - "Streaming Cube doesn't support star-schema so far; only one fact table is allowed."); + context.addResult(ResultLevel.ERROR, "Streaming Cube doesn't support star-schema so far; only one fact table is allowed."); return; } if (cube.getEngineType() == IEngineAware.ID_SPARK) { - context.addResult(ResultLevel.ERROR, - "Spark engine doesn't support streaming source, select MapReduce engine instead."); + context.addResult(ResultLevel.ERROR, "Spark engine doesn't support streaming source, select MapReduce engine instead."); return; } http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-cube/src/main/java/org/apache/kylin/cube/upgrade/V1_5_1/CubeDescUpgrade_v_1_5_1.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/upgrade/V1_5_1/CubeDescUpgrade_v_1_5_1.java b/core-cube/src/main/java/org/apache/kylin/cube/upgrade/V1_5_1/CubeDescUpgrade_v_1_5_1.java index 0740fc0..ec21650 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/upgrade/V1_5_1/CubeDescUpgrade_v_1_5_1.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/upgrade/V1_5_1/CubeDescUpgrade_v_1_5_1.java @@ -226,8 +226,7 @@ public class CubeDescUpgrade_v_1_5_1 { } org.apache.kylin.cube.model.RowKeyDesc newRowKey = new org.apache.kylin.cube.model.RowKeyDesc(); - org.apache.kylin.cube.model.RowKeyColDesc[] cols = new org.apache.kylin.cube.model.RowKeyColDesc[oldRowKey - .getRowKeyColumns().length]; + org.apache.kylin.cube.model.RowKeyColDesc[] cols = new org.apache.kylin.cube.model.RowKeyColDesc[oldRowKey.getRowKeyColumns().length]; int index = 0; for (RowKeyColDesc oldRowKeyCol : oldRowKey.getRowKeyColumns()) { org.apache.kylin.cube.model.RowKeyColDesc newRowKeyCol = new org.apache.kylin.cube.model.RowKeyColDesc(); @@ -243,8 +242,7 @@ public class CubeDescUpgrade_v_1_5_1 { } else if (oldRowKeyCol.getLength() > 0) { newRowKeyCol.setEncoding("fixed_length:" + oldRowKeyCol.getLength()); } else { - throw new IllegalArgumentException("Unknow encoding: Dictionary " + oldRowKeyCol.getDictionary() - + ", length: " + oldRowKeyCol.getLength()); + throw new IllegalArgumentException("Unknow encoding: Dictionary " + oldRowKeyCol.getDictionary() + ", length: " + oldRowKeyCol.getLength()); } cols[index++] = newRowKeyCol; } @@ -262,8 +260,7 @@ public class CubeDescUpgrade_v_1_5_1 { JsonUtil.writeValueIndent(os, hbaseMappingDesc); byte[] blob = os.toByteArray(); ByteArrayInputStream is = new ByteArrayInputStream(blob); - org.apache.kylin.cube.model.HBaseMappingDesc newHBaseMappingDesc = JsonUtil.readValue(is, - org.apache.kylin.cube.model.HBaseMappingDesc.class); + org.apache.kylin.cube.model.HBaseMappingDesc newHBaseMappingDesc = JsonUtil.readValue(is, org.apache.kylin.cube.model.HBaseMappingDesc.class); newModel.setHbaseMapping(newHBaseMappingDesc); } catch (IOException e) { http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-cube/src/main/java/org/apache/kylin/cube/upgrade/V1_5_1/CubeMetadataUpgrade_v_1_5_1.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/upgrade/V1_5_1/CubeMetadataUpgrade_v_1_5_1.java b/core-cube/src/main/java/org/apache/kylin/cube/upgrade/V1_5_1/CubeMetadataUpgrade_v_1_5_1.java index 668f24f..8a85629 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/upgrade/V1_5_1/CubeMetadataUpgrade_v_1_5_1.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/upgrade/V1_5_1/CubeMetadataUpgrade_v_1_5_1.java @@ -90,8 +90,7 @@ public class CubeMetadataUpgrade_v_1_5_1 extends CubeMetadataUpgrade { CubeDescUpgrade_v_1_5_1 upgrade = new CubeDescUpgrade_v_1_5_1(path, store); CubeDesc ndesc = upgrade.upgrade(); - ResourceStore.getStore(config).putResource(ndesc.getResourcePath(), ndesc, - CubeDescManager.CUBE_DESC_SERIALIZER); + ResourceStore.getStore(config).putResource(ndesc.getResourcePath(), ndesc, CubeDescManager.CUBE_DESC_SERIALIZER); updatedResources.add(ndesc.getResourcePath()); } catch (Exception e) { logger.error("error", e); @@ -130,11 +129,8 @@ public class CubeMetadataUpgrade_v_1_5_1 extends CubeMetadataUpgrade { for (CubeInstance cube : cubes) { try { org.apache.kylin.cube.model.CubeDesc cubeDesc = cube.getDescriptor(); - if (cube.getFirstSegment() == null && cubeDesc != null - && cubeDesc.getStorageType() == IStorageAware.ID_HBASE - && cubeDesc.getEngineType() == IEngineAware.ID_MR_V1) { - logger.info("CubeMetadataUpgrade_v_1_5_1 handling in upgradeEngineTypeStorageType {}", - cube.getName()); + if (cube.getFirstSegment() == null && cubeDesc != null && cubeDesc.getStorageType() == IStorageAware.ID_HBASE && cubeDesc.getEngineType() == IEngineAware.ID_MR_V1) { + logger.info("CubeMetadataUpgrade_v_1_5_1 handling in upgradeEngineTypeStorageType {}", cube.getName()); cubeDesc.setEngineType(IEngineAware.ID_MR_V2); cubeDesc.setStorageType(IStorageAware.ID_SHARDED_HBASE); @@ -142,14 +138,11 @@ public class CubeMetadataUpgrade_v_1_5_1 extends CubeMetadataUpgrade { store.putResource(cubeDesc.getResourcePath(), cubeDesc, CubeDescManager.CUBE_DESC_SERIALIZER); updatedResources.add(cubeDesc.getResourcePath()); } else { - logger.info( - "CubeDesc {}'s storage type and engine type will not be upgraded because they're not empty", - cubeDesc.getName()); + logger.info("CubeDesc {}'s storage type and engine type will not be upgraded because they're not empty", cubeDesc.getName()); } } catch (Exception e) { logger.error("error", e); - errorMsgs.add( - "upgradeEngineTypeStorageType [" + cube.getName() + "] failed: " + e.getLocalizedMessage()); + errorMsgs.add("upgradeEngineTypeStorageType [" + cube.getName() + "] failed: " + e.getLocalizedMessage()); } } } http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-cube/src/main/java/org/apache/kylin/cube/upgrade/common/CubeMetadataUpgrade.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/upgrade/common/CubeMetadataUpgrade.java b/core-cube/src/main/java/org/apache/kylin/cube/upgrade/common/CubeMetadataUpgrade.java index d796288..f70286d 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/upgrade/common/CubeMetadataUpgrade.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/upgrade/common/CubeMetadataUpgrade.java @@ -71,8 +71,7 @@ public abstract class CubeMetadataUpgrade { public void verify() { logger.info("================================================================="); - logger.info( - "The changes are applied, now it's time to verify the new metadata store by reloading all metadata:"); + logger.info("The changes are applied, now it's time to verify the new metadata store by reloading all metadata:"); logger.info("================================================================="); MetadataManager.clearCache(); MetadataManager.getInstance(config); @@ -89,14 +88,11 @@ public abstract class CubeMetadataUpgrade { public abstract void upgradeCompatibleMeta(); - public static void upgradeOrVerify(Class upgradeClass, String[] args, boolean firstStepInChain, - boolean lastStepInChain) - throws NoSuchMethodException, IllegalAccessException, InvocationTargetException, InstantiationException { + public static void upgradeOrVerify(Class upgradeClass, String[] args, boolean firstStepInChain, boolean lastStepInChain) throws NoSuchMethodException, IllegalAccessException, InvocationTargetException, InstantiationException { if (!(args != null && (args.length == 1))) { System.out.println("Usage: java CubeMetadataUpgrade <metadata_export_folder>"); - System.out.println( - ", where metadata_export_folder is the folder containing your current metadata's dump (Upgrade program will not modify it directly, relax."); + System.out.println(", where metadata_export_folder is the folder containing your current metadata's dump (Upgrade program will not modify it directly, relax."); return; } http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-cube/src/main/java/org/apache/kylin/cube/upgrade/common/MetadataVersionRefresher.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/upgrade/common/MetadataVersionRefresher.java b/core-cube/src/main/java/org/apache/kylin/cube/upgrade/common/MetadataVersionRefresher.java index f252123..16987c2 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/upgrade/common/MetadataVersionRefresher.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/upgrade/common/MetadataVersionRefresher.java @@ -59,8 +59,7 @@ public class MetadataVersionRefresher { collectFiles(this.store, "/", all); for (String path : all) { - if (path.endsWith(MetadataConstants.FILE_SURFIX) && !(path.startsWith(ResourceStore.DICT_RESOURCE_ROOT) - || path.startsWith(ResourceStore.SNAPSHOT_RESOURCE_ROOT))) { + if (path.endsWith(MetadataConstants.FILE_SURFIX) && !(path.startsWith(ResourceStore.DICT_RESOURCE_ROOT) || path.startsWith(ResourceStore.SNAPSHOT_RESOURCE_ROOT))) { logger.info("Updating metadata version of path {}", path); ObjectNode objectNode = (ObjectNode) mapper.readTree(this.store.getResource(path).inputStream); objectNode.put("version", version);