Repository: incubator-kylin Updated Branches: refs/heads/KYLIN-942 d927ea124 -> b8b3c02cf
remove views Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/b8b3c02c Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/b8b3c02c Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/b8b3c02c Branch: refs/heads/KYLIN-942 Commit: b8b3c02cf5701804005477d5a4d248679f84d845 Parents: d927ea1 Author: honma <ho...@ebay.com> Authored: Tue Sep 29 15:01:25 2015 +0800 Committer: honma <ho...@ebay.com> Committed: Tue Sep 29 15:01:25 2015 +0800 ---------------------------------------------------------------------- .../org/apache/kylin/common/util/BasicTest.java | 6 ++--- .../kylin/engine/mr/common/CuboidShardUtil.java | 26 ++++++++------------ .../storage/hbase/steps/CreateHTableJob.java | 11 ++++----- 3 files changed, 18 insertions(+), 25 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/b8b3c02c/core-common/src/test/java/org/apache/kylin/common/util/BasicTest.java ---------------------------------------------------------------------- diff --git a/core-common/src/test/java/org/apache/kylin/common/util/BasicTest.java b/core-common/src/test/java/org/apache/kylin/common/util/BasicTest.java index 1590c92..cd83d96 100644 --- a/core-common/src/test/java/org/apache/kylin/common/util/BasicTest.java +++ b/core-common/src/test/java/org/apache/kylin/common/util/BasicTest.java @@ -25,6 +25,7 @@ import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.Calendar; import java.util.IdentityHashMap; +import java.util.Map; import org.apache.commons.configuration.ConfigurationException; import org.junit.Assert; @@ -32,7 +33,9 @@ import org.junit.Ignore; import org.junit.Test; import org.slf4j.LoggerFactory; +import com.google.common.base.Predicate; import com.google.common.collect.Lists; +import com.google.common.collect.Maps; import com.google.common.collect.TreeMultiset; /** @@ -64,9 +67,6 @@ public class BasicTest { @Test public void testxx() { - short x = -64; - int y = (int)x; - System.out.println(y); } @Test http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/b8b3c02c/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CuboidShardUtil.java ---------------------------------------------------------------------- diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CuboidShardUtil.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CuboidShardUtil.java index e1af752..d09e4ec 100644 --- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CuboidShardUtil.java +++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CuboidShardUtil.java @@ -28,30 +28,24 @@ import org.apache.kylin.cube.CubeUpdate; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.google.common.base.Predicate; import com.google.common.collect.Maps; public class CuboidShardUtil { protected static final Logger logger = LoggerFactory.getLogger(CuboidShardUtil.class); - // public static Map<Long, Short> loadCuboidShards(CubeSegment segment) { - // return DefaultedMap.defaultedMap(segment.getCuboidShards(), (short) 1); - // } - public static void saveCuboidShards(CubeSegment segment, Map<Long, Short> cuboidShards, int totalShards) throws IOException { CubeManager cubeManager = CubeManager.getInstance(KylinConfig.getInstanceFromEnv()); - Map<Long, Short> filered = Maps.newHashMap(Maps.filterEntries(cuboidShards, new Predicate<Map.Entry<Long, Short>>() { - @Override - public boolean apply(Map.Entry<Long, Short> input) { - if (input.getValue() <= 1) { - logger.info("Cuboid {} has {} shards, skip saving it as an optimization", input.getKey(), input.getValue()); - return false; - } else { - logger.info("Cuboid {} has {} shards, saving it", input.getKey(), input.getValue()); - return true; - } + + Map<Long, Short> filered = Maps.newHashMap(); + for (Map.Entry<Long, Short> entry : cuboidShards.entrySet()) { + if (entry.getValue() <= 1) { + logger.info("Cuboid {} has {} shards, skip saving it as an optimization", entry.getKey(), entry.getValue()); + } else { + logger.info("Cuboid {} has {} shards, saving it", entry.getKey(), entry.getValue()); + filered.put(entry.getKey(), entry.getValue()); } - })); + } + segment.setCuboidShardNums(filered); segment.setTotalShards(totalShards); http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/b8b3c02c/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CreateHTableJob.java ---------------------------------------------------------------------- diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CreateHTableJob.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CreateHTableJob.java index b3b6843..6a4373e 100644 --- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CreateHTableJob.java +++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CreateHTableJob.java @@ -256,12 +256,11 @@ public class CreateHTableJob extends AbstractHadoopJob { allCuboids.addAll(cubeRowCountMap.keySet()); Collections.sort(allCuboids); - Map<Long, Double> cubeSizeMap = Maps.newHashMap(Maps.transformEntries(cubeRowCountMap, new Maps.EntryTransformer<Long, Long, Double>() { - @Override - public Double transformEntry(@Nullable Long key, @Nullable Long value) { - return estimateCuboidStorageSize(cubeDesc, key, value, baseCuboidId, rowkeyColumnSize); - } - })); + Map<Long, Double> cubeSizeMap = Maps.newHashMap(); + for (Map.Entry<Long, Long> entry : cubeRowCountMap.entrySet()) { + cubeSizeMap.put(entry.getKey(),estimateCuboidStorageSize(cubeDesc, entry.getKey(), entry.getValue(), baseCuboidId, rowkeyColumnSize)); + } + for (Double cuboidSize : cubeSizeMap.values()) { totalSizeInM += cuboidSize; }