Repository: kylin Updated Branches: refs/heads/master e21fe2acb -> 658353ad5
reduce UT log size for Travis 4 MB log limit Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/658353ad Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/658353ad Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/658353ad Branch: refs/heads/master Commit: 658353ad57ebb30e6e6a740769a85472354212d9 Parents: e21fe2a Author: Li Yang <liy...@apache.org> Authored: Sat Sep 23 20:18:07 2017 +0800 Committer: Li Yang <liy...@apache.org> Committed: Sat Sep 23 20:18:07 2017 +0800 ---------------------------------------------------------------------- .../common/util/MemoryBudgetController.java | 2 +- .../algorithm/generic/GeneticAlgorithm.java | 18 ++++++------ .../algorithm/greedy/GreedyAlgorithm.java | 16 +++++------ .../cube/inmemcubing/ConcurrentDiskStore.java | 2 +- .../kylin/cube/inmemcubing/MemDiskStore.java | 2 +- .../kylin/cube/AggregationGroupRuleTest.java | 10 +++---- .../org/apache/kylin/cube/CubeDescTest.java | 4 +-- .../org/apache/kylin/cube/CubeManagerTest.java | 3 +- .../kylin/cube/project/ProjectManagerTest.java | 5 ++-- .../apache/kylin/dict/AppendTrieDictionary.java | 30 +++++++++++--------- .../kylin/dict/global/GlobalDictHDFSStore.java | 28 +++++++++--------- .../apache/kylin/dict/NumberDictionaryTest.java | 8 +++--- .../kylin/source/datagen/DataGenTest.java | 4 +-- .../mr/steps/NumberDictionaryForestTest.java | 17 ++++------- .../storage/hbase/steps/CreateHTableJob.java | 6 ++-- 15 files changed, 75 insertions(+), 80 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/658353ad/core-common/src/main/java/org/apache/kylin/common/util/MemoryBudgetController.java ---------------------------------------------------------------------- diff --git a/core-common/src/main/java/org/apache/kylin/common/util/MemoryBudgetController.java b/core-common/src/main/java/org/apache/kylin/common/util/MemoryBudgetController.java index 7a0b919..c19c07e 100644 --- a/core-common/src/main/java/org/apache/kylin/common/util/MemoryBudgetController.java +++ b/core-common/src/main/java/org/apache/kylin/common/util/MemoryBudgetController.java @@ -29,7 +29,7 @@ import com.google.common.base.Preconditions; public class MemoryBudgetController { - private static final boolean debug = true; + private static final boolean debug = false; public interface MemoryConsumer { // return number MB released http://git-wip-us.apache.org/repos/asf/kylin/blob/658353ad/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/generic/GeneticAlgorithm.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/generic/GeneticAlgorithm.java b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/generic/GeneticAlgorithm.java index e2545b9..178b8c5 100755 --- a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/generic/GeneticAlgorithm.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/generic/GeneticAlgorithm.java @@ -24,6 +24,7 @@ import java.util.Random; import org.apache.kylin.cube.cuboid.algorithm.AbstractRecommendAlgorithm; import org.apache.kylin.cube.cuboid.algorithm.BenefitPolicy; +import org.apache.kylin.cube.cuboid.algorithm.CuboidStats; import org.apache.kylin.cube.cuboid.algorithm.generic.lib.BitsMutation; import org.apache.kylin.cube.cuboid.algorithm.generic.lib.Chromosome; import org.apache.kylin.cube.cuboid.algorithm.generic.lib.ChromosomePair; @@ -35,7 +36,6 @@ import org.apache.kylin.cube.cuboid.algorithm.generic.lib.OnePointCrossover; import org.apache.kylin.cube.cuboid.algorithm.generic.lib.Population; import org.apache.kylin.cube.cuboid.algorithm.generic.lib.SelectionPolicy; import org.apache.kylin.cube.cuboid.algorithm.generic.lib.StoppingCondition; -import org.apache.kylin.cube.cuboid.algorithm.CuboidStats; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -87,7 +87,7 @@ public class GeneticAlgorithm extends AbstractRecommendAlgorithm { @Override public List<Long> start(double maxSpaceLimit) { - logger.info("Genetic Algorithm started."); + logger.debug("Genetic Algorithm started."); getBenefitPolicy().initBeforeStart(); @@ -110,24 +110,24 @@ public class GeneticAlgorithm extends AbstractRecommendAlgorithm { //Start the evolution Population current = evolve(initial, stopCondition); BitsChromosome chromosome = (BitsChromosome) current.getFittestChromosome(); - logger.info("Genetic Algorithm finished."); + logger.debug("Genetic Algorithm finished."); List<Long> finalList = Lists.newArrayList(); finalList.addAll(getCuboidStats().getAllCuboidsForMandatory()); finalList.addAll(cuboidEncoder.toCuboidList(chromosome.getKey())); double totalSpace = 0; - if (logger.isInfoEnabled()) { + if (logger.isTraceEnabled()) { for (Long cuboid : finalList) { Double unitSpace = getCuboidStats().getCuboidSize(cuboid); if (unitSpace != null) { - logger.info(String.format("cuboidId %d and Space: %f", cuboid, unitSpace)); + logger.trace(String.format("cuboidId %d and Space: %f", cuboid, unitSpace)); totalSpace += unitSpace; } else { - logger.info(String.format("mandatory cuboidId %d", cuboid)); + logger.trace(String.format("mandatory cuboidId %d", cuboid)); } } - logger.info("Total Space:" + totalSpace); - logger.info("Space Expansion Rate:" + totalSpace / getCuboidStats().getBaseCuboidSize()); + logger.trace("Total Space:" + totalSpace); + logger.trace("Space Expansion Rate:" + totalSpace / getCuboidStats().getBaseCuboidSize()); } return finalList; } @@ -173,7 +173,7 @@ public class GeneticAlgorithm extends AbstractRecommendAlgorithm { while (!condition.isSatisfied(current) && (!shouldCancel())) { current = nextGeneration(current); generationsEvolved++; - logger.info("Generations evolved count:" + generationsEvolved); + logger.trace("Generations evolved count:" + generationsEvolved); } return current; } http://git-wip-us.apache.org/repos/asf/kylin/blob/658353ad/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/greedy/GreedyAlgorithm.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/greedy/GreedyAlgorithm.java b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/greedy/GreedyAlgorithm.java index 0d04cc2..5dbe189 100755 --- a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/greedy/GreedyAlgorithm.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/greedy/GreedyAlgorithm.java @@ -108,8 +108,8 @@ public class GreedyAlgorithm extends AbstractRecommendAlgorithm { remaining.remove(best.getCuboidId()); getBenefitPolicy().propagateAggregationCost(best.getCuboidId(), selected); round++; - if (logger.isInfoEnabled()) { - logger.info(String.format("Recommend in round %d : %s", round, best.toString())); + if (logger.isTraceEnabled()) { + logger.trace(String.format("Recommend in round %d : %s", round, best.toString())); } } @@ -121,15 +121,15 @@ public class GreedyAlgorithm extends AbstractRecommendAlgorithm { "There should be no intersection between excluded list and selected list."); logger.info("Greedy Algorithm finished."); - if (logger.isInfoEnabled()) { - logger.info("Excluded cuboidId size:" + excluded.size()); - logger.info("Excluded cuboidId detail:"); + if (logger.isTraceEnabled()) { + logger.trace("Excluded cuboidId size:" + excluded.size()); + logger.trace("Excluded cuboidId detail:"); for (Long cuboid : excluded) { - logger.info(String.format("cuboidId %d and Cost: %d and Space: %f", cuboid, + logger.trace(String.format("cuboidId %d and Cost: %d and Space: %f", cuboid, getCuboidStats().getCuboidQueryCost(cuboid), getCuboidStats().getCuboidSize(cuboid))); } - logger.info("Total Space:" + (spaceLimit - remainingSpace)); - logger.info("Space Expansion Rate:" + (spaceLimit - remainingSpace) / getCuboidStats().getBaseCuboidSize()); + logger.trace("Total Space:" + (spaceLimit - remainingSpace)); + logger.trace("Space Expansion Rate:" + (spaceLimit - remainingSpace) / getCuboidStats().getBaseCuboidSize()); } return Lists.newArrayList(selected); } http://git-wip-us.apache.org/repos/asf/kylin/blob/658353ad/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/ConcurrentDiskStore.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/ConcurrentDiskStore.java b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/ConcurrentDiskStore.java index 41d2dfb..20ae694 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/ConcurrentDiskStore.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/ConcurrentDiskStore.java @@ -50,7 +50,7 @@ import org.slf4j.LoggerFactory; public class ConcurrentDiskStore implements IGTStore, Closeable { private static final Logger logger = LoggerFactory.getLogger(MemDiskStore.class); - private static final boolean debug = true; + private static final boolean debug = false; private static final int STREAM_BUFFER_SIZE = 8192; http://git-wip-us.apache.org/repos/asf/kylin/blob/658353ad/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/MemDiskStore.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/MemDiskStore.java b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/MemDiskStore.java index a5471df..37cb66c 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/MemDiskStore.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/MemDiskStore.java @@ -50,7 +50,7 @@ import org.slf4j.LoggerFactory; public class MemDiskStore implements IGTStore, Closeable { private static final Logger logger = LoggerFactory.getLogger(MemDiskStore.class); - private static final boolean debug = true; + private static final boolean debug = false; private static final int STREAM_BUFFER_SIZE = 8192; private static final int MEM_CHUNK_SIZE_MB = 5; http://git-wip-us.apache.org/repos/asf/kylin/blob/658353ad/core-cube/src/test/java/org/apache/kylin/cube/AggregationGroupRuleTest.java ---------------------------------------------------------------------- diff --git a/core-cube/src/test/java/org/apache/kylin/cube/AggregationGroupRuleTest.java b/core-cube/src/test/java/org/apache/kylin/cube/AggregationGroupRuleTest.java index ccc3a98..62b9c98 100644 --- a/core-cube/src/test/java/org/apache/kylin/cube/AggregationGroupRuleTest.java +++ b/core-cube/src/test/java/org/apache/kylin/cube/AggregationGroupRuleTest.java @@ -61,7 +61,7 @@ public class AggregationGroupRuleTest extends LocalFileMetadataTestCase { desc.init(getTestConfig()); ValidateContext vContext = new ValidateContext(); rule.validate(desc, vContext); - vContext.print(System.out); + //vContext.print(System.out); assertTrue(vContext.getResults().length == 0); } } @@ -85,7 +85,7 @@ public class AggregationGroupRuleTest extends LocalFileMetadataTestCase { } ValidateContext vContext = new ValidateContext(); rule.validate(desc, vContext); - vContext.print(System.out); + //vContext.print(System.out); assertTrue(vContext.getResults().length > 0); assertTrue(vContext.getResults()[0].getMessage().startsWith("Aggregation group 0 has too many combinations")); } @@ -101,7 +101,7 @@ public class AggregationGroupRuleTest extends LocalFileMetadataTestCase { IValidatorRule<CubeDesc> rule = getAggregationGroupRule(); rule.validate(desc, vContext); - vContext.print(System.out); + //vContext.print(System.out); assertEquals(1, vContext.getResults().length); } @@ -115,7 +115,7 @@ public class AggregationGroupRuleTest extends LocalFileMetadataTestCase { desc.getAggregationGroups().get(0).setIncludes(temp); IValidatorRule<CubeDesc> rule = getAggregationGroupRule(); rule.validate(desc, vContext); - vContext.print(System.out); + //vContext.print(System.out); assertEquals(1, vContext.getResults().length); assertEquals("Aggregation group 0 'includes' dimensions not include all the dimensions:[seller_id, META_CATEG_NAME, lstg_format_name, lstg_site_id, slr_segment_cd]", (vContext.getResults()[0].getMessage())); } @@ -130,7 +130,7 @@ public class AggregationGroupRuleTest extends LocalFileMetadataTestCase { IValidatorRule<CubeDesc> rule = getAggregationGroupRule(); rule.validate(desc, vContext); - vContext.print(System.out); + //vContext.print(System.out); assertEquals(2, vContext.getResults().length); assertEquals("Aggregation group 0 joint dimensions has overlap with more than 1 dimensions in same hierarchy: [CATEG_LVL2_NAME, META_CATEG_NAME]", (vContext.getResults()[0].getMessage())); } http://git-wip-us.apache.org/repos/asf/kylin/blob/658353ad/core-cube/src/test/java/org/apache/kylin/cube/CubeDescTest.java ---------------------------------------------------------------------- diff --git a/core-cube/src/test/java/org/apache/kylin/cube/CubeDescTest.java b/core-cube/src/test/java/org/apache/kylin/cube/CubeDescTest.java index 34ec190..ae7b17c 100644 --- a/core-cube/src/test/java/org/apache/kylin/cube/CubeDescTest.java +++ b/core-cube/src/test/java/org/apache/kylin/cube/CubeDescTest.java @@ -400,7 +400,7 @@ public class CubeDescTest extends LocalFileMetadataTestCase { public void testSerialize() throws Exception { CubeDesc desc = CubeDescManager.getInstance(getTestConfig()).getCubeDesc(CUBE_WITH_SLR_DESC); String str = JsonUtil.writeValueAsIndentString(desc); - System.out.println(str); + //System.out.println(str); @SuppressWarnings("unused") CubeDesc desc2 = JsonUtil.readValue(str, CubeDesc.class); } @@ -435,7 +435,7 @@ public class CubeDescTest extends LocalFileMetadataTestCase { String mapStr = JsonUtil.writeValueAsString(map); - System.out.println(mapStr); + //System.out.println(mapStr); Map<?, ?> map2 = JsonUtil.readValue(mapStr, HashMap.class); http://git-wip-us.apache.org/repos/asf/kylin/blob/658353ad/core-cube/src/test/java/org/apache/kylin/cube/CubeManagerTest.java ---------------------------------------------------------------------- diff --git a/core-cube/src/test/java/org/apache/kylin/cube/CubeManagerTest.java b/core-cube/src/test/java/org/apache/kylin/cube/CubeManagerTest.java index 0188904..d8d48f7 100644 --- a/core-cube/src/test/java/org/apache/kylin/cube/CubeManagerTest.java +++ b/core-cube/src/test/java/org/apache/kylin/cube/CubeManagerTest.java @@ -28,7 +28,6 @@ import java.util.NavigableSet; import org.apache.kylin.common.KylinConfig; import org.apache.kylin.common.persistence.ResourceStore; -import org.apache.kylin.common.util.JsonUtil; import org.apache.kylin.common.util.LocalFileMetadataTestCase; import org.apache.kylin.cube.model.CubeDesc; import org.apache.kylin.metadata.model.SegmentRange; @@ -62,7 +61,7 @@ public class CubeManagerTest extends LocalFileMetadataTestCase { CubeInstance cube = CubeManager.getInstance(getTestConfig()).getCube("test_kylin_cube_without_slr_ready"); CubeDesc desc = cube.getDescriptor(); - System.out.println(JsonUtil.writeValueAsIndentString(desc)); + //System.out.println(JsonUtil.writeValueAsIndentString(desc)); String signature = desc.calculateSignature(); desc.getModel().getPartitionDesc().setPartitionDateColumn("test_column"); http://git-wip-us.apache.org/repos/asf/kylin/blob/658353ad/core-cube/src/test/java/org/apache/kylin/cube/project/ProjectManagerTest.java ---------------------------------------------------------------------- diff --git a/core-cube/src/test/java/org/apache/kylin/cube/project/ProjectManagerTest.java b/core-cube/src/test/java/org/apache/kylin/cube/project/ProjectManagerTest.java index a0fb4a3..c99b683 100644 --- a/core-cube/src/test/java/org/apache/kylin/cube/project/ProjectManagerTest.java +++ b/core-cube/src/test/java/org/apache/kylin/cube/project/ProjectManagerTest.java @@ -25,7 +25,6 @@ import java.io.IOException; import java.util.Set; import org.apache.kylin.common.persistence.ResourceStore; -import org.apache.kylin.common.util.JsonUtil; import org.apache.kylin.common.util.LocalFileMetadataTestCase; import org.apache.kylin.cube.CubeDescManager; import org.apache.kylin.cube.CubeInstance; @@ -85,7 +84,7 @@ public class ProjectManagerTest extends LocalFileMetadataTestCase { Set<IRealization> realizations = proMgr.listAllRealizations("alien"); assertTrue(realizations.contains(createdCube)); - System.out.println(JsonUtil.writeValueAsIndentString(createdCube)); + //System.out.println(JsonUtil.writeValueAsIndentString(createdCube)); assertTrue(prjMgr.listAllProjects().size() == originalProjectCount + 1); assertTrue(prjMgr.listAllRealizations("ALIEN").iterator().next().getName().equalsIgnoreCase("CUBE_IN_ALIEN_PROJECT")); @@ -129,7 +128,7 @@ public class ProjectManagerTest extends LocalFileMetadataTestCase { CubeInstance createdCube = cubeMgr.createCube("new_cube_in_default", ProjectInstance.DEFAULT_PROJECT_NAME, desc, null); assertTrue(createdCube == cubeMgr.getCube("new_cube_in_default")); - System.out.println(JsonUtil.writeValueAsIndentString(createdCube)); + //System.out.println(JsonUtil.writeValueAsIndentString(createdCube)); assertTrue(prjMgr.listAllProjects().size() == originalProjectCount); assertTrue(cubeMgr.listAllCubes().size() == originalCubeCount + 1); http://git-wip-us.apache.org/repos/asf/kylin/blob/658353ad/core-dictionary/src/main/java/org/apache/kylin/dict/AppendTrieDictionary.java ---------------------------------------------------------------------- diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/AppendTrieDictionary.java b/core-dictionary/src/main/java/org/apache/kylin/dict/AppendTrieDictionary.java index 18795b4..1855274 100644 --- a/core-dictionary/src/main/java/org/apache/kylin/dict/AppendTrieDictionary.java +++ b/core-dictionary/src/main/java/org/apache/kylin/dict/AppendTrieDictionary.java @@ -17,11 +17,15 @@ */ package org.apache.kylin.dict; -import com.google.common.cache.CacheBuilder; -import com.google.common.cache.CacheLoader; -import com.google.common.cache.LoadingCache; -import com.google.common.cache.RemovalListener; -import com.google.common.cache.RemovalNotification; +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; +import java.io.PrintStream; +import java.util.Arrays; +import java.util.Objects; +import java.util.TreeMap; +import java.util.concurrent.ExecutionException; + import org.apache.hadoop.fs.Path; import org.apache.kylin.common.KylinConfig; import org.apache.kylin.common.util.Dictionary; @@ -32,14 +36,12 @@ import org.apache.kylin.dict.global.GlobalDictMetadata; import org.apache.kylin.dict.global.GlobalDictStore; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.DataInput; -import java.io.DataOutput; -import java.io.IOException; -import java.io.PrintStream; -import java.util.Arrays; -import java.util.Objects; -import java.util.TreeMap; -import java.util.concurrent.ExecutionException; + +import com.google.common.cache.CacheBuilder; +import com.google.common.cache.CacheLoader; +import com.google.common.cache.LoadingCache; +import com.google.common.cache.RemovalListener; +import com.google.common.cache.RemovalNotification; /** * A dictionary based on Trie data structure that maps enumerations of byte[] to @@ -91,7 +93,7 @@ public class AppendTrieDictionary<T> extends CacheDictionary<T> { @Override public AppendDictSlice load(AppendDictSliceKey key) throws Exception { AppendDictSlice slice = globalDictStore.readSlice(latestVersionPath.toString(), metadata.sliceFileMap.get(key)); - logger.info("Load slice with key {} and value {}", key, slice); + logger.trace("Load slice with key {} and value {}", key, slice); return slice; } }); http://git-wip-us.apache.org/repos/asf/kylin/blob/658353ad/core-dictionary/src/main/java/org/apache/kylin/dict/global/GlobalDictHDFSStore.java ---------------------------------------------------------------------- diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/global/GlobalDictHDFSStore.java b/core-dictionary/src/main/java/org/apache/kylin/dict/global/GlobalDictHDFSStore.java index 4c8ce0f..bad427c 100644 --- a/core-dictionary/src/main/java/org/apache/kylin/dict/global/GlobalDictHDFSStore.java +++ b/core-dictionary/src/main/java/org/apache/kylin/dict/global/GlobalDictHDFSStore.java @@ -18,6 +18,15 @@ package org.apache.kylin.dict.global; +import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.base.Preconditions.checkState; + +import java.io.IOException; +import java.util.Arrays; +import java.util.Map; +import java.util.TreeMap; +import java.util.TreeSet; + import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FSDataOutputStream; @@ -33,15 +42,6 @@ import org.apache.kylin.dict.BytesConverter; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.IOException; -import java.util.Arrays; -import java.util.Map; -import java.util.TreeMap; -import java.util.TreeSet; - -import static com.google.common.base.Preconditions.checkArgument; -import static com.google.common.base.Preconditions.checkState; - public class GlobalDictHDFSStore extends GlobalDictStore { static final Logger logger = LoggerFactory.getLogger(GlobalDictHDFSStore.class); @@ -112,12 +112,12 @@ public class GlobalDictHDFSStore extends GlobalDictStore { migrateOldLayout(); - logger.info("Prepare to write Global dict at {}, isGlobal={}", workingDir, isGlobal); + logger.trace("Prepare to write Global dict at {}, isGlobal={}", workingDir, isGlobal); Path working = new Path(workingDir); if (fileSystem.exists(working)) { fileSystem.delete(working, true); - logger.info("Working directory {} exits, delete it first", working); + logger.trace("Working directory {} exits, delete it first", working); } // when build dict, copy all data into working dir and work on it, avoiding suddenly server crash made data corrupt @@ -182,7 +182,7 @@ public class GlobalDictHDFSStore extends GlobalDictStore { @Override public AppendDictSlice readSlice(String directory, String sliceFileName) throws IOException { Path path = new Path(directory, sliceFileName); - logger.info("read slice from {}", path); + logger.trace("read slice from {}", path); try (FSDataInputStream input = fileSystem.open(path, BUFFER_SIZE)) { return AppendDictSlice.deserializeFrom(input); } @@ -194,7 +194,7 @@ public class GlobalDictHDFSStore extends GlobalDictStore { String sliceFile = IndexFormatV2.sliceFileName(key); Path path = new Path(workingDir, sliceFile); - logger.info("write slice with key {} into file {}", key, path); + logger.trace("write slice with key {} into file {}", key, path); try (FSDataOutputStream out = fileSystem.create(path, true, BUFFER_SIZE)) { byte[] bytes = slice.buildTrieBytes(); out.write(bytes); @@ -205,7 +205,7 @@ public class GlobalDictHDFSStore extends GlobalDictStore { @Override public void deleteSlice(String workingDir, String sliceFileName) throws IOException { Path path = new Path(workingDir, sliceFileName); - logger.info("delete slice at {}", path); + logger.trace("delete slice at {}", path); if (fileSystem.exists(path)) { fileSystem.delete(path, false); } http://git-wip-us.apache.org/repos/asf/kylin/blob/658353ad/core-dictionary/src/test/java/org/apache/kylin/dict/NumberDictionaryTest.java ---------------------------------------------------------------------- diff --git a/core-dictionary/src/test/java/org/apache/kylin/dict/NumberDictionaryTest.java b/core-dictionary/src/test/java/org/apache/kylin/dict/NumberDictionaryTest.java index 8da7208..b2cd86a 100644 --- a/core-dictionary/src/test/java/org/apache/kylin/dict/NumberDictionaryTest.java +++ b/core-dictionary/src/test/java/org/apache/kylin/dict/NumberDictionaryTest.java @@ -141,10 +141,10 @@ public class NumberDictionaryTest extends LocalFileMetadataTestCase { // test exact match NumberDictionary<String> dict = builder.build(0); - for (int i = 0; i < sorted.size(); i++) { - String dictNum = dict.getValueFromId(i); - System.out.println(sorted.get(i) + "\t" + dictNum); - } +// for (int i = 0; i < sorted.size(); i++) { +// String dictNum = dict.getValueFromId(i); +// System.out.println(sorted.get(i) + "\t" + dictNum); +// } for (int i = 0; i < sorted.size(); i++) { String dictNum = dict.getValueFromId(i); http://git-wip-us.apache.org/repos/asf/kylin/blob/658353ad/core-metadata/src/test/java/org/apache/kylin/source/datagen/DataGenTest.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/test/java/org/apache/kylin/source/datagen/DataGenTest.java b/core-metadata/src/test/java/org/apache/kylin/source/datagen/DataGenTest.java index 5e3ca9f..016c591 100644 --- a/core-metadata/src/test/java/org/apache/kylin/source/datagen/DataGenTest.java +++ b/core-metadata/src/test/java/org/apache/kylin/source/datagen/DataGenTest.java @@ -44,7 +44,7 @@ public class DataGenTest extends LocalFileMetadataTestCase { public void testCIConfigured() throws IOException { DataModelDesc model = getModel("ci_inner_join_model"); ModelDataGenerator gen = new ModelDataGenerator(model, 100); - gen.outprint = true; + gen.outprint = false; gen.generate(); } @@ -53,7 +53,7 @@ public class DataGenTest extends LocalFileMetadataTestCase { public void testSSBNoConfig() throws IOException { DataModelDesc model = getModel("ssb"); ModelDataGenerator gen = new ModelDataGenerator(model, 100); - gen.outprint = true; + gen.outprint = false; gen.generate(); } http://git-wip-us.apache.org/repos/asf/kylin/blob/658353ad/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/NumberDictionaryForestTest.java ---------------------------------------------------------------------- diff --git a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/NumberDictionaryForestTest.java b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/NumberDictionaryForestTest.java index 414ab95..60a18cc 100644 --- a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/NumberDictionaryForestTest.java +++ b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/NumberDictionaryForestTest.java @@ -75,7 +75,7 @@ public class NumberDictionaryForestTest { b.addValue(value); } TrieDictionaryForest<String> dict = b.build(); - dict.dump(System.out); + //dict.dump(System.out); ArrayList<Integer> resultIds = new ArrayList<>(); for (int i = 0; i < keyList.size(); i++) { @@ -117,7 +117,7 @@ public class NumberDictionaryForestTest { b.addValue(str); TrieDictionaryForest<String> dict = b.build(); dict = testSerialize(dict); - dict.dump(System.out); + //dict.dump(System.out); for (String str : testData) { assertEquals(str, dict.getValueFromId(dict.getIdFromValue(str))); } @@ -134,13 +134,13 @@ public class NumberDictionaryForestTest { for (String str : testData) b.addValue(str); TrieDictionaryForest<String> dict = b.build(); - dict.dump(System.out); + //dict.dump(System.out); NumberDictionaryBuilder b2 = new NumberDictionaryBuilder(); for (String str : testData) b2.addValue(str); NumberDictionary<String> dict2 = b2.build(0); - dict2.dump(System.out); + //dict2.dump(System.out); } @@ -188,7 +188,7 @@ public class NumberDictionaryForestTest { for (String str : testData) b.addValue(str); TrieDictionaryForest<String> dict = b.build(); - dict.dump(System.out); + //dict.dump(System.out); } private static TrieDictionaryForest<String> testSerialize(TrieDictionaryForest<String> dict) { @@ -274,13 +274,6 @@ public class NumberDictionaryForestTest { return keyList; } - private String printKey(SelfDefineSortableKey key) { - Text data = key.getText(); - String fieldValue = Bytes.toString(data.getBytes(), 1, data.getLength() - 1); - System.out.println("type flag:" + key.getTypeId() + " fieldValue:" + fieldValue); - return fieldValue; - } - private String getFieldValue(SelfDefineSortableKey key) { Text data = key.getText(); return Bytes.toString(data.getBytes(), 1, data.getLength() - 1); http://git-wip-us.apache.org/repos/asf/kylin/blob/658353ad/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CreateHTableJob.java ---------------------------------------------------------------------- diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CreateHTableJob.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CreateHTableJob.java index e1f8bb1..fbd47a8 100644 --- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CreateHTableJob.java +++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CreateHTableJob.java @@ -131,8 +131,10 @@ public class CreateHTableJob extends AbstractHadoopJob { logger.info((rowkeyList.size() + 1) + " regions"); logger.info(rowkeyList.size() + " splits"); - for (byte[] split : rowkeyList) { - logger.info(StringUtils.byteToHexString(split)); + if (logger.isTraceEnabled()) { + for (byte[] split : rowkeyList) { + logger.trace(StringUtils.byteToHexString(split)); + } } byte[][] retValue = rowkeyList.toArray(new byte[rowkeyList.size()][]);
