Repository: incubator-kylin Updated Branches: refs/heads/2.x-staging 975733630 -> b12d86783
KYLIN-942 add more backdoor toggles to help profiling Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/ef2cc71c Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/ef2cc71c Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/ef2cc71c Branch: refs/heads/2.x-staging Commit: ef2cc71ca23999bbb9ea28b337319107dfca9e65 Parents: 9757336 Author: honma <ho...@ebay.com> Authored: Wed Oct 28 15:19:51 2015 +0800 Committer: honma <ho...@ebay.com> Committed: Wed Oct 28 15:19:51 2015 +0800 ---------------------------------------------------------------------- .../kylin/common/debug/BackdoorToggles.java | 29 ++++++++++++++++++++ .../kylin/storage/hbase/HBaseStorage.java | 23 ++++++++++++---- .../hbase/cube/v2/CubeSegmentScanner.java | 9 ++++-- 3 files changed, 54 insertions(+), 7 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/ef2cc71c/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java ---------------------------------------------------------------------- diff --git a/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java b/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java index 53e5864..bac7258 100644 --- a/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java +++ b/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java @@ -34,6 +34,14 @@ public class BackdoorToggles { return getString(DEBUG_TOGGLE_OBSERVER_BEHAVIOR); } + public static String getHbaseCubeQueryVersion() { + return getString(DEBUG_TOGGLE_HBASE_CUBE_QUERY_VERSION); + } + + public static String getHbaseCubeQueryProtocol() { + return getString(DEBUG_TOGGLE_HBASE_CUBE_QUERY_PROTOCOL); + } + public static boolean getDisableFuzzyKey() { return getBoolean(DEBUG_TOGGLE_DISABLE_FUZZY_KEY); } @@ -72,6 +80,27 @@ public class BackdoorToggles { public final static String DEBUG_TOGGLE_DISABLE_FUZZY_KEY = "DEBUG_TOGGLE_DISABLE_FUZZY_KEY"; /** + * set DEBUG_TOGGLE_HBASE_CUBE_QUERY_VERSION=v1/v2 to control which version CubeStorageQuery to use + * + example:(put it into request body) + "backdoorToggles": { + "DEBUG_TOGGLE_HBASE_CUBE_QUERY_VERSION": "v1" + } + */ + public final static String DEBUG_TOGGLE_HBASE_CUBE_QUERY_VERSION = "DEBUG_TOGGLE_HBASE_CUBE_QUERY_VERSION"; + + /** + * set DEBUG_TOGGLE_HBASE_CUBE_QUERY_PROTOCOL=endpoint/scan to control how to visit hbase cube + * this param is only valid when DEBUG_TOGGLE_HBASE_CUBE_QUERY_VERSION set to v2(bdefault) + * + example:(put it into request body) + "backdoorToggles": { + "DEBUG_TOGGLE_HBASE_CUBE_QUERY_PROTOCOL": "scan" + } + */ + public final static String DEBUG_TOGGLE_HBASE_CUBE_QUERY_PROTOCOL = "DEBUG_TOGGLE_HBASE_CUBE_QUERY_PROTOCOL"; + + /** * set DEBUG_TOGGLE_OBSERVER_BEHAVIOR=SCAN/SCAN_FILTER/SCAN_FILTER_AGGR to control observer behavior for debug/profile usage * example:(put it into request body) http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/ef2cc71c/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseStorage.java ---------------------------------------------------------------------- diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseStorage.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseStorage.java index 53465d8..2e2000d 100644 --- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseStorage.java +++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseStorage.java @@ -19,6 +19,7 @@ package org.apache.kylin.storage.hbase; import org.apache.kylin.common.KylinConfig; +import org.apache.kylin.common.debug.BackdoorToggles; import org.apache.kylin.cube.CubeInstance; import org.apache.kylin.engine.mr.IMROutput; import org.apache.kylin.engine.mr.IMROutput2; @@ -39,11 +40,15 @@ import org.apache.kylin.storage.hbase.steps.HBaseMROutput2Transition; import com.google.common.base.Preconditions; -@SuppressWarnings("unused")//used by reflection +@SuppressWarnings("unused") +//used by reflection public class HBaseStorage implements IStorage { private final static boolean allowStorageLayerCache = true; - private final static String defaultCubeStorageQuery = "org.apache.kylin.storage.hbase.cube.v2.CubeStorageQuery"; + + private final static String v2CubeStorageQuery = "org.apache.kylin.storage.hbase.cube.v2.CubeStorageQuery"; + private final static String v1CubeStorageQuery = "org.apache.kylin.storage.hbase.cube.v1.CubeStorageQuery"; + private final static String defaultIIStorageQuery = "org.apache.kylin.storage.hbase.ii.InvertedIndexStorageQuery"; @Override @@ -62,11 +67,19 @@ public class HBaseStorage implements IStorage { return ret; } } else if (realization.getType() == RealizationType.CUBE) { + + String cubeStorageQuery; + if ("v1".equalsIgnoreCase(BackdoorToggles.getHbaseCubeQueryVersion())) { + cubeStorageQuery = v1CubeStorageQuery; + } else { + cubeStorageQuery = v2CubeStorageQuery;//by default use v2 + } + ICachableStorageQuery ret; try { - ret = (ICachableStorageQuery) Class.forName(defaultCubeStorageQuery).getConstructor(CubeInstance.class).newInstance((CubeInstance) realization); + ret = (ICachableStorageQuery) Class.forName(cubeStorageQuery).getConstructor(CubeInstance.class).newInstance((CubeInstance) realization); } catch (Exception e) { - throw new RuntimeException("Failed to initialize storage query for " + defaultCubeStorageQuery, e); + throw new RuntimeException("Failed to initialize storage query for " + cubeStorageQuery, e); } if (allowStorageLayerCache) { @@ -78,7 +91,7 @@ public class HBaseStorage implements IStorage { throw new IllegalArgumentException("Unknown realization type " + realization.getType()); } } - + private static IStorageQuery wrapWithCache(ICachableStorageQuery underlyingStorageEngine, IRealization realization) { if (underlyingStorageEngine.isDynamic()) { return new CacheFledgedDynamicQuery(underlyingStorageEngine, getPartitionCol(realization)); http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/ef2cc71c/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeSegmentScanner.java ---------------------------------------------------------------------- diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeSegmentScanner.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeSegmentScanner.java index 814a359..18ba6fb 100644 --- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeSegmentScanner.java +++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeSegmentScanner.java @@ -11,6 +11,7 @@ import java.util.List; import java.util.NoSuchElementException; import java.util.Set; +import org.apache.kylin.common.debug.BackdoorToggles; import org.apache.kylin.common.util.ByteArray; import org.apache.kylin.common.util.DateFormat; import org.apache.kylin.common.util.ImmutableBitSet; @@ -224,8 +225,12 @@ public class CubeSegmentScanner implements IGTScanner { try { - CubeHBaseRPC rpc = new CubeHBaseEndpointRPC(cubeSeg, cuboid, info); - //CubeHBaseRPC rpc = new CubeHBaseScanRPC(cubeSeg, cuboid, info); + CubeHBaseRPC rpc; + if ("scan".equalsIgnoreCase(BackdoorToggles.getHbaseCubeQueryProtocol())) { + rpc = new CubeHBaseScanRPC(cubeSeg, cuboid, info); + } else { + rpc = new CubeHBaseEndpointRPC(cubeSeg, cuboid, info);//default behavior + } //change previous line to CubeHBaseRPC rpc = new CubeHBaseScanRPC(cubeSeg, cuboid, info); //to debug locally
