Repository: kylin Updated Branches: refs/heads/master c52b6981f -> d7ecd6e2f
KYLIN-2451 rework to respect cube-level coprocessor-timeout-seconds Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/d7ecd6e2 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/d7ecd6e2 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/d7ecd6e2 Branch: refs/heads/master Commit: d7ecd6e2f1a87bf82d3cf6645eab013b873dde8d Parents: c52b698 Author: Li Yang <liy...@apache.org> Authored: Mon Feb 20 17:21:21 2017 +0800 Committer: Li Yang <liy...@apache.org> Committed: Mon Feb 20 17:21:21 2017 +0800 ---------------------------------------------------------------------- .../kylin/storage/hbase/HBaseConnection.java | 6 ---- .../storage/hbase/cube/v2/CubeHBaseRPC.java | 37 ++++++++++++-------- 2 files changed, 22 insertions(+), 21 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/d7ecd6e2/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java ---------------------------------------------------------------------- diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java index e3d2308..73f31c5 100644 --- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java +++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java @@ -37,7 +37,6 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.HColumnDescriptor; -import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.HTableDescriptor; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.Admin; @@ -163,11 +162,6 @@ public class HBaseConnection { conf.set("hbase.fs.tmp.dir", "/tmp"); } - // set RPC timeout - if (kylinConf.getQueryCoprocessorTimeoutSeconds() > 0) { - conf.set(HConstants.HBASE_RPC_TIMEOUT_KEY, "" + (1000 * kylinConf.getQueryCoprocessorTimeoutSeconds())); - } - return conf; } http://git-wip-us.apache.org/repos/asf/kylin/blob/d7ecd6e2/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseRPC.java ---------------------------------------------------------------------- diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseRPC.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseRPC.java index f24290c..88e7176 100644 --- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseRPC.java +++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseRPC.java @@ -35,7 +35,6 @@ import org.apache.kylin.common.util.Bytes; import org.apache.kylin.common.util.ImmutableBitSet; import org.apache.kylin.common.util.Pair; import org.apache.kylin.cube.CubeSegment; -import org.apache.kylin.metadata.model.ISegment; import org.apache.kylin.cube.cuboid.Cuboid; import org.apache.kylin.cube.kv.FuzzyKeyEncoder; import org.apache.kylin.cube.kv.FuzzyMaskEncoder; @@ -49,6 +48,7 @@ import org.apache.kylin.gridtable.GTInfo; import org.apache.kylin.gridtable.GTRecord; import org.apache.kylin.gridtable.GTScanRange; import org.apache.kylin.gridtable.IGTStorage; +import org.apache.kylin.metadata.model.ISegment; import org.apache.kylin.storage.hbase.HBaseConnection; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -283,23 +283,30 @@ public abstract class CubeHBaseRPC implements IGTStorage { } protected int getCoprocessorTimeoutMillis() { - int configTimeout = cubeSeg.getConfig().getQueryCoprocessorTimeoutSeconds() * 1000; - if (configTimeout == 0) { - configTimeout = Integer.MAX_VALUE; + int coopTimeout; + if (BackdoorToggles.getQueryTimeout() != -1) { + coopTimeout = BackdoorToggles.getQueryTimeout(); + } else { + coopTimeout = cubeSeg.getConfig().getQueryCoprocessorTimeoutSeconds() * 1000; } - + + int rpcTimeout; Configuration hconf = HBaseConnection.getCurrentHBaseConfiguration(); - int rpcTimeout = hconf.getInt(HConstants.HBASE_RPC_TIMEOUT_KEY, HConstants.DEFAULT_HBASE_RPC_TIMEOUT); - // final timeout should be smaller than rpc timeout - int upper = (int) (rpcTimeout * 0.9); - - int timeout = Math.min(upper, configTimeout); - if (BackdoorToggles.getQueryTimeout() != -1) { - timeout = Math.min(upper, BackdoorToggles.getQueryTimeout()); + rpcTimeout = hconf.getInt(HConstants.HBASE_RPC_TIMEOUT_KEY, HConstants.DEFAULT_HBASE_RPC_TIMEOUT); + + // HBase rpc timeout must be longer than coprocessor timeout + if ((int) (coopTimeout * 1.1) > rpcTimeout) { + rpcTimeout = (int) (coopTimeout * 1.1); + hconf.setInt(HConstants.HBASE_RPC_TIMEOUT_KEY, rpcTimeout); } - - logger.debug("{} = {} ms, use {} ms as timeout for coprocessor", HConstants.HBASE_RPC_TIMEOUT_KEY, rpcTimeout, timeout); - return timeout; + + // coprocessor timeout is 0 by default + if (coopTimeout <= 0) { + coopTimeout = (int) (rpcTimeout * 0.9); + } + + logger.debug("{} = {} ms, use {} ms as timeout for coprocessor", HConstants.HBASE_RPC_TIMEOUT_KEY, rpcTimeout, coopTimeout); + return coopTimeout; } }
