Repository: incubator-kylin Updated Branches: refs/heads/KYLIN-942 18e7999dc -> ca4552565
minor Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/ca455256 Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/ca455256 Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/ca455256 Branch: refs/heads/KYLIN-942 Commit: ca45525651af4e099bfd48b7634c04abee3f8884 Parents: 18e7999 Author: honma <ho...@ebay.com> Authored: Wed Oct 14 20:20:45 2015 +0800 Committer: honma <ho...@ebay.com> Committed: Wed Oct 14 20:20:45 2015 +0800 ---------------------------------------------------------------------- .../org/apache/kylin/cube/CubeInstance.java | 10 ++------ .../kylin/gridtable/GTScanRangePlanner.java | 26 ++++++++++++++------ .../kylin/gridtable/DictGridTableTest.java | 2 +- .../kylin/query/test/ITKylinQueryTest.java | 2 +- query/src/test/resources/query/sql/query85.sql | 26 ++++++++++++++++++++ .../hbase/cube/v1/CubeSegmentTupleIterator.java | 2 +- .../hbase/cube/v2/CubeHBaseEndpointRPC.java | 3 ++- .../storage/hbase/cube/v2/CubeHBaseRPC.java | 2 +- .../hbase/cube/v2/CubeSegmentScanner.java | 4 +-- 9 files changed, 54 insertions(+), 23 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/ca455256/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java index fd16434..7452539 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java @@ -20,7 +20,6 @@ package org.apache.kylin.cube; import java.util.ArrayList; import java.util.List; -import java.util.Map; import org.apache.kylin.common.KylinConfig; import org.apache.kylin.common.persistence.ResourceStore; @@ -49,10 +48,9 @@ import com.google.common.collect.Lists; @SuppressWarnings("serial") @JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE) public class CubeInstance extends RootPersistentEntity implements IRealization, IBuildable { - private static final int COST_WEIGHT_DIMENSION = 1; private static final int COST_WEIGHT_MEASURE = 1; - private static final int COST_WEIGHT_LOOKUP_TABLE = 1; - private static final int COST_WEIGHT_INNER_JOIN = 2; + private static final int COST_WEIGHT_DIMENSION = 10; + private static final int COST_WEIGHT_INNER_JOIN = 100; public static CubeInstance create(String cubeName, String projectName, CubeDesc cubeDesc) { CubeInstance cubeInstance = new CubeInstance(); @@ -92,8 +90,6 @@ public class CubeInstance extends RootPersistentEntity implements IRealization, @JsonProperty("create_time_utc") private long createTimeUTC; - - private String projectName; // default constructor for jackson @@ -358,7 +354,6 @@ public class CubeInstance extends RootPersistentEntity implements IRealization, for (LookupDesc lookupDesc : this.getDescriptor().getModel().getLookups()) { // more tables, more cost - calculatedCost += COST_WEIGHT_LOOKUP_TABLE; if ("inner".equals(lookupDesc.getJoin().getType())) { // inner join cost is bigger than left join, as it will filter some records calculatedCost += COST_WEIGHT_INNER_JOIN; @@ -447,5 +442,4 @@ public class CubeInstance extends RootPersistentEntity implements IRealization, return getDescriptor().getEngineType(); } - } http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/ca455256/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRangePlanner.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRangePlanner.java b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRangePlanner.java index d01fe2c..3068225 100644 --- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRangePlanner.java +++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRangePlanner.java @@ -491,12 +491,17 @@ public class GTScanRangePlanner { return new ComparatorEx<ByteArray>() { @Override public int compare(ByteArray a, ByteArray b) { - if (a.array() == null) - return -1; - else if (b.array() == null) + if (a.array() == null) { + if (b.array() == null) { + return 0; + } else { + return -1; + } + } else if (b.array() == null) { return 1; - else + } else { return comp.compare(a, b); + } } }; } @@ -505,12 +510,17 @@ public class GTScanRangePlanner { return new ComparatorEx<ByteArray>() { @Override public int compare(ByteArray a, ByteArray b) { - if (a.array() == null) - return 1; - else if (b.array() == null) + if (a.array() == null) { + if (b.array() == null) { + return 0; + } else { + return 1; + } + } else if (b.array() == null) { return -1; - else + } else { return comp.compare(a, b); + } } }; } http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/ca455256/core-cube/src/test/java/org/apache/kylin/gridtable/DictGridTableTest.java ---------------------------------------------------------------------- diff --git a/core-cube/src/test/java/org/apache/kylin/gridtable/DictGridTableTest.java b/core-cube/src/test/java/org/apache/kylin/gridtable/DictGridTableTest.java index c991e66..283c6e9 100644 --- a/core-cube/src/test/java/org/apache/kylin/gridtable/DictGridTableTest.java +++ b/core-cube/src/test/java/org/apache/kylin/gridtable/DictGridTableTest.java @@ -86,7 +86,7 @@ public class DictGridTableTest { LogicalTupleFilter filter = and(timeComp1, or(ageComp1, ageComp2)); List<GTScanRange> r = planner.planScanRanges(filter); assertEquals(1, r.size()); - assertEquals("[1421193600000, 10]-[null, null]", r.get(0).toString()); + assertEquals("[1421193600000, 10]-[null, 20]", r.get(0).toString()); assertEquals("[[10], [20]]", r.get(0).fuzzyKeys.toString()); } http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/ca455256/query/src/test/java/org/apache/kylin/query/test/ITKylinQueryTest.java ---------------------------------------------------------------------- diff --git a/query/src/test/java/org/apache/kylin/query/test/ITKylinQueryTest.java b/query/src/test/java/org/apache/kylin/query/test/ITKylinQueryTest.java index 72d7c4a..6425f11 100644 --- a/query/src/test/java/org/apache/kylin/query/test/ITKylinQueryTest.java +++ b/query/src/test/java/org/apache/kylin/query/test/ITKylinQueryTest.java @@ -95,7 +95,7 @@ public class ITKylinQueryTest extends KylinTestBase { @Test public void testSingleRunQuery() throws Exception { - String queryFileName = "src/test/resources/query/sql/query44.sql"; + String queryFileName = "src/test/resources/query/sql/query85.sql"; File sqlFile = new File(queryFileName); if (sqlFile.exists()) { http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/ca455256/query/src/test/resources/query/sql/query85.sql ---------------------------------------------------------------------- diff --git a/query/src/test/resources/query/sql/query85.sql b/query/src/test/resources/query/sql/query85.sql new file mode 100644 index 0000000..1a51a02 --- /dev/null +++ b/query/src/test/resources/query/sql/query85.sql @@ -0,0 +1,26 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- + +select sum(price) as GMV, count(*) as TRANS_CNT FROM test_kylin_fact + inner JOIN edw.test_cal_dt as test_cal_dt + ON test_kylin_fact.cal_dt = test_cal_dt.cal_dt + inner JOIN test_category_groupings + ON test_kylin_fact.leaf_categ_id = test_category_groupings.leaf_categ_id + AND test_kylin_fact.lstg_site_id = test_category_groupings.site_id +where test_kylin_fact.cal_dt < DATE '2012-05-01' or test_kylin_fact.cal_dt > DATE '2013-05-01' + http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/ca455256/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeSegmentTupleIterator.java ---------------------------------------------------------------------- diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeSegmentTupleIterator.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeSegmentTupleIterator.java index 0110fbe..17fac5e 100644 --- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeSegmentTupleIterator.java +++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeSegmentTupleIterator.java @@ -183,7 +183,7 @@ public class CubeSegmentTupleIterator implements ITupleIterator { info.append(keyRange.getCuboid().getId()); info.append("\nStart: "); info.append(keyRange.getStartKeyAsString()); - info.append(" - "); + info.append(" - "); info.append(Bytes.toStringBinary(keyRange.getStartKey())); info.append("\nStop: "); info.append(keyRange.getStopKeyAsString()); http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/ca455256/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java ---------------------------------------------------------------------- diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java index 2e58644..e0f6922 100644 --- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java +++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java @@ -140,10 +140,11 @@ public class CubeHBaseEndpointRPC extends CubeHBaseRPC { ExecutorService executorService = Executors.newFixedThreadPool(rawScans.size()); final List<byte[]> rowBlocks = Collections.synchronizedList(Lists.<byte[]> newArrayList()); + logger.info("Total RawScan range count: " + rawScans.size()); for (RawScan rawScan : rawScans) { logScan(rawScan, cubeSeg.getStorageLocationIdentifier()); } - + for (final RawScan rawScan : rawScans) { executorService.submit(new Runnable() { @Override http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/ca455256/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseRPC.java ---------------------------------------------------------------------- diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseRPC.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseRPC.java index cc7ec4f..97a7284 100644 --- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseRPC.java +++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseRPC.java @@ -229,7 +229,7 @@ public abstract class CubeHBaseRPC { info.append(cuboid.getId()); info.append("\nStart: "); info.append(rawScan.getStartKeyAsString()); - info.append(" - "); + info.append(" - "); info.append(Bytes.toStringBinary(rawScan.startKey)); info.append("\nStop: "); info.append(rawScan.getEndKeyAsString()); http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/ca455256/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeSegmentScanner.java ---------------------------------------------------------------------- diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeSegmentScanner.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeSegmentScanner.java index d49de56..21881a6 100644 --- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeSegmentScanner.java +++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeSegmentScanner.java @@ -221,8 +221,8 @@ public class CubeSegmentScanner implements IGTScanner { try { - //CubeHBaseRPC rpc = new CubeHBaseEndpointRPC(cubeSeg, cuboid, info); - CubeHBaseRPC rpc = new CubeHBaseScanRPC(cubeSeg, cuboid, info); + CubeHBaseRPC rpc = new CubeHBaseEndpointRPC(cubeSeg, cuboid, info); + //CubeHBaseRPC rpc = new CubeHBaseScanRPC(cubeSeg, cuboid, info); //change previous line to CubeHBaseRPC rpc = new CubeHBaseScanRPC(cubeSeg, cuboid, info); //to debug locally