[2/4] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API
http://git-wip-us.apache.org/repos/asf/kylin/blob/24e0e9af/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java -- diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java index 5b2441c..2f7e164 100644 --- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java +++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java @@ -24,11 +24,11 @@ import java.util.Collections; import java.util.List; import org.apache.commons.lang.StringUtils; -import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.HTableDescriptor; -import org.apache.hadoop.hbase.client.HBaseAdmin; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.Admin; +import org.apache.hadoop.hbase.client.Connection; import org.apache.kylin.common.KylinConfig; -import org.apache.kylin.common.util.Bytes; import org.apache.kylin.job.exception.ExecuteException; import org.apache.kylin.job.execution.AbstractExecutable; import org.apache.kylin.job.execution.ExecutableContext; @@ -69,19 +69,20 @@ public class MergeGCStep extends AbstractExecutable { List oldTables = getOldHTables(); if (oldTables != null && oldTables.size() > 0) { String metadataUrlPrefix = KylinConfig.getInstanceFromEnv().getMetadataUrlPrefix(); -Configuration conf = HBaseConnection.getCurrentHBaseConfiguration(); -HBaseAdmin admin = null; +Admin admin = null; try { -admin = new HBaseAdmin(conf); +Connection conn = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl()); +admin = conn.getAdmin(); + for (String table : oldTables) { -if (admin.tableExists(table)) { -HTableDescriptor tableDescriptor = admin.getTableDescriptor(Bytes.toBytes(table)); +if (admin.tableExists(TableName.valueOf(table))) { +HTableDescriptor tableDescriptor = admin.getTableDescriptor(TableName.valueOf((table))); String host = tableDescriptor.getValue(IRealizationConstants.HTableTag); if (metadataUrlPrefix.equalsIgnoreCase(host)) { -if (admin.isTableEnabled(table)) { -admin.disableTable(table); +if (admin.isTableEnabled(TableName.valueOf(table))) { +admin.disableTable(TableName.valueOf(table)); } -admin.deleteTable(table); +admin.deleteTable(TableName.valueOf(table)); logger.debug("Dropped htable: " + table); output.append("HBase table " + table + " is dropped. \n"); } else { http://git-wip-us.apache.org/repos/asf/kylin/blob/24e0e9af/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java -- diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java index a150607..56f867a 100644 --- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java +++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java @@ -21,9 +21,11 @@ package org.apache.kylin.storage.hbase.util; import java.io.IOException; import org.apache.commons.cli.Options; -import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.HTableDescriptor; -import org.apache.hadoop.hbase.client.HBaseAdmin; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.Admin; +import org.apache.hadoop.hbase.client.Connection; +import org.apache.kylin.common.KylinConfig; import org.apache.kylin.common.util.AbstractApplication; import org.apache.kylin.common.util.OptionsHelper; import org.apache.kylin.metadata.realization.IRealizationConstants; @@ -38,8 +40,8 @@ public class CleanHtableCLI extends AbstractApplication { protected static final Logger logger = LoggerFactory.getLogger(CleanHtableCLI.class); private void clean() throws IOException { -Configuration conf = HBaseConnection.getCurrentHBaseConfiguration(); -HBaseAdmin hbaseAdmin = new HBaseAdmin(conf); +Connection conn = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl()); +Admin hbaseAdmin = conn.getAdmin(); for (HTableDescriptor descriptor : hbaseAdmin.listTables()) { String name =
[3/4] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API
KYLIN-1528 Create a branch for v1.5 with HBase 1.x API Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/24e0e9af Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/24e0e9af Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/24e0e9af Branch: refs/heads/yang22-hbase102 Commit: 24e0e9afff255d74b3ff995cccd2a7c81000649c Parents: d4dc6ee Author: shaofengshiAuthored: Wed Mar 23 17:07:05 2016 +0800 Committer: lidongsjtu Committed: Mon Feb 27 12:08:13 2017 +0800 -- examples/test_case_data/sandbox/hbase-site.xml | 19 +--- .../kylin/provision/BuildCubeWithEngine.java| 12 +-- pom.xml | 12 +-- .../kylin/rest/security/AclHBaseStorage.java| 4 +- .../rest/security/MockAclHBaseStorage.java | 8 +- .../apache/kylin/rest/security/MockHTable.java | 95 .../rest/security/RealAclHBaseStorage.java | 9 +- .../apache/kylin/rest/service/AclService.java | 25 +++--- .../apache/kylin/rest/service/CubeService.java | 35 +++- .../apache/kylin/rest/service/QueryService.java | 24 +++-- .../apache/kylin/rest/service/UserService.java | 17 ++-- .../kylin/storage/hbase/HBaseConnection.java| 44 - .../kylin/storage/hbase/HBaseResourceStore.java | 31 +++ .../kylin/storage/hbase/HBaseStorage.java | 3 +- .../storage/hbase/cube/SimpleHBaseStore.java| 20 ++--- .../hbase/cube/v1/CubeSegmentTupleIterator.java | 11 +-- .../storage/hbase/cube/v1/CubeStorageQuery.java | 4 +- .../hbase/cube/v1/RegionScannerAdapter.java | 10 ++- .../cube/v1/SerializedHBaseTupleIterator.java | 4 +- .../observer/AggregateRegionObserver.java | 4 +- .../observer/AggregationScanner.java| 14 ++- .../observer/ObserverAggregationCache.java | 10 ++- .../coprocessor/observer/ObserverEnabler.java | 4 +- .../hbase/cube/v2/CubeHBaseEndpointRPC.java | 13 +-- .../storage/hbase/cube/v2/CubeHBaseScanRPC.java | 9 +- .../coprocessor/endpoint/CubeVisitService.java | 4 +- .../storage/hbase/steps/CubeHTableUtil.java | 16 ++-- .../storage/hbase/steps/DeprecatedGCStep.java | 26 +++--- .../storage/hbase/steps/HBaseCuboidWriter.java | 7 +- .../kylin/storage/hbase/steps/MergeGCStep.java | 23 ++--- .../storage/hbase/util/CleanHtableCLI.java | 12 +-- .../storage/hbase/util/CubeMigrationCLI.java| 37 .../hbase/util/CubeMigrationCheckCLI.java | 17 ++-- .../hbase/util/DeployCoprocessorCLI.java| 27 +++--- .../hbase/util/ExtendCubeToHybridCLI.java | 8 +- .../hbase/util/GridTableHBaseBenchmark.java | 34 +++ .../kylin/storage/hbase/util/HBaseClean.java| 18 ++-- .../hbase/util/HBaseRegionSizeCalculator.java | 35 .../kylin/storage/hbase/util/HBaseUsage.java| 9 +- .../storage/hbase/util/HbaseStreamingInput.java | 30 +++ .../hbase/util/HtableAlterMetadataCLI.java | 9 +- .../storage/hbase/util/OrphanHBaseCleanJob.java | 19 ++-- .../kylin/storage/hbase/util/PingHBaseCLI.java | 15 ++-- .../kylin/storage/hbase/util/RowCounterCLI.java | 11 +-- .../storage/hbase/util/StorageCleanupJob.java | 20 +++-- .../storage/hbase/util/UpdateHTableHostCLI.java | 17 ++-- .../observer/AggregateRegionObserverTest.java | 26 ++ .../v1/filter/TestFuzzyRowFilterV2EndToEnd.java | 5 +- .../org/apache/kylin/tool/CubeMigrationCLI.java | 19 ++-- .../kylin/tool/ExtendCubeToHybridCLI.java | 8 +- 50 files changed, 418 insertions(+), 475 deletions(-) -- http://git-wip-us.apache.org/repos/asf/kylin/blob/24e0e9af/examples/test_case_data/sandbox/hbase-site.xml -- diff --git a/examples/test_case_data/sandbox/hbase-site.xml b/examples/test_case_data/sandbox/hbase-site.xml index 46d5345..734908e 100644 --- a/examples/test_case_data/sandbox/hbase-site.xml +++ b/examples/test_case_data/sandbox/hbase-site.xml @@ -190,22 +190,5 @@ zookeeper.znode.parent /hbase-unsecure - -hbase.client.pause -100 -General client pause value. Used mostly as value to wait -before running a retry of a failed get, region lookup, etc. -See hbase.client.retries.number for description of how we backoff from -this initial pause amount and how this pause works w/ retries. - - -hbase.client.retries.number -5 -Maximum retries. Used as maximum for all retryable -operations such as the getting of a cell's value, starting a row update, -etc. Retry interval is a rough function based on hbase.client.pause. At -first we retry at this interval but then with backoff, we pretty quickly reach -
[4/4] kylin git commit: KYLIN-2233 Support HBase 1.0.2
KYLIN-2233 Support HBase 1.0.2 Signed-off-by: Yang LiProject: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/7e5f6d65 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/7e5f6d65 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/7e5f6d65 Branch: refs/heads/yang22-hbase102 Commit: 7e5f6d65c1dd3d2b57d68f25789a0f2860ebb7f2 Parents: 24e0e9a Author: Cheng Wang Authored: Wed Oct 12 14:39:24 2016 +0800 Committer: lidongsjtu Committed: Mon Feb 27 12:14:01 2017 +0800 -- pom.xml | 2 +- .../storage/hbase/cube/v1/RegionScannerAdapter.java | 13 +++-- .../coprocessor/observer/AggregateRegionObserver.java | 4 ++-- .../v1/coprocessor/observer/AggregationScanner.java | 14 -- .../observer/ObserverAggregationCache.java| 10 ++ .../observer/AggregateRegionObserverTest.java | 10 ++ .../cube/v1/filter/TestFuzzyRowFilterV2EndToEnd.java | 3 +-- 7 files changed, 15 insertions(+), 41 deletions(-) -- http://git-wip-us.apache.org/repos/asf/kylin/blob/7e5f6d65/pom.xml -- diff --git a/pom.xml b/pom.xml index dc23a2f..099316a 100644 --- a/pom.xml +++ b/pom.xml @@ -54,7 +54,7 @@ 1.2.1 -1.1.1 +1.0.2 0.10.0.0 http://git-wip-us.apache.org/repos/asf/kylin/blob/7e5f6d65/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/RegionScannerAdapter.java -- diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/RegionScannerAdapter.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/RegionScannerAdapter.java index 3d30767..c07d47f 100644 --- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/RegionScannerAdapter.java +++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/RegionScannerAdapter.java @@ -26,7 +26,6 @@ import org.apache.hadoop.hbase.HRegionInfo; import org.apache.hadoop.hbase.client.Result; import org.apache.hadoop.hbase.client.ResultScanner; import org.apache.hadoop.hbase.regionserver.RegionScanner; -import org.apache.hadoop.hbase.regionserver.ScannerContext; /** * @author yangli9 @@ -51,7 +50,7 @@ public class RegionScannerAdapter implements RegionScanner { } @Override -public boolean next(List result, ScannerContext scannerContext) throws IOException { +public boolean next(List result, int limit) throws IOException { return next(result); } @@ -61,7 +60,7 @@ public class RegionScannerAdapter implements RegionScanner { } @Override -public boolean nextRaw(List result, ScannerContext scannerContext) throws IOException { +public boolean nextRaw(List result, int limit) throws IOException { return next(result); } @@ -94,10 +93,4 @@ public class RegionScannerAdapter implements RegionScanner { public long getMvccReadPoint() { return Long.MAX_VALUE; } - -@Override -public int getBatch() { -return -1; -} - -} +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/kylin/blob/7e5f6d65/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserver.java -- diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserver.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserver.java index 7e25e4c..7139ca7 100644 --- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserver.java +++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserver.java @@ -26,7 +26,7 @@ import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.coprocessor.BaseRegionObserver; import org.apache.hadoop.hbase.coprocessor.ObserverContext; import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment; -import org.apache.hadoop.hbase.regionserver.Region; +import org.apache.hadoop.hbase.regionserver.HRegion; import org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost; import org.apache.hadoop.hbase.regionserver.RegionScanner; import org.apache.kylin.gridtable.StorageSideBehavior; @@ -99,7 +99,7 @@ public class AggregateRegionObserver extends BaseRegionObserver { // start/end region operation & sync on scanner is suggested by the // javadoc of
[3/4] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API
KYLIN-1528 Create a branch for v1.5 with HBase 1.x API Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/24e0e9af Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/24e0e9af Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/24e0e9af Branch: refs/heads/yang22-cdh5.7 Commit: 24e0e9afff255d74b3ff995cccd2a7c81000649c Parents: d4dc6ee Author: shaofengshiAuthored: Wed Mar 23 17:07:05 2016 +0800 Committer: lidongsjtu Committed: Mon Feb 27 12:08:13 2017 +0800 -- examples/test_case_data/sandbox/hbase-site.xml | 19 +--- .../kylin/provision/BuildCubeWithEngine.java| 12 +-- pom.xml | 12 +-- .../kylin/rest/security/AclHBaseStorage.java| 4 +- .../rest/security/MockAclHBaseStorage.java | 8 +- .../apache/kylin/rest/security/MockHTable.java | 95 .../rest/security/RealAclHBaseStorage.java | 9 +- .../apache/kylin/rest/service/AclService.java | 25 +++--- .../apache/kylin/rest/service/CubeService.java | 35 +++- .../apache/kylin/rest/service/QueryService.java | 24 +++-- .../apache/kylin/rest/service/UserService.java | 17 ++-- .../kylin/storage/hbase/HBaseConnection.java| 44 - .../kylin/storage/hbase/HBaseResourceStore.java | 31 +++ .../kylin/storage/hbase/HBaseStorage.java | 3 +- .../storage/hbase/cube/SimpleHBaseStore.java| 20 ++--- .../hbase/cube/v1/CubeSegmentTupleIterator.java | 11 +-- .../storage/hbase/cube/v1/CubeStorageQuery.java | 4 +- .../hbase/cube/v1/RegionScannerAdapter.java | 10 ++- .../cube/v1/SerializedHBaseTupleIterator.java | 4 +- .../observer/AggregateRegionObserver.java | 4 +- .../observer/AggregationScanner.java| 14 ++- .../observer/ObserverAggregationCache.java | 10 ++- .../coprocessor/observer/ObserverEnabler.java | 4 +- .../hbase/cube/v2/CubeHBaseEndpointRPC.java | 13 +-- .../storage/hbase/cube/v2/CubeHBaseScanRPC.java | 9 +- .../coprocessor/endpoint/CubeVisitService.java | 4 +- .../storage/hbase/steps/CubeHTableUtil.java | 16 ++-- .../storage/hbase/steps/DeprecatedGCStep.java | 26 +++--- .../storage/hbase/steps/HBaseCuboidWriter.java | 7 +- .../kylin/storage/hbase/steps/MergeGCStep.java | 23 ++--- .../storage/hbase/util/CleanHtableCLI.java | 12 +-- .../storage/hbase/util/CubeMigrationCLI.java| 37 .../hbase/util/CubeMigrationCheckCLI.java | 17 ++-- .../hbase/util/DeployCoprocessorCLI.java| 27 +++--- .../hbase/util/ExtendCubeToHybridCLI.java | 8 +- .../hbase/util/GridTableHBaseBenchmark.java | 34 +++ .../kylin/storage/hbase/util/HBaseClean.java| 18 ++-- .../hbase/util/HBaseRegionSizeCalculator.java | 35 .../kylin/storage/hbase/util/HBaseUsage.java| 9 +- .../storage/hbase/util/HbaseStreamingInput.java | 30 +++ .../hbase/util/HtableAlterMetadataCLI.java | 9 +- .../storage/hbase/util/OrphanHBaseCleanJob.java | 19 ++-- .../kylin/storage/hbase/util/PingHBaseCLI.java | 15 ++-- .../kylin/storage/hbase/util/RowCounterCLI.java | 11 +-- .../storage/hbase/util/StorageCleanupJob.java | 20 +++-- .../storage/hbase/util/UpdateHTableHostCLI.java | 17 ++-- .../observer/AggregateRegionObserverTest.java | 26 ++ .../v1/filter/TestFuzzyRowFilterV2EndToEnd.java | 5 +- .../org/apache/kylin/tool/CubeMigrationCLI.java | 19 ++-- .../kylin/tool/ExtendCubeToHybridCLI.java | 8 +- 50 files changed, 418 insertions(+), 475 deletions(-) -- http://git-wip-us.apache.org/repos/asf/kylin/blob/24e0e9af/examples/test_case_data/sandbox/hbase-site.xml -- diff --git a/examples/test_case_data/sandbox/hbase-site.xml b/examples/test_case_data/sandbox/hbase-site.xml index 46d5345..734908e 100644 --- a/examples/test_case_data/sandbox/hbase-site.xml +++ b/examples/test_case_data/sandbox/hbase-site.xml @@ -190,22 +190,5 @@ zookeeper.znode.parent /hbase-unsecure - -hbase.client.pause -100 -General client pause value. Used mostly as value to wait -before running a retry of a failed get, region lookup, etc. -See hbase.client.retries.number for description of how we backoff from -this initial pause amount and how this pause works w/ retries. - - -hbase.client.retries.number -5 -Maximum retries. Used as maximum for all retryable -operations such as the getting of a cell's value, starting a row update, -etc. Retry interval is a rough function based on hbase.client.pause. At -first we retry at this interval but then with backoff, we pretty quickly reach -
[1/4] kylin git commit: KYLIN-2471 queries with parenthesized sub-clause in JOIN will fail [Forced Update!]
Repository: kylin Updated Branches: refs/heads/yang22-cdh5.7 9322c202f -> 5b4584fd4 (forced update) KYLIN-2471 queries with parenthesized sub-clause in JOIN will fail Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/d4dc6ee0 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/d4dc6ee0 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/d4dc6ee0 Branch: refs/heads/yang22-cdh5.7 Commit: d4dc6ee09a1396d2c9408a6bc677746811e683e0 Parents: cdb33ca Author: Hongbin MaAuthored: Sat Feb 25 16:36:19 2017 +0800 Committer: lidongsjtu Committed: Mon Feb 27 10:50:46 2017 +0800 -- .../rest/util/CognosParenthesesEscape.java | 83 .../rest/util/CognosParentesesEscapeTest.java | 61 ++ .../src/test/resources/query/cognos/query01.sql | 28 +++ .../resources/query/cognos/query01.sql.expected | 28 +++ 4 files changed, 200 insertions(+) -- http://git-wip-us.apache.org/repos/asf/kylin/blob/d4dc6ee0/server-base/src/main/java/org/apache/kylin/rest/util/CognosParenthesesEscape.java -- diff --git a/server-base/src/main/java/org/apache/kylin/rest/util/CognosParenthesesEscape.java b/server-base/src/main/java/org/apache/kylin/rest/util/CognosParenthesesEscape.java new file mode 100644 index 000..33f4b63 --- /dev/null +++ b/server-base/src/main/java/org/apache/kylin/rest/util/CognosParenthesesEscape.java @@ -0,0 +1,83 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +package org.apache.kylin.rest.util; + +import java.util.LinkedList; +import java.util.List; +import java.util.UUID; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import org.apache.kylin.common.util.Pair; +import org.apache.kylin.rest.util.QueryUtil.IQueryTransformer; + +/** + * from (a join b on a.x = b.y) join c + * + * similar in https://issues.apache.org/jira/browse/CALCITE-35 + * + * we'll find such pattern and remove the parentheses + */ +public class CognosParenthesesEscape implements IQueryTransformer { + +private static final String S0 = "\\s*"; +private static final String S1 = "\\s"; +private static final String SM = "\\s+"; +private static final String TABLE_OR_COLUMN_NAME = "[\\w\\\"\\'\\.]+"; +private static final String TABLE_NAME_WITH_OPTIONAL_ALIAS = TABLE_OR_COLUMN_NAME + "((\\s+as)?\\s+" + TABLE_OR_COLUMN_NAME + ")?"; +private static final String JOIN = "(\\s+inner|\\s+((left|right|full)(\\s+outer)?))?\\s+join";// as per http://stackoverflow.com/questions/406294/left-join-vs-left-outer-join-in-sql-server +private static final String EQUAL_CONDITION = SM + TABLE_OR_COLUMN_NAME + S0 + "=" + S0 + TABLE_OR_COLUMN_NAME; +private static final String PARENTHESE_PATTERN_STR = "\\(" + S0 + // ( +TABLE_NAME_WITH_OPTIONAL_ALIAS + // a +JOIN + SM + // join +TABLE_NAME_WITH_OPTIONAL_ALIAS + //b +SM + "on" + EQUAL_CONDITION + "(\\s+and" + EQUAL_CONDITION + ")*" + // on a.x = b.y [and a.x2 = b.y2] +S0 + "\\)";// ) +private static final Pattern PARENTTHESES_PATTERN = Pattern.compile(PARENTHESE_PATTERN_STR, Pattern.CASE_INSENSITIVE); + +private static int identifierNum = 0; + +@Override +public String transform(String sql) { +Matcher m; +List > matches = new LinkedList<>(); +while (true) { +m = PARENTTHESES_PATTERN.matcher(sql); +if (!m.find()) +break; + +String oneParentheses = m.group(0); +String identifier = generateRandomName(); +matches.add(new Pair (identifier, oneParentheses.substring(1, oneParentheses.length() - 1))); +sql = sql.substring(0, m.start()) + identifier + sql.substring(m.end()); +} + +for (int i = matches.size() - 1; i >= 0; i--) { +sql = sql.replaceAll(matches.get(i).getKey(),
[4/4] kylin git commit: KYLIN-1672 support kylin on cdh 5.7
KYLIN-1672 support kylin on cdh 5.7 Signed-off-by: Li YangProject: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/5b4584fd Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/5b4584fd Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/5b4584fd Branch: refs/heads/yang22-cdh5.7 Commit: 5b4584fd4e5c6c402832c2c26c18b7a1cbe0391f Parents: 24e0e9a Author: Lynne Jiang Authored: Mon May 16 03:33:27 2016 -0700 Committer: lidongsjtu Committed: Mon Feb 27 12:12:02 2017 +0800 -- dev-support/test_all_against_hdp_2_2_4_2_2.sh | 0 .../kylin/engine/mr/steps/MockupMapContext.java | 15 +- examples/test_case_data/sandbox/core-site.xml | 146 +++--- examples/test_case_data/sandbox/hbase-site.xml | 162 ++ examples/test_case_data/sandbox/hdfs-site.xml | 259 ++ examples/test_case_data/sandbox/mapred-site.xml | 398 ++- examples/test_case_data/sandbox/yarn-site.xml | 496 ++- pom.xml | 16 +- server/pom.xml | 36 ++ .../storage/hbase/steps/MockupMapContext.java | 19 +- tool/pom.xml| 12 + 11 files changed, 428 insertions(+), 1131 deletions(-) -- http://git-wip-us.apache.org/repos/asf/kylin/blob/5b4584fd/dev-support/test_all_against_hdp_2_2_4_2_2.sh -- diff --git a/dev-support/test_all_against_hdp_2_2_4_2_2.sh b/dev-support/test_all_against_hdp_2_2_4_2_2.sh old mode 100644 new mode 100755 http://git-wip-us.apache.org/repos/asf/kylin/blob/5b4584fd/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java -- diff --git a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java index 847071d..9900465 100644 --- a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java +++ b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java @@ -77,6 +77,7 @@ public class MockupMapContext { outKV[0] = key; outKV[1] = value; } + } @Override @@ -99,6 +100,7 @@ public class MockupMapContext { throw new NotImplementedException(); } + @Override public float getProgress() { throw new NotImplementedException(); @@ -195,17 +197,17 @@ public class MockupMapContext { } @Override -public RawComparator getSortComparator() { +public boolean userClassesTakesPrecedence() { throw new NotImplementedException(); } @Override -public String getJar() { +public RawComparator getSortComparator() { throw new NotImplementedException(); } @Override -public RawComparator getGroupingComparator() { +public String getJar() { throw new NotImplementedException(); } @@ -221,7 +223,7 @@ public class MockupMapContext { @Override public boolean getProfileEnabled() { -throw new NotImplementedException(); +return false; } @Override @@ -308,6 +310,11 @@ public class MockupMapContext { public RawComparator getCombinerKeyGroupingComparator() { throw new NotImplementedException(); } + +@Override +public RawComparator getGroupingComparator() { +return null; +} }); } } http://git-wip-us.apache.org/repos/asf/kylin/blob/5b4584fd/examples/test_case_data/sandbox/core-site.xml -- diff --git a/examples/test_case_data/sandbox/core-site.xml b/examples/test_case_data/sandbox/core-site.xml index 9aa588c..6162406 100644 --- a/examples/test_case_data/sandbox/core-site.xml +++ b/examples/test_case_data/sandbox/core-site.xml @@ -14,152 +14,146 @@ See the License for the specific language governing permissions and limitations under the License. --> + - fs.defaultFS -hdfs://sandbox.hortonworks.com:8020 -true +hdfs://quickstart.cloudera:8020 - fs.trash.interval -360 +1 - - ha.failover-controller.active-standby-elector.zk.op.retries -120 +io.compression.codecs +
[1/3] kylin git commit: KYLIN-2471 queries with parenthesized sub-clause in JOIN will fail [Forced Update!]
Repository: kylin Updated Branches: refs/heads/yang22-hbase1.x 1c57f3e0d -> 24e0e9aff (forced update) KYLIN-2471 queries with parenthesized sub-clause in JOIN will fail Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/d4dc6ee0 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/d4dc6ee0 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/d4dc6ee0 Branch: refs/heads/yang22-hbase1.x Commit: d4dc6ee09a1396d2c9408a6bc677746811e683e0 Parents: cdb33ca Author: Hongbin MaAuthored: Sat Feb 25 16:36:19 2017 +0800 Committer: lidongsjtu Committed: Mon Feb 27 10:50:46 2017 +0800 -- .../rest/util/CognosParenthesesEscape.java | 83 .../rest/util/CognosParentesesEscapeTest.java | 61 ++ .../src/test/resources/query/cognos/query01.sql | 28 +++ .../resources/query/cognos/query01.sql.expected | 28 +++ 4 files changed, 200 insertions(+) -- http://git-wip-us.apache.org/repos/asf/kylin/blob/d4dc6ee0/server-base/src/main/java/org/apache/kylin/rest/util/CognosParenthesesEscape.java -- diff --git a/server-base/src/main/java/org/apache/kylin/rest/util/CognosParenthesesEscape.java b/server-base/src/main/java/org/apache/kylin/rest/util/CognosParenthesesEscape.java new file mode 100644 index 000..33f4b63 --- /dev/null +++ b/server-base/src/main/java/org/apache/kylin/rest/util/CognosParenthesesEscape.java @@ -0,0 +1,83 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +package org.apache.kylin.rest.util; + +import java.util.LinkedList; +import java.util.List; +import java.util.UUID; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import org.apache.kylin.common.util.Pair; +import org.apache.kylin.rest.util.QueryUtil.IQueryTransformer; + +/** + * from (a join b on a.x = b.y) join c + * + * similar in https://issues.apache.org/jira/browse/CALCITE-35 + * + * we'll find such pattern and remove the parentheses + */ +public class CognosParenthesesEscape implements IQueryTransformer { + +private static final String S0 = "\\s*"; +private static final String S1 = "\\s"; +private static final String SM = "\\s+"; +private static final String TABLE_OR_COLUMN_NAME = "[\\w\\\"\\'\\.]+"; +private static final String TABLE_NAME_WITH_OPTIONAL_ALIAS = TABLE_OR_COLUMN_NAME + "((\\s+as)?\\s+" + TABLE_OR_COLUMN_NAME + ")?"; +private static final String JOIN = "(\\s+inner|\\s+((left|right|full)(\\s+outer)?))?\\s+join";// as per http://stackoverflow.com/questions/406294/left-join-vs-left-outer-join-in-sql-server +private static final String EQUAL_CONDITION = SM + TABLE_OR_COLUMN_NAME + S0 + "=" + S0 + TABLE_OR_COLUMN_NAME; +private static final String PARENTHESE_PATTERN_STR = "\\(" + S0 + // ( +TABLE_NAME_WITH_OPTIONAL_ALIAS + // a +JOIN + SM + // join +TABLE_NAME_WITH_OPTIONAL_ALIAS + //b +SM + "on" + EQUAL_CONDITION + "(\\s+and" + EQUAL_CONDITION + ")*" + // on a.x = b.y [and a.x2 = b.y2] +S0 + "\\)";// ) +private static final Pattern PARENTTHESES_PATTERN = Pattern.compile(PARENTHESE_PATTERN_STR, Pattern.CASE_INSENSITIVE); + +private static int identifierNum = 0; + +@Override +public String transform(String sql) { +Matcher m; +List > matches = new LinkedList<>(); +while (true) { +m = PARENTTHESES_PATTERN.matcher(sql); +if (!m.find()) +break; + +String oneParentheses = m.group(0); +String identifier = generateRandomName(); +matches.add(new Pair (identifier, oneParentheses.substring(1, oneParentheses.length() - 1))); +sql = sql.substring(0, m.start()) + identifier + sql.substring(m.end()); +} + +for (int i = matches.size() - 1; i >= 0; i--) { +sql = sql.replaceAll(matches.get(i).getKey(),
[2/3] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API
http://git-wip-us.apache.org/repos/asf/kylin/blob/24e0e9af/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java -- diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java index 5b2441c..2f7e164 100644 --- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java +++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java @@ -24,11 +24,11 @@ import java.util.Collections; import java.util.List; import org.apache.commons.lang.StringUtils; -import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.HTableDescriptor; -import org.apache.hadoop.hbase.client.HBaseAdmin; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.Admin; +import org.apache.hadoop.hbase.client.Connection; import org.apache.kylin.common.KylinConfig; -import org.apache.kylin.common.util.Bytes; import org.apache.kylin.job.exception.ExecuteException; import org.apache.kylin.job.execution.AbstractExecutable; import org.apache.kylin.job.execution.ExecutableContext; @@ -69,19 +69,20 @@ public class MergeGCStep extends AbstractExecutable { List oldTables = getOldHTables(); if (oldTables != null && oldTables.size() > 0) { String metadataUrlPrefix = KylinConfig.getInstanceFromEnv().getMetadataUrlPrefix(); -Configuration conf = HBaseConnection.getCurrentHBaseConfiguration(); -HBaseAdmin admin = null; +Admin admin = null; try { -admin = new HBaseAdmin(conf); +Connection conn = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl()); +admin = conn.getAdmin(); + for (String table : oldTables) { -if (admin.tableExists(table)) { -HTableDescriptor tableDescriptor = admin.getTableDescriptor(Bytes.toBytes(table)); +if (admin.tableExists(TableName.valueOf(table))) { +HTableDescriptor tableDescriptor = admin.getTableDescriptor(TableName.valueOf((table))); String host = tableDescriptor.getValue(IRealizationConstants.HTableTag); if (metadataUrlPrefix.equalsIgnoreCase(host)) { -if (admin.isTableEnabled(table)) { -admin.disableTable(table); +if (admin.isTableEnabled(TableName.valueOf(table))) { +admin.disableTable(TableName.valueOf(table)); } -admin.deleteTable(table); +admin.deleteTable(TableName.valueOf(table)); logger.debug("Dropped htable: " + table); output.append("HBase table " + table + " is dropped. \n"); } else { http://git-wip-us.apache.org/repos/asf/kylin/blob/24e0e9af/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java -- diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java index a150607..56f867a 100644 --- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java +++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java @@ -21,9 +21,11 @@ package org.apache.kylin.storage.hbase.util; import java.io.IOException; import org.apache.commons.cli.Options; -import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.HTableDescriptor; -import org.apache.hadoop.hbase.client.HBaseAdmin; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.Admin; +import org.apache.hadoop.hbase.client.Connection; +import org.apache.kylin.common.KylinConfig; import org.apache.kylin.common.util.AbstractApplication; import org.apache.kylin.common.util.OptionsHelper; import org.apache.kylin.metadata.realization.IRealizationConstants; @@ -38,8 +40,8 @@ public class CleanHtableCLI extends AbstractApplication { protected static final Logger logger = LoggerFactory.getLogger(CleanHtableCLI.class); private void clean() throws IOException { -Configuration conf = HBaseConnection.getCurrentHBaseConfiguration(); -HBaseAdmin hbaseAdmin = new HBaseAdmin(conf); +Connection conn = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl()); +Admin hbaseAdmin = conn.getAdmin(); for (HTableDescriptor descriptor : hbaseAdmin.listTables()) { String name =
svn commit: r1784444 - in /kylin/site: ./ blog/ blog/2017/02/23/ blog/2017/02/23/by-layer-spark-cubing/ images/blog/
Author: lidong Date: Sun Feb 26 13:59:58 2017 New Revision: 178 URL: http://svn.apache.org/viewvc?rev=178=rev Log: add post for spark cubing Added: kylin/site/blog/2017/02/23/ kylin/site/blog/2017/02/23/by-layer-spark-cubing/ kylin/site/blog/2017/02/23/by-layer-spark-cubing/index.html kylin/site/images/blog/spark-cubing-layer.png (with props) kylin/site/images/blog/spark-dag.png (with props) kylin/site/images/blog/spark-mr-layer.png (with props) kylin/site/images/blog/spark-mr-performance.png (with props) Modified: kylin/site/blog/index.html kylin/site/feed.xml Added: kylin/site/blog/2017/02/23/by-layer-spark-cubing/index.html URL: http://svn.apache.org/viewvc/kylin/site/blog/2017/02/23/by-layer-spark-cubing/index.html?rev=178=auto == --- kylin/site/blog/2017/02/23/by-layer-spark-cubing/index.html (added) +++ kylin/site/blog/2017/02/23/by-layer-spark-cubing/index.html Sun Feb 26 13:59:58 2017 @@ -0,0 +1,307 @@ + + + + + + + + + + + Apache Kylin | By-layer Spark Cubing + + + + + + + + + + + + + + + + + + + + + + http://kylin.apache.org/blog/2017/02/23/by-layer-spark-cubing/;> + http://kylin.apache.org/feed.xml; /> + + + + (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){ + (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o), + m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m) + })(window,document,'script','//www.google-analytics.com/analytics.js','ga'); + + //oringal tracker for kylin.io + ga('create', 'UA-55534813-1', 'auto'); + //new tracker for kylin.apache.org + ga('create', 'UA-55534813-2', 'auto', {'name':'toplevel'}); + + ga('send', 'pageview'); + ga('toplevel.send', 'pageview'); + + + + + + + + + + + + + + + Apache Kylin⢠Extreme OLAP Engine for Big Data + + + + + + + + + + + +Toggle navigation + + + + + + + + + + + Home + Docs + Download + Community + Development + Blog + ä¸æç + https://twitter.com/apachekylin; target="_blank" class="fa fa-twitter fa-lg" title="Twitter: @ApacheKylin" > + https://github.com/apache/kylin; target="_blank" class="fa fa-github-alt fa-lg" title="Github: apache/kylin" > + https://www.facebook.com/kylinio; target="_blank" class="fa fa-facebook fa-lg" title="Facebook: kylin.io" > + + + + + + + + + + Apache Kylin⢠Technical Blog + + + + + + + + + + + +By-layer Spark Cubing +Feb 23, 2017 ⢠Shaofeng Shi + + + +Before v2.0, Apache Kylin uses Hadoop MapReduce as the framework to build Cubes over huge dataset. The MapReduce framework is simple, stable and can fulfill Kylinâs need very well except the performance. In order to get better performance, we introduced the âfast cubingâ algorithm in Kylin v1.5, tries to do as much as possible aggregations at map side within memory, so to avoid the disk and network I/O; but not all data models can benefit from it, and it still runs on MR which means on-disk sorting and shuffling. + +Now Spark comes; Apache Spark is an open-source cluster-computing framework, which provides programmers with an application programming interface centered on a data structure called RDD; it runs in-memory on the cluster, this makes repeated access to the same data much faster. Spark provides flexible and fancy APIs. You are not tied to Hadoopâs MapReduce two-stage paradigm. + +Before introducing how calculate Cube with Spark, letâs see how Kylin do that with MR; Figure 1 illustrates how a 4-dimension Cube get calculated with the classic âby-layerâ algorithm: the first round MR aggregates the base (4-D) cuboid from source data; the second MR aggregates on the base cuboid to get the 3-D cuboids; With N+1 round MR all layersâ cuboids get calculated. + + + +The âby-layerâ Cubing divides a big task into a couple steps, and each step bases on the previous stepâs output, so it can reuse the previous calculation and also avoid calculating from very beginning when there is a failure in between. These makes it as a reliable algorithm. When moving to Spark, we decide to keep this algorithm, thatâs why we call this feature as âBy layer Spark Cubingâ. + +As we know, RDD (Resilient Distributed Dataset) is a basic concept in Spark. A collection of N-Dimension cuboids can be well described as an RDD, a N-Dimension Cube will have N+1 RDD. These RDDs have the