[1/3] kylin git commit: KYLIN-2428 fix commons-lang3 shade in tool assembly [Forced Update!]
Repository: kylin Updated Branches: refs/heads/master-hbase0.98 363108f01 -> 0bcd399b0 (forced update) KYLIN-2428 fix commons-lang3 shade in tool assembly Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/385d8011 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/385d8011 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/385d8011 Branch: refs/heads/master-hbase0.98 Commit: 385d8011a97fdb66cf921933881d3f3e690d75f6 Parents: 4f94001 Author: Billy Liu Authored: Sun Feb 19 11:41:58 2017 +0800 Committer: Billy Liu Committed: Sun Feb 19 11:41:58 2017 +0800 -- tool-assembly/pom.xml | 5 + 1 file changed, 5 insertions(+) -- http://git-wip-us.apache.org/repos/asf/kylin/blob/385d8011/tool-assembly/pom.xml -- diff --git a/tool-assembly/pom.xml b/tool-assembly/pom.xml index 67d92d4..126c0bc 100644 --- a/tool-assembly/pom.xml +++ b/tool-assembly/pom.xml @@ -59,6 +59,10 @@ compile +org.apache.commons +commons-lang3 + + org.slf4j slf4j-log4j12 compile @@ -93,6 +97,7 @@ log4j:log4j commons-io:commons-io commons-lang:commons-lang + org.apache.commons:commons-lang3 com.google.guava:guava org.apache.kylin:*
[3/3] kylin git commit: KYLIN-2307 Create a branch for master with HBase 0.98 API
KYLIN-2307 Create a branch for master with HBase 0.98 API Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/0bcd399b Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/0bcd399b Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/0bcd399b Branch: refs/heads/master-hbase0.98 Commit: 0bcd399b0ab8949d173a465dc3cc7480fce48ff7 Parents: 385d801 Author: lidongsjtu Authored: Mon Jan 23 13:17:37 2017 +0800 Committer: lidongsjtu Committed: Mon Feb 20 00:49:10 2017 + -- dev-support/test_all_against_hdp_2_2_4_2_2.sh | 25 dev-support/test_all_against_hdp_2_4_0_0_169.sh | 25 .../sandbox/capacity-scheduler.xml | 17 ++- examples/test_case_data/sandbox/core-site.xml | 28 +--- examples/test_case_data/sandbox/hbase-site.xml | 119 + examples/test_case_data/sandbox/hdfs-site.xml | 84 +--- examples/test_case_data/sandbox/hive-site.xml | 89 + examples/test_case_data/sandbox/mapred-site.xml | 57 +++-- examples/test_case_data/sandbox/yarn-site.xml | 127 +++ .../kylin/provision/BuildCubeWithEngine.java| 17 +-- pom.xml | 117 + .../kylin/rest/security/AclHBaseStorage.java| 4 +- .../rest/security/MockAclHBaseStorage.java | 8 +- .../apache/kylin/rest/security/MockHTable.java | 95 +++--- .../rest/security/RealAclHBaseStorage.java | 9 +- .../apache/kylin/rest/service/AclService.java | 25 ++-- .../apache/kylin/rest/service/CubeService.java | 35 +++-- .../apache/kylin/rest/service/QueryService.java | 24 ++-- .../apache/kylin/rest/service/UserService.java | 17 +-- .../kylin/storage/hbase/HBaseConnection.java| 44 +++ .../kylin/storage/hbase/HBaseResourceStore.java | 31 +++-- .../storage/hbase/cube/SimpleHBaseStore.java| 20 +-- .../hbase/cube/v2/CubeHBaseEndpointRPC.java | 13 +- .../storage/hbase/cube/v2/CubeHBaseScanRPC.java | 9 +- .../coprocessor/endpoint/CubeVisitService.java | 4 +- .../storage/hbase/steps/CubeHTableUtil.java | 16 +-- .../storage/hbase/steps/DeprecatedGCStep.java | 24 ++-- .../storage/hbase/steps/HBaseCuboidWriter.java | 7 +- .../kylin/storage/hbase/steps/MergeGCStep.java | 23 ++-- .../storage/hbase/util/CleanHtableCLI.java | 12 +- .../storage/hbase/util/CubeMigrationCLI.java| 37 +++--- .../hbase/util/CubeMigrationCheckCLI.java | 17 +-- .../hbase/util/DeployCoprocessorCLI.java| 27 ++-- .../hbase/util/ExtendCubeToHybridCLI.java | 8 +- .../hbase/util/GridTableHBaseBenchmark.java | 34 ++--- .../kylin/storage/hbase/util/HBaseClean.java| 18 ++- .../hbase/util/HBaseRegionSizeCalculator.java | 35 +++-- .../kylin/storage/hbase/util/HBaseUsage.java| 9 +- .../storage/hbase/util/HbaseStreamingInput.java | 30 ++--- .../hbase/util/HtableAlterMetadataCLI.java | 9 +- .../storage/hbase/util/OrphanHBaseCleanJob.java | 19 +-- .../kylin/storage/hbase/util/PingHBaseCLI.java | 15 +-- .../kylin/storage/hbase/util/RowCounterCLI.java | 11 +- .../storage/hbase/util/StorageCleanupJob.java | 20 ++- .../storage/hbase/util/UpdateHTableHostCLI.java | 17 +-- tool/pom.xml| 10 -- .../org/apache/kylin/tool/CubeMigrationCLI.java | 19 +-- .../kylin/tool/ExtendCubeToHybridCLI.java | 8 +- 48 files changed, 596 insertions(+), 872 deletions(-) -- http://git-wip-us.apache.org/repos/asf/kylin/blob/0bcd399b/dev-support/test_all_against_hdp_2_2_4_2_2.sh -- diff --git a/dev-support/test_all_against_hdp_2_2_4_2_2.sh b/dev-support/test_all_against_hdp_2_2_4_2_2.sh new file mode 100755 index 000..f7780dd --- /dev/null +++ b/dev-support/test_all_against_hdp_2_2_4_2_2.sh @@ -0,0 +1,25 @@ +#!/bin/bash + +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +#http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +dir=$(dirname ${0}) +cd ${dir} +cd .. + +mvn clean install -DskipTests 2>&1 | tee mci.log +mvn verif
[2/3] kylin git commit: KYLIN-2307 Create a branch for master with HBase 0.98 API
http://git-wip-us.apache.org/repos/asf/kylin/blob/0bcd399b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/SimpleHBaseStore.java -- diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/SimpleHBaseStore.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/SimpleHBaseStore.java index f63d9c2..b141190 100644 --- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/SimpleHBaseStore.java +++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/SimpleHBaseStore.java @@ -26,13 +26,12 @@ import java.util.NoSuchElementException; import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.client.BufferedMutator; -import org.apache.hadoop.hbase.client.Connection; +import org.apache.hadoop.hbase.client.HConnection; +import org.apache.hadoop.hbase.client.HTableInterface; import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.client.Result; import org.apache.hadoop.hbase.client.ResultScanner; import org.apache.hadoop.hbase.client.Scan; -import org.apache.hadoop.hbase.client.Table; import org.apache.hadoop.hbase.util.Bytes; import org.apache.kylin.common.KylinConfig; import org.apache.kylin.cube.kv.RowConstants; @@ -87,13 +86,14 @@ public class SimpleHBaseStore implements IGTStore { } private class Writer implements IGTWriter { -final BufferedMutator table; +final HTableInterface table; final ByteBuffer rowkey = ByteBuffer.allocate(50); final ByteBuffer value = ByteBuffer.allocate(50); Writer() throws IOException { -Connection conn = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl()); -table = conn.getBufferedMutator(htableName); +HConnection conn = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl()); +table = conn.getTable(htableName); +table.setAutoFlush(false, true); } @Override @@ -113,24 +113,24 @@ public class SimpleHBaseStore implements IGTStore { Put put = new Put(rowkey); put.addImmutable(CF_B, ByteBuffer.wrap(COL_B), HConstants.LATEST_TIMESTAMP, value); -table.mutate(put); +table.put(put); } @Override public void close() throws IOException { -table.flush(); +table.flushCommits(); table.close(); } } class Reader implements IGTScanner { -final Table table; +final HTableInterface table; final ResultScanner scanner; int count = 0; Reader() throws IOException { -Connection conn = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl()); +HConnection conn = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl()); table = conn.getTable(htableName); Scan scan = new Scan(); http://git-wip-us.apache.org/repos/asf/kylin/blob/0bcd399b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java -- diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java index abc3437..a251031 100644 --- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java +++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java @@ -27,9 +27,8 @@ import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicReference; import java.util.zip.DataFormatException; -import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.client.Connection; -import org.apache.hadoop.hbase.client.Table; +import org.apache.hadoop.hbase.client.HConnection; +import org.apache.hadoop.hbase.client.HTableInterface; import org.apache.hadoop.hbase.client.coprocessor.Batch; import org.apache.hadoop.hbase.ipc.BlockingRpcCallback; import org.apache.hadoop.hbase.ipc.ServerRpcController; @@ -53,10 +52,10 @@ import org.apache.kylin.storage.gtrecord.StorageResponseGTScatter; import org.apache.kylin.storage.hbase.HBaseConnection; import org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos; import org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest; -import org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntList; import org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitResponse; -import org.apache.kylin.storage.hbase.cub
[1/2] kylin git commit: added Flink and SQquirreL docs
Repository: kylin Updated Branches: refs/heads/document c3417443e -> e3d43f725 added Flink and SQquirreL docs Signed-off-by: shaofengshi Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/159678cd Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/159678cd Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/159678cd Branch: refs/heads/document Commit: 159678cd10f7f42dd7869f1c5df3f78b9dcd094f Parents: c341744 Author: albertoRamon Authored: Sat Feb 18 12:21:55 2017 +0100 Committer: shaofengshi Committed: Sun Feb 19 22:15:43 2017 +0800 -- website/_data/docs16.yml| 2 + website/_docs16/index.md| 2 + website/_docs16/tutorial/flink.md | 277 +++ website/_docs16/tutorial/squirrel.md| 132 + website/images/Flink-Tutorial/00.png| Bin 0 -> 1464 bytes website/images/Flink-Tutorial/01.png| Bin 0 -> 4572 bytes website/images/Flink-Tutorial/02.png| Bin 0 -> 4611 bytes website/images/Flink-Tutorial/03.png| Bin 0 -> 4499 bytes website/images/Flink-Tutorial/03b.png | Bin 0 -> 7954 bytes website/images/Flink-Tutorial/04.png| Bin 0 -> 12933 bytes website/images/Flink-Tutorial/05.png| Bin 0 -> 8127 bytes website/images/Flink-Tutorial/06.png| Bin 0 -> 7748 bytes website/images/Flink-Tutorial/07.png| Bin 0 -> 9189 bytes website/images/Flink-Tutorial/08.png| Bin 0 -> 9774 bytes website/images/Flink-Tutorial/09.png| Bin 0 -> 6800 bytes website/images/Flink-Tutorial/10.png| Bin 0 -> 2241 bytes website/images/Flink-Tutorial/11.png| Bin 0 -> 5916 bytes website/images/Flink-Tutorial/12.png| Bin 0 -> 544 bytes website/images/Flink-Tutorial/13.png| Bin 0 -> 4849 bytes website/images/Flink-Tutorial/14.png| Bin 0 -> 1071 bytes website/images/Flink-Tutorial/15.png| Bin 0 -> 610 bytes website/images/Flink-Tutorial/16.png| Bin 0 -> 4800 bytes website/images/Flink-Tutorial/17.png| Bin 0 -> 53995 bytes website/images/Flink-Tutorial/18.png| Bin 0 -> 14276 bytes website/images/Flink-Tutorial/19.png| Bin 0 -> 11980 bytes website/images/Flink-Tutorial/20.png| Bin 0 -> 16603 bytes website/images/Flink-Tutorial/21.png| Bin 0 -> 2132 bytes website/images/Flink-Tutorial/22.png| Bin 0 -> 4917 bytes website/images/Flink-Tutorial/23.png| Bin 0 -> 15461 bytes website/images/Flink-Tutorial/24.png| Bin 0 -> 10660 bytes website/images/Flink-Tutorial/25.png| Bin 0 -> 23750 bytes website/images/SQuirreL-Tutorial/00.png | Bin 0 -> 1464 bytes website/images/SQuirreL-Tutorial/01.png | Bin 0 -> 9774 bytes website/images/SQuirreL-Tutorial/02.png | Bin 0 -> 4572 bytes website/images/SQuirreL-Tutorial/03.png | Bin 0 -> 749 bytes website/images/SQuirreL-Tutorial/04.png | Bin 0 -> 237 bytes website/images/SQuirreL-Tutorial/05.png | Bin 0 -> 1050 bytes website/images/SQuirreL-Tutorial/06.png | Bin 0 -> 578 bytes website/images/SQuirreL-Tutorial/07.png | Bin 0 -> 1992 bytes website/images/SQuirreL-Tutorial/08.png | Bin 0 -> 1058 bytes website/images/SQuirreL-Tutorial/09.png | Bin 0 -> 2989 bytes website/images/SQuirreL-Tutorial/10.png | Bin 0 -> 2825 bytes website/images/SQuirreL-Tutorial/11.png | Bin 0 -> 656 bytes website/images/SQuirreL-Tutorial/12.png | Bin 0 -> 987 bytes website/images/SQuirreL-Tutorial/13.png | Bin 0 -> 237 bytes website/images/SQuirreL-Tutorial/14.png | Bin 0 -> 9177 bytes website/images/SQuirreL-Tutorial/15.png | Bin 0 -> 12174 bytes website/images/SQuirreL-Tutorial/16.png | Bin 0 -> 42855 bytes website/images/SQuirreL-Tutorial/17.png | Bin 0 -> 11729 bytes website/images/SQuirreL-Tutorial/18.png | Bin 0 -> 356 bytes website/images/SQuirreL-Tutorial/19.png | Bin 0 -> 15611 bytes website/images/SQuirreL-Tutorial/20.png | Bin 0 -> 4917 bytes website/images/SQuirreL-Tutorial/21.png | Bin 0 -> 3190 bytes website/images/SQuirreL-Tutorial/22.png | Bin 0 -> 8275 bytes website/images/SQuirreL-Tutorial/23.png | Bin 0 -> 8516 bytes website/images/SQuirreL-Tutorial/24.png | Bin 0 -> 10339 bytes website/images/SQuirreL-Tutorial/25.png | Bin 0 -> 11061 bytes 57 files changed, 413 insertions(+) -- http://git-wip-us.apache.org/repos/asf/kylin/blob/159678cd/website/_data/docs16.yml -- diff --git a/website/_data/docs16.yml b/website/_data/docs16.yml index 22f9141..3cb084e 100644 --- a/website/_data/docs16.yml +++ b/website/_data/docs16.yml @@ -45,6 +45,8 @@ - tutorial/tableau_91 - tutorial/powerbi - tutorial/odbc + - tutorial/flink + - tutorial/squirrel - title: How To docs: http://git-wip-us.apache.org/repos/asf/kylin/blob/159678cd/website/_docs16/index.md
[2/2] kylin git commit: Document format review on flink.md and squirrel.md
Document format review on flink.md and squirrel.md Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/e3d43f72 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/e3d43f72 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/e3d43f72 Branch: refs/heads/document Commit: e3d43f725e7c9e9e28290b44d4390e990916c58c Parents: 159678c Author: shaofengshi Authored: Mon Feb 20 13:26:09 2017 +0800 Committer: shaofengshi Committed: Mon Feb 20 13:26:09 2017 +0800 -- website/_docs16/tutorial/flink.md| 146 -- website/_docs16/tutorial/squirrel.md | 66 +- 2 files changed, 82 insertions(+), 130 deletions(-) -- http://git-wip-us.apache.org/repos/asf/kylin/blob/e3d43f72/website/_docs16/tutorial/flink.md -- diff --git a/website/_docs16/tutorial/flink.md b/website/_docs16/tutorial/flink.md index 579bb08..f3cb99f 100644 --- a/website/_docs16/tutorial/flink.md +++ b/website/_docs16/tutorial/flink.md @@ -1,45 +1,39 @@ --- layout: docs16 -title: Apache Flink +title: Connect from Apache Flink categories: tutorial permalink: /docs16/tutorial/flink.html --- -### Intro -There are several Attempts to use this in Scala and JDBC [Attempt1](http://apache-flink-user-mailing-list-archive.2336050.n4.nabble.com/JDBCInputFormat-preparation-with-Flink-1-1-SNAPSHOT-and-Scala-2-11-td5371.html) [Attempt2](http://apache-flink-user-mailing-list-archive.2336050.n4.nabble.com/Type-of-TypeVariable-OT-in-class-org-apache-flink-api-common-io-RichInputFormat-could-not-be-determi-td7287.html) [Attempt3](http://stackoverflow.com/questions/36067881/create-dataset-from-jdbc-source-in-flink-using-scala) [Attempt4](https://codegists.com/snippet/scala/jdbcissuescala_zeitgeist_scala) but none works ... +### Introduction -**Problem 1:** There arenât any doc about connect Kylin with Flink +This document describes how to use Kylin as a data source in Apache Flink; -**Problem 2:** We will try use CreateInput and [JDBCInputFormat](https://ci.apache.org/projects/flink/flink-docs-release-1.2/dev/batch/index.html) in batch mode and access via JDBC to Kylin. But isnât implemented in Scala is only in Java [MailList](http://apache-flink-user-mailing-list-archive.2336050.n4.nabble.com/jdbc-JDBCInputFormat-td9393.html) - +There were several attempts to do this in Scala and JDBC, but none of them works: -Then , We will go step by step solving problems +* [attempt1](http://apache-flink-user-mailing-list-archive.2336050.n4.nabble.com/JDBCInputFormat-preparation-with-Flink-1-1-SNAPSHOT-and-Scala-2-11-td5371.html) +* [attempt2](http://apache-flink-user-mailing-list-archive.2336050.n4.nabble.com/Type-of-TypeVariable-OT-in-class-org-apache-flink-api-common-io-RichInputFormat-could-not-be-determi-td7287.html) +* [attempt3](http://stackoverflow.com/questions/36067881/create-dataset-from-jdbc-source-in-flink-using-scala) +* [attempt4](https://codegists.com/snippet/scala/jdbcissuescala_zeitgeist_scala); - -### Pre-requisites -* We need an instance of Kylin, with a cube: [Quick Start with Sample Cube](http://kylin.apache.org/docs15/tutorial/kylin_sample.html), will be enough - - You can check: - - ![](/images/Flink-Tutorial/01.png) +We will try use CreateInput and [JDBCInputFormat](https://ci.apache.org/projects/flink/flink-docs-release-1.2/dev/batch/index.html) in batch mode and access via JDBC to Kylin. But it isnât implemented in Scala, is only in Java [MailList](http://apache-flink-user-mailing-list-archive.2336050.n4.nabble.com/jdbc-JDBCInputFormat-td9393.html). This doc will go step by step solving these problems. +### Pre-requisites +* Need an instance of Kylin, with a Cube; [Sample Cube](/docs16/tutorial/kylin_sample.html) will be good enough. * [Scala](http://www.scala-lang.org/) and [Apache Flink](http://flink.apache.org/) Installed +* [IntelliJ](https://www.jetbrains.com/idea/) Installed and configured for Scala/Flink (see [Flink IDE setup guide](https://ci.apache.org/projects/flink/flink-docs-release-1.1/internals/ide_setup.html) ) -* [IntelliJ](https://www.jetbrains.com/idea/) Installed and configured for Scala / Flink (See [Flink IDE setup guide](https://ci.apache.org/projects/flink/flink-docs-release-1.1/internals/ide_setup.html) ) +### Used software: - -### Used Software: * [Apache Flink](http://flink.apache.org/downloads.html) v1.2-SNAPSHOT -* [Apache Kylin](http://kylin.apache.org/download/) v1.5.2 - - (In Nov 2016, only works with 1.5.4 or more by dependences with Calcite, I Tested with v1.6.0) +* [Apache Kylin](http://kylin.apache.org/download/) v1.5.2 (v1.6.0 also works) * [IntelliJ](https://www.jetbrains.com/idea/download/#section=linux)
svn commit: r1783713 [1/3] - in /kylin/site: ./ cn/docs16/ cn/docs16/install/ cn/docs16/tutorial/ docs16/ docs16/gettingstarted/ docs16/howto/ docs16/install/ docs16/tutorial/ images/Flink-Tutorial/ i
Author: lidong Date: Mon Feb 20 05:29:13 2017 New Revision: 1783713 URL: http://svn.apache.org/viewvc?rev=1783713&view=rev Log: Document format review on flink.md and squirrel.md Added: kylin/site/docs16/tutorial/flink.html kylin/site/docs16/tutorial/squirrel.html kylin/site/images/Flink-Tutorial/ kylin/site/images/Flink-Tutorial/00.png (with props) kylin/site/images/Flink-Tutorial/01.png (with props) kylin/site/images/Flink-Tutorial/02.png (with props) kylin/site/images/Flink-Tutorial/03.png (with props) kylin/site/images/Flink-Tutorial/03b.png (with props) kylin/site/images/Flink-Tutorial/04.png (with props) kylin/site/images/Flink-Tutorial/05.png (with props) kylin/site/images/Flink-Tutorial/06.png (with props) kylin/site/images/Flink-Tutorial/07.png (with props) kylin/site/images/Flink-Tutorial/08.png (with props) kylin/site/images/Flink-Tutorial/09.png (with props) kylin/site/images/Flink-Tutorial/10.png (with props) kylin/site/images/Flink-Tutorial/11.png (with props) kylin/site/images/Flink-Tutorial/12.png (with props) kylin/site/images/Flink-Tutorial/13.png (with props) kylin/site/images/Flink-Tutorial/14.png (with props) kylin/site/images/Flink-Tutorial/15.png (with props) kylin/site/images/Flink-Tutorial/16.png (with props) kylin/site/images/Flink-Tutorial/17.png (with props) kylin/site/images/Flink-Tutorial/18.png (with props) kylin/site/images/Flink-Tutorial/19.png (with props) kylin/site/images/Flink-Tutorial/20.png (with props) kylin/site/images/Flink-Tutorial/21.png (with props) kylin/site/images/Flink-Tutorial/22.png (with props) kylin/site/images/Flink-Tutorial/23.png (with props) kylin/site/images/Flink-Tutorial/24.png (with props) kylin/site/images/Flink-Tutorial/25.png (with props) kylin/site/images/SQuirreL-Tutorial/ kylin/site/images/SQuirreL-Tutorial/00.png (with props) kylin/site/images/SQuirreL-Tutorial/01.png (with props) kylin/site/images/SQuirreL-Tutorial/02.png (with props) kylin/site/images/SQuirreL-Tutorial/03.png (with props) kylin/site/images/SQuirreL-Tutorial/04.png (with props) kylin/site/images/SQuirreL-Tutorial/05.png (with props) kylin/site/images/SQuirreL-Tutorial/06.png (with props) kylin/site/images/SQuirreL-Tutorial/07.png (with props) kylin/site/images/SQuirreL-Tutorial/08.png (with props) kylin/site/images/SQuirreL-Tutorial/09.png (with props) kylin/site/images/SQuirreL-Tutorial/10.png (with props) kylin/site/images/SQuirreL-Tutorial/11.png (with props) kylin/site/images/SQuirreL-Tutorial/12.png (with props) kylin/site/images/SQuirreL-Tutorial/13.png (with props) kylin/site/images/SQuirreL-Tutorial/14.png (with props) kylin/site/images/SQuirreL-Tutorial/15.png (with props) kylin/site/images/SQuirreL-Tutorial/16.png (with props) kylin/site/images/SQuirreL-Tutorial/17.png (with props) kylin/site/images/SQuirreL-Tutorial/18.png (with props) kylin/site/images/SQuirreL-Tutorial/19.png (with props) kylin/site/images/SQuirreL-Tutorial/20.png (with props) kylin/site/images/SQuirreL-Tutorial/21.png (with props) kylin/site/images/SQuirreL-Tutorial/22.png (with props) kylin/site/images/SQuirreL-Tutorial/23.png (with props) kylin/site/images/SQuirreL-Tutorial/24.png (with props) kylin/site/images/SQuirreL-Tutorial/25.png (with props) Modified: kylin/site/cn/docs16/index.html kylin/site/cn/docs16/install/index.html kylin/site/cn/docs16/install/manual_install_guide.html kylin/site/cn/docs16/tutorial/acl.html kylin/site/cn/docs16/tutorial/create_cube.html kylin/site/cn/docs16/tutorial/cube_build_job.html kylin/site/cn/docs16/tutorial/kylin_client_tool.html kylin/site/cn/docs16/tutorial/odbc.html kylin/site/cn/docs16/tutorial/powerbi.html kylin/site/cn/docs16/tutorial/tableau.html kylin/site/cn/docs16/tutorial/tableau_91.html kylin/site/cn/docs16/tutorial/web.html kylin/site/docs16/gettingstarted/best_practices.html kylin/site/docs16/gettingstarted/concepts.html kylin/site/docs16/gettingstarted/events.html kylin/site/docs16/gettingstarted/faq.html kylin/site/docs16/gettingstarted/terminology.html kylin/site/docs16/howto/howto_backup_metadata.html kylin/site/docs16/howto/howto_build_cube_with_restapi.html kylin/site/docs16/howto/howto_cleanup_storage.html kylin/site/docs16/howto/howto_jdbc.html kylin/site/docs16/howto/howto_ldap_and_sso.html kylin/site/docs16/howto/howto_optimize_build.html kylin/site/docs16/howto/howto_optimize_cubes.html kylin/site/docs16/howto/howto_update_coprocessor.html kylin/site/docs16/howto/howto_upgrade.html kylin/site/docs16/howto/howto_use_beeline.html kylin/site/docs16/howto/howto_use_distributed_scheduler.html
svn commit: r1783713 [2/3] - in /kylin/site: ./ cn/docs16/ cn/docs16/install/ cn/docs16/tutorial/ docs16/ docs16/gettingstarted/ docs16/howto/ docs16/install/ docs16/tutorial/ images/Flink-Tutorial/ i
Modified: kylin/site/docs16/install/kylin_docker.html URL: http://svn.apache.org/viewvc/kylin/site/docs16/install/kylin_docker.html?rev=1783713&r1=1783712&r2=1783713&view=diff == --- kylin/site/docs16/install/kylin_docker.html (original) +++ kylin/site/docs16/install/kylin_docker.html Mon Feb 20 05:29:13 2017 @@ -651,6 +651,10 @@ + + + + @@ -941,6 +945,8 @@ + + Quick Start with Sample Cube @@ -1283,6 +1289,10 @@ + + + + Kylin Web Interface @@ -1383,6 +1393,10 @@ + + + + Tableau 8 @@ -1487,6 +1501,10 @@ + + + + Tableau 9 @@ -1583,6 +1601,8 @@ + + MS Excel and Power BI @@ -1675,10 +1695,198 @@ + + Kylin ODBC Driver + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Connect from Apache Flink + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Connect from SQuirreL + + + Modified: kylin/site/docs16/release_notes.html URL: http://svn.apache.org/viewvc/kylin/site/docs16/release_notes.html?rev=1783713&r1=1783712&r2=1783713&view=diff == --- kylin/site/docs16/release_notes.html (original) +++ kylin/site/docs16/release_notes.html Mon Feb 20 05:29:13 2017 @@ -651,6 +651,10 @@ + + + + @@ -941,6 +945,8 @@ + + Quick Start with Sample Cube @@ -1283,6 +1289,10 @@ + + + + Kylin Web Interface @@ -1383,6 +1393,10 @@ + + + + Tableau 8 @@ -1487,6 +1501,10 @@ + + + + Tableau 9 @@ -1583,6 +1601,8 @@ + + MS Excel and Power BI @@ -1675,10 +1695,198 @@ + + Kylin ODBC Driver + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Connect from Apache Flink + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Connect from SQuirreL + + + Modified: kylin/site/docs16/tutorial/acl.html URL: http://svn.apache.org/viewvc/kylin/site/docs16/tutorial/acl.html?rev=1783713&r1=1783712&r2=1783713&view=diff == --- kylin/site/docs16/tutorial/acl.html (original) +++ kylin/site/docs16/tutorial/acl.html Mon Feb 20 05:29:13 2017 @@ -651,6 +651,10 @@ + + + + @@ -941,6 +945,8 @@ + + Quick Start with Sample Cube @@ -1283,6 +1289,10 @@ + + + + Kylin Web Interface @@ -1383,6 +1393,10 @@ + + + + Tableau 8 @@ -1487,6 +1501,10 @@ + + + + Tableau 9 @@ -1583,6 +1601,8 @@ + + MS Excel and Power BI @@ -1675,10 +1695,198 @@ + + Kylin ODBC Driver + + + + + + + + + + + + + + + +
svn commit: r1783713 [3/3] - in /kylin/site: ./ cn/docs16/ cn/docs16/install/ cn/docs16/tutorial/ docs16/ docs16/gettingstarted/ docs16/howto/ docs16/install/ docs16/tutorial/ images/Flink-Tutorial/ i
Added: kylin/site/docs16/tutorial/squirrel.html URL: http://svn.apache.org/viewvc/kylin/site/docs16/tutorial/squirrel.html?rev=1783713&view=auto == --- kylin/site/docs16/tutorial/squirrel.html (added) +++ kylin/site/docs16/tutorial/squirrel.html Mon Feb 20 05:29:13 2017 @@ -0,0 +1,2536 @@ + + + + + + + + + + + + Apache Kylin | Connect from SQuirreL + + + + + + + + + + + + + + + + + + + + + + http://kylin.apache.org/docs16/tutorial/squirrel.html";> + http://kylin.apache.org/feed.xml"; /> + + + + (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){ + (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o), + m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m) + })(window,document,'script','//www.google-analytics.com/analytics.js','ga'); + + //oringal tracker for kylin.io + ga('create', 'UA-55534813-1', 'auto'); + //new tracker for kylin.apache.org + ga('create', 'UA-55534813-2', 'auto', {'name':'toplevel'}); + + ga('send', 'pageview'); + ga('toplevel.send', 'pageview'); + + + + + + + + + + + + + + + Apache Kylin⢠Extreme OLAP Engine for Big Data + + + + + + + + + + + +Toggle navigation + + + + + + + + + + + Home + Docs + Download + Community + Development + Blog + ä¸æç + https://twitter.com/apachekylin"; target="_blank" class="fa fa-twitter fa-lg" title="Twitter: @ApacheKylin" > + https://github.com/apache/kylin"; target="_blank" class="fa fa-github-alt fa-lg" title="Github: apache/kylin" > + https://www.facebook.com/kylinio"; target="_blank" class="fa fa-facebook fa-lg" title="Facebook: kylin.io" > + + + + + + + + + + + + + + +Getting Started + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Overview + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Apache Kylin Release Notes + + + + + + + + + + + + + + + + + + + + + + FAQ + + + + + + + + + + + + + + + + + + + + Events and Conferences + + + + + + + + + + + + + + + + Community Best Practices + + + + + + + + + + + + + + + + + + + + + + + + Terminology + + + + + + + + + + + + + + + + + + Technical Concepts + + + + + + + + + + + + +Installation + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Installation Guide + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Hadoop Environment + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Deploy in Cluster Mode + + + + + + + + + + + + + + + + + + + + + + + + + +
kylin git commit: default kylin.engine.mr.uhc-reducer-count to 1 in code, while set it to 3 in CI
Repository: kylin Updated Branches: refs/heads/master 385d8011a -> 92533c777 default kylin.engine.mr.uhc-reducer-count to 1 in code, while set it to 3 in CI Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/92533c77 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/92533c77 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/92533c77 Branch: refs/heads/master Commit: 92533c7771f676634a37710a1a0fa4ce68ec162a Parents: 385d801 Author: Li Yang Authored: Mon Feb 20 14:03:59 2017 +0800 Committer: Li Yang Committed: Mon Feb 20 14:03:59 2017 +0800 -- .../src/main/java/org/apache/kylin/common/KylinConfigBase.java | 2 +- examples/test_case_data/localmeta/kylin.properties | 2 +- examples/test_case_data/sandbox/kylin.properties | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) -- http://git-wip-us.apache.org/repos/asf/kylin/blob/92533c77/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java -- diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java index 2025a37..50d87a4 100644 --- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java +++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java @@ -754,7 +754,7 @@ abstract public class KylinConfigBase implements Serializable { //UHC: ultra high cardinality columns, contain the ShardByColumns and the GlobalDictionaryColumns public int getUHCReducerCount() { -return Integer.parseInt(getOptional("kylin.engine.mr.uhc-reducer-count", "3")); +return Integer.parseInt(getOptional("kylin.engine.mr.uhc-reducer-count", "1")); } public boolean isBuildDictInReducerEnabled() { http://git-wip-us.apache.org/repos/asf/kylin/blob/92533c77/examples/test_case_data/localmeta/kylin.properties -- diff --git a/examples/test_case_data/localmeta/kylin.properties b/examples/test_case_data/localmeta/kylin.properties index e89c767..eb4f35d 100644 --- a/examples/test_case_data/localmeta/kylin.properties +++ b/examples/test_case_data/localmeta/kylin.properties @@ -73,7 +73,7 @@ kylin.job.max-concurrent-jobs=10 kylin.engine.mr.yarn-check-interval-seconds=10 # for test -kylin.engine.mr.uhc-reducer-count=1 +kylin.engine.mr.uhc-reducer-count=3 ### CUBE ### http://git-wip-us.apache.org/repos/asf/kylin/blob/92533c77/examples/test_case_data/sandbox/kylin.properties -- diff --git a/examples/test_case_data/sandbox/kylin.properties b/examples/test_case_data/sandbox/kylin.properties index 91566ae..7040cf3 100644 --- a/examples/test_case_data/sandbox/kylin.properties +++ b/examples/test_case_data/sandbox/kylin.properties @@ -114,7 +114,7 @@ kylin.query.udf.version=org.apache.kylin.query.udf.VersionUDF # for test kylin.job.lock=org.apache.kylin.job.lock.MockJobLock -kylin.engine.mr.uhc-reducer-count=1 +kylin.engine.mr.uhc-reducer-count=3 ### CUBE ###
kylin git commit: KYLIN-2331 update IEngineAware.ID_SPARK to be consistent with KylinConfig
Repository: kylin Updated Branches: refs/heads/master 92533c777 -> bbcb0ed52 KYLIN-2331 update IEngineAware.ID_SPARK to be consistent with KylinConfig Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/bbcb0ed5 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/bbcb0ed5 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/bbcb0ed5 Branch: refs/heads/master Commit: bbcb0ed52c121bc68bea835e0fa0be7d9397b986 Parents: 92533c7 Author: shaofengshi Authored: Mon Feb 20 14:23:12 2017 +0800 Committer: shaofengshi Committed: Mon Feb 20 14:23:17 2017 +0800 -- .../src/main/java/org/apache/kylin/common/KylinConfigBase.java | 6 +++--- .../java/org/apache/kylin/metadata/model/IEngineAware.java | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) -- http://git-wip-us.apache.org/repos/asf/kylin/blob/bbcb0ed5/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java -- diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java index 50d87a4..f7d8452 100644 --- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java +++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java @@ -692,9 +692,9 @@ abstract public class KylinConfigBase implements Serializable { public Map getJobEngines() { Map r = Maps.newLinkedHashMap(); // ref constants in IEngineAware -r.put(0, "org.apache.kylin.engine.mr.MRBatchCubingEngine"); -r.put(2, "org.apache.kylin.engine.mr.MRBatchCubingEngine2"); -r.put(4, "org.apache.kylin.engine.spark.SparkBatchCubingEngine2"); +r.put(0, "org.apache.kylin.engine.mr.MRBatchCubingEngine"); //IEngineAware.ID_MR_V1 +r.put(2, "org.apache.kylin.engine.mr.MRBatchCubingEngine2"); //IEngineAware.ID_MR_V2 +r.put(4, "org.apache.kylin.engine.spark.SparkBatchCubingEngine2"); //IEngineAware.ID_SPARK r.putAll(convertKeyToInteger(getPropertiesByPrefix("kylin.engine.provider."))); return r; } http://git-wip-us.apache.org/repos/asf/kylin/blob/bbcb0ed5/core-metadata/src/main/java/org/apache/kylin/metadata/model/IEngineAware.java -- diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/IEngineAware.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/IEngineAware.java index 882b2e3..78f7faa 100644 --- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/IEngineAware.java +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/IEngineAware.java @@ -23,7 +23,7 @@ public interface IEngineAware { public static final int ID_MR_V1 = 0; public static final int ID_MR_V2 = 2; public static final int ID_MR_II = 3; -public static final int ID_SPARK = 5; +public static final int ID_SPARK = 4; int getEngineType(); }
[kylin] Git Push Summary
Repository: kylin Updated Branches: refs/heads/KYLIN-2428 [deleted] 303dea95d
kylin git commit: KYLIN-2428 Move SparkEntry to common-core
Repository: kylin Updated Branches: refs/heads/KYLIN-2428 [created] 4f17fa802 KYLIN-2428 Move SparkEntry to common-core Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/4f17fa80 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/4f17fa80 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/4f17fa80 Branch: refs/heads/KYLIN-2428 Commit: 4f17fa8022064532fda9f939439bdaaa74ac37dc Parents: 385d801 Author: Billy Liu Authored: Mon Feb 20 14:32:35 2017 +0800 Committer: Billy Liu Committed: Mon Feb 20 14:32:35 2017 +0800 -- .../apache/kylin/common/util/SparkEntry.java| 46 +++ .../kylin/engine/spark/SparkExecutable.java | 2 +- .../kylin/engine/spark/util/SparkEntry.java | 48 3 files changed, 47 insertions(+), 49 deletions(-) -- http://git-wip-us.apache.org/repos/asf/kylin/blob/4f17fa80/core-common/src/main/java/org/apache/kylin/common/util/SparkEntry.java -- diff --git a/core-common/src/main/java/org/apache/kylin/common/util/SparkEntry.java b/core-common/src/main/java/org/apache/kylin/common/util/SparkEntry.java new file mode 100644 index 000..ce11b91 --- /dev/null +++ b/core-common/src/main/java/org/apache/kylin/common/util/SparkEntry.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kylin.common.util; + +import org.apache.commons.lang.StringUtils; + +/** + */ +public final class SparkEntry { + +public static void main(String[] args) throws Exception { +System.out.println("SparkEntry args:" + StringUtils.join(args, " ")); +if (!(args.length >= 2)) { +throw new IllegalArgumentException(String.valueOf("-className is required")); +} +if (!(args[0].equals("-className"))) { +throw new IllegalArgumentException(String.valueOf("-className is required")); +} +final String className = args[1]; +final Object o = Class. forName(className).newInstance(); +if (!(o instanceof AbstractApplication)) { +throw new IllegalArgumentException(String.valueOf(className + " is not a subClass of AbstractApplication")); +} +String[] appArgs = new String[args.length - 2]; +for (int i = 2; i < args.length; i++) { +appArgs[i - 2] = args[i]; +} +AbstractApplication abstractApplication = (AbstractApplication) o; +abstractApplication.execute(appArgs); +} +} http://git-wip-us.apache.org/repos/asf/kylin/blob/4f17fa80/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkExecutable.java -- diff --git a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkExecutable.java b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkExecutable.java index 015b480..c671a91 100644 --- a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkExecutable.java +++ b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkExecutable.java @@ -105,7 +105,7 @@ public class SparkExecutable extends AbstractExecutable { } StringBuilder stringBuilder = new StringBuilder(); -stringBuilder.append("export HADOOP_CONF_DIR=%s && %s/bin/spark-submit --class org.apache.kylin.engine.spark.util.SparkEntry "); +stringBuilder.append("export HADOOP_CONF_DIR=%s && %s/bin/spark-submit --class org.apache.kylin.common.util.SparkEntry "); Map sparkConfs = config.getSparkConfigOverride(); for (Map.Entry entry : sparkConfs.entrySet()) { http://git-wip-us.apache.org/repos/asf/kylin/blob/4f17fa80/engine-spark/src/main/java/org/apache/kylin/engine/spark/util/SparkEntry.java -- diff --git a/engine-spark/src/main/java/org/apache/kylin/engine/spark/util/SparkEntry.java b/engine-spark/src/main/java/org/apache/kylin/engine/spark/util/SparkEntry.j
[1/2] kylin git commit: KYLIN-2452 validate there is at least one AggregationGroup
Repository: kylin Updated Branches: refs/heads/master bbcb0ed52 -> bf8134010 KYLIN-2452 validate there is at least one AggregationGroup Signed-off-by: shaofengshi Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/42d09a79 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/42d09a79 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/42d09a79 Branch: refs/heads/master Commit: 42d09a79848f2becf57deab8742b896d017791b3 Parents: bbcb0ed Author: Cheng Wang Authored: Fri Feb 17 10:05:28 2017 +0800 Committer: shaofengshi Committed: Mon Feb 20 15:20:31 2017 +0800 -- .../org/apache/kylin/cube/model/CubeDesc.java | 18 ++ .../kylin/rest/controller/CubeController.java | 13 + 2 files changed, 19 insertions(+), 12 deletions(-) -- http://git-wip-us.apache.org/repos/asf/kylin/blob/42d09a79/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java -- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java index 0c1419d..0f804d1 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java @@ -440,6 +440,8 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware { } public int getBuildLevel() { +if (aggregationGroups.size() <= 0) +throw new IllegalStateException("AggregationGroup size is: " + aggregationGroups.size() + ", there should be at least one AggregationGroup!"); return Collections.max(Collections2.transform(aggregationGroups, new Function() { @Nullable @Override @@ -554,7 +556,7 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware { initMeasureColumns(); rowkey.init(this); - + validateAggregationGroups(); // check if aggregation group is valid for (AggregationGroup agg : this.aggregationGroups) { agg.init(this, rowkey); @@ -752,13 +754,13 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware { } initDerivedMap(dimColArray, DeriveType.LOOKUP, join, derivedCols, derivedExtra); } - + if (join != null) { allColumns.addAll(Arrays.asList(join.getForeignKeyColumns())); allColumns.addAll(Arrays.asList(join.getPrimaryKeyColumns())); } } - + // PK-FK derive the other side Set realDimensions = new HashSet<>(listDimensionColumnsExcludingDerived(true)); for (JoinTableDesc joinTable : model.getJoinTables()) { @@ -777,7 +779,7 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware { } } } - + private String[][] splitDerivedColumnAndExtra(String[] derived) { String[] cols = new String[derived.length]; String[] extra = new String[derived.length]; @@ -810,7 +812,7 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware { i--; } } - + if (derivedCols.length == 0) return; @@ -826,7 +828,7 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware { infoList = new ArrayList(); hostToDerivedMap.put(hostColArray, infoList); } - + // Merged duplicated derived column List whatsLeft = new ArrayList<>(); for (TblColRef derCol : derivedCols) { @@ -966,7 +968,7 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware { allColumns.addAll(Arrays.asList(join.getPrimaryKeyColumns())); } } - + for (TblColRef col : allColumns) { allColumnDescs.add(col.getColumnDesc()); } @@ -1142,7 +1144,7 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware { return null; } - + /** Get a column which can be used to cluster the source table. * To reduce memory footprint in base cuboid for global dict */ // TODO handle more than one ultra high cardinality columns use global dict in one cube http://git-wip-us.apache.org/repos/asf/kylin/blob/42d09a79/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java -- diff --git a/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java b/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java index f905
[2/2] kylin git commit: KYLIN-2452 code review
KYLIN-2452 code review Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/bf813401 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/bf813401 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/bf813401 Branch: refs/heads/master Commit: bf8134010a927b35b942957080466bce7ccac6c9 Parents: 42d09a7 Author: shaofengshi Authored: Mon Feb 20 15:30:59 2017 +0800 Committer: shaofengshi Committed: Mon Feb 20 15:30:59 2017 +0800 -- .../src/main/java/org/apache/kylin/cube/model/CubeDesc.java | 5 +++-- .../kylin/cube/model/validation/rule/AggregationGroupRule.java | 5 + .../java/org/apache/kylin/rest/controller/CubeController.java | 4 3 files changed, 8 insertions(+), 6 deletions(-) -- http://git-wip-us.apache.org/repos/asf/kylin/blob/bf813401/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java -- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java index 0f804d1..95e3343 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java @@ -440,8 +440,9 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware { } public int getBuildLevel() { -if (aggregationGroups.size() <= 0) -throw new IllegalStateException("AggregationGroup size is: " + aggregationGroups.size() + ", there should be at least one AggregationGroup!"); +if (aggregationGroups == null || aggregationGroups.size() == 0) +throw new IllegalStateException("Cube has no aggregation group."); + return Collections.max(Collections2.transform(aggregationGroups, new Function() { @Nullable @Override http://git-wip-us.apache.org/repos/asf/kylin/blob/bf813401/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/AggregationGroupRule.java -- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/AggregationGroupRule.java b/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/AggregationGroupRule.java index 3f865ea..c9e2d28 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/AggregationGroupRule.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/AggregationGroupRule.java @@ -50,6 +50,11 @@ public class AggregationGroupRule implements IValidatorRule { private void inner(CubeDesc cube, ValidateContext context) { +if (cube.getAggregationGroups() == null || cube.getAggregationGroups().size() == 0) { +context.addResult(ResultLevel.ERROR, "Cube should have at least one Aggregation group."); +return; +} + int index = 0; for (AggregationGroup agg : cube.getAggregationGroups()) { if (agg.getIncludes() == null) { http://git-wip-us.apache.org/repos/asf/kylin/blob/bf813401/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java -- diff --git a/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java b/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java index fb4be5a..da44aec 100644 --- a/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java +++ b/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java @@ -441,10 +441,6 @@ public class CubeController extends BasicController { throw new BadRequestException("Invalid Cube name, only letters, numbers and underline supported."); } -int aggSize = desc.getAggregationGroups().size(); -if (aggSize <= 0) -throw new IllegalStateException("AggregationGroup size is: " + aggSize + ", there should be at least one AggregationGroup!"); - try { desc.setUuid(UUID.randomUUID().toString()); String projectName = (null == cubeRequest.getProject()) ? ProjectInstance.DEFAULT_PROJECT_NAME : cubeRequest.getProject();