[1/3] kylin git commit: KYLIN-2428 fix commons-lang3 shade in tool assembly [Forced Update!]
Repository: kylin Updated Branches: refs/heads/master-hbase0.98 363108f01 -> 0bcd399b0 (forced update) KYLIN-2428 fix commons-lang3 shade in tool assembly Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/385d8011 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/385d8011 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/385d8011 Branch: refs/heads/master-hbase0.98 Commit: 385d8011a97fdb66cf921933881d3f3e690d75f6 Parents: 4f94001 Author: Billy Liu Authored: Sun Feb 19 11:41:58 2017 +0800 Committer: Billy Liu Committed: Sun Feb 19 11:41:58 2017 +0800 -- tool-assembly/pom.xml | 5 + 1 file changed, 5 insertions(+) -- http://git-wip-us.apache.org/repos/asf/kylin/blob/385d8011/tool-assembly/pom.xml -- diff --git a/tool-assembly/pom.xml b/tool-assembly/pom.xml index 67d92d4..126c0bc 100644 --- a/tool-assembly/pom.xml +++ b/tool-assembly/pom.xml @@ -59,6 +59,10 @@ compile +org.apache.commons +commons-lang3 + + org.slf4j slf4j-log4j12 compile @@ -93,6 +97,7 @@ log4j:log4j commons-io:commons-io commons-lang:commons-lang + org.apache.commons:commons-lang3 com.google.guava:guava org.apache.kylin:*
[3/3] kylin git commit: KYLIN-2307 Create a branch for master with HBase 0.98 API
KYLIN-2307 Create a branch for master with HBase 0.98 API
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/0bcd399b
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/0bcd399b
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/0bcd399b
Branch: refs/heads/master-hbase0.98
Commit: 0bcd399b0ab8949d173a465dc3cc7480fce48ff7
Parents: 385d801
Author: lidongsjtu
Authored: Mon Jan 23 13:17:37 2017 +0800
Committer: lidongsjtu
Committed: Mon Feb 20 00:49:10 2017 +
--
dev-support/test_all_against_hdp_2_2_4_2_2.sh | 25
dev-support/test_all_against_hdp_2_4_0_0_169.sh | 25
.../sandbox/capacity-scheduler.xml | 17 ++-
examples/test_case_data/sandbox/core-site.xml | 28 +---
examples/test_case_data/sandbox/hbase-site.xml | 119 +
examples/test_case_data/sandbox/hdfs-site.xml | 84 +---
examples/test_case_data/sandbox/hive-site.xml | 89 +
examples/test_case_data/sandbox/mapred-site.xml | 57 +++--
examples/test_case_data/sandbox/yarn-site.xml | 127 +++
.../kylin/provision/BuildCubeWithEngine.java| 17 +--
pom.xml | 117 +
.../kylin/rest/security/AclHBaseStorage.java| 4 +-
.../rest/security/MockAclHBaseStorage.java | 8 +-
.../apache/kylin/rest/security/MockHTable.java | 95 +++---
.../rest/security/RealAclHBaseStorage.java | 9 +-
.../apache/kylin/rest/service/AclService.java | 25 ++--
.../apache/kylin/rest/service/CubeService.java | 35 +++--
.../apache/kylin/rest/service/QueryService.java | 24 ++--
.../apache/kylin/rest/service/UserService.java | 17 +--
.../kylin/storage/hbase/HBaseConnection.java| 44 +++
.../kylin/storage/hbase/HBaseResourceStore.java | 31 +++--
.../storage/hbase/cube/SimpleHBaseStore.java| 20 +--
.../hbase/cube/v2/CubeHBaseEndpointRPC.java | 13 +-
.../storage/hbase/cube/v2/CubeHBaseScanRPC.java | 9 +-
.../coprocessor/endpoint/CubeVisitService.java | 4 +-
.../storage/hbase/steps/CubeHTableUtil.java | 16 +--
.../storage/hbase/steps/DeprecatedGCStep.java | 24 ++--
.../storage/hbase/steps/HBaseCuboidWriter.java | 7 +-
.../kylin/storage/hbase/steps/MergeGCStep.java | 23 ++--
.../storage/hbase/util/CleanHtableCLI.java | 12 +-
.../storage/hbase/util/CubeMigrationCLI.java| 37 +++---
.../hbase/util/CubeMigrationCheckCLI.java | 17 +--
.../hbase/util/DeployCoprocessorCLI.java| 27 ++--
.../hbase/util/ExtendCubeToHybridCLI.java | 8 +-
.../hbase/util/GridTableHBaseBenchmark.java | 34 ++---
.../kylin/storage/hbase/util/HBaseClean.java| 18 ++-
.../hbase/util/HBaseRegionSizeCalculator.java | 35 +++--
.../kylin/storage/hbase/util/HBaseUsage.java| 9 +-
.../storage/hbase/util/HbaseStreamingInput.java | 30 ++---
.../hbase/util/HtableAlterMetadataCLI.java | 9 +-
.../storage/hbase/util/OrphanHBaseCleanJob.java | 19 +--
.../kylin/storage/hbase/util/PingHBaseCLI.java | 15 +--
.../kylin/storage/hbase/util/RowCounterCLI.java | 11 +-
.../storage/hbase/util/StorageCleanupJob.java | 20 ++-
.../storage/hbase/util/UpdateHTableHostCLI.java | 17 +--
tool/pom.xml| 10 --
.../org/apache/kylin/tool/CubeMigrationCLI.java | 19 +--
.../kylin/tool/ExtendCubeToHybridCLI.java | 8 +-
48 files changed, 596 insertions(+), 872 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/0bcd399b/dev-support/test_all_against_hdp_2_2_4_2_2.sh
--
diff --git a/dev-support/test_all_against_hdp_2_2_4_2_2.sh
b/dev-support/test_all_against_hdp_2_2_4_2_2.sh
new file mode 100755
index 000..f7780dd
--- /dev/null
+++ b/dev-support/test_all_against_hdp_2_2_4_2_2.sh
@@ -0,0 +1,25 @@
+#!/bin/bash
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+#http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+dir=$(dirname ${0})
+cd ${dir}
+cd ..
+
+mvn clean install -DskipTests 2>&1 | tee mci.log
+mvn verif
[2/3] kylin git commit: KYLIN-2307 Create a branch for master with HBase 0.98 API
http://git-wip-us.apache.org/repos/asf/kylin/blob/0bcd399b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/SimpleHBaseStore.java
--
diff --git
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/SimpleHBaseStore.java
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/SimpleHBaseStore.java
index f63d9c2..b141190 100644
---
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/SimpleHBaseStore.java
+++
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/SimpleHBaseStore.java
@@ -26,13 +26,12 @@ import java.util.NoSuchElementException;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.BufferedMutator;
-import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.HConnection;
+import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Scan;
-import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.cube.kv.RowConstants;
@@ -87,13 +86,14 @@ public class SimpleHBaseStore implements IGTStore {
}
private class Writer implements IGTWriter {
-final BufferedMutator table;
+final HTableInterface table;
final ByteBuffer rowkey = ByteBuffer.allocate(50);
final ByteBuffer value = ByteBuffer.allocate(50);
Writer() throws IOException {
-Connection conn =
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
-table = conn.getBufferedMutator(htableName);
+HConnection conn =
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+table = conn.getTable(htableName);
+table.setAutoFlush(false, true);
}
@Override
@@ -113,24 +113,24 @@ public class SimpleHBaseStore implements IGTStore {
Put put = new Put(rowkey);
put.addImmutable(CF_B, ByteBuffer.wrap(COL_B),
HConstants.LATEST_TIMESTAMP, value);
-table.mutate(put);
+table.put(put);
}
@Override
public void close() throws IOException {
-table.flush();
+table.flushCommits();
table.close();
}
}
class Reader implements IGTScanner {
-final Table table;
+final HTableInterface table;
final ResultScanner scanner;
int count = 0;
Reader() throws IOException {
-Connection conn =
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+HConnection conn =
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
table = conn.getTable(htableName);
Scan scan = new Scan();
http://git-wip-us.apache.org/repos/asf/kylin/blob/0bcd399b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
--
diff --git
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
index abc3437..a251031 100644
---
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
+++
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
@@ -27,9 +27,8 @@ import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.atomic.AtomicReference;
import java.util.zip.DataFormatException;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.Connection;
-import org.apache.hadoop.hbase.client.Table;
+import org.apache.hadoop.hbase.client.HConnection;
+import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.hadoop.hbase.client.coprocessor.Batch;
import org.apache.hadoop.hbase.ipc.BlockingRpcCallback;
import org.apache.hadoop.hbase.ipc.ServerRpcController;
@@ -53,10 +52,10 @@ import
org.apache.kylin.storage.gtrecord.StorageResponseGTScatter;
import org.apache.kylin.storage.hbase.HBaseConnection;
import
org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos;
import
org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest;
-import
org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntList;
import
org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitResponse;
-import
org.apache.kylin.storage.hbase.cub
[1/2] kylin git commit: added Flink and SQquirreL docs
Repository: kylin Updated Branches: refs/heads/document c3417443e -> e3d43f725 added Flink and SQquirreL docs Signed-off-by: shaofengshi Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/159678cd Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/159678cd Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/159678cd Branch: refs/heads/document Commit: 159678cd10f7f42dd7869f1c5df3f78b9dcd094f Parents: c341744 Author: albertoRamon Authored: Sat Feb 18 12:21:55 2017 +0100 Committer: shaofengshi Committed: Sun Feb 19 22:15:43 2017 +0800 -- website/_data/docs16.yml| 2 + website/_docs16/index.md| 2 + website/_docs16/tutorial/flink.md | 277 +++ website/_docs16/tutorial/squirrel.md| 132 + website/images/Flink-Tutorial/00.png| Bin 0 -> 1464 bytes website/images/Flink-Tutorial/01.png| Bin 0 -> 4572 bytes website/images/Flink-Tutorial/02.png| Bin 0 -> 4611 bytes website/images/Flink-Tutorial/03.png| Bin 0 -> 4499 bytes website/images/Flink-Tutorial/03b.png | Bin 0 -> 7954 bytes website/images/Flink-Tutorial/04.png| Bin 0 -> 12933 bytes website/images/Flink-Tutorial/05.png| Bin 0 -> 8127 bytes website/images/Flink-Tutorial/06.png| Bin 0 -> 7748 bytes website/images/Flink-Tutorial/07.png| Bin 0 -> 9189 bytes website/images/Flink-Tutorial/08.png| Bin 0 -> 9774 bytes website/images/Flink-Tutorial/09.png| Bin 0 -> 6800 bytes website/images/Flink-Tutorial/10.png| Bin 0 -> 2241 bytes website/images/Flink-Tutorial/11.png| Bin 0 -> 5916 bytes website/images/Flink-Tutorial/12.png| Bin 0 -> 544 bytes website/images/Flink-Tutorial/13.png| Bin 0 -> 4849 bytes website/images/Flink-Tutorial/14.png| Bin 0 -> 1071 bytes website/images/Flink-Tutorial/15.png| Bin 0 -> 610 bytes website/images/Flink-Tutorial/16.png| Bin 0 -> 4800 bytes website/images/Flink-Tutorial/17.png| Bin 0 -> 53995 bytes website/images/Flink-Tutorial/18.png| Bin 0 -> 14276 bytes website/images/Flink-Tutorial/19.png| Bin 0 -> 11980 bytes website/images/Flink-Tutorial/20.png| Bin 0 -> 16603 bytes website/images/Flink-Tutorial/21.png| Bin 0 -> 2132 bytes website/images/Flink-Tutorial/22.png| Bin 0 -> 4917 bytes website/images/Flink-Tutorial/23.png| Bin 0 -> 15461 bytes website/images/Flink-Tutorial/24.png| Bin 0 -> 10660 bytes website/images/Flink-Tutorial/25.png| Bin 0 -> 23750 bytes website/images/SQuirreL-Tutorial/00.png | Bin 0 -> 1464 bytes website/images/SQuirreL-Tutorial/01.png | Bin 0 -> 9774 bytes website/images/SQuirreL-Tutorial/02.png | Bin 0 -> 4572 bytes website/images/SQuirreL-Tutorial/03.png | Bin 0 -> 749 bytes website/images/SQuirreL-Tutorial/04.png | Bin 0 -> 237 bytes website/images/SQuirreL-Tutorial/05.png | Bin 0 -> 1050 bytes website/images/SQuirreL-Tutorial/06.png | Bin 0 -> 578 bytes website/images/SQuirreL-Tutorial/07.png | Bin 0 -> 1992 bytes website/images/SQuirreL-Tutorial/08.png | Bin 0 -> 1058 bytes website/images/SQuirreL-Tutorial/09.png | Bin 0 -> 2989 bytes website/images/SQuirreL-Tutorial/10.png | Bin 0 -> 2825 bytes website/images/SQuirreL-Tutorial/11.png | Bin 0 -> 656 bytes website/images/SQuirreL-Tutorial/12.png | Bin 0 -> 987 bytes website/images/SQuirreL-Tutorial/13.png | Bin 0 -> 237 bytes website/images/SQuirreL-Tutorial/14.png | Bin 0 -> 9177 bytes website/images/SQuirreL-Tutorial/15.png | Bin 0 -> 12174 bytes website/images/SQuirreL-Tutorial/16.png | Bin 0 -> 42855 bytes website/images/SQuirreL-Tutorial/17.png | Bin 0 -> 11729 bytes website/images/SQuirreL-Tutorial/18.png | Bin 0 -> 356 bytes website/images/SQuirreL-Tutorial/19.png | Bin 0 -> 15611 bytes website/images/SQuirreL-Tutorial/20.png | Bin 0 -> 4917 bytes website/images/SQuirreL-Tutorial/21.png | Bin 0 -> 3190 bytes website/images/SQuirreL-Tutorial/22.png | Bin 0 -> 8275 bytes website/images/SQuirreL-Tutorial/23.png | Bin 0 -> 8516 bytes website/images/SQuirreL-Tutorial/24.png | Bin 0 -> 10339 bytes website/images/SQuirreL-Tutorial/25.png | Bin 0 -> 11061 bytes 57 files changed, 413 insertions(+) -- http://git-wip-us.apache.org/repos/asf/kylin/blob/159678cd/website/_data/docs16.yml -- diff --git a/website/_data/docs16.yml b/website/_data/docs16.yml index 22f9141..3cb084e 100644 --- a/website/_data/docs16.yml +++ b/website/_data/docs16.yml @@ -45,6 +45,8 @@ - tutorial/tableau_91 - tutorial/powerbi - tutorial/odbc + - tutorial/flink + - tutorial/squirrel - title: How To docs: http://git-wip-us.apache.org/repos/asf/kylin/blob/159678cd/website/_docs16/index.md
[2/2] kylin git commit: Document format review on flink.md and squirrel.md
Document format review on flink.md and squirrel.md Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/e3d43f72 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/e3d43f72 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/e3d43f72 Branch: refs/heads/document Commit: e3d43f725e7c9e9e28290b44d4390e990916c58c Parents: 159678c Author: shaofengshi Authored: Mon Feb 20 13:26:09 2017 +0800 Committer: shaofengshi Committed: Mon Feb 20 13:26:09 2017 +0800 -- website/_docs16/tutorial/flink.md| 146 -- website/_docs16/tutorial/squirrel.md | 66 +- 2 files changed, 82 insertions(+), 130 deletions(-) -- http://git-wip-us.apache.org/repos/asf/kylin/blob/e3d43f72/website/_docs16/tutorial/flink.md -- diff --git a/website/_docs16/tutorial/flink.md b/website/_docs16/tutorial/flink.md index 579bb08..f3cb99f 100644 --- a/website/_docs16/tutorial/flink.md +++ b/website/_docs16/tutorial/flink.md @@ -1,45 +1,39 @@ --- layout: docs16 -title: Apache Flink +title: Connect from Apache Flink categories: tutorial permalink: /docs16/tutorial/flink.html --- -### Intro -There are several Attempts to use this in Scala and JDBC [Attempt1](http://apache-flink-user-mailing-list-archive.2336050.n4.nabble.com/JDBCInputFormat-preparation-with-Flink-1-1-SNAPSHOT-and-Scala-2-11-td5371.html) [Attempt2](http://apache-flink-user-mailing-list-archive.2336050.n4.nabble.com/Type-of-TypeVariable-OT-in-class-org-apache-flink-api-common-io-RichInputFormat-could-not-be-determi-td7287.html) [Attempt3](http://stackoverflow.com/questions/36067881/create-dataset-from-jdbc-source-in-flink-using-scala) [Attempt4](https://codegists.com/snippet/scala/jdbcissuescala_zeitgeist_scala) but none works ... +### Introduction -**Problem 1:** There arenât any doc about connect Kylin with Flink +This document describes how to use Kylin as a data source in Apache Flink; -**Problem 2:** We will try use CreateInput and [JDBCInputFormat](https://ci.apache.org/projects/flink/flink-docs-release-1.2/dev/batch/index.html) in batch mode and access via JDBC to Kylin. But isnât implemented in Scala is only in Java [MailList](http://apache-flink-user-mailing-list-archive.2336050.n4.nabble.com/jdbc-JDBCInputFormat-td9393.html) - +There were several attempts to do this in Scala and JDBC, but none of them works: -Then , We will go step by step solving problems +* [attempt1](http://apache-flink-user-mailing-list-archive.2336050.n4.nabble.com/JDBCInputFormat-preparation-with-Flink-1-1-SNAPSHOT-and-Scala-2-11-td5371.html) +* [attempt2](http://apache-flink-user-mailing-list-archive.2336050.n4.nabble.com/Type-of-TypeVariable-OT-in-class-org-apache-flink-api-common-io-RichInputFormat-could-not-be-determi-td7287.html) +* [attempt3](http://stackoverflow.com/questions/36067881/create-dataset-from-jdbc-source-in-flink-using-scala) +* [attempt4](https://codegists.com/snippet/scala/jdbcissuescala_zeitgeist_scala); - -### Pre-requisites -* We need an instance of Kylin, with a cube: [Quick Start with Sample Cube](http://kylin.apache.org/docs15/tutorial/kylin_sample.html), will be enough - - You can check: - -  +We will try use CreateInput and [JDBCInputFormat](https://ci.apache.org/projects/flink/flink-docs-release-1.2/dev/batch/index.html) in batch mode and access via JDBC to Kylin. But it isnât implemented in Scala, is only in Java [MailList](http://apache-flink-user-mailing-list-archive.2336050.n4.nabble.com/jdbc-JDBCInputFormat-td9393.html). This doc will go step by step solving these problems. +### Pre-requisites +* Need an instance of Kylin, with a Cube; [Sample Cube](/docs16/tutorial/kylin_sample.html) will be good enough. * [Scala](http://www.scala-lang.org/) and [Apache Flink](http://flink.apache.org/) Installed +* [IntelliJ](https://www.jetbrains.com/idea/) Installed and configured for Scala/Flink (see [Flink IDE setup guide](https://ci.apache.org/projects/flink/flink-docs-release-1.1/internals/ide_setup.html) ) -* [IntelliJ](https://www.jetbrains.com/idea/) Installed and configured for Scala / Flink (See [Flink IDE setup guide](https://ci.apache.org/projects/flink/flink-docs-release-1.1/internals/ide_setup.html) ) +### Used software: - -### Used Software: * [Apache Flink](http://flink.apache.org/downloads.html) v1.2-SNAPSHOT -* [Apache Kylin](http://kylin.apache.org/download/) v1.5.2 - - (In Nov 2016, only works with 1.5.4 or more by dependences with Calcite, I Tested with v1.6.0) +* [Apache Kylin](http://kylin.apache.org/download/) v1.5.2 (v1.6.0 also works) * [IntelliJ](https://www.jetbrains.com/idea/download/#section=linux)
svn commit: r1783713 [1/3] - in /kylin/site: ./ cn/docs16/ cn/docs16/install/ cn/docs16/tutorial/ docs16/ docs16/gettingstarted/ docs16/howto/ docs16/install/ docs16/tutorial/ images/Flink-Tutorial/ i
Author: lidong Date: Mon Feb 20 05:29:13 2017 New Revision: 1783713 URL: http://svn.apache.org/viewvc?rev=1783713&view=rev Log: Document format review on flink.md and squirrel.md Added: kylin/site/docs16/tutorial/flink.html kylin/site/docs16/tutorial/squirrel.html kylin/site/images/Flink-Tutorial/ kylin/site/images/Flink-Tutorial/00.png (with props) kylin/site/images/Flink-Tutorial/01.png (with props) kylin/site/images/Flink-Tutorial/02.png (with props) kylin/site/images/Flink-Tutorial/03.png (with props) kylin/site/images/Flink-Tutorial/03b.png (with props) kylin/site/images/Flink-Tutorial/04.png (with props) kylin/site/images/Flink-Tutorial/05.png (with props) kylin/site/images/Flink-Tutorial/06.png (with props) kylin/site/images/Flink-Tutorial/07.png (with props) kylin/site/images/Flink-Tutorial/08.png (with props) kylin/site/images/Flink-Tutorial/09.png (with props) kylin/site/images/Flink-Tutorial/10.png (with props) kylin/site/images/Flink-Tutorial/11.png (with props) kylin/site/images/Flink-Tutorial/12.png (with props) kylin/site/images/Flink-Tutorial/13.png (with props) kylin/site/images/Flink-Tutorial/14.png (with props) kylin/site/images/Flink-Tutorial/15.png (with props) kylin/site/images/Flink-Tutorial/16.png (with props) kylin/site/images/Flink-Tutorial/17.png (with props) kylin/site/images/Flink-Tutorial/18.png (with props) kylin/site/images/Flink-Tutorial/19.png (with props) kylin/site/images/Flink-Tutorial/20.png (with props) kylin/site/images/Flink-Tutorial/21.png (with props) kylin/site/images/Flink-Tutorial/22.png (with props) kylin/site/images/Flink-Tutorial/23.png (with props) kylin/site/images/Flink-Tutorial/24.png (with props) kylin/site/images/Flink-Tutorial/25.png (with props) kylin/site/images/SQuirreL-Tutorial/ kylin/site/images/SQuirreL-Tutorial/00.png (with props) kylin/site/images/SQuirreL-Tutorial/01.png (with props) kylin/site/images/SQuirreL-Tutorial/02.png (with props) kylin/site/images/SQuirreL-Tutorial/03.png (with props) kylin/site/images/SQuirreL-Tutorial/04.png (with props) kylin/site/images/SQuirreL-Tutorial/05.png (with props) kylin/site/images/SQuirreL-Tutorial/06.png (with props) kylin/site/images/SQuirreL-Tutorial/07.png (with props) kylin/site/images/SQuirreL-Tutorial/08.png (with props) kylin/site/images/SQuirreL-Tutorial/09.png (with props) kylin/site/images/SQuirreL-Tutorial/10.png (with props) kylin/site/images/SQuirreL-Tutorial/11.png (with props) kylin/site/images/SQuirreL-Tutorial/12.png (with props) kylin/site/images/SQuirreL-Tutorial/13.png (with props) kylin/site/images/SQuirreL-Tutorial/14.png (with props) kylin/site/images/SQuirreL-Tutorial/15.png (with props) kylin/site/images/SQuirreL-Tutorial/16.png (with props) kylin/site/images/SQuirreL-Tutorial/17.png (with props) kylin/site/images/SQuirreL-Tutorial/18.png (with props) kylin/site/images/SQuirreL-Tutorial/19.png (with props) kylin/site/images/SQuirreL-Tutorial/20.png (with props) kylin/site/images/SQuirreL-Tutorial/21.png (with props) kylin/site/images/SQuirreL-Tutorial/22.png (with props) kylin/site/images/SQuirreL-Tutorial/23.png (with props) kylin/site/images/SQuirreL-Tutorial/24.png (with props) kylin/site/images/SQuirreL-Tutorial/25.png (with props) Modified: kylin/site/cn/docs16/index.html kylin/site/cn/docs16/install/index.html kylin/site/cn/docs16/install/manual_install_guide.html kylin/site/cn/docs16/tutorial/acl.html kylin/site/cn/docs16/tutorial/create_cube.html kylin/site/cn/docs16/tutorial/cube_build_job.html kylin/site/cn/docs16/tutorial/kylin_client_tool.html kylin/site/cn/docs16/tutorial/odbc.html kylin/site/cn/docs16/tutorial/powerbi.html kylin/site/cn/docs16/tutorial/tableau.html kylin/site/cn/docs16/tutorial/tableau_91.html kylin/site/cn/docs16/tutorial/web.html kylin/site/docs16/gettingstarted/best_practices.html kylin/site/docs16/gettingstarted/concepts.html kylin/site/docs16/gettingstarted/events.html kylin/site/docs16/gettingstarted/faq.html kylin/site/docs16/gettingstarted/terminology.html kylin/site/docs16/howto/howto_backup_metadata.html kylin/site/docs16/howto/howto_build_cube_with_restapi.html kylin/site/docs16/howto/howto_cleanup_storage.html kylin/site/docs16/howto/howto_jdbc.html kylin/site/docs16/howto/howto_ldap_and_sso.html kylin/site/docs16/howto/howto_optimize_build.html kylin/site/docs16/howto/howto_optimize_cubes.html kylin/site/docs16/howto/howto_update_coprocessor.html kylin/site/docs16/howto/howto_upgrade.html kylin/site/docs16/howto/howto_use_beeline.html kylin/site/docs16/howto/howto_use_distributed_scheduler.html
svn commit: r1783713 [2/3] - in /kylin/site: ./ cn/docs16/ cn/docs16/install/ cn/docs16/tutorial/ docs16/ docs16/gettingstarted/ docs16/howto/ docs16/install/ docs16/tutorial/ images/Flink-Tutorial/ i
Modified: kylin/site/docs16/install/kylin_docker.html URL: http://svn.apache.org/viewvc/kylin/site/docs16/install/kylin_docker.html?rev=1783713&r1=1783712&r2=1783713&view=diff == --- kylin/site/docs16/install/kylin_docker.html (original) +++ kylin/site/docs16/install/kylin_docker.html Mon Feb 20 05:29:13 2017 @@ -651,6 +651,10 @@ + + + + @@ -941,6 +945,8 @@ + + Quick Start with Sample Cube @@ -1283,6 +1289,10 @@ + + + + Kylin Web Interface @@ -1383,6 +1393,10 @@ + + + + Tableau 8 @@ -1487,6 +1501,10 @@ + + + + Tableau 9 @@ -1583,6 +1601,8 @@ + + MS Excel and Power BI @@ -1675,10 +1695,198 @@ + + Kylin ODBC Driver + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Connect from Apache Flink + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Connect from SQuirreL + + + Modified: kylin/site/docs16/release_notes.html URL: http://svn.apache.org/viewvc/kylin/site/docs16/release_notes.html?rev=1783713&r1=1783712&r2=1783713&view=diff == --- kylin/site/docs16/release_notes.html (original) +++ kylin/site/docs16/release_notes.html Mon Feb 20 05:29:13 2017 @@ -651,6 +651,10 @@ + + + + @@ -941,6 +945,8 @@ + + Quick Start with Sample Cube @@ -1283,6 +1289,10 @@ + + + + Kylin Web Interface @@ -1383,6 +1393,10 @@ + + + + Tableau 8 @@ -1487,6 +1501,10 @@ + + + + Tableau 9 @@ -1583,6 +1601,8 @@ + + MS Excel and Power BI @@ -1675,10 +1695,198 @@ + + Kylin ODBC Driver + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Connect from Apache Flink + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Connect from SQuirreL + + + Modified: kylin/site/docs16/tutorial/acl.html URL: http://svn.apache.org/viewvc/kylin/site/docs16/tutorial/acl.html?rev=1783713&r1=1783712&r2=1783713&view=diff == --- kylin/site/docs16/tutorial/acl.html (original) +++ kylin/site/docs16/tutorial/acl.html Mon Feb 20 05:29:13 2017 @@ -651,6 +651,10 @@ + + + + @@ -941,6 +945,8 @@ + + Quick Start with Sample Cube @@ -1283,6 +1289,10 @@ + + + + Kylin Web Interface @@ -1383,6 +1393,10 @@ + + + + Tableau 8 @@ -1487,6 +1501,10 @@ + + + + Tableau 9 @@ -1583,6 +1601,8 @@ + + MS Excel and Power BI @@ -1675,10 +1695,198 @@ + + Kylin ODBC Driver + + + + + + + + + + + + + + + +
svn commit: r1783713 [3/3] - in /kylin/site: ./ cn/docs16/ cn/docs16/install/ cn/docs16/tutorial/ docs16/ docs16/gettingstarted/ docs16/howto/ docs16/install/ docs16/tutorial/ images/Flink-Tutorial/ i
Added: kylin/site/docs16/tutorial/squirrel.html
URL:
http://svn.apache.org/viewvc/kylin/site/docs16/tutorial/squirrel.html?rev=1783713&view=auto
==
--- kylin/site/docs16/tutorial/squirrel.html (added)
+++ kylin/site/docs16/tutorial/squirrel.html Mon Feb 20 05:29:13 2017
@@ -0,0 +1,2536 @@
+
+
+
+
+
+
+
+
+
+
+
+ Apache Kylin | Connect from SQuirreL
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ http://kylin.apache.org/docs16/tutorial/squirrel.html";>
+ http://kylin.apache.org/feed.xml"; />
+
+
+
+ (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
+ (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new
Date();a=s.createElement(o),
+
m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
+ })(window,document,'script','//www.google-analytics.com/analytics.js','ga');
+
+ //oringal tracker for kylin.io
+ ga('create', 'UA-55534813-1', 'auto');
+ //new tracker for kylin.apache.org
+ ga('create', 'UA-55534813-2', 'auto', {'name':'toplevel'});
+
+ ga('send', 'pageview');
+ ga('toplevel.send', 'pageview');
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ Apache Kylin⢠Extreme OLAP Engine for Big
Data
+
+
+
+
+
+
+
+
+
+
+
+Toggle navigation
+
+
+
+
+
+
+
+
+
+
+ Home
+ Docs
+ Download
+ Community
+ Development
+ Blog
+ ä¸æç
+ https://twitter.com/apachekylin"; target="_blank"
class="fa fa-twitter fa-lg" title="Twitter: @ApacheKylin" >
+ https://github.com/apache/kylin"; target="_blank"
class="fa fa-github-alt fa-lg" title="Github: apache/kylin" >
+ https://www.facebook.com/kylinio"; target="_blank"
class="fa fa-facebook fa-lg" title="Facebook: kylin.io" >
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+Getting Started
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ Overview
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ Apache Kylin Release Notes
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ FAQ
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ Events and
Conferences
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ Community Best
Practices
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ Terminology
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ Technical Concepts
+
+
+
+
+
+
+
+
+
+
+
+
+Installation
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ Installation Guide
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ Hadoop Environment
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ Deploy in Cluster
Mode
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
kylin git commit: default kylin.engine.mr.uhc-reducer-count to 1 in code, while set it to 3 in CI
Repository: kylin
Updated Branches:
refs/heads/master 385d8011a -> 92533c777
default kylin.engine.mr.uhc-reducer-count to 1 in code, while set it to 3 in CI
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/92533c77
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/92533c77
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/92533c77
Branch: refs/heads/master
Commit: 92533c7771f676634a37710a1a0fa4ce68ec162a
Parents: 385d801
Author: Li Yang
Authored: Mon Feb 20 14:03:59 2017 +0800
Committer: Li Yang
Committed: Mon Feb 20 14:03:59 2017 +0800
--
.../src/main/java/org/apache/kylin/common/KylinConfigBase.java | 2 +-
examples/test_case_data/localmeta/kylin.properties | 2 +-
examples/test_case_data/sandbox/kylin.properties | 2 +-
3 files changed, 3 insertions(+), 3 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/92533c77/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
--
diff --git
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 2025a37..50d87a4 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -754,7 +754,7 @@ abstract public class KylinConfigBase implements
Serializable {
//UHC: ultra high cardinality columns, contain the ShardByColumns and the
GlobalDictionaryColumns
public int getUHCReducerCount() {
-return
Integer.parseInt(getOptional("kylin.engine.mr.uhc-reducer-count", "3"));
+return
Integer.parseInt(getOptional("kylin.engine.mr.uhc-reducer-count", "1"));
}
public boolean isBuildDictInReducerEnabled() {
http://git-wip-us.apache.org/repos/asf/kylin/blob/92533c77/examples/test_case_data/localmeta/kylin.properties
--
diff --git a/examples/test_case_data/localmeta/kylin.properties
b/examples/test_case_data/localmeta/kylin.properties
index e89c767..eb4f35d 100644
--- a/examples/test_case_data/localmeta/kylin.properties
+++ b/examples/test_case_data/localmeta/kylin.properties
@@ -73,7 +73,7 @@ kylin.job.max-concurrent-jobs=10
kylin.engine.mr.yarn-check-interval-seconds=10
# for test
-kylin.engine.mr.uhc-reducer-count=1
+kylin.engine.mr.uhc-reducer-count=3
### CUBE ###
http://git-wip-us.apache.org/repos/asf/kylin/blob/92533c77/examples/test_case_data/sandbox/kylin.properties
--
diff --git a/examples/test_case_data/sandbox/kylin.properties
b/examples/test_case_data/sandbox/kylin.properties
index 91566ae..7040cf3 100644
--- a/examples/test_case_data/sandbox/kylin.properties
+++ b/examples/test_case_data/sandbox/kylin.properties
@@ -114,7 +114,7 @@
kylin.query.udf.version=org.apache.kylin.query.udf.VersionUDF
# for test
kylin.job.lock=org.apache.kylin.job.lock.MockJobLock
-kylin.engine.mr.uhc-reducer-count=1
+kylin.engine.mr.uhc-reducer-count=3
### CUBE ###
kylin git commit: KYLIN-2331 update IEngineAware.ID_SPARK to be consistent with KylinConfig
Repository: kylin
Updated Branches:
refs/heads/master 92533c777 -> bbcb0ed52
KYLIN-2331 update IEngineAware.ID_SPARK to be consistent with KylinConfig
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/bbcb0ed5
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/bbcb0ed5
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/bbcb0ed5
Branch: refs/heads/master
Commit: bbcb0ed52c121bc68bea835e0fa0be7d9397b986
Parents: 92533c7
Author: shaofengshi
Authored: Mon Feb 20 14:23:12 2017 +0800
Committer: shaofengshi
Committed: Mon Feb 20 14:23:17 2017 +0800
--
.../src/main/java/org/apache/kylin/common/KylinConfigBase.java | 6 +++---
.../java/org/apache/kylin/metadata/model/IEngineAware.java | 2 +-
2 files changed, 4 insertions(+), 4 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/bbcb0ed5/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
--
diff --git
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 50d87a4..f7d8452 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -692,9 +692,9 @@ abstract public class KylinConfigBase implements
Serializable {
public Map getJobEngines() {
Map r = Maps.newLinkedHashMap();
// ref constants in IEngineAware
-r.put(0, "org.apache.kylin.engine.mr.MRBatchCubingEngine");
-r.put(2, "org.apache.kylin.engine.mr.MRBatchCubingEngine2");
-r.put(4, "org.apache.kylin.engine.spark.SparkBatchCubingEngine2");
+r.put(0, "org.apache.kylin.engine.mr.MRBatchCubingEngine");
//IEngineAware.ID_MR_V1
+r.put(2, "org.apache.kylin.engine.mr.MRBatchCubingEngine2");
//IEngineAware.ID_MR_V2
+r.put(4, "org.apache.kylin.engine.spark.SparkBatchCubingEngine2");
//IEngineAware.ID_SPARK
r.putAll(convertKeyToInteger(getPropertiesByPrefix("kylin.engine.provider.")));
return r;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/bbcb0ed5/core-metadata/src/main/java/org/apache/kylin/metadata/model/IEngineAware.java
--
diff --git
a/core-metadata/src/main/java/org/apache/kylin/metadata/model/IEngineAware.java
b/core-metadata/src/main/java/org/apache/kylin/metadata/model/IEngineAware.java
index 882b2e3..78f7faa 100644
---
a/core-metadata/src/main/java/org/apache/kylin/metadata/model/IEngineAware.java
+++
b/core-metadata/src/main/java/org/apache/kylin/metadata/model/IEngineAware.java
@@ -23,7 +23,7 @@ public interface IEngineAware {
public static final int ID_MR_V1 = 0;
public static final int ID_MR_V2 = 2;
public static final int ID_MR_II = 3;
-public static final int ID_SPARK = 5;
+public static final int ID_SPARK = 4;
int getEngineType();
}
[kylin] Git Push Summary
Repository: kylin Updated Branches: refs/heads/KYLIN-2428 [deleted] 303dea95d
kylin git commit: KYLIN-2428 Move SparkEntry to common-core
Repository: kylin
Updated Branches:
refs/heads/KYLIN-2428 [created] 4f17fa802
KYLIN-2428 Move SparkEntry to common-core
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/4f17fa80
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/4f17fa80
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/4f17fa80
Branch: refs/heads/KYLIN-2428
Commit: 4f17fa8022064532fda9f939439bdaaa74ac37dc
Parents: 385d801
Author: Billy Liu
Authored: Mon Feb 20 14:32:35 2017 +0800
Committer: Billy Liu
Committed: Mon Feb 20 14:32:35 2017 +0800
--
.../apache/kylin/common/util/SparkEntry.java| 46 +++
.../kylin/engine/spark/SparkExecutable.java | 2 +-
.../kylin/engine/spark/util/SparkEntry.java | 48
3 files changed, 47 insertions(+), 49 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/4f17fa80/core-common/src/main/java/org/apache/kylin/common/util/SparkEntry.java
--
diff --git
a/core-common/src/main/java/org/apache/kylin/common/util/SparkEntry.java
b/core-common/src/main/java/org/apache/kylin/common/util/SparkEntry.java
new file mode 100644
index 000..ce11b91
--- /dev/null
+++ b/core-common/src/main/java/org/apache/kylin/common/util/SparkEntry.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kylin.common.util;
+
+import org.apache.commons.lang.StringUtils;
+
+/**
+ */
+public final class SparkEntry {
+
+public static void main(String[] args) throws Exception {
+System.out.println("SparkEntry args:" + StringUtils.join(args, " "));
+if (!(args.length >= 2)) {
+throw new IllegalArgumentException(String.valueOf("-className is
required"));
+}
+if (!(args[0].equals("-className"))) {
+throw new IllegalArgumentException(String.valueOf("-className is
required"));
+}
+final String className = args[1];
+final Object o = Class.
forName(className).newInstance();
+if (!(o instanceof AbstractApplication)) {
+throw new IllegalArgumentException(String.valueOf(className + " is
not a subClass of AbstractApplication"));
+}
+String[] appArgs = new String[args.length - 2];
+for (int i = 2; i < args.length; i++) {
+appArgs[i - 2] = args[i];
+}
+AbstractApplication abstractApplication = (AbstractApplication) o;
+abstractApplication.execute(appArgs);
+}
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/4f17fa80/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkExecutable.java
--
diff --git
a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkExecutable.java
b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkExecutable.java
index 015b480..c671a91 100644
---
a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkExecutable.java
+++
b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkExecutable.java
@@ -105,7 +105,7 @@ public class SparkExecutable extends AbstractExecutable {
}
StringBuilder stringBuilder = new StringBuilder();
-stringBuilder.append("export HADOOP_CONF_DIR=%s && %s/bin/spark-submit
--class org.apache.kylin.engine.spark.util.SparkEntry ");
+stringBuilder.append("export HADOOP_CONF_DIR=%s && %s/bin/spark-submit
--class org.apache.kylin.common.util.SparkEntry ");
Map sparkConfs = config.getSparkConfigOverride();
for (Map.Entry entry : sparkConfs.entrySet()) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/4f17fa80/engine-spark/src/main/java/org/apache/kylin/engine/spark/util/SparkEntry.java
--
diff --git
a/engine-spark/src/main/java/org/apache/kylin/engine/spark/util/SparkEntry.java
b/engine-spark/src/main/java/org/apache/kylin/engine/spark/util/SparkEntry.j
[1/2] kylin git commit: KYLIN-2452 validate there is at least one AggregationGroup
Repository: kylin
Updated Branches:
refs/heads/master bbcb0ed52 -> bf8134010
KYLIN-2452 validate there is at least one AggregationGroup
Signed-off-by: shaofengshi
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/42d09a79
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/42d09a79
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/42d09a79
Branch: refs/heads/master
Commit: 42d09a79848f2becf57deab8742b896d017791b3
Parents: bbcb0ed
Author: Cheng Wang
Authored: Fri Feb 17 10:05:28 2017 +0800
Committer: shaofengshi
Committed: Mon Feb 20 15:20:31 2017 +0800
--
.../org/apache/kylin/cube/model/CubeDesc.java | 18 ++
.../kylin/rest/controller/CubeController.java | 13 +
2 files changed, 19 insertions(+), 12 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/42d09a79/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
--
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
index 0c1419d..0f804d1 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
@@ -440,6 +440,8 @@ public class CubeDesc extends RootPersistentEntity
implements IEngineAware {
}
public int getBuildLevel() {
+if (aggregationGroups.size() <= 0)
+throw new IllegalStateException("AggregationGroup size is: " +
aggregationGroups.size() + ", there should be at least one AggregationGroup!");
return Collections.max(Collections2.transform(aggregationGroups, new
Function() {
@Nullable
@Override
@@ -554,7 +556,7 @@ public class CubeDesc extends RootPersistentEntity
implements IEngineAware {
initMeasureColumns();
rowkey.init(this);
-
+
validateAggregationGroups(); // check if aggregation group is valid
for (AggregationGroup agg : this.aggregationGroups) {
agg.init(this, rowkey);
@@ -752,13 +754,13 @@ public class CubeDesc extends RootPersistentEntity
implements IEngineAware {
}
initDerivedMap(dimColArray, DeriveType.LOOKUP, join,
derivedCols, derivedExtra);
}
-
+
if (join != null) {
allColumns.addAll(Arrays.asList(join.getForeignKeyColumns()));
allColumns.addAll(Arrays.asList(join.getPrimaryKeyColumns()));
}
}
-
+
// PK-FK derive the other side
Set realDimensions = new
HashSet<>(listDimensionColumnsExcludingDerived(true));
for (JoinTableDesc joinTable : model.getJoinTables()) {
@@ -777,7 +779,7 @@ public class CubeDesc extends RootPersistentEntity
implements IEngineAware {
}
}
}
-
+
private String[][] splitDerivedColumnAndExtra(String[] derived) {
String[] cols = new String[derived.length];
String[] extra = new String[derived.length];
@@ -810,7 +812,7 @@ public class CubeDesc extends RootPersistentEntity
implements IEngineAware {
i--;
}
}
-
+
if (derivedCols.length == 0)
return;
@@ -826,7 +828,7 @@ public class CubeDesc extends RootPersistentEntity
implements IEngineAware {
infoList = new ArrayList();
hostToDerivedMap.put(hostColArray, infoList);
}
-
+
// Merged duplicated derived column
List whatsLeft = new ArrayList<>();
for (TblColRef derCol : derivedCols) {
@@ -966,7 +968,7 @@ public class CubeDesc extends RootPersistentEntity
implements IEngineAware {
allColumns.addAll(Arrays.asList(join.getPrimaryKeyColumns()));
}
}
-
+
for (TblColRef col : allColumns) {
allColumnDescs.add(col.getColumnDesc());
}
@@ -1142,7 +1144,7 @@ public class CubeDesc extends RootPersistentEntity
implements IEngineAware {
return null;
}
-
+
/** Get a column which can be used to cluster the source table.
* To reduce memory footprint in base cuboid for global dict */
// TODO handle more than one ultra high cardinality columns use global
dict in one cube
http://git-wip-us.apache.org/repos/asf/kylin/blob/42d09a79/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
--
diff --git
a/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
b/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
index f905
[2/2] kylin git commit: KYLIN-2452 code review
KYLIN-2452 code review
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/bf813401
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/bf813401
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/bf813401
Branch: refs/heads/master
Commit: bf8134010a927b35b942957080466bce7ccac6c9
Parents: 42d09a7
Author: shaofengshi
Authored: Mon Feb 20 15:30:59 2017 +0800
Committer: shaofengshi
Committed: Mon Feb 20 15:30:59 2017 +0800
--
.../src/main/java/org/apache/kylin/cube/model/CubeDesc.java | 5 +++--
.../kylin/cube/model/validation/rule/AggregationGroupRule.java | 5 +
.../java/org/apache/kylin/rest/controller/CubeController.java | 4
3 files changed, 8 insertions(+), 6 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/bf813401/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
--
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
index 0f804d1..95e3343 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
@@ -440,8 +440,9 @@ public class CubeDesc extends RootPersistentEntity
implements IEngineAware {
}
public int getBuildLevel() {
-if (aggregationGroups.size() <= 0)
-throw new IllegalStateException("AggregationGroup size is: " +
aggregationGroups.size() + ", there should be at least one AggregationGroup!");
+if (aggregationGroups == null || aggregationGroups.size() == 0)
+throw new IllegalStateException("Cube has no aggregation group.");
+
return Collections.max(Collections2.transform(aggregationGroups, new
Function() {
@Nullable
@Override
http://git-wip-us.apache.org/repos/asf/kylin/blob/bf813401/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/AggregationGroupRule.java
--
diff --git
a/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/AggregationGroupRule.java
b/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/AggregationGroupRule.java
index 3f865ea..c9e2d28 100644
---
a/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/AggregationGroupRule.java
+++
b/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/AggregationGroupRule.java
@@ -50,6 +50,11 @@ public class AggregationGroupRule implements
IValidatorRule {
private void inner(CubeDesc cube, ValidateContext context) {
+if (cube.getAggregationGroups() == null ||
cube.getAggregationGroups().size() == 0) {
+context.addResult(ResultLevel.ERROR, "Cube should have at least
one Aggregation group.");
+return;
+}
+
int index = 0;
for (AggregationGroup agg : cube.getAggregationGroups()) {
if (agg.getIncludes() == null) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/bf813401/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
--
diff --git
a/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
b/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
index fb4be5a..da44aec 100644
---
a/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
+++
b/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
@@ -441,10 +441,6 @@ public class CubeController extends BasicController {
throw new BadRequestException("Invalid Cube name, only letters,
numbers and underline supported.");
}
-int aggSize = desc.getAggregationGroups().size();
-if (aggSize <= 0)
-throw new IllegalStateException("AggregationGroup size is: " +
aggSize + ", there should be at least one AggregationGroup!");
-
try {
desc.setUuid(UUID.randomUUID().toString());
String projectName = (null == cubeRequest.getProject()) ?
ProjectInstance.DEFAULT_PROJECT_NAME : cubeRequest.getProject();
