[4/4] kylin git commit: KYLIN-2506 Refactor Global Dictionary
KYLIN-2506 Refactor Global Dictionary
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/9b36e268
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/9b36e268
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/9b36e268
Branch: refs/heads/master
Commit: 9b36e268d17fae688620faeaf6fabf9e660c04b2
Parents: 9131704
Author: kangkaisen
Authored: Mon Feb 20 21:06:44 2017 +0800
Committer: kangkaisen
Committed: Thu Apr 20 20:01:24 2017 +0800
--
.../apache/kylin/common/KylinConfigBase.java|7 +
.../kylin/common/lock/DistributedLock.java | 37 +
.../apache/kylin/dict/AppendTrieDictionary.java | 1204 +-
.../kylin/dict/AppendTrieDictionaryChecker.java | 101 --
.../org/apache/kylin/dict/CachedTreeMap.java| 481 ---
.../kylin/dict/GlobalDictionaryBuilder.java | 119 +-
.../kylin/dict/global/AppendDictNode.java | 378 ++
.../kylin/dict/global/AppendDictSlice.java | 283
.../kylin/dict/global/AppendDictSliceKey.java | 75 ++
.../global/AppendTrieDictionaryBuilder.java | 283
.../global/AppendTrieDictionaryChecker.java | 102 ++
.../kylin/dict/global/GlobalDictHDFSStore.java | 413 ++
.../kylin/dict/global/GlobalDictMetadata.java | 51 +
.../kylin/dict/global/GlobalDictStore.java | 102 ++
.../kylin/dict/AppendTrieDictionaryTest.java| 367 --
.../apache/kylin/dict/CachedTreeMapTest.java| 378 --
.../dict/global/AppendTrieDictionaryTest.java | 523
.../job/impl/threadpool/DefaultScheduler.java |8 +-
.../impl/threadpool/DistributedScheduler.java | 52 +-
.../kylin/job/lock/DistributedJobLock.java | 16 +-
.../java/org/apache/kylin/job/lock/JobLock.java |7 +-
.../org/apache/kylin/job/lock/MockJobLock.java |4 +-
.../engine/spark/KylinKryoRegistrator.java |3 -
.../test_case_data/localmeta/kylin.properties |2 +-
.../dict/ITGlobalDictionaryBuilderTest.java | 121 ++
.../kylin/job/BaseTestDistributedScheduler.java |4 +-
.../hbase/util/ZookeeperDistributedJobLock.java | 167 ++-
.../storage/hbase/util/ZookeeperJobLock.java|4 +-
28 files changed, 2687 insertions(+), 2605 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/9b36e268/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
--
diff --git
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 4361242..bf6cdb8 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -29,6 +29,8 @@ import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.apache.commons.lang.StringUtils;
+import org.apache.kylin.common.lock.DistributedLock;
+import org.apache.kylin.common.util.ClassUtil;
import org.apache.kylin.common.util.CliCommandExecutor;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -235,6 +237,11 @@ abstract public class KylinConfigBase implements
Serializable {
public Map getCubeCustomMeasureTypes() {
return getPropertiesByPrefix("kylin.metadata.custom-measure-types.");
}
+
+public DistributedLock getDistributedLock() {
+String clsName = getOptional("kylin.metadata.distributed-lock-impl",
"org.apache.kylin.storage.hbase.util.ZookeeperDistributedJobLock");
+return (DistributedLock) ClassUtil.newInstance(clsName);
+}
//
// DICTIONARY & SNAPSHOT
http://git-wip-us.apache.org/repos/asf/kylin/blob/9b36e268/core-common/src/main/java/org/apache/kylin/common/lock/DistributedLock.java
--
diff --git
a/core-common/src/main/java/org/apache/kylin/common/lock/DistributedLock.java
b/core-common/src/main/java/org/apache/kylin/common/lock/DistributedLock.java
new file mode 100644
index 000..9f04f05
--- /dev/null
+++
b/core-common/src/main/java/org/apache/kylin/common/lock/DistributedLock.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Un
[1/4] kylin git commit: KYLIN-2506 Refactor Global Dictionary
Repository: kylin
Updated Branches:
refs/heads/master 913170468 -> 9b36e268d
http://git-wip-us.apache.org/repos/asf/kylin/blob/9b36e268/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/ZookeeperDistributedJobLock.java
--
diff --git
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/ZookeeperDistributedJobLock.java
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/ZookeeperDistributedJobLock.java
index 983bfd9..c65b8da 100644
---
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/ZookeeperDistributedJobLock.java
+++
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/ZookeeperDistributedJobLock.java
@@ -19,13 +19,14 @@
package org.apache.kylin.storage.hbase.util;
import java.nio.charset.Charset;
-import java.util.concurrent.ExecutorService;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.Executor;
import org.apache.commons.lang3.StringUtils;
import org.apache.curator.RetryPolicy;
import org.apache.curator.framework.CuratorFramework;
import org.apache.curator.framework.CuratorFrameworkFactory;
-import org.apache.curator.framework.imps.CuratorFrameworkState;
import org.apache.curator.framework.recipes.cache.PathChildrenCache;
import org.apache.curator.framework.recipes.cache.PathChildrenCacheEvent;
import org.apache.curator.framework.recipes.cache.PathChildrenCacheListener;
@@ -36,18 +37,14 @@ import org.apache.zookeeper.CreateMode;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-/**
- * the jobLock is specially used to support distributed scheduler.
- */
-
public class ZookeeperDistributedJobLock implements DistributedJobLock {
private static Logger logger =
LoggerFactory.getLogger(ZookeeperDistributedJobLock.class);
-public static final String ZOOKEEPER_LOCK_PATH = "/kylin/job_engine/lock";
+@SuppressWarnings("unused")
+private final KylinConfig config;
-final private KylinConfig config;
-final CuratorFramework zkClient;
-final PathChildrenCache childrenCache;
+private static final ConcurrentMap CACHE =
new ConcurrentHashMap();
+private final CuratorFramework zkClient;
public ZookeeperDistributedJobLock() {
this(KylinConfig.getInstanceFromEnv());
@@ -57,16 +54,12 @@ public class ZookeeperDistributedJobLock implements
DistributedJobLock {
this.config = config;
String zkConnectString = ZookeeperUtil.getZKConnectString();
-logger.info("zk connection string:" + zkConnectString);
if (StringUtils.isEmpty(zkConnectString)) {
throw new IllegalArgumentException("ZOOKEEPER_QUORUM is empty!");
}
-RetryPolicy retryPolicy = new ExponentialBackoffRetry(1000, 3);
-zkClient = CuratorFrameworkFactory.newClient(zkConnectString,
retryPolicy);
-zkClient.start();
+zkClient = getZKClient(config, zkConnectString);
-childrenCache = new PathChildrenCache(zkClient, getWatchPath(), true);
Runtime.getRuntime().addShutdownHook(new Thread(new Runnable() {
@Override
public void run() {
@@ -75,97 +68,104 @@ public class ZookeeperDistributedJobLock implements
DistributedJobLock {
}));
}
+//make the zkClient to be singleton
+private static CuratorFramework getZKClient(KylinConfig config, String
zkConnectString) {
+CuratorFramework zkClient = CACHE.get(config);
+if (zkClient == null) {
+synchronized (ZookeeperDistributedJobLock.class) {
+zkClient = CACHE.get(config);
+if (zkClient == null) {
+RetryPolicy retryPolicy = new
ExponentialBackoffRetry(1000, 3);
+zkClient =
CuratorFrameworkFactory.newClient(zkConnectString, 12, 15000, retryPolicy);
+zkClient.start();
+CACHE.put(config, zkClient);
+if (CACHE.size() > 1) {
+logger.warn("More than one singleton exist");
+}
+}
+}
+}
+return zkClient;
+}
+
/**
- * Lock the segment with the segmentId and serverName.
- *
- * if the segment related job want to be scheduled,
- * it must acquire the segment lock. segmentId is used to get the lock
path,
- * serverName marked which job server keep the segment lock.
+ * Try locking the path with the lockPath and lockClient, if lock
successfully,
+ * the lockClient will write into the data of lockPath.
*
- * @param segmentId the id of segment need to lock
+ * @param lockPath the path will create in zookeeper
*
- * @param serverName the hostname of job server
+ * @param lockClient the mark of client
*
- * @return true if the segment locked successfully
+ * @return
[2/4] kylin git commit: KYLIN-2506 Refactor Global Dictionary
http://git-wip-us.apache.org/repos/asf/kylin/blob/9b36e268/core-dictionary/src/test/java/org/apache/kylin/dict/AppendTrieDictionaryTest.java
--
diff --git
a/core-dictionary/src/test/java/org/apache/kylin/dict/AppendTrieDictionaryTest.java
b/core-dictionary/src/test/java/org/apache/kylin/dict/AppendTrieDictionaryTest.java
deleted file mode 100644
index e2af338..000
---
a/core-dictionary/src/test/java/org/apache/kylin/dict/AppendTrieDictionaryTest.java
+++ /dev/null
@@ -1,367 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.dict;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertNotEquals;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
-
-import java.io.BufferedReader;
-import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
-import java.io.DataInputStream;
-import java.io.DataOutputStream;
-import java.io.FileInputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.InputStreamReader;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.Random;
-import java.util.TreeMap;
-import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.TimeUnit;
-
-import org.apache.hadoop.fs.Path;
-import org.apache.kylin.common.util.HadoopUtil;
-import org.apache.kylin.common.util.LocalFileMetadataTestCase;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Ignore;
-import org.junit.Test;
-
-/**
- * Created by sunyerui on 16/4/28.
- */
-public class AppendTrieDictionaryTest extends LocalFileMetadataTestCase {
-
-public static final String BASE_DIR = "file:///tmp/kylin_append_dict";
-public static final String RESOURCE_DIR = "/dict/append_dict_test";
-
-@Before
-public void setUp() {
-staticCreateTestMetadata();
-System.setProperty("kylin.dictionary.append-entry-size", "5");
-System.setProperty("kylin.env.hdfs-working-dir", BASE_DIR);
-}
-
-@After
-public void after() {
-cleanup();
-staticCleanupTestMetadata();
-}
-
-public static void cleanup() {
-Path basePath = new Path(BASE_DIR);
-try {
-HadoopUtil.getFileSystem(basePath).delete(basePath, true);
-} catch (IOException e) {}
-}
-
-public static final String[] words = new String[] { "paint", "par",
"part", "parts", "partition", "partitions", "party", "partie", "parties",
"patient", "taste", "tar", "trie", "try", "tries", "åå
¸", "åå
¸æ ",
"åæ¯", // non-ascii characters
-"", // empty
-
"paiii",
-
"paiiipaiii",
-
"paintjkjdfklajkdljfkdsajklfjklsadjkjekjrklewjrklewjklrjklewjkljkljkljkljweklrjewkljrklewjrlkjewkljrkljkljkjlkjjkljkljkljkljlkjlkjlkjljdfadfads"
+
"dkfjadslkfjdsakljflksadjklfjklsjfkljwelkrjewkljrklewjklrjelkwjrklewjrlkjwkljerklkljlkjrlkwejrk"
+
"dkfjadslkfjdsakljflksadjklfjklsjfkljwelkrjewkljrklewjklrjelkwjrklewjrlkjwkljerklkljlkjrlkwejrk"
+
"dkfjadslkfjdsakljflksadjklfjklsjfkljwelkrjewk
kylin git commit: KYLIN-2506 Fix IT
Repository: kylin
Updated Branches:
refs/heads/master f9791172e -> 139aa1886
KYLIN-2506 Fix IT
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/139aa188
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/139aa188
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/139aa188
Branch: refs/heads/master
Commit: 139aa1886145c9210a1a752d1036766cfa00b73f
Parents: f979117
Author: kangkaisen
Authored: Fri Apr 21 14:36:40 2017 +0800
Committer: kangkaisen
Committed: Fri Apr 21 17:19:35 2017 +0800
--
.../java/org/apache/kylin/job/BaseTestDistributedScheduler.java | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/139aa188/kylin-it/src/test/java/org/apache/kylin/job/BaseTestDistributedScheduler.java
--
diff --git
a/kylin-it/src/test/java/org/apache/kylin/job/BaseTestDistributedScheduler.java
b/kylin-it/src/test/java/org/apache/kylin/job/BaseTestDistributedScheduler.java
index b4ac42f..aa96e2e 100644
---
a/kylin-it/src/test/java/org/apache/kylin/job/BaseTestDistributedScheduler.java
+++
b/kylin-it/src/test/java/org/apache/kylin/job/BaseTestDistributedScheduler.java
@@ -54,6 +54,7 @@ public class BaseTestDistributedScheduler extends
HBaseMetadataTestCase {
static KylinConfig kylinConfig2;
static CuratorFramework zkClient;
static File localMetaDir;
+static String backup;
static final String SEGMENT_ID = "segmentId";
static final String segmentId1 = "seg1" + UUID.randomUUID();
@@ -75,11 +76,10 @@ public class BaseTestDistributedScheduler extends
HBaseMetadataTestCase {
KylinConfig srcConfig = KylinConfig.getInstanceFromEnv();
localMetaDir = Files.createTempDir();
-String backup = srcConfig.getMetadataUrl();
+backup = srcConfig.getMetadataUrl();
srcConfig.setProperty("kylin.metadata.url",
localMetaDir.getAbsolutePath());
srcConfig.writeProperties(new File(confDstPath1));
srcConfig.writeProperties(new File(confDstPath2));
-srcConfig.setProperty("kylin.metadata.url", backup);
kylinConfig1 = KylinConfig.createInstanceFromUri(new
File(confDstPath1).getAbsolutePath());
kylinConfig2 = KylinConfig.createInstanceFromUri(new
File(confDstPath2).getAbsolutePath());
@@ -132,6 +132,7 @@ public class BaseTestDistributedScheduler extends
HBaseMetadataTestCase {
FileUtils.deleteDirectory(localMetaDir);
System.clearProperty("kylin.job.lock");
+System.clearProperty("kylin.metadata.url");
staticCleanupTestMetadata();
}
kylin git commit: KYLIN-2563 Fix bug in query authorization
Repository: kylin
Updated Branches:
refs/heads/master 2d11fbcea -> 85b4b2c7c
KYLIN-2563 Fix bug in query authorization
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/85b4b2c7
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/85b4b2c7
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/85b4b2c7
Branch: refs/heads/master
Commit: 85b4b2c7c0dc7d2052cd92cc50ac3b6b94f88668
Parents: 2d11fbc
Author: kangkaisen
Authored: Fri Apr 21 17:10:19 2017 +0800
Committer: kangkaisen
Committed: Wed Apr 26 18:35:17 2017 +0800
--
.../rest/controller/ProjectController.java | 59 ++
.../apache/kylin/rest/service/QueryService.java | 9 +-
.../org/apache/kylin/rest/util/AclUtil.java | 116 +++
3 files changed, 134 insertions(+), 50 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/85b4b2c7/server-base/src/main/java/org/apache/kylin/rest/controller/ProjectController.java
--
diff --git
a/server-base/src/main/java/org/apache/kylin/rest/controller/ProjectController.java
b/server-base/src/main/java/org/apache/kylin/rest/controller/ProjectController.java
index a6edece..40e798a 100644
---
a/server-base/src/main/java/org/apache/kylin/rest/controller/ProjectController.java
+++
b/server-base/src/main/java/org/apache/kylin/rest/controller/ProjectController.java
@@ -34,6 +34,7 @@ import org.apache.kylin.rest.request.ProjectRequest;
import org.apache.kylin.rest.service.AccessService;
import org.apache.kylin.rest.service.CubeService;
import org.apache.kylin.rest.service.ProjectService;
+import org.apache.kylin.rest.util.AclUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
@@ -41,9 +42,6 @@ import
org.springframework.security.acls.domain.GrantedAuthoritySid;
import org.springframework.security.acls.domain.PrincipalSid;
import org.springframework.security.acls.model.AccessControlEntry;
import org.springframework.security.acls.model.Acl;
-import org.springframework.security.core.Authentication;
-import org.springframework.security.core.GrantedAuthority;
-import org.springframework.security.core.context.SecurityContextHolder;
import org.springframework.security.core.userdetails.UserDetails;
import org.springframework.stereotype.Controller;
import org.springframework.web.bind.annotation.PathVariable;
@@ -69,6 +67,8 @@ public class ProjectController extends BasicController {
private AccessService accessService;
@Autowired
private CubeService cubeService;
+@Autowired
+private AclUtil aclUtil;
/**
* Get available project list
@@ -90,29 +90,16 @@ public class ProjectController extends BasicController {
List projectInstances =
projectService.listAllProjects(limit, offset);
//get user infomation
-Authentication authentication =
SecurityContextHolder.getContext().getAuthentication();
-UserDetails userDetails = null;
-if (authentication == null) {
-logger.debug("authentication is null.");
-throw new InternalErrorException("Can not find authentication
infomation.");
-}
-if (authentication.getPrincipal() instanceof UserDetails) {
-logger.debug("authentication.getPrincipal() is " +
authentication.getPrincipal());
-userDetails = (UserDetails) authentication.getPrincipal();
-}
-if (authentication.getDetails() instanceof UserDetails) {
-logger.debug("authentication.getDetails() is " +
authentication.getDetails());
-userDetails = (UserDetails) authentication.getDetails();
-}
+UserDetails userDetails = aclUtil.getCurrentUser();
+String userName = userDetails.getUsername();
//check if ROLE_ADMIN return all,also get user role list
-List userAuthority = new ArrayList<>();
-for (GrantedAuthority auth : authentication.getAuthorities()) {
-userAuthority.add(auth.getAuthority());
-if (auth.getAuthority().equals(Constant.ROLE_ADMIN))
+List userAuthority = aclUtil.getAuthorityList();
+for (String auth : userAuthority) {
+if (auth.equals(Constant.ROLE_ADMIN))
return projectInstances;
}
-String userName = userDetails.getUsername();
+
for (ProjectInstance projectInstance : projectInstances) {
if (projectInstance == null) {
continue;
@@ -157,32 +144,8 @@ public class ProjectController extends BasicController {
if (cubeInstance == null) {
continue;
}
-
kylin git commit: The return type of tinyint for sum measure should be bigint
Repository: kylin
Updated Branches:
refs/heads/master 901b10cee -> 1264ff39b
The return type of tinyint for sum measure should be bigint
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/1264ff39
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/1264ff39
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/1264ff39
Branch: refs/heads/master
Commit: 1264ff39b740f9515653d30e0712739bc717d438
Parents: 901b10c
Author: kangkaisen
Authored: Wed May 10 20:03:40 2017 +0800
Committer: kangkaisen
Committed: Wed May 10 20:03:40 2017 +0800
--
webapp/app/js/controllers/cubeMeasures.js | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/1264ff39/webapp/app/js/controllers/cubeMeasures.js
--
diff --git a/webapp/app/js/controllers/cubeMeasures.js
b/webapp/app/js/controllers/cubeMeasures.js
index 73a6cd0..824e2ca 100644
--- a/webapp/app/js/controllers/cubeMeasures.js
+++ b/webapp/app/js/controllers/cubeMeasures.js
@@ -367,7 +367,7 @@ KylinApp.controller('CubeMeasuresCtrl', function ($scope,
$modal,MetaModel,cubes
switch($scope.newMeasure.function.expression){
case "SUM":
-
if(colType==="smallint"||colType==="int"||colType==="bigint"||colType==="integer"){
+
if(colType==="tinyint"||colType==="smallint"||colType==="int"||colType==="bigint"||colType==="integer"){
$scope.newMeasure.function.returntype= 'bigint';
}else{
if(colType.indexOf('decimal')!=-1||colType==="double"||colType==="float"){
kylin git commit: KYLIN-2602 Add optional job threshold arg for MetadataCleanupJob
Repository: kylin
Updated Branches:
refs/heads/master 1264ff39b -> b3310dcdc
KYLIN-2602 Add optional job threshold arg for MetadataCleanupJob
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/b3310dcd
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/b3310dcd
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/b3310dcd
Branch: refs/heads/master
Commit: b3310dcdc0f9ac99006eb4d1a0910de9c43f22fd
Parents: 1264ff3
Author: kangkaisen
Authored: Thu May 11 10:11:40 2017 +0800
Committer: kangkaisen
Committed: Thu May 11 10:13:58 2017 +0800
--
.../org/apache/kylin/tool/MetadataCleanupJob.java | 14 +++---
1 file changed, 11 insertions(+), 3 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/b3310dcd/tool/src/main/java/org/apache/kylin/tool/MetadataCleanupJob.java
--
diff --git a/tool/src/main/java/org/apache/kylin/tool/MetadataCleanupJob.java
b/tool/src/main/java/org/apache/kylin/tool/MetadataCleanupJob.java
index 7040dbb..8e17645 100644
--- a/tool/src/main/java/org/apache/kylin/tool/MetadataCleanupJob.java
+++ b/tool/src/main/java/org/apache/kylin/tool/MetadataCleanupJob.java
@@ -48,14 +48,17 @@ public class MetadataCleanupJob extends AbstractHadoopJob {
@SuppressWarnings("static-access")
private static final Option OPTION_DELETE =
OptionBuilder.withArgName("delete").hasArg().isRequired(false).withDescription("Delete
the unused metadata").create("delete");
+@SuppressWarnings("static-access")
+private static final Option OPTION_THRESHOLD_FOR_JOB =
OptionBuilder.withArgName("jobThreshold").hasArg().isRequired(false).withDescription("Specify
how many days of job metadata keeping. Default 30
days").create("jobThreshold");
+
protected static final Logger logger =
LoggerFactory.getLogger(MetadataCleanupJob.class);
boolean delete = false;
private KylinConfig config = null;
-public static final long TIME_THREADSHOLD = 1 * 3600 * 1000L; // 1 hour
-public static final long TIME_THREADSHOLD_FOR_JOB = 30 * 24 * 3600 *
1000L; // 30 days
+private static final long TIME_THREADSHOLD = 1 * 3600 * 1000L; // 1 hour
+private static final int DEFAULT_DAY_THREADSHOLD_FOR_JOB = 30 ; // 30 days
/*
* (non-Javadoc)
@@ -68,10 +71,12 @@ public class MetadataCleanupJob extends AbstractHadoopJob {
logger.info("jobs args: " + Arrays.toString(args));
options.addOption(OPTION_DELETE);
+options.addOption(OPTION_THRESHOLD_FOR_JOB);
parseOptions(options, args);
logger.info("options: '" + getOptionsAsString() + "'");
logger.info("delete option value: '" + getOptionValue(OPTION_DELETE) +
"'");
+logger.info("jobThreshold option value: '" +
getOptionValue(OPTION_THRESHOLD_FOR_JOB) + "'");
delete = Boolean.parseBoolean(getOptionValue(OPTION_DELETE));
config = KylinConfig.getInstanceFromEnv();
@@ -148,7 +153,10 @@ public class MetadataCleanupJob extends AbstractHadoopJob {
for (ExecutablePO executable : allExecutable) {
long lastModified = executable.getLastModified();
ExecutableOutputPO output =
executableDao.getJobOutput(executable.getUuid());
-if (System.currentTimeMillis() - lastModified >
TIME_THREADSHOLD_FOR_JOB &&
(ExecutableState.SUCCEED.toString().equals(output.getStatus()) ||
ExecutableState.DISCARDED.toString().equals(output.getStatus( {
+int jobThresholdDay =
optionsHelper.hasOption(OPTION_THRESHOLD_FOR_JOB) ?
Integer.valueOf(optionsHelper.getOptionValue(OPTION_THRESHOLD_FOR_JOB)) :
DEFAULT_DAY_THREADSHOLD_FOR_JOB;
+long jobThresholdTime = jobThresholdDay * 24 * 3600 * 1000L;
+
+if (System.currentTimeMillis() - lastModified > jobThresholdTime
&& (ExecutableState.SUCCEED.toString().equals(output.getStatus()) ||
ExecutableState.DISCARDED.toString().equals(output.getStatus( {
toDeleteResource.add(ResourceStore.EXECUTE_RESOURCE_ROOT + "/"
+ executable.getUuid());
toDeleteResource.add(ResourceStore.EXECUTE_OUTPUT_RESOURCE_ROOT + "/" +
executable.getUuid());
kylin git commit: KYLIN-2606 Only return counter for precise count_distinct if query is exactAggregate
Repository: kylin
Updated Branches:
refs/heads/KYLIN-2606 [created] f8336d4bb
KYLIN-2606 Only return counter for precise count_distinct if query is
exactAggregate
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/f8336d4b
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/f8336d4b
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/f8336d4b
Branch: refs/heads/KYLIN-2606
Commit: f8336d4bb85cba257c6487332c876436cdf9ef95
Parents: d449c40
Author: kangkaisen
Authored: Wed Feb 15 19:53:17 2017 +0800
Committer: kangkaisen
Committed: Thu May 11 14:24:49 2017 +0800
--
.../kylin/cube/gridtable/CubeCodeSystem.java| 4 +
.../org/apache/kylin/gridtable/GTRecord.java| 8 +
.../kylin/gridtable/GTSampleCodeSystem.java | 4 +
.../apache/kylin/gridtable/IGTCodeSystem.java | 3 +
.../measure/bitmap/BitmapCounterFactory.java| 2 +
.../kylin/measure/bitmap/BitmapSerializer.java | 37 -
.../measure/bitmap/RoaringBitmapCounter.java| 10 ++
.../bitmap/RoaringBitmapCounterFactory.java | 5 +
.../metadata/datatype/DataTypeSerializer.java | 13 ++
.../gtrecord/GTCubeStorageQueryBase.java| 43 ++
.../hbase/cube/v2/CubeHBaseEndpointRPC.java | 1 +
.../storage/hbase/cube/v2/CubeHBaseScanRPC.java | 2 +-
.../hbase/cube/v2/HBaseReadonlyStore.java | 33 +++-
.../coprocessor/endpoint/CubeVisitService.java | 2 +-
.../endpoint/generated/CubeVisitProtos.java | 151 +++
.../endpoint/protobuf/CubeVisit.proto | 1 +
16 files changed, 282 insertions(+), 37 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/f8336d4b/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CubeCodeSystem.java
--
diff --git
a/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CubeCodeSystem.java
b/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CubeCodeSystem.java
index aaa12a7..9eae6f3 100644
---
a/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CubeCodeSystem.java
+++
b/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CubeCodeSystem.java
@@ -177,4 +177,8 @@ public class CubeCodeSystem implements IGTCodeSystem {
return result;
}
+@Override
+public DataTypeSerializer getSerializer(int col) {
+return serializers[col];
+}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/f8336d4b/core-cube/src/main/java/org/apache/kylin/gridtable/GTRecord.java
--
diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/GTRecord.java
b/core-cube/src/main/java/org/apache/kylin/gridtable/GTRecord.java
index 3e62ea7..f65e4b5 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTRecord.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTRecord.java
@@ -296,4 +296,12 @@ public class GTRecord implements Comparable,
Cloneable {
}
}
+/** change pointers to point to data in given buffer, this
+ * method allows to defined specific column to load */
+public void loadColumns(int selectedCol, ByteBuffer buf) {
+int pos = buf.position();
+int len = info.codeSystem.codeLength(selectedCol, buf);
+cols[selectedCol].set(buf.array(), buf.arrayOffset() + pos, len);
+}
+
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/f8336d4b/core-cube/src/main/java/org/apache/kylin/gridtable/GTSampleCodeSystem.java
--
diff --git
a/core-cube/src/main/java/org/apache/kylin/gridtable/GTSampleCodeSystem.java
b/core-cube/src/main/java/org/apache/kylin/gridtable/GTSampleCodeSystem.java
index 3f3c844..2a5e791 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTSampleCodeSystem.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTSampleCodeSystem.java
@@ -118,4 +118,8 @@ public class GTSampleCodeSystem implements IGTCodeSystem {
}
};
+@Override
+public DataTypeSerializer getSerializer(int col) {
+return serializers[col];
+}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/f8336d4b/core-cube/src/main/java/org/apache/kylin/gridtable/IGTCodeSystem.java
--
diff --git
a/core-cube/src/main/java/org/apache/kylin/gridtable/IGTCodeSystem.java
b/core-cube/src/main/java/org/apache/kylin/gridtable/IGTCodeSystem.java
index 89dfc99..9c8ad6b 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/IGTCodeSystem.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/IGTCodeSystem.java
@@ -23,6 +23,7 @@ import java.nio.ByteBuffer;
import org.apache.kylin.common.util.ImmutableBitSet
kylin git commit: KYLIN-2607 Add http timeout for RestClient
Repository: kylin
Updated Branches:
refs/heads/master b3ba338ea -> 320e56a53
KYLIN-2607 Add http timeout for RestClient
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/320e56a5
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/320e56a5
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/320e56a5
Branch: refs/heads/master
Commit: 320e56a532cf6d4968430c8111d239961a07c7f8
Parents: b3ba338
Author: kangkaisen
Authored: Thu May 11 14:57:29 2017 +0800
Committer: kangkaisen
Committed: Fri May 12 16:47:39 2017 +0800
--
.../apache/kylin/common/restclient/RestClient.java | 16 +---
1 file changed, 13 insertions(+), 3 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/320e56a5/core-common/src/main/java/org/apache/kylin/common/restclient/RestClient.java
--
diff --git
a/core-common/src/main/java/org/apache/kylin/common/restclient/RestClient.java
b/core-common/src/main/java/org/apache/kylin/common/restclient/RestClient.java
index 825c67f..33a4e7a 100644
---
a/core-common/src/main/java/org/apache/kylin/common/restclient/RestClient.java
+++
b/core-common/src/main/java/org/apache/kylin/common/restclient/RestClient.java
@@ -40,6 +40,9 @@ import org.apache.http.client.methods.HttpRequestBase;
import org.apache.http.entity.StringEntity;
import org.apache.http.impl.client.BasicCredentialsProvider;
import org.apache.http.impl.client.DefaultHttpClient;
+import org.apache.http.params.BasicHttpParams;
+import org.apache.http.params.HttpConnectionParams;
+import org.apache.http.params.HttpParams;
import org.apache.http.util.EntityUtils;
import org.apache.kylin.common.util.JsonUtil;
@@ -59,6 +62,9 @@ public class RestClient {
protected static Pattern fullRestPattern =
Pattern.compile("(?:([^:]+)[:]([^@]+)[@])?([^:]+)(?:[:](\\d+))?");
+private static final int HTTP_CONNECTION_TIMEOUT_MS = 3;
+private static final int HTTP_SOCKET_TIMEOUT_MS = 12;
+
public static boolean matchFullRestPattern(String uri) {
Matcher m = fullRestPattern.matcher(uri);
return m.matches();
@@ -93,7 +99,11 @@ public class RestClient {
this.password = password;
this.baseUrl = "http://"; + host + ":" + port + "/kylin/api";
-client = new DefaultHttpClient();
+final HttpParams httpParams = new BasicHttpParams();
+HttpConnectionParams.setSoTimeout(httpParams, HTTP_SOCKET_TIMEOUT_MS);
+HttpConnectionParams.setConnectionTimeout(httpParams,
HTTP_CONNECTION_TIMEOUT_MS);
+
+client = new DefaultHttpClient(httpParams);
if (userName != null && password != null) {
CredentialsProvider provider = new BasicCredentialsProvider();
@@ -263,14 +273,14 @@ public class RestClient {
InputStreamReader reader = null;
BufferedReader rd = null;
StringBuffer result = new StringBuffer();
-try{
+try {
reader = new InputStreamReader(response.getEntity().getContent());
rd = new BufferedReader(reader);
String line = null;
while ((line = rd.readLine()) != null) {
result.append(line);
}
-}finally {
+} finally {
IOUtils.closeQuietly(reader);
IOUtils.closeQuietly(rd);
}
kylin git commit: KYLIN-2619 Use newCachedThreadPool instead of newFixedThreadPool in Broadcaster
Repository: kylin
Updated Branches:
refs/heads/master 1c80c29b2 -> 1c3ed36f7
KYLIN-2619 Use newCachedThreadPool instead of newFixedThreadPool in Broadcaster
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/1c3ed36f
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/1c3ed36f
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/1c3ed36f
Branch: refs/heads/master
Commit: 1c3ed36f7dcf2152b17966bb69dd3211ce4e8aba
Parents: 1c80c29
Author: gaodayue
Authored: Sat May 6 05:21:22 2017 +0800
Committer: kangkaisen
Committed: Mon May 15 16:02:01 2017 +0800
--
.../java/org/apache/kylin/common/restclient/RestClient.java | 4
.../java/org/apache/kylin/metadata/cachesync/Broadcaster.java | 7 +--
2 files changed, 9 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/1c3ed36f/core-common/src/main/java/org/apache/kylin/common/restclient/RestClient.java
--
diff --git
a/core-common/src/main/java/org/apache/kylin/common/restclient/RestClient.java
b/core-common/src/main/java/org/apache/kylin/common/restclient/RestClient.java
index 33a4e7a..470386b 100644
---
a/core-common/src/main/java/org/apache/kylin/common/restclient/RestClient.java
+++
b/core-common/src/main/java/org/apache/kylin/common/restclient/RestClient.java
@@ -287,4 +287,8 @@ public class RestClient {
return result.toString();
}
+@Override
+public String toString() {
+return String.format("RestClient(%s:%d)", host, port);
+}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/1c3ed36f/core-metadata/src/main/java/org/apache/kylin/metadata/cachesync/Broadcaster.java
--
diff --git
a/core-metadata/src/main/java/org/apache/kylin/metadata/cachesync/Broadcaster.java
b/core-metadata/src/main/java/org/apache/kylin/metadata/cachesync/Broadcaster.java
index 1394f7b..e787dfd 100644
---
a/core-metadata/src/main/java/org/apache/kylin/metadata/cachesync/Broadcaster.java
+++
b/core-metadata/src/main/java/org/apache/kylin/metadata/cachesync/Broadcaster.java
@@ -35,6 +35,7 @@ import org.apache.commons.lang.StringUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.restclient.RestClient;
import org.apache.kylin.common.util.DaemonThreadFactory;
+import org.apache.kylin.common.util.SetThreadName;
import org.apache.kylin.metadata.project.ProjectManager;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -116,7 +117,7 @@ public class Broadcaster {
for (String node : config.getRestServers()) {
restClients.add(new RestClient(node));
}
-final ExecutorService wipingCachePool =
Executors.newFixedThreadPool(restClients.size(), new DaemonThreadFactory());
+final ExecutorService wipingCachePool =
Executors.newCachedThreadPool(new DaemonThreadFactory());
while (true) {
try {
final BroadcastEvent broadcastEvent =
broadcastEvents.takeFirst();
@@ -125,8 +126,10 @@ public class Broadcaster {
wipingCachePool.execute(new Runnable() {
@Override
public void run() {
-try {
+try (SetThreadName ignored = new
SetThreadName("CacheWiper %s %s", restClient, broadcastEvent)) {
+logger.info("{} wipe cache {}",
restClient, broadcastEvent);
restClient.wipeCache(broadcastEvent.getEntity(), broadcastEvent.getEvent(),
broadcastEvent.getCacheKey());
+logger.info("{} wipe cache {}
success", restClient, broadcastEvent);
} catch (IOException e) {
logger.warn("Thread failed during wipe
cache at " + broadcastEvent, e);
}
kylin git commit: Revert "KYLIN-2619 Use newCachedThreadPool instead of newFixedThreadPool in Broadcaster"
Repository: kylin
Updated Branches:
refs/heads/master 1c3ed36f7 -> 858d94751
Revert "KYLIN-2619 Use newCachedThreadPool instead of newFixedThreadPool in
Broadcaster"
This reverts commit 1c3ed36f7dcf2152b17966bb69dd3211ce4e8aba.
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/858d9475
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/858d9475
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/858d9475
Branch: refs/heads/master
Commit: 858d947518b308c1a33b832ea8ef49144285c02e
Parents: 1c3ed36
Author: kangkaisen
Authored: Tue May 16 09:48:22 2017 +0800
Committer: kangkaisen
Committed: Tue May 16 09:48:22 2017 +0800
--
.../java/org/apache/kylin/common/restclient/RestClient.java | 4
.../java/org/apache/kylin/metadata/cachesync/Broadcaster.java | 7 ++-
2 files changed, 2 insertions(+), 9 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/858d9475/core-common/src/main/java/org/apache/kylin/common/restclient/RestClient.java
--
diff --git
a/core-common/src/main/java/org/apache/kylin/common/restclient/RestClient.java
b/core-common/src/main/java/org/apache/kylin/common/restclient/RestClient.java
index 470386b..33a4e7a 100644
---
a/core-common/src/main/java/org/apache/kylin/common/restclient/RestClient.java
+++
b/core-common/src/main/java/org/apache/kylin/common/restclient/RestClient.java
@@ -287,8 +287,4 @@ public class RestClient {
return result.toString();
}
-@Override
-public String toString() {
-return String.format("RestClient(%s:%d)", host, port);
-}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/858d9475/core-metadata/src/main/java/org/apache/kylin/metadata/cachesync/Broadcaster.java
--
diff --git
a/core-metadata/src/main/java/org/apache/kylin/metadata/cachesync/Broadcaster.java
b/core-metadata/src/main/java/org/apache/kylin/metadata/cachesync/Broadcaster.java
index e787dfd..1394f7b 100644
---
a/core-metadata/src/main/java/org/apache/kylin/metadata/cachesync/Broadcaster.java
+++
b/core-metadata/src/main/java/org/apache/kylin/metadata/cachesync/Broadcaster.java
@@ -35,7 +35,6 @@ import org.apache.commons.lang.StringUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.restclient.RestClient;
import org.apache.kylin.common.util.DaemonThreadFactory;
-import org.apache.kylin.common.util.SetThreadName;
import org.apache.kylin.metadata.project.ProjectManager;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -117,7 +116,7 @@ public class Broadcaster {
for (String node : config.getRestServers()) {
restClients.add(new RestClient(node));
}
-final ExecutorService wipingCachePool =
Executors.newCachedThreadPool(new DaemonThreadFactory());
+final ExecutorService wipingCachePool =
Executors.newFixedThreadPool(restClients.size(), new DaemonThreadFactory());
while (true) {
try {
final BroadcastEvent broadcastEvent =
broadcastEvents.takeFirst();
@@ -126,10 +125,8 @@ public class Broadcaster {
wipingCachePool.execute(new Runnable() {
@Override
public void run() {
-try (SetThreadName ignored = new
SetThreadName("CacheWiper %s %s", restClient, broadcastEvent)) {
-logger.info("{} wipe cache {}",
restClient, broadcastEvent);
+try {
restClient.wipeCache(broadcastEvent.getEntity(), broadcastEvent.getEvent(),
broadcastEvent.getCacheKey());
-logger.info("{} wipe cache {}
success", restClient, broadcastEvent);
} catch (IOException e) {
logger.warn("Thread failed during wipe
cache at " + broadcastEvent, e);
}
kylin git commit: KYLIN-2547 Fix multi-process concurrence bug in UpdateCubeInfoXXStep
Repository: kylin
Updated Branches:
refs/heads/KYLIN-2547 [created] d0f7f1882
KYLIN-2547 Fix multi-process concurrence bug in UpdateCubeInfoXXStep
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/d0f7f188
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/d0f7f188
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/d0f7f188
Branch: refs/heads/KYLIN-2547
Commit: d0f7f1882ef02e84c112ea358152e4976ee081e0
Parents: a34db26
Author: kangkaisen
Authored: Fri May 12 15:44:23 2017 +0800
Committer: kangkaisen
Committed: Wed May 17 18:32:19 2017 +0800
--
.../java/org/apache/kylin/cube/CubeManager.java | 14 ++
.../mr/steps/UpdateCubeInfoAfterBuildStep.java | 7 +-
.../mr/steps/UpdateCubeInfoAfterMergeStep.java | 7 +-
.../engine/mr/steps/UpdateCubeInfoStep.java | 144 +++
.../apache/kylin/rest/service/CacheService.java | 17 ---
.../apache/kylin/rest/service/CubeService.java | 80 ---
6 files changed, 163 insertions(+), 106 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/d0f7f188/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
--
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
index 0a94fb2..dc50e8a 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
@@ -183,6 +183,20 @@ public class CubeManager implements IRealizationProvider {
return cubeMap.get(cubeName);
}
+//get CubeInstance from store to ensure the cube metadata is latest
+public CubeInstance getCubeFromStore(String cubeName) throws IOException {
+String storePath = CubeInstance.concatResourcePath(cubeName);
+CubeInstance cube = getStore().getResource(storePath,
CubeInstance.class, CUBE_SERIALIZER);
+
+CubeDesc cubeDesc =
CubeDescManager.getInstance(config).getCubeDesc(cube.getDescName());
+cube.setConfig((KylinConfigExt) cubeDesc.getConfig());
+
+//update local cache
+cubeMap.putLocal(cube.getName(), cube);
+
+return cube;
+}
+
public CubeInstance getCubeByUuid(String uuid) {
Collection copy = new
ArrayList(cubeMap.values());
for (CubeInstance cube : copy) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/d0f7f188/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateCubeInfoAfterBuildStep.java
--
diff --git
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateCubeInfoAfterBuildStep.java
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateCubeInfoAfterBuildStep.java
index 2efd718..8df4a80 100644
---
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateCubeInfoAfterBuildStep.java
+++
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateCubeInfoAfterBuildStep.java
@@ -33,7 +33,6 @@ import org.apache.kylin.cube.CubeSegment;
import org.apache.kylin.engine.mr.CubingJob;
import org.apache.kylin.engine.mr.common.BatchConstants;
import org.apache.kylin.job.exception.ExecuteException;
-import org.apache.kylin.job.execution.AbstractExecutable;
import org.apache.kylin.job.execution.ExecutableContext;
import org.apache.kylin.job.execution.ExecuteResult;
import org.apache.kylin.metadata.model.TblColRef;
@@ -42,7 +41,7 @@ import org.slf4j.LoggerFactory;
/**
*/
-public class UpdateCubeInfoAfterBuildStep extends AbstractExecutable {
+public class UpdateCubeInfoAfterBuildStep extends UpdateCubeInfoStep {
private static final Logger logger =
LoggerFactory.getLogger(UpdateCubeInfoAfterBuildStep.class);
public UpdateCubeInfoAfterBuildStep() {
@@ -50,9 +49,9 @@ public class UpdateCubeInfoAfterBuildStep extends
AbstractExecutable {
}
@Override
-protected ExecuteResult doWork(ExecutableContext context) throws
ExecuteException {
+protected ExecuteResult work(ExecutableContext context, CubeInstance cube)
throws ExecuteException {
final CubeManager cubeManager =
CubeManager.getInstance(context.getConfig());
-final CubeInstance cube =
cubeManager.getCube(CubingExecutableUtil.getCubeName(this.getParams()));
+
final CubeSegment segment =
cube.getSegmentById(CubingExecutableUtil.getSegmentId(this.getParams()));
CubingJob cubingJob = (CubingJob)
getManager().getJob(CubingExecutableUtil.getCubingJobId(this.getParams()));
http://git-wip-us.apache.org/repos/asf/kylin/blob/d0f7f188/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateCubeInfoAfterMergeStep.java
[10/50] [abbrv] kylin git commit: KYLIN-2526 refine find-hive-dependency.sh to support Ubuntu
KYLIN-2526 refine find-hive-dependency.sh to support Ubuntu
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/0061900b
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/0061900b
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/0061900b
Branch: refs/heads/KYLIN-2606
Commit: 0061900bb5bd34c8f0cc1c6e955d34bf34ca48e2
Parents: 5f52cea
Author: Cheng Wang
Authored: Thu May 18 11:04:47 2017 +0800
Committer: liyang-gmt8
Committed: Thu May 18 15:27:36 2017 +0800
--
build/bin/find-hive-dependency.sh | 10 ++
1 file changed, 6 insertions(+), 4 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/0061900b/build/bin/find-hive-dependency.sh
--
diff --git a/build/bin/find-hive-dependency.sh
b/build/bin/find-hive-dependency.sh
index 02a47ef..c4c7523 100644
--- a/build/bin/find-hive-dependency.sh
+++ b/build/bin/find-hive-dependency.sh
@@ -100,10 +100,10 @@ fi
function checkFileExist()
{
-files=$1
+files=(`echo $1 | cut -d ":" -f 1- | sed 's/:/ /g'`)
misFiles=0
outputMissFiles=
-for file in ${files//:/ }
+for file in ${files}
do
let allFiles++
if [ ! -f "${file}" ]; then
@@ -111,8 +111,10 @@ function checkFileExist()
let misFiles++
fi
done
-ratio=`echo "scale=3; ${misFiles}/${allFiles}" | bc`
-[[ `echo "$ratio < 0.01" | bc ` -eq 1 ]] || quit "A couple of hive jars
can't be found: ${outputMisFiles}!"
+if [ 0 != ${misFiles} ]; then
+times=`expr ${allFiles} / ${misFiles}`
+[[ ${times} -gt 10 ]] || quit "A couple of hive jars can't be found:
${outputMisFiles}, please export HIVE_LIB='YOUR_LOCAL_HIVE_LIB'"
+fi
}
function validateDirectory()
[07/50] [abbrv] kylin git commit: KYLIN-2627 add simple rollback on ResourceStore
KYLIN-2627 add simple rollback on ResourceStore
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/ea5cabac
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/ea5cabac
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/ea5cabac
Branch: refs/heads/KYLIN-2606
Commit: ea5cabac95b3d27aa60d6b16263904142e3d2daa
Parents: c9dc7cc
Author: Li Yang
Authored: Wed May 17 17:46:08 2017 +0800
Committer: Roger Shi
Committed: Wed May 17 19:40:34 2017 +0800
--
.../kylin/common/persistence/ResourceStore.java | 103 ++-
.../persistence/LocalFileResourceStoreTest.java | 44
2 files changed, 145 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/ea5cabac/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java
--
diff --git
a/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java
b/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java
index d5fbc2e..0565c66 100644
---
a/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java
+++
b/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java
@@ -18,7 +18,9 @@
package org.apache.kylin.common.persistence;
+import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
+import java.io.Closeable;
import java.io.DataInputStream;
import java.io.DataOutputStream;
import java.io.File;
@@ -27,6 +29,7 @@ import java.io.InputStream;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
+import java.util.LinkedHashMap;
import java.util.List;
import java.util.NavigableSet;
import java.util.UUID;
@@ -237,6 +240,11 @@ abstract public class ResourceStore {
final public void putResource(String resPath, InputStream content, long
ts) throws IOException {
resPath = norm(resPath);
logger.trace("Directly saving resource " + resPath + " (Store " +
kylinConfig.getMetadataUrl() + ")");
+putResourceCheckpoint(resPath, content, ts);
+}
+
+private void putResourceCheckpoint(String resPath, InputStream content,
long ts) throws IOException {
+beforeChange(resPath);
putResourceImpl(resPath, content, ts);
}
@@ -266,7 +274,7 @@ abstract public class ResourceStore {
dout.close();
buf.close();
-newTS = checkAndPutResourceImpl(resPath, buf.toByteArray(), oldTS,
newTS);
+newTS = checkAndPutResourceCheckpoint(resPath, buf.toByteArray(),
oldTS, newTS);
obj.setLastModified(newTS); // update again the confirmed TS
return newTS;
} catch (IOException e) {
@@ -278,6 +286,11 @@ abstract public class ResourceStore {
}
}
+private long checkAndPutResourceCheckpoint(String resPath, byte[] content,
long oldTS, long newTS) throws IOException {
+beforeChange(resPath);
+return checkAndPutResourceImpl(resPath, content, oldTS, newTS);
+}
+
/**
* checks old timestamp when overwriting existing
*/
@@ -288,7 +301,12 @@ abstract public class ResourceStore {
*/
final public void deleteResource(String resPath) throws IOException {
logger.trace("Deleting resource " + resPath + " (Store " +
kylinConfig.getMetadataUrl() + ")");
-deleteResourceImpl(norm(resPath));
+deleteResourceCheckpoint(norm(resPath));
+}
+
+private void deleteResourceCheckpoint(String resPath) throws IOException {
+beforeChange(resPath);
+deleteResourceImpl(resPath);
}
abstract protected void deleteResourceImpl(String resPath) throws
IOException;
@@ -315,6 +333,87 @@ abstract public class ResourceStore {
//
+ThreadLocal checkpointing = new ThreadLocal<>();
+
+public Checkpoint checkpoint() {
+Checkpoint cp = checkpointing.get();
+if (cp != null)
+throw new IllegalStateException("A checkpoint has been open for
this thread: " + cp);
+
+cp = new Checkpoint();
+checkpointing.set(cp);
+return cp;
+}
+
+private void beforeChange(String resPath) throws IOException {
+Checkpoint cp = checkpointing.get();
+if (cp != null)
+cp.beforeChange(resPath);
+}
+
+public class Checkpoint implements Closeable {
+
+LinkedHashMap origResData = new LinkedHashMap<>();
+LinkedHashMap origResTimestamp = new LinkedHashMap<>();
+
+private void beforeChange(String resPath) throws IOException {
+if (origResData.containsKey(resPath))
+return;
+
+
[03/50] [abbrv] kylin git commit: Revert "KYLIN-2619 Use newCachedThreadPool instead of newFixedThreadPool in Broadcaster"
Revert "KYLIN-2619 Use newCachedThreadPool instead of newFixedThreadPool in
Broadcaster"
This reverts commit 1c3ed36f7dcf2152b17966bb69dd3211ce4e8aba.
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/858d9475
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/858d9475
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/858d9475
Branch: refs/heads/KYLIN-2606
Commit: 858d947518b308c1a33b832ea8ef49144285c02e
Parents: 1c3ed36
Author: kangkaisen
Authored: Tue May 16 09:48:22 2017 +0800
Committer: kangkaisen
Committed: Tue May 16 09:48:22 2017 +0800
--
.../java/org/apache/kylin/common/restclient/RestClient.java | 4
.../java/org/apache/kylin/metadata/cachesync/Broadcaster.java | 7 ++-
2 files changed, 2 insertions(+), 9 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/858d9475/core-common/src/main/java/org/apache/kylin/common/restclient/RestClient.java
--
diff --git
a/core-common/src/main/java/org/apache/kylin/common/restclient/RestClient.java
b/core-common/src/main/java/org/apache/kylin/common/restclient/RestClient.java
index 470386b..33a4e7a 100644
---
a/core-common/src/main/java/org/apache/kylin/common/restclient/RestClient.java
+++
b/core-common/src/main/java/org/apache/kylin/common/restclient/RestClient.java
@@ -287,8 +287,4 @@ public class RestClient {
return result.toString();
}
-@Override
-public String toString() {
-return String.format("RestClient(%s:%d)", host, port);
-}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/858d9475/core-metadata/src/main/java/org/apache/kylin/metadata/cachesync/Broadcaster.java
--
diff --git
a/core-metadata/src/main/java/org/apache/kylin/metadata/cachesync/Broadcaster.java
b/core-metadata/src/main/java/org/apache/kylin/metadata/cachesync/Broadcaster.java
index e787dfd..1394f7b 100644
---
a/core-metadata/src/main/java/org/apache/kylin/metadata/cachesync/Broadcaster.java
+++
b/core-metadata/src/main/java/org/apache/kylin/metadata/cachesync/Broadcaster.java
@@ -35,7 +35,6 @@ import org.apache.commons.lang.StringUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.restclient.RestClient;
import org.apache.kylin.common.util.DaemonThreadFactory;
-import org.apache.kylin.common.util.SetThreadName;
import org.apache.kylin.metadata.project.ProjectManager;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -117,7 +116,7 @@ public class Broadcaster {
for (String node : config.getRestServers()) {
restClients.add(new RestClient(node));
}
-final ExecutorService wipingCachePool =
Executors.newCachedThreadPool(new DaemonThreadFactory());
+final ExecutorService wipingCachePool =
Executors.newFixedThreadPool(restClients.size(), new DaemonThreadFactory());
while (true) {
try {
final BroadcastEvent broadcastEvent =
broadcastEvents.takeFirst();
@@ -126,10 +125,8 @@ public class Broadcaster {
wipingCachePool.execute(new Runnable() {
@Override
public void run() {
-try (SetThreadName ignored = new
SetThreadName("CacheWiper %s %s", restClient, broadcastEvent)) {
-logger.info("{} wipe cache {}",
restClient, broadcastEvent);
+try {
restClient.wipeCache(broadcastEvent.getEntity(), broadcastEvent.getEvent(),
broadcastEvent.getCacheKey());
-logger.info("{} wipe cache {}
success", restClient, broadcastEvent);
} catch (IOException e) {
logger.warn("Thread failed during wipe
cache at " + broadcastEvent, e);
}
[23/50] [abbrv] kylin git commit: KYLIN-2632 Refactor REST API for better convention
http://git-wip-us.apache.org/repos/asf/kylin/blob/73a78dbe/server-base/src/main/java/org/apache/kylin/rest/model/TableMeta.java
--
diff --git
a/server-base/src/main/java/org/apache/kylin/rest/model/TableMeta.java
b/server-base/src/main/java/org/apache/kylin/rest/model/TableMeta.java
index 9a48851..47002b5 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/model/TableMeta.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/model/TableMeta.java
@@ -27,16 +27,16 @@ import java.util.List;
public class TableMeta implements Serializable {
private static final long serialVersionUID = 1L;
-private String TABLE_CAT;
-private String TABLE_SCHEM;
-private String TABLE_NAME;
-private String TABLE_TYPE;
-private String REMARKS;
-private String TYPE_CAT;
-private String TYPE_SCHEM;
-private String TYPE_NAME;
-private String SELF_REFERENCING_COL_NAME;
-private String REF_GENERATION;
+protected String TABLE_CAT;
+protected String TABLE_SCHEM;
+protected String TABLE_NAME;
+protected String TABLE_TYPE;
+protected String REMARKS;
+protected String TYPE_CAT;
+protected String TYPE_SCHEM;
+protected String TYPE_NAME;
+protected String SELF_REFERENCING_COL_NAME;
+protected String REF_GENERATION;
private List columns = new ArrayList();
public TableMeta() {
http://git-wip-us.apache.org/repos/asf/kylin/blob/73a78dbe/server-base/src/main/java/org/apache/kylin/rest/model/TableMetaWithType.java
--
diff --git
a/server-base/src/main/java/org/apache/kylin/rest/model/TableMetaWithType.java
b/server-base/src/main/java/org/apache/kylin/rest/model/TableMetaWithType.java
new file mode 100644
index 000..55198fe
--- /dev/null
+++
b/server-base/src/main/java/org/apache/kylin/rest/model/TableMetaWithType.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.rest.model;
+
+import java.io.Serializable;
+import java.util.HashSet;
+
+/**
+ * Created by luwei on 17-4-26.
+ */
+public class TableMetaWithType extends TableMeta {
+public static enum tableTypeEnum implements Serializable {
+
+LOOKUP, FACT
+
+}
+
+private HashSet TYPE;
+
+public TableMetaWithType(String tABLE_CAT, String tABLE_SCHEM, String
tABLE_NAME, String tABLE_TYPE, String rEMARKS, String tYPE_CAT, String
tYPE_SCHEM, String tYPE_NAME, String sELF_REFERENCING_COL_NAME, String
rEF_GENERATION) {
+TABLE_CAT = tABLE_CAT;
+TABLE_SCHEM = tABLE_SCHEM;
+TABLE_NAME = tABLE_NAME;
+TABLE_TYPE = tABLE_TYPE;
+REMARKS = rEMARKS;
+TYPE_CAT = tYPE_CAT;
+TYPE_SCHEM = tYPE_SCHEM;
+TYPE_NAME = tYPE_NAME;
+SELF_REFERENCING_COL_NAME = sELF_REFERENCING_COL_NAME;
+REF_GENERATION = rEF_GENERATION;
+TYPE = new HashSet();
+}
+
+public HashSet getTYPE() {
+return TYPE;
+}
+
+public void setTYPE(HashSet TYPE) {
+this.TYPE = TYPE;
+}
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/73a78dbe/server-base/src/main/java/org/apache/kylin/rest/msg/CnMessage.java
--
diff --git a/server-base/src/main/java/org/apache/kylin/rest/msg/CnMessage.java
b/server-base/src/main/java/org/apache/kylin/rest/msg/CnMessage.java
new file mode 100644
index 000..53cbaba
--- /dev/null
+++ b/server-base/src/main/java/org/apache/kylin/rest/msg/CnMessage.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the Li
[46/50] [abbrv] kylin git commit: KYLIN-2636 optimize case when in group by
KYLIN-2636 optimize case when in group by KYLIN-2636 bug fix Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/79c75015 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/79c75015 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/79c75015 Branch: refs/heads/KYLIN-2606 Commit: 79c75015e49b2216f6e8ca9ec0cedbff72737821 Parents: 2c54989 Author: Hongbin Ma Authored: Mon May 22 16:45:24 2017 +0800 Committer: Hongbin Ma Committed: Tue May 23 20:30:06 2017 +0800 -- .../resources/query/sql_subquery/query32.sql| 14 + .../resources/query/sql_subquery/query33.sql| 14 + .../resources/query/sql_subquery/query34.sql| 14 + .../kylin/query/relnode/OLAPProjectRel.java | 61 +++- 4 files changed, 102 insertions(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/kylin/blob/79c75015/kylin-it/src/test/resources/query/sql_subquery/query32.sql -- diff --git a/kylin-it/src/test/resources/query/sql_subquery/query32.sql b/kylin-it/src/test/resources/query/sql_subquery/query32.sql new file mode 100644 index 000..394ea15 --- /dev/null +++ b/kylin-it/src/test/resources/query/sql_subquery/query32.sql @@ -0,0 +1,14 @@ + + select sum(price) as sum_price, (case when '1'='1' then test_cal_dt.week_beg_dt when '1'='2' then test_kylin_fact.lstg_site_id else test_kylin_fact.leaf_categ_id end) as xxx + from test_kylin_fact + inner JOIN edw.test_cal_dt as test_cal_dt + ON test_kylin_fact.cal_dt = test_cal_dt.cal_dt + inner JOIN test_category_groupings + ON test_kylin_fact.leaf_categ_id = test_category_groupings.leaf_categ_id AND test_kylin_fact.lstg_site_id = test_category_groupings.site_id + inner JOIN edw.test_sites as test_sites + ON test_kylin_fact.lstg_site_id = test_sites.site_id + + where case when '1'='1' then test_kylin_fact.cal_dt < date'2012-04-01' when '1'='2' then test_cal_dt.week_beg_dt > date'2012-04-01' else test_kylin_fact.lstg_site_id is not null end and lstg_format_name='FP-GTC' + + group by case when '1'='1' then test_cal_dt.week_beg_dt when '1'='2' then test_kylin_fact.lstg_site_id else test_kylin_fact.leaf_categ_id end + http://git-wip-us.apache.org/repos/asf/kylin/blob/79c75015/kylin-it/src/test/resources/query/sql_subquery/query33.sql -- diff --git a/kylin-it/src/test/resources/query/sql_subquery/query33.sql b/kylin-it/src/test/resources/query/sql_subquery/query33.sql new file mode 100644 index 000..08ecc8e --- /dev/null +++ b/kylin-it/src/test/resources/query/sql_subquery/query33.sql @@ -0,0 +1,14 @@ + + select sum(price) as sum_price, lstg_format_name,(case when '2'='1' then test_kylin_fact.lstg_site_id when '2'='2' then test_cal_dt.week_beg_dt else test_kylin_fact.leaf_categ_id end) as xxx + from test_kylin_fact + inner JOIN edw.test_cal_dt as test_cal_dt + ON test_kylin_fact.cal_dt = test_cal_dt.cal_dt + inner JOIN test_category_groupings + ON test_kylin_fact.leaf_categ_id = test_category_groupings.leaf_categ_id AND test_kylin_fact.lstg_site_id = test_category_groupings.site_id + inner JOIN edw.test_sites as test_sites + ON test_kylin_fact.lstg_site_id = test_sites.site_id + + where case when '2'='1' then test_cal_dt.week_beg_dt > date'2012-04-01' when '2'='2' then test_kylin_fact.cal_dt < date'2012-04-01' else test_kylin_fact.lstg_site_id is not null end + + group by case when '2'='1' then test_kylin_fact.lstg_site_id when '2'='2' then test_cal_dt.week_beg_dt else test_kylin_fact.leaf_categ_id end,lstg_format_name + http://git-wip-us.apache.org/repos/asf/kylin/blob/79c75015/kylin-it/src/test/resources/query/sql_subquery/query34.sql -- diff --git a/kylin-it/src/test/resources/query/sql_subquery/query34.sql b/kylin-it/src/test/resources/query/sql_subquery/query34.sql new file mode 100644 index 000..0dda9aa --- /dev/null +++ b/kylin-it/src/test/resources/query/sql_subquery/query34.sql @@ -0,0 +1,14 @@ + + select sum(price) as sum_price, (case when '3'='1' then test_cal_dt.week_beg_dt else test_kylin_fact.cal_dt end) as xxx + from test_kylin_fact + inner JOIN edw.test_cal_dt as test_cal_dt + ON test_kylin_fact.cal_dt = test_cal_dt.cal_dt + inner JOIN test_category_groupings + ON test_kylin_fact.leaf_categ_id = test_category_groupings.leaf_categ_id AND test_kylin_fact.lstg_site_id = test_category_groupings.site_id + inner JOIN edw.test_sites as test_sites + ON test_kylin_fact.lstg_site_id = test_sites.site_id + + where case when '3'='1' then test_kylin_fact.cal_dt < date'2012-04-01' else test_cal_dt.week_beg_dt > date'2012-04-01' end and lstg_format_n
[42/50] [abbrv] kylin git commit: Merge commit '9fef04d9a98e11454d00247d8cd2f4d501ba66a2'
Merge commit '9fef04d9a98e11454d00247d8cd2f4d501ba66a2' Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/34a65bae Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/34a65bae Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/34a65bae Branch: refs/heads/KYLIN-2606 Commit: 34a65baec0d47de0d91aa8a98b8e1a70db7b6d65 Parents: ed0430b 9fef04d Author: Hongbin Ma Authored: Tue May 23 20:22:54 2017 +0800 Committer: Hongbin Ma Committed: Tue May 23 20:22:54 2017 +0800 -- webapp/app/js/model/projectModel.js | 18 -- 1 file changed, 12 insertions(+), 6 deletions(-) --
[37/50] [abbrv] kylin git commit: minor, fix 'CREATE-TABLE-AS-SELECT cannot create external table'
minor, fix 'CREATE-TABLE-AS-SELECT cannot create external table'
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/83fb144f
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/83fb144f
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/83fb144f
Branch: refs/heads/KYLIN-2606
Commit: 83fb144f2c3df127a2b893f96a05507b8de01e7f
Parents: 81ed301
Author: Cheng Wang
Authored: Tue May 23 19:41:21 2017 +0800
Committer: hongbin ma
Committed: Tue May 23 19:46:02 2017 +0800
--
.../java/org/apache/kylin/source/hive/HiveMRInput.java | 12 +++-
1 file changed, 7 insertions(+), 5 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/83fb144f/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
--
diff --git
a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
index dffba8a..15d4456 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
@@ -190,14 +190,16 @@ public class HiveMRInput implements IMRInput {
HiveCmdBuilder hiveCmdBuilder = new HiveCmdBuilder();
hiveCmdBuilder.addStatement(hiveInitStatements);
for (TableDesc lookUpTableDesc : lookupViewsTables) {
+String identity = lookUpTableDesc.getIdentity();
+String intermediate = lookUpTableDesc.getMaterializedName();
if (lookUpTableDesc.isView()) {
StringBuilder createIntermediateTableHql = new
StringBuilder();
-createIntermediateTableHql.append("DROP TABLE IF EXISTS "
+ lookUpTableDesc.getMaterializedName() + ";\n");
-createIntermediateTableHql.append("CREATE EXTERNAL TABLE
IF NOT EXISTS " + lookUpTableDesc.getMaterializedName() + "\n");
-createIntermediateTableHql.append("LOCATION '" +
jobWorkingDir + "/" + lookUpTableDesc.getMaterializedName() + "'\n");
-createIntermediateTableHql.append("AS SELECT * FROM " +
lookUpTableDesc.getIdentity() + ";\n");
+createIntermediateTableHql.append("DROP TABLE IF EXISTS "
+ intermediate + ";\n");
+createIntermediateTableHql.append("CREATE EXTERNAL TABLE
IF NOT EXISTS " + intermediate + " LIKE " + identity + "\n");
+createIntermediateTableHql.append("LOCATION '" +
jobWorkingDir + "/" + intermediate + "';\n");
+createIntermediateTableHql.append("INSERT OVERWRITE TABLE
" + intermediate + " SELECT * FROM " + identity + ";\n");
hiveCmdBuilder.addStatement(createIntermediateTableHql.toString());
-hiveViewIntermediateTables = hiveViewIntermediateTables +
lookUpTableDesc.getMaterializedName() + ";";
+hiveViewIntermediateTables = hiveViewIntermediateTables +
intermediate + ";";
}
}
[17/50] [abbrv] kylin git commit: KYLIN-2624 pass IT
KYLIN-2624 pass IT
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/0bdd1c21
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/0bdd1c21
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/0bdd1c21
Branch: refs/heads/KYLIN-2606
Commit: 0bdd1c21981cef4d9e6edafd63b9c5a0fb2a6725
Parents: be32f75
Author: Yang Li
Authored: Sat May 20 07:56:24 2017 +0800
Committer: Yang Li
Committed: Sat May 20 09:38:51 2017 +0800
--
examples/test_case_data/localmeta/UUID | 1 +
.../kylin/job/BaseTestDistributedScheduler.java | 2 +
.../hbase/ITZookeeperDistributedLockTest.java | 257 +++
.../hbase/util/ZookeeperDistributedLock.java| 11 +-
.../util/ITZookeeperDistributedLockTest.java| 254 --
5 files changed, 268 insertions(+), 257 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/0bdd1c21/examples/test_case_data/localmeta/UUID
--
diff --git a/examples/test_case_data/localmeta/UUID
b/examples/test_case_data/localmeta/UUID
new file mode 100644
index 000..e3f5e6b
--- /dev/null
+++ b/examples/test_case_data/localmeta/UUID
@@ -0,0 +1 @@
+$3591078f-9401-481a-a233-e7755c1142d0
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/0bdd1c21/kylin-it/src/test/java/org/apache/kylin/job/BaseTestDistributedScheduler.java
--
diff --git
a/kylin-it/src/test/java/org/apache/kylin/job/BaseTestDistributedScheduler.java
b/kylin-it/src/test/java/org/apache/kylin/job/BaseTestDistributedScheduler.java
index 6343dfa..1ea6507 100644
---
a/kylin-it/src/test/java/org/apache/kylin/job/BaseTestDistributedScheduler.java
+++
b/kylin-it/src/test/java/org/apache/kylin/job/BaseTestDistributedScheduler.java
@@ -113,6 +113,8 @@ public class BaseTestDistributedScheduler extends
HBaseMetadataTestCase {
@AfterClass
public static void after() throws Exception {
+jobLock1.purgeLocks("");
+
if (scheduler1 != null) {
scheduler1.shutdown();
scheduler1 = null;
http://git-wip-us.apache.org/repos/asf/kylin/blob/0bdd1c21/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ITZookeeperDistributedLockTest.java
--
diff --git
a/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ITZookeeperDistributedLockTest.java
b/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ITZookeeperDistributedLockTest.java
new file mode 100644
index 000..8460cb9
--- /dev/null
+++
b/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ITZookeeperDistributedLockTest.java
@@ -0,0 +1,257 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.storage.hbase;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.util.Random;
+import java.util.concurrent.Executors;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.apache.kylin.common.lock.DistributedLock;
+import org.apache.kylin.common.lock.DistributedLock.Watcher;
+import org.apache.kylin.common.util.HBaseMetadataTestCase;
+import org.apache.kylin.storage.hbase.util.ZookeeperDistributedLock;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class ITZookeeperDistributedLockTest extends HBaseMetadataTestCase {
+private static final Logger logger =
LoggerFactory.getLogger(ITZookeeperDistributedLockTest.class);
+private static final String ZK_PFX = "/test/ZookeeperDistributedLockTest/"
+ new Random().nextInt(1000);
+
+static ZookeeperDistributedLock.Factory factory;
+
+@BeforeClass
+public static void setup() throws Exception {
+staticCreateTestMetadata();
+factory = new Z
[41/50] [abbrv] kylin git commit: KYLIN-2603 enable GTAggregateScanner when there is having filter
KYLIN-2603 enable GTAggregateScanner when there is having filter
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/0e44a195
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/0e44a195
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/0e44a195
Branch: refs/heads/KYLIN-2606
Commit: 0e44a195b5b2a0074896ffae0636b82a1d7c1355
Parents: e1bc72e
Author: Li Yang
Authored: Thu May 11 17:26:10 2017 +0800
Committer: hongbin ma
Committed: Tue May 23 20:18:17 2017 +0800
--
.../src/main/java/org/apache/kylin/gridtable/GTScanRequest.java| 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/0e44a195/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java
--
diff --git
a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java
b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java
index 9b6b2a6..ffaa8bd 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java
@@ -195,7 +195,7 @@ public class GTScanRequest {
return new EmptyGTScanner();
}
-if (!this.isAllowStorageAggregation()) {
+if (!this.isAllowStorageAggregation() && havingFilterPushDown ==
null) {
logger.info("pre aggregation is not beneficial, skip it");
} else if (this.hasAggregation()) {
logger.info("pre aggregating results before returning");
[02/50] [abbrv] kylin git commit: KYLIN-2619 Use newCachedThreadPool instead of newFixedThreadPool in Broadcaster
KYLIN-2619 Use newCachedThreadPool instead of newFixedThreadPool in Broadcaster
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/1c3ed36f
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/1c3ed36f
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/1c3ed36f
Branch: refs/heads/KYLIN-2606
Commit: 1c3ed36f7dcf2152b17966bb69dd3211ce4e8aba
Parents: 1c80c29
Author: gaodayue
Authored: Sat May 6 05:21:22 2017 +0800
Committer: kangkaisen
Committed: Mon May 15 16:02:01 2017 +0800
--
.../java/org/apache/kylin/common/restclient/RestClient.java | 4
.../java/org/apache/kylin/metadata/cachesync/Broadcaster.java | 7 +--
2 files changed, 9 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/1c3ed36f/core-common/src/main/java/org/apache/kylin/common/restclient/RestClient.java
--
diff --git
a/core-common/src/main/java/org/apache/kylin/common/restclient/RestClient.java
b/core-common/src/main/java/org/apache/kylin/common/restclient/RestClient.java
index 33a4e7a..470386b 100644
---
a/core-common/src/main/java/org/apache/kylin/common/restclient/RestClient.java
+++
b/core-common/src/main/java/org/apache/kylin/common/restclient/RestClient.java
@@ -287,4 +287,8 @@ public class RestClient {
return result.toString();
}
+@Override
+public String toString() {
+return String.format("RestClient(%s:%d)", host, port);
+}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/1c3ed36f/core-metadata/src/main/java/org/apache/kylin/metadata/cachesync/Broadcaster.java
--
diff --git
a/core-metadata/src/main/java/org/apache/kylin/metadata/cachesync/Broadcaster.java
b/core-metadata/src/main/java/org/apache/kylin/metadata/cachesync/Broadcaster.java
index 1394f7b..e787dfd 100644
---
a/core-metadata/src/main/java/org/apache/kylin/metadata/cachesync/Broadcaster.java
+++
b/core-metadata/src/main/java/org/apache/kylin/metadata/cachesync/Broadcaster.java
@@ -35,6 +35,7 @@ import org.apache.commons.lang.StringUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.restclient.RestClient;
import org.apache.kylin.common.util.DaemonThreadFactory;
+import org.apache.kylin.common.util.SetThreadName;
import org.apache.kylin.metadata.project.ProjectManager;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -116,7 +117,7 @@ public class Broadcaster {
for (String node : config.getRestServers()) {
restClients.add(new RestClient(node));
}
-final ExecutorService wipingCachePool =
Executors.newFixedThreadPool(restClients.size(), new DaemonThreadFactory());
+final ExecutorService wipingCachePool =
Executors.newCachedThreadPool(new DaemonThreadFactory());
while (true) {
try {
final BroadcastEvent broadcastEvent =
broadcastEvents.takeFirst();
@@ -125,8 +126,10 @@ public class Broadcaster {
wipingCachePool.execute(new Runnable() {
@Override
public void run() {
-try {
+try (SetThreadName ignored = new
SetThreadName("CacheWiper %s %s", restClient, broadcastEvent)) {
+logger.info("{} wipe cache {}",
restClient, broadcastEvent);
restClient.wipeCache(broadcastEvent.getEntity(), broadcastEvent.getEvent(),
broadcastEvent.getCacheKey());
+logger.info("{} wipe cache {}
success", restClient, broadcastEvent);
} catch (IOException e) {
logger.warn("Thread failed during wipe
cache at " + broadcastEvent, e);
}
[18/50] [abbrv] kylin git commit: minor, more stable ITZookeeperDistributedLockTest
minor, more stable ITZookeeperDistributedLockTest
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/fd3bfedb
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/fd3bfedb
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/fd3bfedb
Branch: refs/heads/KYLIN-2606
Commit: fd3bfedb50f488d6679b9a01bc17e214622b8838
Parents: 0bdd1c2
Author: Yang Li
Authored: Sat May 20 14:00:06 2017 +0800
Committer: Yang Li
Committed: Sat May 20 14:00:06 2017 +0800
--
.../kylin/storage/hbase/ITZookeeperDistributedLockTest.java | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/fd3bfedb/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ITZookeeperDistributedLockTest.java
--
diff --git
a/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ITZookeeperDistributedLockTest.java
b/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ITZookeeperDistributedLockTest.java
index 8460cb9..48d6736 100644
---
a/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ITZookeeperDistributedLockTest.java
+++
b/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ITZookeeperDistributedLockTest.java
@@ -113,12 +113,12 @@ public class ITZookeeperDistributedLockTest extends
HBaseMetadataTestCase {
new Thread() {
@Override
public void run() {
-d.lock(path, 1);
+d.lock(path, 15000);
}
}.start();
c.unlock(path);
-Thread.sleep(1);
+Thread.sleep(2);
assertTrue(c.isLocked(path));
assertEquals(d.getClient(), d.peekLock(path));
[30/50] [abbrv] kylin git commit: minor, apply first found hive_lib_dir
minor, apply first found hive_lib_dir
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/8cd807f7
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/8cd807f7
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/8cd807f7
Branch: refs/heads/KYLIN-2606
Commit: 8cd807f778eeafbb54effb0dd1c7ab552e68c730
Parents: 11a0b57
Author: Cheng Wang
Authored: Mon May 22 17:03:45 2017 +0800
Committer: æ
Committed: Tue May 23 11:30:25 2017 +0800
--
build/bin/find-hive-dependency.sh | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/8cd807f7/build/bin/find-hive-dependency.sh
--
diff --git a/build/bin/find-hive-dependency.sh
b/build/bin/find-hive-dependency.sh
index 8ebdfbd..6f28fd1 100644
--- a/build/bin/find-hive-dependency.sh
+++ b/build/bin/find-hive-dependency.sh
@@ -46,7 +46,8 @@ fi
for data in ${arr[@]}
do
result=`echo $data | grep -e 'hive-exec[a-z0-9A-Z\.-]*.jar'`
-if [ $result ]
+# In some cases there are more than one lib dirs, only the first one will
be applied.
+if [ $result ] && [ -z "$hive_exec_path" ]
then
hive_exec_path=$data
fi
[19/50] [abbrv] kylin git commit: KYLIN-2624 pass IT
KYLIN-2624 pass IT
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/1b864f04
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/1b864f04
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/1b864f04
Branch: refs/heads/KYLIN-2606
Commit: 1b864f04e3fbf50c08978c2f8533ead4ca9784c6
Parents: d21c817
Author: Yang Li
Authored: Sat May 20 14:14:43 2017 +0800
Committer: hongbin ma
Committed: Sat May 20 14:19:07 2017 +0800
--
examples/test_case_data/localmeta/UUID | 1 +
.../kylin/job/BaseTestDistributedScheduler.java | 2 +
.../hbase/ITZookeeperDistributedLockTest.java | 257 +++
.../hbase/util/ZookeeperDistributedLock.java| 11 +-
.../util/ITZookeeperDistributedLockTest.java| 254 --
5 files changed, 268 insertions(+), 257 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/1b864f04/examples/test_case_data/localmeta/UUID
--
diff --git a/examples/test_case_data/localmeta/UUID
b/examples/test_case_data/localmeta/UUID
new file mode 100644
index 000..e3f5e6b
--- /dev/null
+++ b/examples/test_case_data/localmeta/UUID
@@ -0,0 +1 @@
+$3591078f-9401-481a-a233-e7755c1142d0
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/1b864f04/kylin-it/src/test/java/org/apache/kylin/job/BaseTestDistributedScheduler.java
--
diff --git
a/kylin-it/src/test/java/org/apache/kylin/job/BaseTestDistributedScheduler.java
b/kylin-it/src/test/java/org/apache/kylin/job/BaseTestDistributedScheduler.java
index 6343dfa..1ea6507 100644
---
a/kylin-it/src/test/java/org/apache/kylin/job/BaseTestDistributedScheduler.java
+++
b/kylin-it/src/test/java/org/apache/kylin/job/BaseTestDistributedScheduler.java
@@ -113,6 +113,8 @@ public class BaseTestDistributedScheduler extends
HBaseMetadataTestCase {
@AfterClass
public static void after() throws Exception {
+jobLock1.purgeLocks("");
+
if (scheduler1 != null) {
scheduler1.shutdown();
scheduler1 = null;
http://git-wip-us.apache.org/repos/asf/kylin/blob/1b864f04/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ITZookeeperDistributedLockTest.java
--
diff --git
a/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ITZookeeperDistributedLockTest.java
b/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ITZookeeperDistributedLockTest.java
new file mode 100644
index 000..48d6736
--- /dev/null
+++
b/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ITZookeeperDistributedLockTest.java
@@ -0,0 +1,257 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.storage.hbase;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.util.Random;
+import java.util.concurrent.Executors;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.apache.kylin.common.lock.DistributedLock;
+import org.apache.kylin.common.lock.DistributedLock.Watcher;
+import org.apache.kylin.common.util.HBaseMetadataTestCase;
+import org.apache.kylin.storage.hbase.util.ZookeeperDistributedLock;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class ITZookeeperDistributedLockTest extends HBaseMetadataTestCase {
+private static final Logger logger =
LoggerFactory.getLogger(ITZookeeperDistributedLockTest.class);
+private static final String ZK_PFX = "/test/ZookeeperDistributedLockTest/"
+ new Random().nextInt(1000);
+
+static ZookeeperDistributedLock.Factory factory;
+
+@BeforeClass
+public static void setup() throws Exception {
+staticCreateTestMetadata();
+factory = ne
[22/50] [abbrv] kylin git commit: KYLIN-2632 Refactor REST API for better convention
http://git-wip-us.apache.org/repos/asf/kylin/blob/73a78dbe/server-base/src/main/java/org/apache/kylin/rest/service/CubeServiceV2.java
--
diff --git
a/server-base/src/main/java/org/apache/kylin/rest/service/CubeServiceV2.java
b/server-base/src/main/java/org/apache/kylin/rest/service/CubeServiceV2.java
new file mode 100644
index 000..2fae789
--- /dev/null
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/CubeServiceV2.java
@@ -0,0 +1,497 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.rest.service;
+
+import static org.apache.kylin.cube.model.CubeDesc.STATUS_DRAFT;
+import static
org.apache.kylin.rest.controller2.CubeControllerV2.VALID_CUBENAME;
+
+import java.io.IOException;
+import java.util.EnumSet;
+import java.util.List;
+import java.util.UUID;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.kylin.cube.CubeInstance;
+import org.apache.kylin.cube.CubeManager;
+import org.apache.kylin.cube.CubeSegment;
+import org.apache.kylin.cube.CubeUpdate;
+import org.apache.kylin.cube.cuboid.CuboidCLI;
+import org.apache.kylin.cube.model.CubeDesc;
+import org.apache.kylin.engine.mr.CubingJob;
+import org.apache.kylin.job.exception.JobException;
+import org.apache.kylin.job.execution.ExecutableState;
+import org.apache.kylin.metadata.model.DataModelDesc;
+import org.apache.kylin.metadata.model.SegmentStatusEnum;
+import org.apache.kylin.metadata.project.ProjectInstance;
+import org.apache.kylin.metadata.project.ProjectManager;
+import org.apache.kylin.metadata.realization.RealizationStatusEnum;
+import org.apache.kylin.metadata.realization.RealizationType;
+import org.apache.kylin.rest.constant.Constant;
+import org.apache.kylin.rest.exception.BadRequestException;
+import org.apache.kylin.rest.exception.ForbiddenException;
+import org.apache.kylin.rest.msg.Message;
+import org.apache.kylin.rest.msg.MsgPicker;
+import org.apache.kylin.rest.security.AclPermission;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Qualifier;
+import org.springframework.security.access.AccessDeniedException;
+import org.springframework.security.access.prepost.PreAuthorize;
+import org.springframework.security.core.context.SecurityContextHolder;
+import org.springframework.stereotype.Component;
+
+/**
+ * Created by luwei on 17-4-17.
+ */
+
+@Component("cubeMgmtServiceV2")
+public class CubeServiceV2 extends CubeService {
+
+private static final Logger logger =
LoggerFactory.getLogger(CubeServiceV2.class);
+
+@Autowired
+@Qualifier("accessService")
+private AccessService accessService;
+
+@Autowired
+@Qualifier("jobService")
+private JobService jobService;
+
+@Autowired
+@Qualifier("modelMgmtServiceV2")
+private ModelServiceV2 modelServiceV2;
+
+
+@PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN + " or hasPermission(#cube,
'ADMINISTRATION') or hasPermission(#cube, 'OPERATION') or hasPermission(#cube,
'MANAGEMENT')")
+public CubeInstance deleteSegment(CubeInstance cube, String segmentName)
throws IOException {
+Message msg = MsgPicker.getMsg();
+
+if (!segmentName.equals(cube.getSegments().get(0).getName()) &&
!segmentName.equals(cube.getSegments().get(cube.getSegments().size() -
1).getName())) {
+throw new
BadRequestException(String.format(msg.getDELETE_NOT_FIRST_LAST_SEG(),
segmentName));
+}
+CubeSegment toDelete = null;
+for (CubeSegment seg : cube.getSegments()) {
+if (seg.getName().equals(segmentName)) {
+toDelete = seg;
+}
+}
+
+if (toDelete == null) {
+throw new
BadRequestException(String.format(msg.getSEG_NOT_FOUND(), segmentName));
+}
+
+if (toDelete.getStatus() != SegmentStatusEnum.READY) {
+throw new
BadRequestException(String.format(msg.getDELETE_NOT_READY_SEG(), segmentName));
+}
+
+CubeUpdate update = new CubeUpdate(cube);
+update.setToRemoveSegs(new CubeSegment[] { toDelete });
+
[26/50] [abbrv] kylin git commit: KYLIN-2632 Refactor REST API for better convention
KYLIN-2632 Refactor REST API for better convention Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/73a78dbe Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/73a78dbe Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/73a78dbe Branch: refs/heads/KYLIN-2606 Commit: 73a78dbee84808de42925ca5a0b8e16761840233 Parents: 429dfbd Author: Luwei-Chen Authored: Fri May 19 21:49:31 2017 +0800 Committer: liyang-gmt8 Committed: Sun May 21 09:18:59 2017 +0800 -- .../org/apache/kylin/cube/CubeDescManager.java | 91 ++- .../org/apache/kylin/cube/model/CubeDesc.java | 56 +- .../validation/rule/DictionaryRuleTest.java | 8 +- .../model/validation/rule/FunctionRuleTest.java | 6 +- .../apache/kylin/metadata/MetadataManager.java | 16 +- .../kylin/metadata/model/DataModelDesc.java | 49 +- .../kylin/metadata/model/JoinTableDesc.java | 6 +- .../apache/kylin/metadata/model/JoinsTree.java | 10 + .../kylin/metadata/model/TableExtDesc.java | 10 + .../org/apache/kylin/engine/mr/IMRInput.java| 3 - .../java/org/apache/kylin/engine/mr/MRUtil.java | 14 +- .../kylin/rest/controller/AccessController.java | 12 +- .../kylin/rest/controller/AdminController.java | 34 +- .../kylin/rest/controller/CacheController.java | 8 +- .../kylin/rest/controller/CubeController.java | 67 +- .../rest/controller/CubeDescController.java | 6 +- .../rest/controller/DiagnosisController.java| 9 +- .../rest/controller/EncodingController.java | 5 +- .../controller/ExternalFilterController.java| 10 +- .../kylin/rest/controller/HybridController.java | 13 +- .../kylin/rest/controller/JobController.java| 21 +- .../kylin/rest/controller/ModelController.java | 17 +- .../rest/controller/ModelDescController.java| 4 +- .../rest/controller/ProjectController.java | 19 +- .../kylin/rest/controller/QueryController.java | 14 +- .../rest/controller/StreamingController.java| 16 +- .../kylin/rest/controller/TableController.java | 16 +- .../kylin/rest/controller/UserController.java | 11 +- .../rest/controller2/AccessControllerV2.java| 135 .../rest/controller2/AdminControllerV2.java | 106 +++ .../rest/controller2/CacheControllerV2.java | 91 +++ .../rest/controller2/CubeControllerV2.java | 755 +++ .../rest/controller2/CubeDescControllerV2.java | 141 .../rest/controller2/DiagnosisControllerV2.java | 121 +++ .../rest/controller2/EncodingControllerV2.java | 80 ++ .../controller2/ExternalFilterControllerV2.java | 101 +++ .../rest/controller2/HybridControllerV2.java| 97 +++ .../kylin/rest/controller2/JobControllerV2.java | 222 ++ .../rest/controller2/ModelControllerV2.java | 284 +++ .../rest/controller2/ModelDescControllerV2.java | 107 +++ .../rest/controller2/ProjectControllerV2.java | 269 +++ .../rest/controller2/QueryControllerV2.java | 186 + .../rest/controller2/StreamingControllerV2.java | 288 +++ .../rest/controller2/TableControllerV2.java | 159 .../rest/controller2/UserControllerV2.java | 103 +++ .../rest/exception/BadRequestException.java | 1 + .../org/apache/kylin/rest/model/ColumnMeta.java | 46 +- .../kylin/rest/model/ColumnMetaWithType.java| 70 ++ .../org/apache/kylin/rest/model/TableMeta.java | 20 +- .../kylin/rest/model/TableMetaWithType.java | 57 ++ .../org/apache/kylin/rest/msg/CnMessage.java| 38 + .../java/org/apache/kylin/rest/msg/Message.java | 588 +++ .../org/apache/kylin/rest/msg/MsgPicker.java| 42 ++ .../kylin/rest/request/HiveTableRequestV2.java | 59 ++ .../rest/response/CubeInstanceResponse.java | 72 ++ .../rest/response/DataModelDescResponse.java| 62 ++ .../kylin/rest/response/HBaseResponse.java | 9 + .../kylin/rest/response/ResponseCode.java | 2 +- .../security/KylinAuthenticationProvider.java | 2 + .../kylin/rest/security/LdapProvider.java | 2 + .../kylin/rest/service/AccessService.java | 59 +- .../apache/kylin/rest/service/AclService.java | 49 +- .../apache/kylin/rest/service/AdminService.java | 63 +- .../apache/kylin/rest/service/CacheService.java | 10 +- .../apache/kylin/rest/service/CubeService.java | 55 +- .../kylin/rest/service/CubeServiceV2.java | 497 .../kylin/rest/service/DiagnosisService.java| 18 +- .../kylin/rest/service/EncodingService.java | 7 +- .../kylin/rest/service/ExtFilterService.java| 12 +- .../apache/kylin/rest/service/JobService.java | 122 +-- .../kylin/rest/service/KafkaConfigService.java | 8 +- .../apache/kylin/rest/service/ModelService.java | 20 +- .../kylin/rest/service/ModelServiceV2.java | 351 + .../kylin/rest/service/ProjectService.java | 2 + .../kylin/rest/service/P
[04/50] [abbrv] kylin git commit: KYLIN-2623 Move output(Hbase) code to outputside
KYLIN-2623 Move output(Hbase) code to outputside
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/a38b02df
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/a38b02df
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/a38b02df
Branch: refs/heads/KYLIN-2606
Commit: a38b02df0387541684aa1689e044927af650f1c7
Parents: edc4d4c
Author: Roger Shi
Authored: Tue May 16 13:50:23 2017 +0800
Committer: liyang-gmt8
Committed: Tue May 16 14:03:50 2017 +0800
--
.../org/apache/kylin/engine/mr/IMROutput2.java | 33 +-
.../engine/mr/common/AbstractHadoopJob.java | 6 +-
.../apache/kylin/engine/mr/steps/CuboidJob.java | 37 +++
.../kylin/engine/mr/steps/InMemCuboidJob.java | 55 ++
.../engine/mr/steps/LayerReducerNumSizing.java | 80 --
.../kylin/engine/mr/steps/MergeCuboidJob.java | 37 ---
.../engine/mr/steps/MergeCuboidMapper.java | 35 +-
.../kylin/engine/mr/steps/ReducerNumSizing.java | 106 +++
.../apache/kylin/source/hive/HiveMRInput.java | 2 -
.../apache/kylin/source/kafka/KafkaMRInput.java | 1 -
.../hbase/steps/HBaseMROutput2Transition.java | 97 +
11 files changed, 284 insertions(+), 205 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/a38b02df/engine-mr/src/main/java/org/apache/kylin/engine/mr/IMROutput2.java
--
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/IMROutput2.java
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/IMROutput2.java
index 603f207..69bba0a 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/IMROutput2.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/IMROutput2.java
@@ -18,11 +18,14 @@
package org.apache.kylin.engine.mr;
+import java.util.List;
+
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapreduce.lib.input.FileSplit;
+import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeSegment;
import org.apache.kylin.job.execution.DefaultChainedExecutable;
-import java.util.List;
-
public interface IMROutput2 {
/** Return a helper to participate in batch cubing job flow. */
@@ -53,6 +56,19 @@ public interface IMROutput2 {
/** Add step that does any necessary clean up. */
public void addStepPhase4_Cleanup(DefaultChainedExecutable jobFlow);
+
+public IMROutputFormat getOuputFormat();
+
+}
+
+public interface IMROutputFormat {
+
+/** Configure the InputFormat of given job. */
+public void configureJobInput(Job job, String input) throws Exception;
+
+/** Configure the OutputFormat of given job. */
+public void configureJobOutput(Job job, String output, CubeSegment
segment, int level) throws Exception;
+
}
/** Return a helper to participate in batch merge job flow. */
@@ -82,6 +98,19 @@ public interface IMROutput2 {
/** Add step that does any necessary clean up. */
public void addStepPhase3_Cleanup(DefaultChainedExecutable jobFlow);
+
+public IMRMergeOutputFormat getOuputFormat();
+}
+
+public interface IMRMergeOutputFormat {
+
+/** Configure the InputFormat of given job. */
+public void configureJobInput(Job job, String input) throws Exception;
+
+/** Configure the OutputFormat of given job. */
+public void configureJobOutput(Job job, String output, CubeSegment
segment) throws Exception;
+
+public CubeSegment findSourceSegment(FileSplit fileSplit, CubeInstance
cube);
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/a38b02df/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
--
diff --git
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
index 44686d6..764cbdd 100644
---
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
+++
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
@@ -559,7 +559,7 @@ public abstract class AbstractHadoopJob extends Configured
implements Tool {
HadoopUtil.deletePath(conf, path);
}
-protected double getTotalMapInputMB() throws ClassNotFoundException,
IOException, InterruptedException, JobException {
+public static double getTotalMapInputMB(Job job) throws
ClassNotFoundException, IOException, InterruptedException, JobException {
if (job == null) {
throw new JobException("Job is null");
}
@@ -576,6 +576,10 @@ public abstract class AbstractHadoopJob extends Configured
imple
[36/50] [abbrv] kylin git commit: Merge commit 'ce6e3ede2f670bf037a55234630b85098265075a'
Merge commit 'ce6e3ede2f670bf037a55234630b85098265075a'
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/81ed3011
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/81ed3011
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/81ed3011
Branch: refs/heads/KYLIN-2606
Commit: 81ed30110186e2c13511a3fc50f6098645412aab
Parents: 2471d5e ce6e3ed
Author: Hongbin Ma
Authored: Tue May 23 19:31:36 2017 +0800
Committer: Hongbin Ma
Committed: Tue May 23 19:31:36 2017 +0800
--
build/bin/load-hive-conf.sh | 2 +-
.../impl/threadpool/DistributedScheduler.java | 1 +
.../apache/kylin/metadata/MetadataManager.java | 2 ++
.../kylin/metadata/model/DataModelDesc.java | 1 +
.../security/KylinAuthenticationProvider.java | 20 +++-
.../apache/kylin/rest/service/CacheService.java | 1 -
.../kylin/rest/service/ModelServiceTest.java| 2 +-
.../hbase/util/ZookeeperDistributedLock.java| 2 --
webapp/app/js/controllers/page.js | 16 +---
9 files changed, 26 insertions(+), 21 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/81ed3011/build/bin/load-hive-conf.sh
--
diff --cc build/bin/load-hive-conf.sh
index c7317c6,a42cd0c..0937719
--- a/build/bin/load-hive-conf.sh
+++ b/build/bin/load-hive-conf.sh
@@@ -12,4 -12,4 +12,4 @@@ rm -rf ${hive_conf_prop
export ENABLE_CHECK_ENV=false
${dir}/kylin.sh org.apache.kylin.tool.HiveConfigCLI ${hive_conf_dir}
${hive_conf_prop}
[[ 0 == $? ]] || quit "Can not parse xml file: ${hive_conf_dir}, please check
it."
--hive_conf_properties=`cat ${hive_conf_prop}`
++hive_conf_properties=`cat ${hive_conf_prop}`
http://git-wip-us.apache.org/repos/asf/kylin/blob/81ed3011/core-metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java
--
diff --cc
core-metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java
index 00fafaf,b3ca14a..39d7830
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java
@@@ -603,9 -600,7 +603,10 @@@ public class MetadataManager
ResourceStore store = getStore();
try {
DataModelDesc dataModelDesc = store.getResource(path,
DataModelDesc.class, MODELDESC_SERIALIZER);
-dataModelDesc.init(config, this.getAllTablesMap(),
this.ccInfoMap);
++
+if (dataModelDesc.getStatus() == null)
+dataModelDesc.init(config, this.getAllTablesMap(),
this.ccInfoMap);
+
dataModelDescMap.putLocal(dataModelDesc.getName(), dataModelDesc);
return dataModelDesc;
} catch (Exception e) {
@@@ -653,8 -648,7 +654,9 @@@
}
private DataModelDesc saveDataModelDesc(DataModelDesc dataModelDesc)
throws IOException {
-dataModelDesc.init(config, this.getAllTablesMap(), this.ccInfoMap);
++
+if (dataModelDesc.getStatus() == null)
+dataModelDesc.init(config, this.getAllTablesMap(),
this.ccInfoMap);
String path = dataModelDesc.getResourcePath();
getStore().putResource(path, dataModelDesc, MODELDESC_SERIALIZER);
http://git-wip-us.apache.org/repos/asf/kylin/blob/81ed3011/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
--
diff --cc
core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
index c3554f2,2174b7a..a689979
---
a/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
+++
b/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
@@@ -415,10 -385,9 +415,11 @@@ public class DataModelDesc extends Root
TableDesc tableDesc = tables.get(join.getTable());
String alias = join.getAlias();
-if (alias == null)
+if (alias == null) {
alias = tableDesc.getName();
+join.setAlias(alias);
+}
+
TableRef ref = new TableRef(this, alias, tableDesc);
join.setTableRef(ref);
http://git-wip-us.apache.org/repos/asf/kylin/blob/81ed3011/server-base/src/main/java/org/apache/kylin/rest/security/KylinAuthenticationProvider.java
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/81ed3011/server-base/src/main/java/org/apache/kylin/rest/service/CacheService.java
--
diff --cc
server-base/src/main/java/org/apache/kylin/rest/service
[25/50] [abbrv] kylin git commit: KYLIN-2632 Refactor REST API for better convention
http://git-wip-us.apache.org/repos/asf/kylin/blob/73a78dbe/server-base/src/main/java/org/apache/kylin/rest/controller/QueryController.java
--
diff --git
a/server-base/src/main/java/org/apache/kylin/rest/controller/QueryController.java
b/server-base/src/main/java/org/apache/kylin/rest/controller/QueryController.java
index 99447a1..b6e2d45 100644
---
a/server-base/src/main/java/org/apache/kylin/rest/controller/QueryController.java
+++
b/server-base/src/main/java/org/apache/kylin/rest/controller/QueryController.java
@@ -39,6 +39,7 @@ import org.apache.kylin.rest.service.QueryService;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.security.core.context.SecurityContextHolder;
import org.springframework.stereotype.Controller;
import org.springframework.web.bind.annotation.PathVariable;
@@ -61,9 +62,10 @@ public class QueryController extends BasicController {
private static final Logger logger =
LoggerFactory.getLogger(QueryController.class);
@Autowired
+@Qualifier("queryService")
private QueryService queryService;
-@RequestMapping(value = "/query", method = RequestMethod.POST)
+@RequestMapping(value = "/query", method = RequestMethod.POST, produces =
{ "application/json" })
@ResponseBody
public SQLResponse query(@RequestBody SQLRequest sqlRequest) {
return queryService.doQueryWithCache(sqlRequest);
@@ -76,7 +78,7 @@ public class QueryController extends BasicController {
return queryService.doQueryWithCache(sqlRequest);
}
-@RequestMapping(value = "/saved_queries", method = RequestMethod.POST)
+@RequestMapping(value = "/saved_queries", method = RequestMethod.POST,
produces = { "application/json" })
@ResponseBody
public void saveQuery(@RequestBody SaveSqlRequest sqlRequest) throws
IOException {
String creator =
SecurityContextHolder.getContext().getAuthentication().getName();
@@ -85,21 +87,21 @@ public class QueryController extends BasicController {
queryService.saveQuery(creator, newQuery);
}
-@RequestMapping(value = "/saved_queries/{id}", method =
RequestMethod.DELETE)
+@RequestMapping(value = "/saved_queries/{id}", method =
RequestMethod.DELETE, produces = { "application/json" })
@ResponseBody
public void removeQuery(@PathVariable String id) throws IOException {
String creator =
SecurityContextHolder.getContext().getAuthentication().getName();
queryService.removeQuery(creator, id);
}
-@RequestMapping(value = "/saved_queries", method = RequestMethod.GET)
+@RequestMapping(value = "/saved_queries", method = RequestMethod.GET,
produces = { "application/json" })
@ResponseBody
public List getQueries() throws IOException {
String creator =
SecurityContextHolder.getContext().getAuthentication().getName();
return queryService.getQueries(creator);
}
-@RequestMapping(value = "/query/format/{format}", method =
RequestMethod.GET)
+@RequestMapping(value = "/query/format/{format}", method =
RequestMethod.GET, produces = { "application/json" })
@ResponseBody
public void downloadQueryResult(@PathVariable String format, SQLRequest
sqlRequest, HttpServletResponse response) {
SQLResponse result = queryService.doQueryWithCache(sqlRequest);
@@ -129,7 +131,7 @@ public class QueryController extends BasicController {
}
}
-@RequestMapping(value = "/tables_and_columns", method = RequestMethod.GET)
+@RequestMapping(value = "/tables_and_columns", method = RequestMethod.GET,
produces = { "application/json" })
@ResponseBody
public List getMetadata(MetaRequest metaRequest) {
try {
http://git-wip-us.apache.org/repos/asf/kylin/blob/73a78dbe/server-base/src/main/java/org/apache/kylin/rest/controller/StreamingController.java
--
diff --git
a/server-base/src/main/java/org/apache/kylin/rest/controller/StreamingController.java
b/server-base/src/main/java/org/apache/kylin/rest/controller/StreamingController.java
index 407ee2e..b0bb02a 100644
---
a/server-base/src/main/java/org/apache/kylin/rest/controller/StreamingController.java
+++
b/server-base/src/main/java/org/apache/kylin/rest/controller/StreamingController.java
@@ -39,6 +39,7 @@ import org.apache.kylin.source.kafka.config.KafkaConfig;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.security.access.AccessDeniedException;
import org.springframework.stereotype.Controller;
import org.springframework.web.bind.annotation.PathVariable;
@@ -63,13 +64,18
[38/50] [abbrv] kylin git commit: KYLIN-2641 - The current selected project is incorrect after we delete a project.
KYLIN-2641 - The current selected project is incorrect after we delete a
project.
Signed-off-by: Billy Liu
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/9fef04d9
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/9fef04d9
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/9fef04d9
Branch: refs/heads/KYLIN-2606
Commit: 9fef04d9a98e11454d00247d8cd2f4d501ba66a2
Parents: 81ed301
Author: 10069681
Authored: Tue May 23 18:34:26 2017 +0800
Committer: Billy Liu
Committed: Tue May 23 20:02:21 2017 +0800
--
webapp/app/js/model/projectModel.js | 18 --
1 file changed, 12 insertions(+), 6 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/9fef04d9/webapp/app/js/model/projectModel.js
--
diff --git a/webapp/app/js/model/projectModel.js
b/webapp/app/js/model/projectModel.js
index d2d32b4..77f47e8 100644
--- a/webapp/app/js/model/projectModel.js
+++ b/webapp/app/js/model/projectModel.js
@@ -21,7 +21,6 @@ KylinApp.service('ProjectModel', function () {
this.projects = [];
this.selectedProject = "_null";
-
this.setSelectedProject = function (project) {
var _projects =[];
angular.forEach(this.projects,function(pro){
@@ -31,6 +30,7 @@ KylinApp.service('ProjectModel', function () {
this.selectedProject = project;
}
};
+
this.getSelectedProject = function (project) {
if (this.selectedProject == "_null") {
return null;
@@ -56,10 +56,10 @@ KylinApp.service('ProjectModel', function () {
this.sortProjects();
}
- this.removeProject = function (project) {
+ this.removeProject = function (projectName) {
var index = -1;
for (var i = 0; i < this.projects.length; i++) {
- if (this.projects[i].name == project) {
+ if (this.projects[i].name == projectName) {
index = i;
break;
}
@@ -67,7 +67,15 @@ KylinApp.service('ProjectModel', function () {
if (index > -1) {
this.projects.splice(index, 1);
}
-this.selectedProject = this.projects[0];
+
+if (this.selectedProject == projectName) {
+if (this.projects.length) {
+ this.selectedProject = this.projects[0].name;
+} else {
+ this.selectedProject = "_null";
+}
+}
+
this.sortProjects();
}
@@ -96,7 +104,6 @@ KylinApp.service('ProjectModel', function () {
}
}
}
-;
return this.getSelectedProject();
}
@@ -124,7 +131,6 @@ KylinApp.service('ProjectModel', function () {
}
}
return index;
-
}
})
[34/50] [abbrv] kylin git commit: KYLIN-2640 refactor ISource API
KYLIN-2640 refactor ISource API
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/0a0edfef
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/0a0edfef
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/0a0edfef
Branch: refs/heads/KYLIN-2606
Commit: 0a0edfefb96689ba06bb89a657d7c65711894ebb
Parents: 4766c78
Author: Li Yang
Authored: Tue May 23 14:55:09 2017 +0800
Committer: hongbin ma
Committed: Tue May 23 17:30:18 2017 +0800
--
.../apache/kylin/common/KylinConfigBase.java| 4 +
.../java/org/apache/kylin/cube/CubeManager.java | 24 +-
.../java/org/apache/kylin/cube/CubeSegment.java | 14 +-
.../kylin/cube/cli/DictionaryGeneratorCLI.java | 8 +-
.../org/apache/kylin/cube/util/CubingUtils.java | 4 +-
.../org/apache/kylin/dict/DictionaryInfo.java | 2 +-
.../apache/kylin/dict/DictionaryManager.java| 14 +-
.../dict/DistinctColumnValuesProvider.java | 4 +-
.../kylin/dict/TableColumnValueEnumerator.java | 6 +-
.../dict/TableColumnValueSortedEnumerator.java | 14 +-
.../kylin/dict/lookup/LookupStringTable.java| 4 +-
.../apache/kylin/dict/lookup/LookupTable.java | 8 +-
.../kylin/dict/lookup/SnapshotManager.java | 8 +-
.../apache/kylin/dict/lookup/SnapshotTable.java | 8 +-
.../apache/kylin/dict/MockupReadableTable.java | 8 +-
.../org/apache/kylin/source/IReadableTable.java | 146
.../java/org/apache/kylin/source/ISource.java | 29 ++-
.../kylin/source/ISourceMetadataExplorer.java | 36 +++
.../org/apache/kylin/source/ReadableTable.java | 145
.../org/apache/kylin/source/SourceFactory.java | 15 +-
.../apache/kylin/source/SourcePartition.java| 8 +
.../storage/gtrecord/CubeTupleConverter.java| 4 +-
.../apache/kylin/engine/mr/DFSFileTable.java| 4 +-
.../kylin/engine/mr/DFSFileTableReader.java | 2 +-
.../org/apache/kylin/engine/mr/IMRInput.java| 6 +-
.../kylin/engine/mr/SortedColumnDFSFile.java| 4 +-
.../engine/mr/SortedColumnDFSFileReader.java| 16 +-
.../engine/mr/steps/CreateDictionaryJob.java| 4 +-
.../engine/mr/steps/MergeCuboidMapperTest.java | 2 +-
.../engine/spark/KylinKryoRegistrator.java | 2 +-
.../kylin/cube/ITDictionaryManagerTest.java | 4 +-
.../kylin/provision/BuildCubeWithEngine.java| 4 +-
.../kylin/provision/BuildCubeWithStream.java| 4 +-
.../hive/ITHiveSourceTableLoaderTest.java | 29 ++-
.../source/hive/ITSnapshotManagerTest.java | 6 +-
server-base/pom.xml | 2 +
.../kylin/rest/controller/CubeController.java | 2 +-
.../kylin/rest/controller/TableController.java | 2 +-
.../rest/controller2/TableControllerV2.java | 8 +-
.../kylin/rest/job/StorageCleanupJob.java | 8 +-
.../apache/kylin/rest/service/JobService.java | 4 +-
.../rest/service/TableSchemaUpdateChecker.java | 207
.../apache/kylin/rest/service/TableService.java | 105 +++--
.../kylin/rest/service/TableServiceV2.java | 12 +-
.../kylin/source/hive/HiveMetadataExplorer.java | 116 +
.../apache/kylin/source/hive/HiveSource.java| 38 +--
.../source/hive/HiveSourceTableLoader.java | 147
.../org/apache/kylin/source/hive/HiveTable.java | 4 +-
.../kylin/source/hive/HiveTableReader.java | 2 +-
.../apache/kylin/source/hive/SchemaChecker.java | 234 ---
.../apache/kylin/source/kafka/KafkaSource.java | 46 +++-
51 files changed, 816 insertions(+), 721 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/0a0edfef/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
--
diff --git
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 8930129..abe142a 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -474,6 +474,10 @@ abstract public class KylinConfigBase implements
Serializable {
// SOURCE.HIVE
//
+public int getDefaultSource() {
+return Integer.parseInt(getOptional("kylin.source.default", "0"));
+}
+
public Map getSourceEngines() {
Map r = Maps.newLinkedHashMap();
// ref constants in ISourceAware
http://git-wip-us.apache.org/repos/asf/kylin/blob/0a0edfef/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
--
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
[11/50] [abbrv] kylin git commit: KYLIN-2624 introduce StorageURL and set timeout for metadata access
KYLIN-2624 introduce StorageURL and set timeout for metadata access
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/2dfc5bb3
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/2dfc5bb3
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/2dfc5bb3
Branch: refs/heads/KYLIN-2606
Commit: 2dfc5bb36a00245b95deeebb5bb3bca9c353c934
Parents: 7c152c5
Author: Li Yang
Authored: Thu May 18 14:16:17 2017 +0800
Committer: nichunen
Committed: Fri May 19 13:52:53 2017 +0800
--
.../apache/kylin/common/KylinConfigBase.java| 30 ++-
.../org/apache/kylin/common/StorageURL.java | 183 +++
.../common/persistence/FileResourceStore.java | 2 +-
.../apache/kylin/common/KylinConfigTest.java| 9 +-
.../org/apache/kylin/common/StorageURLTest.java | 103 +++
.../persistence/LocalFileResourceStoreTest.java | 3 +-
.../common/persistence/ResourceStoreTest.java | 18 +-
.../common/util/LocalFileMetadataTestCase.java | 7 +
.../apache/kylin/dict/DictionaryManager.java| 18 +-
.../dict/global/AppendTrieDictionaryTest.java | 5 +-
.../impl/threadpool/DistributedScheduler.java | 11 +-
.../apache/kylin/storage/StorageContext.java| 7 +-
.../kylin/engine/mr/steps/CubeReducerTest.java | 2 +-
.../engine/mr/steps/NDCuboidMapperTest.java | 2 +-
.../kylin/job/BaseTestDistributedScheduler.java | 5 +-
.../kylin/storage/hbase/ITStorageTest.java | 3 +-
.../apache/kylin/query/schema/OLAPSchema.java | 5 +-
.../rest/security/MockAclHBaseStorage.java | 11 +-
.../rest/security/RealAclHBaseStorage.java | 15 +-
.../apache/kylin/rest/service/QueryService.java | 11 +-
.../kylin/storage/hbase/HBaseConnection.java| 51 +++---
.../kylin/storage/hbase/HBaseResourceStore.java | 46 +++--
.../storage/hbase/util/CubeMigrationCLI.java| 20 +-
.../hbase/util/GridTableHBaseBenchmark.java | 3 +-
.../kylin/storage/hdfs/HDFSResourceStore.java | 50 ++---
.../hbase/steps/CubeHFileMapper2Test.java | 4 +-
.../storage/hbase/steps/MockupMapContext.java | 2 +-
.../org/apache/kylin/tool/CubeMigrationCLI.java | 20 +-
28 files changed, 461 insertions(+), 185 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/2dfc5bb3/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
--
diff --git
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index b620924..caef95e 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -169,7 +169,7 @@ abstract public class KylinConfigBase implements
Serializable {
}
public String toString() {
-return getMetadataUrl();
+return getMetadataUrl().toString();
}
//
@@ -205,8 +205,8 @@ abstract public class KylinConfigBase implements
Serializable {
// METADATA
//
-public String getMetadataUrl() {
-return getOptional("kylin.metadata.url");
+public StorageURL getMetadataUrl() {
+return StorageURL.valueOf(getOptional("kylin.metadata.url", ""));
}
// for test only
@@ -215,14 +215,7 @@ abstract public class KylinConfigBase implements
Serializable {
}
public String getMetadataUrlPrefix() {
-String metadataUrl = getMetadataUrl();
-String defaultPrefix = "kylin_metadata";
-
-int cut = metadataUrl.lastIndexOf('@');
-if (cut > 0) {
-return metadataUrl.substring(0, cut);
-}
-return defaultPrefix;
+return getMetadataUrl().getIdentifier();
}
public String[] getRealizationProviders() {
@@ -558,8 +551,14 @@ abstract public class KylinConfigBase implements
Serializable {
return Integer.parseInt(getOptional("kylin.storage.default", "2"));
}
-public String getStorageUrl() {
-return getOptional("kylin.storage.url");
+public StorageURL getStorageUrl() {
+String url = getOptional("kylin.storage.url", "default@hbase");
+
+// for backward compatibility
+if ("hbase".equals(url))
+url = "default@hbase";
+
+return StorageURL.valueOf(url);
}
public String getHBaseClusterFs() {
@@ -987,10 +986,7 @@ abstract public class KylinConfigBase implements
Serializable {
}
public String getResourceStoreImpl() {
-String metadataUrl = KylinConfig.getInstanceFromEnv().getMetadataUrl();
-
[13/50] [abbrv] kylin git commit: KAP-733 minor UT fix
KAP-733 minor UT fix
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/8ce972fd
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/8ce972fd
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/8ce972fd
Branch: refs/heads/KYLIN-2606
Commit: 8ce972fd6475e923af2a5e525d8ddb698d0dbadf
Parents: 2dfc5bb
Author: Li Yang
Authored: Fri May 19 15:50:32 2017 +0800
Committer: liyang-gmt8
Committed: Fri May 19 17:25:55 2017 +0800
--
.../apache/kylin/common/util/LocalFileMetadataTestCase.java| 6 +-
1 file changed, 5 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/8ce972fd/core-common/src/test/java/org/apache/kylin/common/util/LocalFileMetadataTestCase.java
--
diff --git
a/core-common/src/test/java/org/apache/kylin/common/util/LocalFileMetadataTestCase.java
b/core-common/src/test/java/org/apache/kylin/common/util/LocalFileMetadataTestCase.java
index e5e4f8b..a44a55d 100644
---
a/core-common/src/test/java/org/apache/kylin/common/util/LocalFileMetadataTestCase.java
+++
b/core-common/src/test/java/org/apache/kylin/common/util/LocalFileMetadataTestCase.java
@@ -81,7 +81,11 @@ public class LocalFileMetadataTestCase extends
AbstractKylinTestCase {
String dir =
KylinConfig.getInstanceFromEnv().getHdfsWorkingDirectory();
if (dir.startsWith("file://"))
dir = dir.substring("file://".length());
-return dir;
+try {
+return new File(dir).getCanonicalPath();
+} catch (IOException e) {
+throw new RuntimeException(e);
+}
}
protected ResourceStore getStore() {
[15/50] [abbrv] kylin git commit: KYLIN-2624 fix IT with refactored DistributedLock
KYLIN-2624 fix IT with refactored DistributedLock
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/be32f758
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/be32f758
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/be32f758
Branch: refs/heads/KYLIN-2606
Commit: be32f75883c21bf32545d1b06158ad7cf36d676b
Parents: f36372b
Author: Yang Li
Authored: Fri May 19 22:02:38 2017 +0800
Committer: Yang Li
Committed: Fri May 19 22:02:38 2017 +0800
--
.../job/impl/threadpool/DistributedScheduler.java | 1 +
.../hbase/util/ZookeeperDistributedLock.java| 16
2 files changed, 13 insertions(+), 4 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/be32f758/core-job/src/main/java/org/apache/kylin/job/impl/threadpool/DistributedScheduler.java
--
diff --git
a/core-job/src/main/java/org/apache/kylin/job/impl/threadpool/DistributedScheduler.java
b/core-job/src/main/java/org/apache/kylin/job/impl/threadpool/DistributedScheduler.java
index d01cb5a..8812dad 100644
---
a/core-job/src/main/java/org/apache/kylin/job/impl/threadpool/DistributedScheduler.java
+++
b/core-job/src/main/java/org/apache/kylin/job/impl/threadpool/DistributedScheduler.java
@@ -319,6 +319,7 @@ public class DistributedScheduler implements
Scheduler, Conn
}
return path;
}
+
@Override
public void shutdown() throws SchedulerException {
logger.info("Will shut down Job Engine ");
http://git-wip-us.apache.org/repos/asf/kylin/blob/be32f758/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/ZookeeperDistributedLock.java
--
diff --git
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/ZookeeperDistributedLock.java
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/ZookeeperDistributedLock.java
index d181d81..dc2a23e 100644
---
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/ZookeeperDistributedLock.java
+++
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/ZookeeperDistributedLock.java
@@ -281,10 +281,18 @@ public class ZookeeperDistributedLock implements
DistributedLock, JobLock {
// normalize lock path
private String norm(String lockPath) {
-if (lockPath.startsWith(zkPathBase))
-return lockPath;
-else
-return zkPathBase + (lockPath.startsWith("/") ? "" : "/") +
lockPath;
+if (!lockPath.startsWith(zkPathBase))
+lockPath = zkPathBase + (lockPath.startsWith("/") ? "" : "/") +
lockPath;
+
+return dropDoubleSlash(lockPath);
+}
+
+public static String dropDoubleSlash(String path) {
+for (int n = Integer.MAX_VALUE; n > path.length();) {
+n = path.length();
+path = path.replace("//", "/");
+}
+return path;
}
//
[44/50] [abbrv] kylin git commit: KYLIN-2631 Seek to next model when no cube in current model satisfies query
KYLIN-2631 Seek to next model when no cube in current model satisfies query
KYLIN-2631 bug fix
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/f870c6c4
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/f870c6c4
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/f870c6c4
Branch: refs/heads/KYLIN-2606
Commit: f870c6c4d64e4aa2ae5f3495534d97e04466b843
Parents: ada37a9
Author: Hongbin Ma
Authored: Fri May 19 21:12:26 2017 +0800
Committer: Hongbin Ma
Committed: Tue May 23 20:30:03 2017 +0800
--
.../apache/kylin/metadata/model/TblColRef.java | 20 +-
.../kylin/metadata/realization/SQLDigest.java | 2 +
.../gtrecord/GTCubeStorageQueryBase.java| 3 -
.../kylin/query/relnode/OLAPTableScan.java | 13 +-
.../relnode/OLAPToEnumerableConverter.java | 11 +-
.../kylin/query/routing/ModelChooser.java | 219 --
.../apache/kylin/query/routing/QueryRouter.java | 2 +-
.../kylin/query/routing/RealizationChooser.java | 224 +++
8 files changed, 260 insertions(+), 234 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/f870c6c4/core-metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java
--
diff --git
a/core-metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java
b/core-metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java
index aa4c056..3bca61c 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java
@@ -87,8 +87,12 @@ public class TblColRef implements Serializable {
checkArgument(col.table.getModel() == UNKNOWN_MODEL ||
col.table.getModel() == model);
TableRef tableRef = model.findTable(alias);
checkArgument(tableRef.getTableDesc().getIdentity().equals(col.column.getTable().getIdentity()));
-col.table = tableRef;
-col.identity = null;
+checkArgument(tableRef.getTableDesc() == col.column.getTable());
+col.fixTableRef(tableRef);
+}
+
+public static void unfixUnknownModel(TblColRef col) {
+col.unfixTableRef();
}
// for test mainly
@@ -105,6 +109,7 @@ public class TblColRef implements Serializable {
//
private TableRef table;
+private TableRef backupTable;// only used in fixTableRef()
private ColumnDesc column;
private String identity;
private String parserDescription;
@@ -119,6 +124,17 @@ public class TblColRef implements Serializable {
this.column = column;
}
+public void fixTableRef(TableRef tableRef) {
+this.backupTable = this.table;
+this.table = tableRef;
+this.identity = null;
+}
+
+public void unfixTableRef() {
+this.table = backupTable;
+this.identity = null;
+}
+
public ColumnDesc getColumnDesc() {
return column;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/f870c6c4/core-metadata/src/main/java/org/apache/kylin/metadata/realization/SQLDigest.java
--
diff --git
a/core-metadata/src/main/java/org/apache/kylin/metadata/realization/SQLDigest.java
b/core-metadata/src/main/java/org/apache/kylin/metadata/realization/SQLDigest.java
index 03ff3ff..9ce65bb 100644
---
a/core-metadata/src/main/java/org/apache/kylin/metadata/realization/SQLDigest.java
+++
b/core-metadata/src/main/java/org/apache/kylin/metadata/realization/SQLDigest.java
@@ -96,6 +96,8 @@ public class SQLDigest {
this.sortOrders = sortOrders;
this.isRawQuery = isRawQuery();
this.limitPrecedesAggr = limitPrecedesAggr;
+
+this.includeSubqueryJoinParticipants();
}
private boolean isRawQuery() {
http://git-wip-us.apache.org/repos/asf/kylin/blob/f870c6c4/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
--
diff --git
a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
index 5faa098..176d73c 100644
---
a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
+++
b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
@@ -99,9 +99,6 @@ public abstract class GTCubeStorageQueryBase implements
IStorageQuery {
protected GTCubeStorageQueryRequest getStorageQueryRequest(StorageContext
context, SQLDigest sqlDigest, TupleInfo re
[06/50] [abbrv] kylin git commit: KYLIN-2621 The user of the LDAP group named admin always has ROLE_ADMIN permission
KYLIN-2621 The user of the LDAP group named admin always has ROLE_ADMIN
permission
Signed-off-by: Billy Liu
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/a34db26f
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/a34db26f
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/a34db26f
Branch: refs/heads/KYLIN-2606
Commit: a34db26f0bd594b55fd4df29b66f15579bb3c8ff
Parents: 858d947
Author: 10069681
Authored: Mon May 15 20:31:26 2017 +0800
Committer: Billy Liu
Committed: Wed May 17 16:23:12 2017 +0800
--
.../rest/security/AuthoritiesPopulator.java | 20 ++--
1 file changed, 10 insertions(+), 10 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/a34db26f/server-base/src/main/java/org/apache/kylin/rest/security/AuthoritiesPopulator.java
--
diff --git
a/server-base/src/main/java/org/apache/kylin/rest/security/AuthoritiesPopulator.java
b/server-base/src/main/java/org/apache/kylin/rest/security/AuthoritiesPopulator.java
index 2b290ce..592791c 100644
---
a/server-base/src/main/java/org/apache/kylin/rest/security/AuthoritiesPopulator.java
+++
b/server-base/src/main/java/org/apache/kylin/rest/security/AuthoritiesPopulator.java
@@ -52,8 +52,11 @@ public class AuthoritiesPopulator extends
DefaultLdapAuthoritiesPopulator {
this.adminRoleAsAuthority = new SimpleGrantedAuthority(adminRole);
String[] defaultRoles = StringUtils.split(defaultRole, ",");
-if (ArrayUtils.contains(defaultRoles, Constant.ROLE_MODELER))
+if (ArrayUtils.contains(defaultRoles, Constant.ROLE_MODELER)) {
this.defaultAuthorities.add(modelerAuthority);
+this.defaultAuthorities.add(analystAuthority);
+}
+
if (ArrayUtils.contains(defaultRoles, Constant.ROLE_ANALYST))
this.defaultAuthorities.add(analystAuthority);
}
@@ -62,19 +65,16 @@ public class AuthoritiesPopulator extends
DefaultLdapAuthoritiesPopulator {
public Set getGroupMembershipRoles(String userDn, String
username) {
Set authorities =
super.getGroupMembershipRoles(userDn, username);
-authorities.addAll(defaultAuthorities);
+Set userAuthorities = new
HashSet();
+userAuthorities.addAll(defaultAuthorities);
if (authorities.contains(adminRoleAsAuthority)) {
-authorities.add(adminAuthority);
-authorities.add(modelerAuthority);
-authorities.add(analystAuthority);
-}
-
-if (authorities.contains(modelerAuthority)) {
-authorities.add(analystAuthority);
+userAuthorities.add(adminAuthority);
+userAuthorities.add(modelerAuthority);
+userAuthorities.add(analystAuthority);
}
-return authorities;
+return userAuthorities;
}
}
[20/50] [abbrv] kylin git commit: KYLIN-2459 fix implementation
KYLIN-2459 fix implementation
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/429dfbdc
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/429dfbdc
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/429dfbdc
Branch: refs/heads/KYLIN-2606
Commit: 429dfbdc93371e8df2095e35298425ccab8ac7de
Parents: 1b864f0
Author: Yang Li
Authored: Sat May 20 19:52:30 2017 +0800
Committer: Roger Shi
Committed: Sat May 20 21:35:36 2017 +0800
--
.../apache/kylin/common/KylinConfigBase.java| 36 ++--
.../kylin/common/persistence/ResourceStore.java | 45 ++--
.../common/persistence/ResourceStoreTest.java | 11 +
.../common/util/LocalFileMetadataTestCase.java | 7 +--
4 files changed, 32 insertions(+), 67 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/429dfbdc/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
--
diff --git
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index df3baf4..8930129 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -218,6 +218,16 @@ abstract public class KylinConfigBase implements
Serializable {
return getMetadataUrl().getIdentifier();
}
+public Map getResourceStoreImpls() {
+Map r = Maps.newLinkedHashMap();
+// ref constants in ISourceAware
+r.put("", "org.apache.kylin.common.persistence.FileResourceStore");
+r.put("hbase", "org.apache.kylin.storage.hbase.HBaseResourceStore");
+r.put("hdfs", "org.apache.kylin.storage.hdfs.HDFSResourceStore");
+
r.putAll(getPropertiesByPrefix("kylin.metadata.resource-store-provider.")); //
note the naming convention --
http://kylin.apache.org/development/coding_naming_convention.html
+return r;
+}
+
public String[] getRealizationProviders() {
return getOptionalStringArray("kylin.metadata.realization-providers",
//
new String[] { "org.apache.kylin.cube.CubeManager",
"org.apache.kylin.storage.hybrid.HybridManager" });
@@ -456,6 +466,10 @@ abstract public class KylinConfigBase implements
Serializable {
return getOptional("kylin.job.advanced-flat-table.class");
}
+public String getJobTrackingURLPattern() {
+return getOptional("kylin.job.tracking-url-pattern", "");
+}
+
//
// SOURCE.HIVE
//
@@ -970,26 +984,4 @@ abstract public class KylinConfigBase implements
Serializable {
return
Boolean.parseBoolean(getOptional("kylin.web.cross-domain-enabled", "true"));
}
-//ResourceStore Impl
-/*public String getResourceStoreImpl() {
-return getOptional("kylin.metadata.default-resource-store-impl",
"org.apache.kylin.storage.hbase.HBaseResourceStore");
-}*/
-
-public Map getResourceStoreImpls() {
-Map r = Maps.newLinkedHashMap();
-// ref constants in ISourceAware
-r.put("", "org.apache.kylin.common.persistence.FileResourceStore");
-r.put("hbase", "org.apache.kylin.storage.hbase.HBaseResourceStore");
-r.put("hdfs", "org.apache.kylin.storage.hdfs.HDFSResourceStore");
-r.putAll(getPropertiesByPrefix("kylin.resource.store.provider."));
-return r;
-}
-
-public String getResourceStoreImpl() {
-return getResourceStoreImpls().get(getMetadataUrl().getScheme());
-}
-
-public String getJobTrackingURLPattern() {
-return getOptional("kylin.job.tracking-url-pattern", "");
-}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/429dfbdc/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java
--
diff --git
a/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java
b/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java
index 0565c66..7fb93e7 100644
---
a/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java
+++
b/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java
@@ -39,6 +39,7 @@ import java.util.concurrent.ConcurrentMap;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.IOUtils;
import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.StorageURL;
import org.apache.kylin.common.util.ClassUtil;
import org.apache.kylin.common.util.Optio
[08/50] [abbrv] kylin git commit: KYLIN-2635 fix IT
KYLIN-2635 fix IT
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/5f52cea4
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/5f52cea4
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/5f52cea4
Branch: refs/heads/KYLIN-2606
Commit: 5f52cea4f7ffbb0fa885df650803156cd1daa5d2
Parents: ea5caba
Author: Hongbin Ma
Authored: Wed May 17 21:26:09 2017 +0800
Committer: Roger Shi
Committed: Thu May 18 11:02:49 2017 +0800
--
.../org/apache/kylin/metadata/filter/CompareTupleFilter.java | 6 ++
1 file changed, 6 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/5f52cea4/core-metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java
--
diff --git
a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java
b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java
index 2771250..c091c7c 100755
---
a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java
+++
b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java
@@ -167,7 +167,13 @@ public class CompareTupleFilter extends TupleFilter {
return true;
else
return false;
+} else {
+if (operator == FilterOperatorEnum.ISNOTNULL)
+return true;
+else if (operator == FilterOperatorEnum.ISNULL)
+return false;
}
+
if (cs.isNull(firstCondValue)) {
return false;
}
[33/50] [abbrv] kylin git commit: KYLIN-2640 refactor ISource API
http://git-wip-us.apache.org/repos/asf/kylin/blob/0a0edfef/server-base/src/main/java/org/apache/kylin/rest/service/TableSchemaUpdateChecker.java
--
diff --git
a/server-base/src/main/java/org/apache/kylin/rest/service/TableSchemaUpdateChecker.java
b/server-base/src/main/java/org/apache/kylin/rest/service/TableSchemaUpdateChecker.java
new file mode 100644
index 000..a87ddd8
--- /dev/null
+++
b/server-base/src/main/java/org/apache/kylin/rest/service/TableSchemaUpdateChecker.java
@@ -0,0 +1,207 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.rest.service;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+import static java.lang.String.format;
+
+import java.util.List;
+import java.util.Set;
+
+import javax.annotation.Nullable;
+
+import org.apache.kylin.cube.CubeInstance;
+import org.apache.kylin.cube.CubeManager;
+import org.apache.kylin.metadata.MetadataManager;
+import org.apache.kylin.metadata.model.ColumnDesc;
+import org.apache.kylin.metadata.model.DataModelDesc;
+import org.apache.kylin.metadata.model.TableDesc;
+import org.apache.kylin.metadata.model.TblColRef;
+
+import com.google.common.base.Predicate;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Sets;
+
+public class TableSchemaUpdateChecker {
+private final MetadataManager metadataManager;
+private final CubeManager cubeManager;
+
+static class CheckResult {
+private final boolean valid;
+private final String reason;
+
+private CheckResult(boolean valid, String reason) {
+this.valid = valid;
+this.reason = reason;
+}
+
+void raiseExceptionWhenInvalid() {
+if (!valid) {
+throw new RuntimeException(reason);
+}
+}
+
+static CheckResult validOnFirstLoad(String tableName) {
+return new CheckResult(true, format("Table '%s' hasn't been loaded
before", tableName));
+}
+
+static CheckResult validOnCompatibleSchema(String tableName) {
+return new CheckResult(true, format("Table '%s' is compatible with
all existing cubes", tableName));
+}
+
+static CheckResult invalidOnFetchSchema(String tableName, Exception e)
{
+return new CheckResult(false, format("Failed to fetch metadata of
'%s': %s", tableName, e.getMessage()));
+}
+
+static CheckResult invalidOnIncompatibleSchema(String tableName,
List reasons) {
+StringBuilder buf = new StringBuilder();
+for (String reason : reasons) {
+buf.append("- ").append(reason).append("\n");
+}
+
+return new CheckResult(false, format("Found %d issue(s) with
'%s':%n%s Please disable and purge related cube(s) first", reasons.size(),
tableName, buf.toString()));
+}
+}
+
+TableSchemaUpdateChecker(MetadataManager metadataManager, CubeManager
cubeManager) {
+this.metadataManager = checkNotNull(metadataManager, "metadataManager
is null");
+this.cubeManager = checkNotNull(cubeManager, "cubeManager is null");
+}
+
+private List findCubeByTable(final String fullTableName) {
+Iterable relatedCubes =
Iterables.filter(cubeManager.listAllCubes(), new Predicate() {
+@Override
+public boolean apply(@Nullable CubeInstance cube) {
+if (cube == null || cube.allowBrokenDescriptor()) {
+return false;
+}
+DataModelDesc model = cube.getModel();
+if (model == null)
+return false;
+return model.containsTable(fullTableName);
+}
+});
+
+return ImmutableList.copyOf(relatedCubes);
+}
+
+private boolean isColumnCompatible(ColumnDesc column, ColumnDesc newCol) {
+if (!column.getName().equalsIgnoreCase(newCol.getName())) {
+return false;
+}
+
+if (column.getType().isIntegerFamily()) {
+// OLAPTable.l
[50/50] [abbrv] kylin git commit: KYLIN-2606 Only return counter for precise count_distinct if query is exactAggregate
KYLIN-2606 Only return counter for precise count_distinct if query is
exactAggregate
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/d610a6dd
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/d610a6dd
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/d610a6dd
Branch: refs/heads/KYLIN-2606
Commit: d610a6dd50b547c0f1f4d1529aaf626760c66549
Parents: fdacf5d
Author: kangkaisen
Authored: Wed Feb 15 19:53:17 2017 +0800
Committer: kangkaisen
Committed: Wed May 24 15:36:38 2017 +0800
--
.../kylin/cube/gridtable/CubeCodeSystem.java| 4 +
.../org/apache/kylin/gridtable/GTRecord.java| 8 +
.../kylin/gridtable/GTSampleCodeSystem.java | 4 +
.../apache/kylin/gridtable/IGTCodeSystem.java | 3 +
.../measure/bitmap/BitmapCounterFactory.java| 2 +
.../kylin/measure/bitmap/BitmapSerializer.java | 37 -
.../measure/bitmap/RoaringBitmapCounter.java| 10 ++
.../bitmap/RoaringBitmapCounterFactory.java | 5 +
.../metadata/datatype/DataTypeSerializer.java | 13 ++
.../gtrecord/GTCubeStorageQueryBase.java| 47 ++
.../hbase/cube/v2/CubeHBaseEndpointRPC.java | 1 +
.../storage/hbase/cube/v2/CubeHBaseScanRPC.java | 2 +-
.../hbase/cube/v2/HBaseReadonlyStore.java | 33 +++-
.../coprocessor/endpoint/CubeVisitService.java | 2 +-
.../endpoint/generated/CubeVisitProtos.java | 151 +++
.../endpoint/protobuf/CubeVisit.proto | 1 +
16 files changed, 286 insertions(+), 37 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/d610a6dd/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CubeCodeSystem.java
--
diff --git
a/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CubeCodeSystem.java
b/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CubeCodeSystem.java
index aaa12a7..9eae6f3 100644
---
a/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CubeCodeSystem.java
+++
b/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CubeCodeSystem.java
@@ -177,4 +177,8 @@ public class CubeCodeSystem implements IGTCodeSystem {
return result;
}
+@Override
+public DataTypeSerializer getSerializer(int col) {
+return serializers[col];
+}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/d610a6dd/core-cube/src/main/java/org/apache/kylin/gridtable/GTRecord.java
--
diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/GTRecord.java
b/core-cube/src/main/java/org/apache/kylin/gridtable/GTRecord.java
index 3e62ea7..f65e4b5 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTRecord.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTRecord.java
@@ -296,4 +296,12 @@ public class GTRecord implements Comparable,
Cloneable {
}
}
+/** change pointers to point to data in given buffer, this
+ * method allows to defined specific column to load */
+public void loadColumns(int selectedCol, ByteBuffer buf) {
+int pos = buf.position();
+int len = info.codeSystem.codeLength(selectedCol, buf);
+cols[selectedCol].set(buf.array(), buf.arrayOffset() + pos, len);
+}
+
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/d610a6dd/core-cube/src/main/java/org/apache/kylin/gridtable/GTSampleCodeSystem.java
--
diff --git
a/core-cube/src/main/java/org/apache/kylin/gridtable/GTSampleCodeSystem.java
b/core-cube/src/main/java/org/apache/kylin/gridtable/GTSampleCodeSystem.java
index 3f3c844..2a5e791 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTSampleCodeSystem.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTSampleCodeSystem.java
@@ -118,4 +118,8 @@ public class GTSampleCodeSystem implements IGTCodeSystem {
}
};
+@Override
+public DataTypeSerializer getSerializer(int col) {
+return serializers[col];
+}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/d610a6dd/core-cube/src/main/java/org/apache/kylin/gridtable/IGTCodeSystem.java
--
diff --git
a/core-cube/src/main/java/org/apache/kylin/gridtable/IGTCodeSystem.java
b/core-cube/src/main/java/org/apache/kylin/gridtable/IGTCodeSystem.java
index 89dfc99..9c8ad6b 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/IGTCodeSystem.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/IGTCodeSystem.java
@@ -23,6 +23,7 @@ import java.nio.ByteBuffer;
import org.apache.kylin.common.util.ImmutableBitSet;
import org.apache.kylin.dimension.DimensionEncoding;
import
[16/50] [abbrv] kylin git commit: KYLIN-2624 fix IT with refactored DistributedLock
KYLIN-2624 fix IT with refactored DistributedLock
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/d21c8170
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/d21c8170
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/d21c8170
Branch: refs/heads/KYLIN-2606
Commit: d21c8170b7bbc8f750aadab5c68d229dee6d252f
Parents: f36372b
Author: Hongbin Ma
Authored: Fri May 19 22:44:50 2017 +0800
Committer: hongbin ma
Committed: Fri May 19 22:48:41 2017 +0800
--
.../hbase/util/ZookeeperDistributedLock.java| 30 +---
1 file changed, 19 insertions(+), 11 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/d21c8170/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/ZookeeperDistributedLock.java
--
diff --git
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/ZookeeperDistributedLock.java
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/ZookeeperDistributedLock.java
index d181d81..ea64bbf 100644
---
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/ZookeeperDistributedLock.java
+++
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/ZookeeperDistributedLock.java
@@ -123,7 +123,7 @@ public class ZookeeperDistributedLock implements
DistributedLock, JobLock {
throw new NullPointerException("client must not be null");
if (zkPathBase == null)
throw new NullPointerException("zkPathBase must not be null");
-
+
this.curator = curator;
this.zkPathBase = zkPathBase;
this.client = client;
@@ -138,7 +138,7 @@ public class ZookeeperDistributedLock implements
DistributedLock, JobLock {
@Override
public boolean lock(String lockPath) {
lockPath = norm(lockPath);
-
+
logger.debug(client + " trying to lock " + lockPath);
try {
@@ -148,7 +148,7 @@ public class ZookeeperDistributedLock implements
DistributedLock, JobLock {
} catch (Exception ex) {
throw new RuntimeException("Error while " + client + " trying to
lock " + lockPath, ex);
}
-
+
String lockOwner = peekLock(lockPath);
if (client.equals(lockOwner)) {
logger.info(client + " acquired lock at " + lockPath);
@@ -214,7 +214,7 @@ public class ZookeeperDistributedLock implements
DistributedLock, JobLock {
public boolean isLockedByMe(String lockPath) {
return client.equals(peekLock(lockPath));
}
-
+
@Override
public void unlock(String lockPath) {
lockPath = norm(lockPath);
@@ -236,7 +236,7 @@ public class ZookeeperDistributedLock implements
DistributedLock, JobLock {
throw new RuntimeException("Error while " + client + " trying to
unlock " + lockPath, ex);
}
}
-
+
@Override
public void purgeLocks(String lockPathRoot) {
lockPathRoot = norm(lockPathRoot);
@@ -245,7 +245,7 @@ public class ZookeeperDistributedLock implements
DistributedLock, JobLock {
curator.delete().guaranteed().deletingChildrenIfNeeded().forPath(lockPathRoot);
logger.info(client + " purged all locks under " + lockPathRoot);
-
+
} catch (Exception ex) {
throw new RuntimeException("Error while " + client + " trying to
purge " + lockPathRoot, ex);
}
@@ -278,13 +278,21 @@ public class ZookeeperDistributedLock implements
DistributedLock, JobLock {
}
return cache;
}
-
+
// normalize lock path
private String norm(String lockPath) {
-if (lockPath.startsWith(zkPathBase))
-return lockPath;
-else
-return zkPathBase + (lockPath.startsWith("/") ? "" : "/") +
lockPath;
+if (!lockPath.startsWith(zkPathBase))
+lockPath = zkPathBase + (lockPath.startsWith("/") ? "" : "/") +
lockPath;
+
+return dropDoubleSlash(lockPath);
+}
+
+public static String dropDoubleSlash(String path) {
+for (int n = Integer.MAX_VALUE; n > path.length();) {
+n = path.length();
+path = path.replace("//", "/");
+}
+return path;
}
//
[27/50] [abbrv] kylin git commit: KYLIN-2589 fix MessageDigest not thread safe in KylinAuthenticationProvider
KYLIN-2589 fix MessageDigest not thread safe in KylinAuthenticationProvider
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/ce09d6c7
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/ce09d6c7
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/ce09d6c7
Branch: refs/heads/KYLIN-2606
Commit: ce09d6c7f027cdc55c10fb96ea21a70c4002f244
Parents: fd3bfed
Author: shaofengshi
Authored: Mon May 22 11:24:38 2017 +0800
Committer: shaofengshi
Committed: Mon May 22 11:25:13 2017 +0800
--
.../security/KylinAuthenticationProvider.java | 20 +++-
1 file changed, 7 insertions(+), 13 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/ce09d6c7/server-base/src/main/java/org/apache/kylin/rest/security/KylinAuthenticationProvider.java
--
diff --git
a/server-base/src/main/java/org/apache/kylin/rest/security/KylinAuthenticationProvider.java
b/server-base/src/main/java/org/apache/kylin/rest/security/KylinAuthenticationProvider.java
index d0dd06a..ffe9811 100644
---
a/server-base/src/main/java/org/apache/kylin/rest/security/KylinAuthenticationProvider.java
+++
b/server-base/src/main/java/org/apache/kylin/rest/security/KylinAuthenticationProvider.java
@@ -18,10 +18,9 @@
package org.apache.kylin.rest.security;
-import java.security.MessageDigest;
-import java.security.NoSuchAlgorithmException;
-import java.util.Arrays;
-
+import com.google.common.hash.HashFunction;
+import com.google.common.hash.Hashing;
+import org.apache.kylin.common.util.ByteArray;
import org.apache.kylin.rest.service.UserService;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -55,26 +54,21 @@ public class KylinAuthenticationProvider implements
AuthenticationProvider {
//Embedded authentication provider
private AuthenticationProvider authenticationProvider;
-MessageDigest md = null;
+private HashFunction hf = null;
public KylinAuthenticationProvider(AuthenticationProvider
authenticationProvider) {
super();
Assert.notNull(authenticationProvider, "The embedded
authenticationProvider should not be null.");
this.authenticationProvider = authenticationProvider;
-try {
-md = MessageDigest.getInstance("MD5");
-} catch (NoSuchAlgorithmException e) {
-throw new RuntimeException("Failed to init Message Digest ", e);
-}
+hf = Hashing.murmur3_128();
}
@Override
public Authentication authenticate(Authentication authentication) throws
AuthenticationException {
Authentication authed = null;
Cache userCache = cacheManager.getCache("UserCache");
-md.reset();
-byte[] hashKey = md.digest((authentication.getName() +
authentication.getCredentials()).getBytes());
-String userKey = Arrays.toString(hashKey);
+byte[] hashKey = hf.hashString(authentication.getName() +
authentication.getCredentials()).asBytes();
+ByteArray userKey = new ByteArray(hashKey);
Element authedUser = userCache.get(userKey);
if (null != authedUser) {
[43/50] [abbrv] kylin git commit: KYLIN-2630 NPE when a subquery joins another lookup tables
KYLIN-2630 NPE when a subquery joins another lookup tables
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/ada37a91
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/ada37a91
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/ada37a91
Branch: refs/heads/KYLIN-2606
Commit: ada37a9109678abf347d3600b3a19888d6f64056
Parents: 34a65ba
Author: Hongbin Ma
Authored: Thu May 18 20:54:45 2017 +0800
Committer: Hongbin Ma
Committed: Tue May 23 20:24:46 2017 +0800
--
.../resources/query/sql_subquery/query30.sql| 15 ++
.../resources/query/sql_subquery/query31.sql| 19
.../apache/kylin/query/relnode/OLAPJoinRel.java | 12
.../org/apache/kylin/query/relnode/OLAPRel.java | 31 +++-
.../kylin/query/relnode/OLAPTableScan.java | 18 ++--
5 files changed, 72 insertions(+), 23 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/ada37a91/kylin-it/src/test/resources/query/sql_subquery/query30.sql
--
diff --git a/kylin-it/src/test/resources/query/sql_subquery/query30.sql
b/kylin-it/src/test/resources/query/sql_subquery/query30.sql
new file mode 100644
index 000..3b0dfbf
--- /dev/null
+++ b/kylin-it/src/test/resources/query/sql_subquery/query30.sql
@@ -0,0 +1,15 @@
+
+SELECT t1.cal_dt, t1.sum_price,t1.lstg_site_id
+FROM (
+ select cal_dt, lstg_site_id, sum(price) as sum_price
+ from test_kylin_fact
+ group by cal_dt, lstg_site_id
+
+) t1
+
+inner JOIN edw.test_cal_dt as test_cal_dt
+on t1.cal_dt=test_cal_dt.cal_dt
+
+inner JOIN edw.test_sites as test_sites
+on t1.lstg_site_id = test_sites.site_id
+
http://git-wip-us.apache.org/repos/asf/kylin/blob/ada37a91/kylin-it/src/test/resources/query/sql_subquery/query31.sql
--
diff --git a/kylin-it/src/test/resources/query/sql_subquery/query31.sql
b/kylin-it/src/test/resources/query/sql_subquery/query31.sql
new file mode 100644
index 000..fafc01f
--- /dev/null
+++ b/kylin-it/src/test/resources/query/sql_subquery/query31.sql
@@ -0,0 +1,19 @@
+
+SELECT t1.week_beg_dt, t1.sum_price,t1.lstg_site_id
+FROM (
+ select test_cal_dt.week_beg_dt, sum(price) as sum_price, lstg_site_id
+ from test_kylin_fact
+ inner JOIN edw.test_cal_dt as test_cal_dt
+ ON test_kylin_fact.cal_dt = test_cal_dt.cal_dt
+ inner JOIN test_category_groupings
+ ON test_kylin_fact.leaf_categ_id = test_category_groupings.leaf_categ_id AND
test_kylin_fact.lstg_site_id = test_category_groupings.site_id
+ inner JOIN edw.test_sites as test_sites
+ ON test_kylin_fact.lstg_site_id = test_sites.site_id
+ group by test_cal_dt.week_beg_dt, lstg_site_id
+) t1
+inner JOIN edw.test_cal_dt as test_cal_dt
+on t1.week_beg_dt=test_cal_dt.week_beg_dt
+ inner JOIN edw.test_sites as test_sites
+
+ on t1.lstg_site_id = test_sites.site_id
+
http://git-wip-us.apache.org/repos/asf/kylin/blob/ada37a91/query/src/main/java/org/apache/kylin/query/relnode/OLAPJoinRel.java
--
diff --git
a/query/src/main/java/org/apache/kylin/query/relnode/OLAPJoinRel.java
b/query/src/main/java/org/apache/kylin/query/relnode/OLAPJoinRel.java
index 60b5712..1b5970c 100644
--- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPJoinRel.java
+++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPJoinRel.java
@@ -140,12 +140,24 @@ public class OLAPJoinRel extends EnumerableJoin
implements OLAPRel {
implementor.freeContext();
}
}
+
+if (leftHasSubquery) {
+// After KYLIN-2579, leftHasSubquery means right side have to be
separate olap context
+implementor.setNewOLAPContextRequired(true);
+}
+
implementor.fixSharedOlapTableScanOnTheRight(this);
implementor.visitChild(this.right, this);
if (this.context != implementor.getContext() || ((OLAPRel)
this.right).hasSubQuery()) {
this.hasSubQuery = true;
rightHasSubquery = true;
// if child is also an OLAPJoin, then the context has already been
popped
+
+if (leftHasSubquery) {
+
Preconditions.checkState(!implementor.isNewOLAPContextRequired());//should have
been satisfied
+Preconditions.checkState(this.context !=
implementor.getContext(), "missing a new olapcontext");
+}
+
if (this.context != implementor.getContext()) {
implementor.freeContext();
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/ada37a91/query/src/main/java/org/apache/kylin/query/relnode/OLAPRel.java
--
diff --gi
[47/50] [abbrv] kylin git commit: Merge commit '79c75015e49b2216f6e8ca9ec0cedbff72737821'
Merge commit '79c75015e49b2216f6e8ca9ec0cedbff72737821'
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/92b64043
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/92b64043
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/92b64043
Branch: refs/heads/KYLIN-2606
Commit: 92b640437ca09ba75fc7bce6407cf722f4f4dd8b
Parents: 34a65ba 79c7501
Author: Hongbin Ma
Authored: Tue May 23 20:34:28 2017 +0800
Committer: Hongbin Ma
Committed: Tue May 23 20:34:28 2017 +0800
--
.../kylin/dict/BuiltInFunctionTransformer.java | 2 +
.../kylin/metadata/filter/CaseTupleFilter.java | 52 -
.../metadata/filter/CompareTupleFilter.java | 29 ++-
.../filter/FilterOptimizeTransformer.java | 88 +---
.../filter/IOptimizeableTupleFilter.java| 22 ++
.../filter/ITupleFilterTransformer.java | 5 +
.../metadata/filter/LogicalTupleFilter.java | 27 ++-
.../kylin/metadata/filter/TupleFilter.java | 1 -
.../apache/kylin/metadata/model/TblColRef.java | 20 +-
.../kylin/metadata/realization/SQLDigest.java | 2 +
.../gtrecord/GTCubeStorageQueryBase.java| 27 +--
.../resources/query/sql_subquery/query30.sql| 15 ++
.../resources/query/sql_subquery/query31.sql| 19 ++
.../resources/query/sql_subquery/query32.sql| 14 ++
.../resources/query/sql_subquery/query33.sql| 14 ++
.../resources/query/sql_subquery/query34.sql| 14 ++
.../kylin/query/relnode/OLAPFilterRel.java | 2 +-
.../apache/kylin/query/relnode/OLAPJoinRel.java | 13 ++
.../kylin/query/relnode/OLAPProjectRel.java | 61 -
.../org/apache/kylin/query/relnode/OLAPRel.java | 31 +--
.../kylin/query/relnode/OLAPTableScan.java | 29 ++-
.../relnode/OLAPToEnumerableConverter.java | 11 +-
.../kylin/query/routing/ModelChooser.java | 219 --
.../apache/kylin/query/routing/QueryRouter.java | 2 +-
.../kylin/query/routing/RealizationChooser.java | 224 +++
.../kylin/rest/controller/UserController.java | 4 +-
26 files changed, 634 insertions(+), 313 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/92b64043/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
--
diff --cc
core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
index 5faa098,b4e9d47..a8c4872
---
a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
+++
b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
@@@ -85,7 -86,7 +86,6 @@@ public abstract class GTCubeStorageQuer
}
scanner = new CubeSegmentScanner(cubeSeg, request.getCuboid(),
request.getDimensions(), request.getGroups(), request.getMetrics(),
request.getFilter(), request.getHavingFilter(), request.getContext());
-
-
if (!scanner.isSegmentSkipped())
scanners.add(scanner);
}
@@@ -455,7 -458,7 +457,6 @@@
}
aggrOutCol.getColumnDesc().setId("" + (aggrIdxAmongMetrics + 1));
}
-
-
return havingFilter;
}
[45/50] [abbrv] kylin git commit: KYLIN-2635 optimize determined case when filters
KYLIN-2635 optimize determined case when filters
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/2c549895
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/2c549895
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/2c549895
Branch: refs/heads/KYLIN-2606
Commit: 2c549895dab065491faef3a46cf0b710791ef2af
Parents: f870c6c
Author: Hongbin Ma
Authored: Mon May 22 15:05:20 2017 +0800
Committer: Hongbin Ma
Committed: Tue May 23 20:30:06 2017 +0800
--
.../kylin/dict/BuiltInFunctionTransformer.java | 2 +
.../kylin/metadata/filter/CaseTupleFilter.java | 52 ++--
.../metadata/filter/CompareTupleFilter.java | 29 +--
.../filter/FilterOptimizeTransformer.java | 88 ++--
.../filter/IOptimizeableTupleFilter.java| 22 +
.../filter/ITupleFilterTransformer.java | 5 ++
.../metadata/filter/LogicalTupleFilter.java | 27 +-
.../kylin/metadata/filter/TupleFilter.java | 1 -
.../gtrecord/GTCubeStorageQueryBase.java| 26 +++---
.../kylin/query/relnode/OLAPFilterRel.java | 2 +-
.../apache/kylin/query/relnode/OLAPJoinRel.java | 1 +
.../kylin/rest/controller/UserController.java | 4 +-
12 files changed, 203 insertions(+), 56 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/2c549895/core-dictionary/src/main/java/org/apache/kylin/dict/BuiltInFunctionTransformer.java
--
diff --git
a/core-dictionary/src/main/java/org/apache/kylin/dict/BuiltInFunctionTransformer.java
b/core-dictionary/src/main/java/org/apache/kylin/dict/BuiltInFunctionTransformer.java
index f5cbe68..7a48a1c 100755
---
a/core-dictionary/src/main/java/org/apache/kylin/dict/BuiltInFunctionTransformer.java
+++
b/core-dictionary/src/main/java/org/apache/kylin/dict/BuiltInFunctionTransformer.java
@@ -41,6 +41,8 @@ import com.google.common.primitives.Primitives;
/**
* only take effect when the compare filter has function
+ *
+ * is a first type transformer defined in ITupleFilterTransformer
*/
public class BuiltInFunctionTransformer implements ITupleFilterTransformer {
public static final Logger logger =
LoggerFactory.getLogger(BuiltInFunctionTransformer.class);
http://git-wip-us.apache.org/repos/asf/kylin/blob/2c549895/core-metadata/src/main/java/org/apache/kylin/metadata/filter/CaseTupleFilter.java
--
diff --git
a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/CaseTupleFilter.java
b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/CaseTupleFilter.java
index 2b00d69..9083212 100644
---
a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/CaseTupleFilter.java
+++
b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/CaseTupleFilter.java
@@ -26,11 +26,13 @@ import java.util.List;
import org.apache.kylin.metadata.tuple.IEvaluatableTuple;
+import com.google.common.collect.Lists;
+
/**
* @author xjiang
*
*/
-public class CaseTupleFilter extends TupleFilter {
+public class CaseTupleFilter extends TupleFilter implements
IOptimizeableTupleFilter {
private List whenFilters;
private List thenFilters;
@@ -40,6 +42,12 @@ public class CaseTupleFilter extends TupleFilter {
public CaseTupleFilter() {
super(new ArrayList(), FilterOperatorEnum.CASE);
+reinit();
+}
+
+private void reinit() {
+this.children.clear();
+
this.filterIndex = 0;
this.values = Collections.emptyList();
this.whenFilters = new ArrayList();
@@ -47,14 +55,31 @@ public class CaseTupleFilter extends TupleFilter {
this.elseFilter = null;
}
+public List getWhenFilters() {
+return Collections.unmodifiableList(whenFilters);
+}
+
+public List getThenFilters() {
+return Collections.unmodifiableList(thenFilters);
+}
+
+public TupleFilter getElseFilter() {
+return elseFilter;
+}
+
@Override
public void addChild(TupleFilter child) {
-super.addChild(child);
+
if (this.filterIndex % 2 == 0) {
-this.whenFilters.add(child);
+this.elseFilter = child;
} else {
+this.whenFilters.add(this.elseFilter);
this.thenFilters.add(child);
+this.elseFilter = null;
}
+
+super.addChild(child);
+
this.filterIndex++;
}
@@ -65,9 +90,7 @@ public class CaseTupleFilter extends TupleFilter {
@Override
public boolean evaluate(IEvaluatableTuple tuple, IFilterCodeSystem cs) {
-if (whenFilters.size() != thenFilters.size()) {
-elseFilter = whenFilters.remove(whenFilters.size() - 1);
-}
+
boolean mat
[49/50] [abbrv] kylin git commit: minor, fix CI
minor, fix CI Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/fdacf5d3 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/fdacf5d3 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/fdacf5d3 Branch: refs/heads/KYLIN-2606 Commit: fdacf5d3fac8740ab031f35d6317571f11a87c23 Parents: 5d2934b Author: Hongbin Ma Authored: Wed May 24 10:42:30 2017 +0800 Committer: hongbin ma Committed: Wed May 24 10:52:57 2017 +0800 -- kylin-it/src/test/resources/query/sql_subquery/query32.sql | 2 +- kylin-it/src/test/resources/query/sql_subquery/query33.sql | 2 +- kylin-it/src/test/resources/query/sql_subquery/query34.sql | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) -- http://git-wip-us.apache.org/repos/asf/kylin/blob/fdacf5d3/kylin-it/src/test/resources/query/sql_subquery/query32.sql -- diff --git a/kylin-it/src/test/resources/query/sql_subquery/query32.sql b/kylin-it/src/test/resources/query/sql_subquery/query32.sql index 394ea15..6565aa0 100644 --- a/kylin-it/src/test/resources/query/sql_subquery/query32.sql +++ b/kylin-it/src/test/resources/query/sql_subquery/query32.sql @@ -1,5 +1,5 @@ - select sum(price) as sum_price, (case when '1'='1' then test_cal_dt.week_beg_dt when '1'='2' then test_kylin_fact.lstg_site_id else test_kylin_fact.leaf_categ_id end) as xxx + select (case when '1'='1' then test_cal_dt.week_beg_dt when '1'='2' then test_kylin_fact.lstg_site_id else test_kylin_fact.leaf_categ_id end) as xxx , sum(price) as sum_price from test_kylin_fact inner JOIN edw.test_cal_dt as test_cal_dt ON test_kylin_fact.cal_dt = test_cal_dt.cal_dt http://git-wip-us.apache.org/repos/asf/kylin/blob/fdacf5d3/kylin-it/src/test/resources/query/sql_subquery/query33.sql -- diff --git a/kylin-it/src/test/resources/query/sql_subquery/query33.sql b/kylin-it/src/test/resources/query/sql_subquery/query33.sql index 08ecc8e..6ed1e37 100644 --- a/kylin-it/src/test/resources/query/sql_subquery/query33.sql +++ b/kylin-it/src/test/resources/query/sql_subquery/query33.sql @@ -1,5 +1,5 @@ - select sum(price) as sum_price, lstg_format_name,(case when '2'='1' then test_kylin_fact.lstg_site_id when '2'='2' then test_cal_dt.week_beg_dt else test_kylin_fact.leaf_categ_id end) as xxx + selectlstg_format_name,(case when '2'='1' then test_kylin_fact.lstg_site_id when '2'='2' then test_cal_dt.week_beg_dt else test_kylin_fact.leaf_categ_id end) as xxx ,sum(price) as sum_price from test_kylin_fact inner JOIN edw.test_cal_dt as test_cal_dt ON test_kylin_fact.cal_dt = test_cal_dt.cal_dt http://git-wip-us.apache.org/repos/asf/kylin/blob/fdacf5d3/kylin-it/src/test/resources/query/sql_subquery/query34.sql -- diff --git a/kylin-it/src/test/resources/query/sql_subquery/query34.sql b/kylin-it/src/test/resources/query/sql_subquery/query34.sql index 0dda9aa..56d04fa 100644 --- a/kylin-it/src/test/resources/query/sql_subquery/query34.sql +++ b/kylin-it/src/test/resources/query/sql_subquery/query34.sql @@ -1,5 +1,5 @@ - select sum(price) as sum_price, (case when '3'='1' then test_cal_dt.week_beg_dt else test_kylin_fact.cal_dt end) as xxx + select (case when '3'='1' then test_cal_dt.week_beg_dt else test_kylin_fact.cal_dt end) as xxx , sum(price) as sum_price from test_kylin_fact inner JOIN edw.test_cal_dt as test_cal_dt ON test_kylin_fact.cal_dt = test_cal_dt.cal_dt
[28/50] [abbrv] kylin git commit: Revert "minor, fix csv export garbled issue"
Revert "minor, fix csv export garbled issue"
This reverts commit 949237ddca0ab24bf8121ccc002d802df678761c.
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/f9b6db6a
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/f9b6db6a
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/f9b6db6a
Branch: refs/heads/KYLIN-2606
Commit: f9b6db6a582747f58e91d22de76ff7549172f460
Parents: ce09d6c
Author: shaofengshi
Authored: Mon May 22 16:52:47 2017 +0800
Committer: shaofengshi
Committed: Mon May 22 16:52:47 2017 +0800
--
.../java/org/apache/kylin/rest/controller/QueryController.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/f9b6db6a/server-base/src/main/java/org/apache/kylin/rest/controller/QueryController.java
--
diff --git
a/server-base/src/main/java/org/apache/kylin/rest/controller/QueryController.java
b/server-base/src/main/java/org/apache/kylin/rest/controller/QueryController.java
index 99447a1..c5f896d 100644
---
a/server-base/src/main/java/org/apache/kylin/rest/controller/QueryController.java
+++
b/server-base/src/main/java/org/apache/kylin/rest/controller/QueryController.java
@@ -103,7 +103,7 @@ public class QueryController extends BasicController {
@ResponseBody
public void downloadQueryResult(@PathVariable String format, SQLRequest
sqlRequest, HttpServletResponse response) {
SQLResponse result = queryService.doQueryWithCache(sqlRequest);
-response.setContentType("text/" + format + ";charset=ansi");
+response.setContentType("text/" + format + ";charset=utf-8");
response.setHeader("Content-Disposition", "attachment;
filename=\"result." + format + "\"");
ICsvListWriter csvWriter = null;
[31/50] [abbrv] kylin git commit: minor, materialize lookup views as external table
minor, materialize lookup views as external table
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/4766c789
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/4766c789
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/4766c789
Branch: refs/heads/KYLIN-2606
Commit: 4766c789d8298b477edad7a9e86ebaec603f1b52
Parents: 8cd807f
Author: Cheng Wang
Authored: Tue May 23 11:10:41 2017 +0800
Committer: æ
Committed: Tue May 23 11:30:25 2017 +0800
--
.../src/main/java/org/apache/kylin/source/hive/HiveMRInput.java| 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/4766c789/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
--
diff --git
a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
index bdbe321..dffba8a 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
@@ -193,7 +193,7 @@ public class HiveMRInput implements IMRInput {
if (lookUpTableDesc.isView()) {
StringBuilder createIntermediateTableHql = new
StringBuilder();
createIntermediateTableHql.append("DROP TABLE IF EXISTS "
+ lookUpTableDesc.getMaterializedName() + ";\n");
-createIntermediateTableHql.append("CREATE TABLE IF NOT
EXISTS " + lookUpTableDesc.getMaterializedName() + "\n");
+createIntermediateTableHql.append("CREATE EXTERNAL TABLE
IF NOT EXISTS " + lookUpTableDesc.getMaterializedName() + "\n");
createIntermediateTableHql.append("LOCATION '" +
jobWorkingDir + "/" + lookUpTableDesc.getMaterializedName() + "'\n");
createIntermediateTableHql.append("AS SELECT * FROM " +
lookUpTableDesc.getIdentity() + ";\n");
hiveCmdBuilder.addStatement(createIntermediateTableHql.toString());
[24/50] [abbrv] kylin git commit: KYLIN-2632 Refactor REST API for better convention
http://git-wip-us.apache.org/repos/asf/kylin/blob/73a78dbe/server-base/src/main/java/org/apache/kylin/rest/controller2/HybridControllerV2.java
--
diff --git
a/server-base/src/main/java/org/apache/kylin/rest/controller2/HybridControllerV2.java
b/server-base/src/main/java/org/apache/kylin/rest/controller2/HybridControllerV2.java
new file mode 100644
index 000..ddf745a
--- /dev/null
+++
b/server-base/src/main/java/org/apache/kylin/rest/controller2/HybridControllerV2.java
@@ -0,0 +1,97 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.rest.controller2;
+
+import org.apache.kylin.rest.controller.BasicController;
+import org.apache.kylin.rest.msg.MsgPicker;
+import org.apache.kylin.rest.request.HybridRequest;
+import org.apache.kylin.rest.response.EnvelopeResponse;
+import org.apache.kylin.rest.response.ResponseCode;
+import org.apache.kylin.rest.service.HybridService;
+import org.apache.kylin.storage.hybrid.HybridInstance;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Controller;
+import org.springframework.web.bind.annotation.PathVariable;
+import org.springframework.web.bind.annotation.RequestBody;
+import org.springframework.web.bind.annotation.RequestHeader;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RequestMethod;
+import org.springframework.web.bind.annotation.RequestParam;
+import org.springframework.web.bind.annotation.ResponseBody;
+
+@Controller
+@RequestMapping(value = "/hybrids")
+public class HybridControllerV2 extends BasicController {
+
+@Autowired
+private HybridService hybridService;
+
+@RequestMapping(value = "", method = RequestMethod.POST, produces = {
"application/vnd.apache.kylin-v2+json" })
+@ResponseBody
+public EnvelopeResponse createV2(@RequestHeader("Accept-Language") String
lang, @RequestBody HybridRequest request) {
+MsgPicker.setMsg(lang);
+
+checkRequiredArg("hybrid", request.getHybrid());
+checkRequiredArg("project", request.getProject());
+checkRequiredArg("model", request.getModel());
+checkRequiredArg("cubes", request.getCubes());
+HybridInstance instance =
hybridService.createHybridCube(request.getHybrid(), request.getProject(),
request.getModel(), request.getCubes());
+return new EnvelopeResponse(ResponseCode.CODE_SUCCESS, instance, "");
+}
+
+@RequestMapping(value = "", method = RequestMethod.PUT, produces = {
"application/vnd.apache.kylin-v2+json" })
+@ResponseBody
+public EnvelopeResponse updateV2(@RequestHeader("Accept-Language") String
lang, @RequestBody HybridRequest request) {
+MsgPicker.setMsg(lang);
+
+checkRequiredArg("hybrid", request.getHybrid());
+checkRequiredArg("project", request.getProject());
+checkRequiredArg("model", request.getModel());
+checkRequiredArg("cubes", request.getCubes());
+HybridInstance instance =
hybridService.updateHybridCube(request.getHybrid(), request.getProject(),
request.getModel(), request.getCubes());
+return new EnvelopeResponse(ResponseCode.CODE_SUCCESS, instance, "");
+}
+
+@RequestMapping(value = "", method = RequestMethod.DELETE, produces = {
"application/vnd.apache.kylin-v2+json" })
+@ResponseBody
+public void deleteV2(@RequestHeader("Accept-Language") String lang,
@RequestBody HybridRequest request) {
+MsgPicker.setMsg(lang);
+
+checkRequiredArg("hybrid", request.getHybrid());
+checkRequiredArg("project", request.getProject());
+checkRequiredArg("model", request.getModel());
+hybridService.deleteHybridCube(request.getHybrid(),
request.getProject(), request.getModel());
+}
+
+@RequestMapping(value = "", method = RequestMethod.GET, produces = {
"application/vnd.apache.kylin-v2+json" })
+@ResponseBody
+public EnvelopeResponse listV2(@RequestHeader("Accept-Language") String
lang, @RequestParam(required = false) String project, @RequestParam(required =
false) String model) {
+MsgPicker.setMsg(lang);
+
+
[40/50] [abbrv] kylin git commit: KYLIN-2603, push down having clause when possible
KYLIN-2603, push down having clause when possible
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/e1bc72e3
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/e1bc72e3
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/e1bc72e3
Branch: refs/heads/KYLIN-2606
Commit: e1bc72e36d83a42d364e6d9b0ee893fbc018d498
Parents: 83fb144
Author: Li Yang
Authored: Wed May 10 21:01:53 2017 +0800
Committer: hongbin ma
Committed: Tue May 23 20:18:17 2017 +0800
--
.../cube/gridtable/ScanRangePlannerBase.java| 3 +-
.../org/apache/kylin/cube/model/RowKeyDesc.java | 16 +-
.../kylin/gridtable/GTAggregateScanner.java | 223 ++-
.../apache/kylin/gridtable/GTScanRequest.java | 19 +-
.../kylin/gridtable/GTScanRequestBuilder.java | 8 +-
.../java/org/apache/kylin/gridtable/GTUtil.java | 33 ++-
.../apache/kylin/metadata/model/ColumnDesc.java | 2 +
.../apache/kylin/metadata/model/TblColRef.java | 2 +
.../kylin/metadata/realization/SQLDigest.java | 36 ++-
.../storage/gtrecord/CubeScanRangePlanner.java | 12 +-
.../storage/gtrecord/CubeSegmentScanner.java| 10 +-
.../gtrecord/GTCubeStorageQueryBase.java| 51 -
.../gtrecord/GTCubeStorageQueryRequest.java | 14 +-
.../kylin/storage/gtrecord/ScannerWorker.java | 4 +
.../storage/gtrecord/DictGridTableTest.java | 14 ++
.../kylin/storage/hbase/ITStorageTest.java | 6 +-
.../kylin/query/relnode/OLAPAggregateRel.java | 6 +-
.../apache/kylin/query/relnode/OLAPContext.java | 9 +-
.../kylin/query/relnode/OLAPFilterRel.java | 9 +-
19 files changed, 370 insertions(+), 107 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/e1bc72e3/core-cube/src/main/java/org/apache/kylin/cube/gridtable/ScanRangePlannerBase.java
--
diff --git
a/core-cube/src/main/java/org/apache/kylin/cube/gridtable/ScanRangePlannerBase.java
b/core-cube/src/main/java/org/apache/kylin/cube/gridtable/ScanRangePlannerBase.java
index d938f2b..ed0a77a 100644
---
a/core-cube/src/main/java/org/apache/kylin/cube/gridtable/ScanRangePlannerBase.java
+++
b/core-cube/src/main/java/org/apache/kylin/cube/gridtable/ScanRangePlannerBase.java
@@ -44,14 +44,15 @@ import com.google.common.collect.Sets;
public abstract class ScanRangePlannerBase {
//GT
-protected TupleFilter gtFilter;
protected GTInfo gtInfo;
+protected TupleFilter gtFilter;
protected Pair gtStartAndEnd;
protected TblColRef gtPartitionCol;
protected ImmutableBitSet gtDimensions;
protected ImmutableBitSet gtAggrGroups;
protected ImmutableBitSet gtAggrMetrics;
protected String[] gtAggrFuncs;
+protected TupleFilter havingFilter;
protected boolean isPartitionColUsingDatetimeEncoding = true;
protected RecordComparator rangeStartComparator;
http://git-wip-us.apache.org/repos/asf/kylin/blob/e1bc72e3/core-cube/src/main/java/org/apache/kylin/cube/model/RowKeyDesc.java
--
diff --git
a/core-cube/src/main/java/org/apache/kylin/cube/model/RowKeyDesc.java
b/core-cube/src/main/java/org/apache/kylin/cube/model/RowKeyDesc.java
index 00557c5..124f126 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/RowKeyDesc.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/RowKeyDesc.java
@@ -18,21 +18,23 @@
package org.apache.kylin.cube.model;
-import com.fasterxml.jackson.annotation.JsonAutoDetect;
-import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
-import com.fasterxml.jackson.annotation.JsonProperty;
-import com.google.common.base.Objects;
-import org.apache.commons.lang.ArrayUtils;
-import org.apache.kylin.metadata.model.TblColRef;
-
import java.util.Arrays;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
+import org.apache.commons.lang.ArrayUtils;
+import org.apache.kylin.metadata.model.TblColRef;
+
+import com.fasterxml.jackson.annotation.JsonAutoDetect;
+import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.google.common.base.Objects;
+
/**
*/
+@SuppressWarnings("serial")
@JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility =
Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility =
Visibility.NONE)
public class RowKeyDesc implements java.io.Serializable {
http://git-wip-us.apache.org/repos/asf/kylin/blob/e1bc72e3/core-cube/src/main/java/org/apache/kylin/gridtable/GTAggregateScanner.java
--
diff --git
a/core-cube/src/main/java/org/apache/kylin/gridtable/GTAggregateScanner.j
[29/50] [abbrv] kylin git commit: KYLIN-2637 The tips is not shown after creating project successfully.
KYLIN-2637 The tips is not shown after creating project successfully.
Signed-off-by: Billy Liu
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/ce6e3ede
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/ce6e3ede
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/ce6e3ede
Branch: refs/heads/KYLIN-2606
Commit: ce6e3ede2f670bf037a55234630b85098265075a
Parents: f9b6db6
Author: 10069681
Authored: Mon May 22 20:34:53 2017 +0800
Committer: Billy Liu
Committed: Tue May 23 09:43:07 2017 +0800
--
webapp/app/js/controllers/page.js | 16 +---
1 file changed, 13 insertions(+), 3 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/ce6e3ede/webapp/app/js/controllers/page.js
--
diff --git a/webapp/app/js/controllers/page.js
b/webapp/app/js/controllers/page.js
index 056cc48..c426516 100644
--- a/webapp/app/js/controllers/page.js
+++ b/webapp/app/js/controllers/page.js
@@ -188,7 +188,7 @@ KylinApp.controller('PageCtrl', function ($scope, $q,
AccessService, $modal, $lo
});
-var projCtrl = function ($scope, $location, $modalInstance, ProjectService,
MessageService, projects, project, SweetAlert, ProjectModel, $cookieStore,
$route) {
+var projCtrl = function ($scope, $location, $modalInstance, ProjectService,
MessageService, projects, project, SweetAlert, ProjectModel, $cookieStore,
$route, $timeout) {
$scope.state = {
isEdit: false,
oldProjName: null,
@@ -240,10 +240,20 @@ var projCtrl = function ($scope, $location,
$modalInstance, ProjectService, Mess
}
else {
ProjectService.save({}, {projectDescData: angular.toJson($scope.proj)},
function (newProj) {
-SweetAlert.swal('Success!', 'New project created successfully!',
'success');
$modalInstance.dismiss('cancel');
$cookieStore.put("project", newProj.name);
-location.reload();
+SweetAlert.swal({
+ title: "Success!",
+ text: "New project created successfully!",
+ confirmButtonClass: 'btn-primary',
+ type: "success"
+},function(){
+ location.reload();
+});
+
+$timeout(function () {
+ location.reload();
+}, 3000);
}, function (e) {
if (e.data && e.data.exception) {
var message = e.data.exception;
[01/50] [abbrv] kylin git commit: KYLIN-2557 code review [Forced Update!]
Repository: kylin
Updated Branches:
refs/heads/KYLIN-2606 f8336d4bb -> d610a6dd5 (forced update)
KYLIN-2557 code review
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/1c80c29b
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/1c80c29b
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/1c80c29b
Branch: refs/heads/KYLIN-2606
Commit: 1c80c29b22d9c00de1eb2e9a09c21377714248b7
Parents: e6a8a00
Author: Yang Li
Authored: Sun May 14 20:14:29 2017 +0800
Committer: Yang Li
Committed: Sun May 14 20:14:29 2017 +0800
--
.../kylin/storage/hbase/HBaseConnection.java| 54 ++--
1 file changed, 27 insertions(+), 27 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/1c80c29b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java
--
diff --git
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java
index ef82641..5fafa2b 100644
---
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java
+++
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java
@@ -18,7 +18,18 @@
package org.apache.kylin.storage.hbase;
-import com.google.common.collect.Sets;
+import java.io.IOException;
+import java.io.UnsupportedEncodingException;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
@@ -39,17 +50,7 @@ import org.apache.kylin.common.util.HadoopUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.io.IOException;
-import java.io.UnsupportedEncodingException;
-import java.util.Collection;
-import java.util.HashSet;
-import java.util.Map;
-import java.util.Set;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.LinkedBlockingQueue;
-import java.util.concurrent.ThreadPoolExecutor;
-import java.util.concurrent.TimeUnit;
+import com.google.common.collect.Sets;
/**
* @author yangli9
@@ -67,8 +68,6 @@ public class HBaseConnection {
private static ExecutorService coprocessorPool = null;
-private static DistributedLock lock = null;
-
static {
Runtime.getRuntime().addShutdownHook(new Thread() {
@Override
@@ -268,18 +267,20 @@ public class HBaseConnection {
}
public static void createHTableIfNeeded(Connection conn, String table,
String... families) throws IOException {
-Admin hbase = conn.getAdmin();
+Admin admin = conn.getAdmin();
TableName tableName = TableName.valueOf(table);
-boolean hasLock = false;
+DistributedLock lock = null;
+String lockPath = getLockPath(table);
+
try {
if (tableExists(conn, table)) {
logger.debug("HTable '" + table + "' already exists");
-Set existingFamilies =
getFamilyNames(hbase.getTableDescriptor(tableName));
+Set existingFamilies =
getFamilyNames(admin.getTableDescriptor(tableName));
boolean wait = false;
for (String family : families) {
if (existingFamilies.contains(family) == false) {
logger.debug("Adding family '" + family + "' to HTable
'" + table + "'");
-hbase.addColumn(tableName,
newFamilyDescriptor(family));
+admin.addColumn(tableName,
newFamilyDescriptor(family));
// addColumn() is async, is there a way to wait it
finish?
wait = true;
}
@@ -295,7 +296,8 @@ public class HBaseConnection {
}
lock =
KylinConfig.getInstanceFromEnv().getDistributedLockFactory().lockForCurrentProcess();
-hasLock = lock.lock(getLockPath(table), Long.MAX_VALUE);
+if (!lock.lock(lockPath, Long.MAX_VALUE))
+throw new RuntimeException("Cannot acquire lock to create
HTable " + table);
if (tableExists(conn, table)) {
logger.debug("HTable '" + table + "' already exists");
@@ -313,15 +315,13 @@ public class HBaseConnection {
}
}
-//desc.setValue(HTABLE_UUID_TAG, UUID.randomUUID().toString());
-
[09/50] [abbrv] kylin git commit: KYLIN-2577, refine hive configurations during cubing steps
KYLIN-2577, refine hive configurations during cubing steps
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/7c152c51
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/7c152c51
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/7c152c51
Branch: refs/heads/KYLIN-2606
Commit: 7c152c515ce6cebe380c4e1b7c16d109192308db
Parents: 0061900
Author: Cheng Wang
Authored: Thu May 18 11:05:34 2017 +0800
Committer: liyang-gmt8
Committed: Thu May 18 15:27:36 2017 +0800
--
build/bin/load-hive-conf.sh | 2 +-
.../kylin/common/util/HiveCmdBuilder.java | 56 +++-
.../kylin/common/util/HiveCmdBuilderTest.java | 2 +-
.../kylin/job/engine/JobEngineConfig.java | 43 ---
.../source/hive/CreateFlatHiveTableStep.java| 3 --
.../apache/kylin/source/hive/HiveMRInput.java | 4 --
6 files changed, 57 insertions(+), 53 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/7c152c51/build/bin/load-hive-conf.sh
--
diff --git a/build/bin/load-hive-conf.sh b/build/bin/load-hive-conf.sh
index a5046f0..a42cd0c 100644
--- a/build/bin/load-hive-conf.sh
+++ b/build/bin/load-hive-conf.sh
@@ -11,5 +11,5 @@ hive_conf_prop="${KYLIN_HOME}/logs/hive_props"
rm -rf ${hive_conf_prop}
export ENABLE_CHECK_ENV=false
${dir}/kylin.sh org.apache.kylin.tool.HiveConfigCLI ${hive_conf_dir}
${hive_conf_prop}
-[[ 0 == $? ]] || quit "Error, can not parse ${hive_conf_dir} and can not apply
it to hive relevant check."
+[[ 0 == $? ]] || quit "Can not parse xml file: ${hive_conf_dir}, please check
it."
hive_conf_properties=`cat ${hive_conf_prop}`
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/7c152c51/core-common/src/main/java/org/apache/kylin/common/util/HiveCmdBuilder.java
--
diff --git
a/core-common/src/main/java/org/apache/kylin/common/util/HiveCmdBuilder.java
b/core-common/src/main/java/org/apache/kylin/common/util/HiveCmdBuilder.java
index 255867a..2f6b9a0 100644
--- a/core-common/src/main/java/org/apache/kylin/common/util/HiveCmdBuilder.java
+++ b/core-common/src/main/java/org/apache/kylin/common/util/HiveCmdBuilder.java
@@ -27,17 +27,25 @@ import java.util.ArrayList;
import java.util.HashMap;
import java.util.Map;
+import javax.xml.parsers.DocumentBuilder;
+import javax.xml.parsers.DocumentBuilderFactory;
+
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.IOUtils;
+import org.apache.commons.lang.StringUtils;
import org.apache.kylin.common.KylinConfig;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import org.w3c.dom.Document;
+import org.w3c.dom.NodeList;
import com.google.common.collect.Lists;
public class HiveCmdBuilder {
private static final Logger logger =
LoggerFactory.getLogger(HiveCmdBuilder.class);
+public static final String HIVE_CONF_FILENAME = "kylin_hive_conf";
+
public enum HiveClientMode {
CLI, BEELINE
}
@@ -50,6 +58,7 @@ public class HiveCmdBuilder {
public HiveCmdBuilder() {
kylinConfig = KylinConfig.getInstanceFromEnv();
clientMode =
HiveClientMode.valueOf(kylinConfig.getHiveClientMode().toUpperCase());
+loadHiveConfiguration();
}
public String build() {
@@ -75,8 +84,8 @@ public class HiveCmdBuilder {
bw.newLine();
}
buf.append("beeline ");
-buf.append(parseProps());
buf.append(kylinConfig.getHiveBeelineParams());
+buf.append(parseProps());
buf.append(" -f ");
buf.append(tmpHql.getAbsolutePath());
buf.append(";ret_code=$?;rm -f ");
@@ -123,6 +132,7 @@ public class HiveCmdBuilder {
}
public void setHiveConfProps(Map hiveConfProps) {
+this.hiveConfProps.clear();
this.hiveConfProps.putAll(hiveConfProps);
}
@@ -144,4 +154,48 @@ public class HiveCmdBuilder {
public String toString() {
return build();
}
+
+private void loadHiveConfiguration() {
+
+File hiveConfFile;
+String hiveConfFileName = (HIVE_CONF_FILENAME + ".xml");
+String path = System.getProperty(KylinConfig.KYLIN_CONF);
+
+if (StringUtils.isNotEmpty(path)) {
+hiveConfFile = new File(path, hiveConfFileName);
+} else {
+path = KylinConfig.getKylinHome();
+if (StringUtils.isEmpty(path)) {
+logger.error("KYLIN_HOME is not set, can not locate hive conf:
{}.xml", HIVE_CONF_FILENAME);
+return;
+}
+hiveConfFile = new File(path + File.separator + "conf",
hiveConfF
[14/50] [abbrv] kylin git commit: Merge commit '87fedf75e5bbf61a33cf1b31720ad9ac55de6287'
Merge commit '87fedf75e5bbf61a33cf1b31720ad9ac55de6287'
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/f36372bc
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/f36372bc
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/f36372bc
Branch: refs/heads/KYLIN-2606
Commit: f36372bc657b1edb312b36a071b077cde184eab8
Parents: 8ce972f 87fedf7
Author: Hongbin Ma
Authored: Fri May 19 17:55:26 2017 +0800
Committer: Hongbin Ma
Committed: Fri May 19 17:55:26 2017 +0800
--
.../rel/rules/OLAPJoinPushThroughJoinRule.java | 172
.../rel/rules/OLAPJoinPushThroughJoinRule2.java | 205 +
.../java/org/apache/calcite/tools/Programs.java | 438 +++
build/bin/find-hive-dependency.sh | 4 +-
build/bin/kylin.sh | 41 +-
build/bin/sample.sh | 24 +-
.../apache/kylin/common/KylinConfigBase.java| 8 +-
.../kylin/common/lock/DistributedLock.java | 63 ++-
.../common/lock/DistributedLockFactory.java | 43 ++
.../kylin/common/restclient/RestClient.java | 16 +-
.../org/apache/kylin/common/util/CheckUtil.java | 50 ++-
.../apache/kylin/cube/model/DimensionDesc.java | 3 +-
.../kylin/dict/BuiltInFunctionTransformer.java | 8 +-
.../kylin/dict/GlobalDictionaryBuilder.java | 71 +--
.../impl/threadpool/DistributedScheduler.java | 55 +--
.../kylin/job/lock/DistributedJobLock.java | 24 -
.../kylin/metadata/cachesync/Broadcaster.java | 66 ++-
.../metadata/filter/function/BuiltInMethod.java | 39 +-
.../metadata/filter/function/LikeMatchers.java | 141 ++
examples/sample_cube/create_sample_tables.sql | 10 +-
.../kylin/job/BaseTestDistributedScheduler.java | 35 +-
.../job/ITDistributedSchedulerBaseTest.java | 6 +-
.../job/ITDistributedSchedulerTakeOverTest.java | 2 +-
.../resources/query/sql_subquery/query17.sql| 17 +
.../resources/query/sql_subquery/query18.sql| 23 +
.../resources/query/sql_subquery/query19.sql| 20 +
.../resources/query/sql_subquery/query21.sql| 26 ++
.../resources/query/sql_subquery/query22.sql| 25 ++
.../resources/query/sql_subquery/query23.sql| 26 ++
.../resources/query/sql_subquery/query24.sql| 29 ++
.../resources/query/sql_subquery/query25.sql| 22 +
.../resources/query/sql_subquery/query26.sql| 16 +
.../resources/query/sql_subquery/query27.sql| 20 +
.../resources/query/sql_subquery/query28.sql| 21 +
.../query/sql_subquery/query29.sql.todo | 22 +
.../kylin/query/relnode/OLAPAggregateRel.java | 4 +-
.../apache/kylin/query/relnode/OLAPContext.java | 10 +
.../kylin/query/relnode/OLAPFilterRel.java | 2 +-
.../apache/kylin/query/relnode/OLAPJoinRel.java | 12 +-
.../kylin/query/relnode/OLAPProjectRel.java | 15 +-
.../rest/security/AuthoritiesPopulator.java | 20 +-
.../apache/kylin/rest/service/AclService.java | 1 -
.../kylin/rest/service/CacheServiceTest.java| 10 +-
.../kylin/storage/hbase/HBaseConnection.java| 30 +-
.../hbase/util/ZookeeperDistributedJobLock.java | 257 ---
.../hbase/util/ZookeeperDistributedLock.java| 307 +
.../storage/hbase/util/ZookeeperJobLock.java| 144 ++
.../apache/kylin/storage/hdfs/LockManager.java | 8 +-
.../util/ITZookeeperDistributedLockTest.java| 254 +++
.../apache/kylin/tool/MetadataCleanupJob.java | 14 +-
webapp/app/js/controllers/access.js | 1 +
webapp/app/js/controllers/admin.js | 12 +-
webapp/app/js/controllers/cubeMeasures.js | 2 +-
webapp/app/js/controllers/query.js | 4 +-
54 files changed, 2273 insertions(+), 625 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/f36372bc/build/bin/find-hive-dependency.sh
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/f36372bc/build/bin/sample.sh
--
diff --cc build/bin/sample.sh
index 2e13805,10900bb..b9c4ea3
--- a/build/bin/sample.sh
+++ b/build/bin/sample.sh
@@@ -52,10 -62,10 +63,11 @@@ the
else
beeline_params=${beeline_params/${hive2_url}/${hive2_url}${sample_database}}
fi
- beeline ${hive_conf_properties} ${beeline_params} -f
${KYLIN_HOME}/sample_cube/create_sample_tables.sql || { exit 1; }
-beeline ${beeline_params} --hivevar hdfs_tmp_dir=${hdfs_tmp_dir} -f
${KYLIN_HOME}/sample_cube/create_sample_tables.sql || { exit 1; }
++
++beeline ${hive_conf_properties} --hivevar hdfs_tmp_dir=${hdfs_tmp_dir}
${beeline_params} -f ${KYLIN_HOME}/sample_cube/create_sample_tables.sql || {
exit 1; }
else
-hive -e "CREATE DATABASE IF NOT EX
[12/50] [abbrv] kylin git commit: fix KYLIN-2594: After reloading metadata, refresh the project list
fix KYLIN-2594: After reloading metadata, refresh the project list
Signed-off-by: Billy Liu
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/87fedf75
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/87fedf75
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/87fedf75
Branch: refs/heads/KYLIN-2606
Commit: 87fedf75e5bbf61a33cf1b31720ad9ac55de6287
Parents: a34db26
Author: pengjianlin
Authored: Fri May 19 16:24:28 2017 +0800
Committer: Billy Liu
Committed: Fri May 19 16:44:28 2017 +0800
--
webapp/app/js/controllers/admin.js | 7 +--
1 file changed, 5 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/87fedf75/webapp/app/js/controllers/admin.js
--
diff --git a/webapp/app/js/controllers/admin.js
b/webapp/app/js/controllers/admin.js
index 10e2c38..37c1d67 100644
--- a/webapp/app/js/controllers/admin.js
+++ b/webapp/app/js/controllers/admin.js
@@ -18,7 +18,7 @@
'use strict';
-KylinApp.controller('AdminCtrl', function ($scope, AdminService, CacheService,
TableService, loadingRequest, MessageService, $modal,
SweetAlert,kylinConfig,ProjectModel,$window) {
+KylinApp.controller('AdminCtrl', function ($scope, AdminService, CacheService,
TableService, loadingRequest, MessageService, ProjectService, $modal,
SweetAlert,kylinConfig,ProjectModel,$window) {
$scope.configStr = "";
$scope.envStr = "";
@@ -84,7 +84,7 @@ KylinApp.controller('AdminCtrl', function ($scope,
AdminService, CacheService, T
}
});
}
-
+
$scope.reloadMeta = function () {
SweetAlert.swal({
title: '',
@@ -98,6 +98,9 @@ KylinApp.controller('AdminCtrl', function ($scope,
AdminService, CacheService, T
if (isConfirm) {
CacheService.clean({}, function () {
SweetAlert.swal('Success!', 'Cache reload successfully', 'success');
+ ProjectService.listReadable({}, function(projects) {
+ProjectModel.setProjects(projects);
+ });
}, function (e) {
if (e.data && e.data.exception) {
var message = e.data.exception;
[05/50] [abbrv] kylin git commit: KYLIN-2625 not null filter clause should be evaluable in storage
KYLIN-2625 not null filter clause should be evaluable in storage
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/c9dc7cc7
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/c9dc7cc7
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/c9dc7cc7
Branch: refs/heads/KYLIN-2606
Commit: c9dc7cc78c5eed40f4c5a438423b857a145f17b1
Parents: a38b02d
Author: Hongbin Ma
Authored: Tue May 16 18:04:03 2017 +0800
Committer: Roger Shi
Committed: Tue May 16 20:59:12 2017 +0800
--
.../metadata/filter/CompareTupleFilter.java | 3 ++-
.../test/resources/query/sql_limit/query04.sql | 25
2 files changed, 27 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/c9dc7cc7/core-metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java
--
diff --git
a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java
b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java
index d783e52..2771250 100755
---
a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java
+++
b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java
@@ -215,7 +215,8 @@ public class CompareTupleFilter extends TupleFilter {
@Override
public boolean isEvaluable() {
return (column != null || (function != null &&
function.isEvaluable())) //
-&& !conditionValues.isEmpty() && secondColumn == null;
+&& (!conditionValues.isEmpty() || operator ==
FilterOperatorEnum.ISNOTNULL || operator == FilterOperatorEnum.ISNULL) //
+&& secondColumn == null;
}
public boolean alwaysReturnTrue() {
http://git-wip-us.apache.org/repos/asf/kylin/blob/c9dc7cc7/kylin-it/src/test/resources/query/sql_limit/query04.sql
--
diff --git a/kylin-it/src/test/resources/query/sql_limit/query04.sql
b/kylin-it/src/test/resources/query/sql_limit/query04.sql
new file mode 100644
index 000..381ab65
--- /dev/null
+++ b/kylin-it/src/test/resources/query/sql_limit/query04.sql
@@ -0,0 +1,25 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+
+
+select * from (
+select * from test_kylin_fact
+ where lstg_format_name is not null
+ ) limit 20
+
[39/50] [abbrv] kylin git commit: minor, use soft reference in dict cache
minor, use soft reference in dict cache
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/ed0430bc
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/ed0430bc
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/ed0430bc
Branch: refs/heads/KYLIN-2606
Commit: ed0430bc0649ac5b4de7a6c3df0f65b8121d12cd
Parents: 0e44a19
Author: Li Yang
Authored: Fri May 12 15:23:19 2017 +0800
Committer: hongbin ma
Committed: Tue May 23 20:18:17 2017 +0800
--
.../src/main/java/org/apache/kylin/dict/DictionaryManager.java| 3 +--
1 file changed, 1 insertion(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/ed0430bc/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
--
diff --git
a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
index bbe6cac..1628f4e 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
@@ -99,8 +99,7 @@ public class DictionaryManager {
}
})//
.maximumSize(config.getCachedDictMaxEntrySize())//
-.expireAfterWrite(1, TimeUnit.DAYS)//
-.build(new CacheLoader() {
+.expireAfterWrite(1, TimeUnit.DAYS).build(new
CacheLoader() {
@Override
public DictionaryInfo load(String key) throws Exception {
DictionaryInfo dictInfo =
DictionaryManager.this.load(key, true);
[32/50] [abbrv] kylin git commit: minor, move temporary file to folder var
minor, move temporary file to folder var
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/11a0b57a
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/11a0b57a
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/11a0b57a
Branch: refs/heads/KYLIN-2606
Commit: 11a0b57acd434cff426e860a0b4c645b496522f6
Parents: 73a78db
Author: Cheng Wang
Authored: Mon May 22 16:30:38 2017 +0800
Committer: æ
Committed: Tue May 23 11:30:25 2017 +0800
--
build/bin/load-hive-conf.sh | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/11a0b57a/build/bin/load-hive-conf.sh
--
diff --git a/build/bin/load-hive-conf.sh b/build/bin/load-hive-conf.sh
index a42cd0c..c7317c6 100644
--- a/build/bin/load-hive-conf.sh
+++ b/build/bin/load-hive-conf.sh
@@ -5,9 +5,9 @@
source $(cd -P -- "$(dirname -- "$0")" && pwd -P)/header.sh
# source me
-
+mkdir -p ${KYLIN_HOME}/var
hive_conf_dir="${KYLIN_HOME}/conf/kylin_hive_conf.xml"
-hive_conf_prop="${KYLIN_HOME}/logs/hive_props"
+hive_conf_prop="${KYLIN_HOME}/var/hive_props"
rm -rf ${hive_conf_prop}
export ENABLE_CHECK_ENV=false
${dir}/kylin.sh org.apache.kylin.tool.HiveConfigCLI ${hive_conf_dir}
${hive_conf_prop}
[21/50] [abbrv] kylin git commit: KYLIN-2632 Refactor REST API for better convention
http://git-wip-us.apache.org/repos/asf/kylin/blob/73a78dbe/server-base/src/main/java/org/apache/kylin/rest/service/QueryServiceV2.java
--
diff --git
a/server-base/src/main/java/org/apache/kylin/rest/service/QueryServiceV2.java
b/server-base/src/main/java/org/apache/kylin/rest/service/QueryServiceV2.java
new file mode 100644
index 000..368
--- /dev/null
+++
b/server-base/src/main/java/org/apache/kylin/rest/service/QueryServiceV2.java
@@ -0,0 +1,516 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.rest.service;
+
+import static org.apache.kylin.common.util.CheckUtil.checkCondition;
+
+import java.io.IOException;
+import java.sql.Connection;
+import java.sql.DatabaseMetaData;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.ResultSetMetaData;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.sql.Time;
+import java.sql.Timestamp;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+
+import javax.sql.DataSource;
+
+import org.apache.calcite.avatica.ColumnMetaData;
+import org.apache.commons.lang.StringUtils;
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.QueryContext;
+import org.apache.kylin.common.debug.BackdoorToggles;
+import org.apache.kylin.common.exceptions.ResourceLimitExceededException;
+import org.apache.kylin.common.util.SetThreadName;
+import org.apache.kylin.cube.CubeManager;
+import org.apache.kylin.metadata.model.DataModelDesc;
+import org.apache.kylin.metadata.model.JoinDesc;
+import org.apache.kylin.metadata.model.JoinTableDesc;
+import org.apache.kylin.metadata.model.ModelDimensionDesc;
+import org.apache.kylin.metadata.model.TableRef;
+import org.apache.kylin.metadata.project.ProjectInstance;
+import org.apache.kylin.query.relnode.OLAPContext;
+import org.apache.kylin.query.util.QueryUtil;
+import org.apache.kylin.rest.constant.Constant;
+import org.apache.kylin.rest.exception.BadRequestException;
+import org.apache.kylin.rest.exception.InternalErrorException;
+import org.apache.kylin.rest.metrics.QueryMetricsFacade;
+import org.apache.kylin.rest.model.ColumnMetaWithType;
+import org.apache.kylin.rest.model.SelectedColumnMeta;
+import org.apache.kylin.rest.model.TableMetaWithType;
+import org.apache.kylin.rest.msg.Message;
+import org.apache.kylin.rest.msg.MsgPicker;
+import org.apache.kylin.rest.request.PrepareSqlRequest;
+import org.apache.kylin.rest.request.SQLRequest;
+import org.apache.kylin.rest.response.SQLResponse;
+import org.apache.kylin.rest.util.TableauInterceptor;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Qualifier;
+import org.springframework.security.core.GrantedAuthority;
+import org.springframework.security.core.context.SecurityContextHolder;
+import org.springframework.stereotype.Component;
+
+import com.google.common.collect.Lists;
+
+import net.sf.ehcache.Cache;
+import net.sf.ehcache.Element;
+
+/**
+ * Created by luwei on 17-4-24.
+ */
+@Component("queryServiceV2")
+public class QueryServiceV2 extends QueryService {
+private static final Logger logger =
LoggerFactory.getLogger(QueryServiceV2.class);
+
+@Autowired
+@Qualifier("cacheService")
+private CacheService cacheService;
+
+@Autowired
+@Qualifier("modelMgmtServiceV2")
+private ModelServiceV2 modelServiceV2;
+
+public SQLResponse doQueryWithCache(SQLRequest sqlRequest) {
+Message msg = MsgPicker.getMsg();
+
+KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
+String serverMode = kylinConfig.getServerMode();
+if (!(Constant.SERVER_MODE_QUERY.equals(serverMode.toLowerCase()) ||
Constant.SERVER_MODE_ALL.equals(serverMode.toLowerCase( {
+throw new
BadRequestException(String.format(msg.getQUERY_NOT_ALLOWED(), serverMode));
+}
+if (StringUtils.isBlank(sqlRequest.getProject())) {
+throw new BadRequestException(msg.
[48/50] [abbrv] kylin git commit: fix CI
fix CI
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/5d2934b7
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/5d2934b7
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/5d2934b7
Branch: refs/heads/KYLIN-2606
Commit: 5d2934b7e6331e5fef2c18c92d7338932b087b3f
Parents: 92b6404
Author: Hongbin Ma
Authored: Tue May 23 21:26:02 2017 +0800
Committer: hongbin ma
Committed: Tue May 23 21:27:41 2017 +0800
--
.../apache/kylin/metadata/model/TblColRef.java | 1 -
.../org/apache/kylin/rest/DebugDeployCLI.java | 31
2 files changed, 31 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/5d2934b7/core-metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java
--
diff --git
a/core-metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java
b/core-metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java
index 3bca61c..5a28e8b 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java
@@ -87,7 +87,6 @@ public class TblColRef implements Serializable {
checkArgument(col.table.getModel() == UNKNOWN_MODEL ||
col.table.getModel() == model);
TableRef tableRef = model.findTable(alias);
checkArgument(tableRef.getTableDesc().getIdentity().equals(col.column.getTable().getIdentity()));
-checkArgument(tableRef.getTableDesc() == col.column.getTable());
col.fixTableRef(tableRef);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/5d2934b7/server/src/main/java/org/apache/kylin/rest/DebugDeployCLI.java
--
diff --git a/server/src/main/java/org/apache/kylin/rest/DebugDeployCLI.java
b/server/src/main/java/org/apache/kylin/rest/DebugDeployCLI.java
new file mode 100644
index 000..28c1b12
--- /dev/null
+++ b/server/src/main/java/org/apache/kylin/rest/DebugDeployCLI.java
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.rest;
+
+import java.io.IOException;
+
+import org.apache.kylin.storage.hbase.util.DeployCoprocessorCLI;
+
+public class DebugDeployCLI {
+
+public static void main(String[] args) throws IOException {
+DebugTomcat.setupDebugEnv();
+DeployCoprocessorCLI.main(new String[] { "default", "all" });
+}
+}
[35/50] [abbrv] kylin git commit: KYLIN-2632 Refactor REST API and service
KYLIN-2632 Refactor REST API and service
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/2471d5e5
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/2471d5e5
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/2471d5e5
Branch: refs/heads/KYLIN-2606
Commit: 2471d5e56219d7c51342953ae37eb77370ae1e58
Parents: 0a0edfe
Author: Luwei-Chen
Authored: Tue May 23 18:17:29 2017 +0800
Committer: liyang-gmt8
Committed: Tue May 23 18:21:27 2017 +0800
--
.../rest/controller2/CubeControllerV2.java | 74 +
.../rest/controller2/DiagnosisControllerV2.java | 21 ++-
.../rest/controller2/ModelControllerV2.java | 63 +---
.../rest/controller2/ProjectControllerV2.java | 109 +
.../rest/response/CubeInstanceResponse.java | 23 ++-
.../rest/response/DataModelDescResponse.java| 23 ++-
.../apache/kylin/rest/service/CacheService.java | 14 +-
.../kylin/rest/service/CubeServiceV2.java | 159 ++
.../kylin/rest/service/ModelServiceV2.java | 161 +++
.../kylin/rest/service/ProjectServiceV2.java| 91 +++
10 files changed, 319 insertions(+), 419 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/2471d5e5/server-base/src/main/java/org/apache/kylin/rest/controller2/CubeControllerV2.java
--
diff --git
a/server-base/src/main/java/org/apache/kylin/rest/controller2/CubeControllerV2.java
b/server-base/src/main/java/org/apache/kylin/rest/controller2/CubeControllerV2.java
index 05d99f5..a9e00ba 100644
---
a/server-base/src/main/java/org/apache/kylin/rest/controller2/CubeControllerV2.java
+++
b/server-base/src/main/java/org/apache/kylin/rest/controller2/CubeControllerV2.java
@@ -21,6 +21,7 @@ package org.apache.kylin.rest.controller2;
import java.io.IOException;
import java.net.UnknownHostException;
import java.util.ArrayList;
+import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -50,6 +51,7 @@ import org.apache.kylin.rest.request.CubeRequest;
import org.apache.kylin.rest.request.JobBuildRequest;
import org.apache.kylin.rest.request.JobBuildRequest2;
import org.apache.kylin.rest.response.CubeInstanceResponse;
+import org.apache.kylin.rest.response.CubeInstanceResponse.CubeComparator;
import org.apache.kylin.rest.response.EnvelopeResponse;
import org.apache.kylin.rest.response.GeneralResponse;
import org.apache.kylin.rest.response.HBaseResponse;
@@ -74,7 +76,6 @@ import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.ResponseBody;
import com.fasterxml.jackson.core.JsonParseException;
-import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.JsonMappingException;
import com.google.common.collect.Lists;
@@ -149,6 +150,8 @@ public class CubeControllerV2 extends BasicController {
cubeInstanceResponses.add(cubeInstanceResponse);
}
+CubeComparator cubeComparator = new CubeComparator();
+Collections.sort(cubeInstanceResponses, cubeComparator);
data.put("cubes", cubeInstanceResponses);
data.put("size", cubes.size());
@@ -468,58 +471,6 @@ public class CubeControllerV2 extends BasicController {
}
/**
- * update CubDesc
- *
- * @return Table metadata array
- * @throws JsonProcessingException
- * @throws IOException
- */
-
-@RequestMapping(value = "", method = { RequestMethod.PUT }, produces = {
"application/vnd.apache.kylin-v2+json" })
-@ResponseBody
-public EnvelopeResponse updateCubeDescV2(@RequestHeader("Accept-Language")
String lang, @RequestBody CubeRequest cubeRequest) throws IOException {
-MsgPicker.setMsg(lang);
-
-CubeDesc desc = deserializeCubeDescV2(cubeRequest);
-cubeServiceV2.validateCubeDesc(desc, false);
-
-boolean createNew = cubeServiceV2.unifyCubeDesc(desc, false);
-
-String projectName = (null == cubeRequest.getProject()) ?
ProjectInstance.DEFAULT_PROJECT_NAME : cubeRequest.getProject();
-
-desc = cubeServiceV2.updateCubeToResourceStore(desc, projectName,
createNew, false);
-
-String descData = JsonUtil.writeValueAsIndentString(desc);
-GeneralResponse data = new GeneralResponse();
-data.setProperty("uuid", desc.getUuid());
-data.setProperty("cubeDescData", descData);
-
-return new EnvelopeResponse(ResponseCode.CODE_SUCCESS, data, "");
-}
-
-@RequestMapping(value = "/draft", method = { RequestMethod.PUT }, produces
= { "application/vnd.apache.kylin-v2+json" })
-@ResponseBody
-public EnvelopeResponse
updateCubeDescDraftV2(@RequestHeader("Accept-Language") Stri
kylin git commit: KYLIN-2378 Set job thread name with job uuid
Repository: kylin
Updated Branches:
refs/heads/master a11a31da0 -> 0f36b9b37
KYLIN-2378 Set job thread name with job uuid
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/0f36b9b3
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/0f36b9b3
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/0f36b9b3
Branch: refs/heads/master
Commit: 0f36b9b37097b95c16dd62cbe32781e5af8911dc
Parents: a11a31d
Author: kangkaisen
Authored: Mon Jan 9 17:21:38 2017 +0800
Committer: kangkaisen
Committed: Thu Jan 12 17:42:54 2017 +0800
--
.../org/apache/kylin/job/impl/threadpool/DefaultScheduler.java| 3 ++-
.../apache/kylin/job/impl/threadpool/DistributedScheduler.java| 3 ++-
2 files changed, 4 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/0f36b9b3/core-job/src/main/java/org/apache/kylin/job/impl/threadpool/DefaultScheduler.java
--
diff --git
a/core-job/src/main/java/org/apache/kylin/job/impl/threadpool/DefaultScheduler.java
b/core-job/src/main/java/org/apache/kylin/job/impl/threadpool/DefaultScheduler.java
index ddb3a21..403abc4 100644
---
a/core-job/src/main/java/org/apache/kylin/job/impl/threadpool/DefaultScheduler.java
+++
b/core-job/src/main/java/org/apache/kylin/job/impl/threadpool/DefaultScheduler.java
@@ -29,6 +29,7 @@ import java.util.concurrent.TimeUnit;
import org.apache.curator.framework.CuratorFramework;
import org.apache.curator.framework.state.ConnectionState;
import org.apache.curator.framework.state.ConnectionStateListener;
+import org.apache.kylin.common.util.SetThreadName;
import org.apache.kylin.job.Scheduler;
import org.apache.kylin.job.constant.ExecutableConstants;
import org.apache.kylin.job.engine.JobEngineConfig;
@@ -137,7 +138,7 @@ public class DefaultScheduler implements
Scheduler, Connecti
@Override
public void run() {
-try {
+try (SetThreadName ignored = new SetThreadName("Job %s",
executable.getId())) {
executable.execute(context);
// trigger the next step asap
fetcherPool.schedule(fetcher, 0, TimeUnit.SECONDS);
http://git-wip-us.apache.org/repos/asf/kylin/blob/0f36b9b3/core-job/src/main/java/org/apache/kylin/job/impl/threadpool/DistributedScheduler.java
--
diff --git
a/core-job/src/main/java/org/apache/kylin/job/impl/threadpool/DistributedScheduler.java
b/core-job/src/main/java/org/apache/kylin/job/impl/threadpool/DistributedScheduler.java
index be07d76..3436529 100644
---
a/core-job/src/main/java/org/apache/kylin/job/impl/threadpool/DistributedScheduler.java
+++
b/core-job/src/main/java/org/apache/kylin/job/impl/threadpool/DistributedScheduler.java
@@ -35,6 +35,7 @@ import org.apache.curator.framework.CuratorFramework;
import org.apache.curator.framework.state.ConnectionState;
import org.apache.curator.framework.state.ConnectionStateListener;
import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.util.SetThreadName;
import org.apache.kylin.job.Scheduler;
import org.apache.kylin.job.constant.ExecutableConstants;
import org.apache.kylin.job.engine.JobEngineConfig;
@@ -173,7 +174,7 @@ public class DistributedScheduler implements
Scheduler, Conn
@Override
public void run() {
-try {
+try (SetThreadName ignored = new SetThreadName("Job %s",
executable.getId())) {
String segmentId = executable.getParam(SEGMENT_ID);
if (jobLock.lockWithName(segmentId, serverName)) {
logger.info(executable.toString() + " scheduled in server:
" + serverName);
kylin git commit: KYLIN-2238 Add query server scan threshold
Repository: kylin
Updated Branches:
refs/heads/master 20c329423 -> 4b00ec26d
KYLIN-2238 Add query server scan threshold
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/4b00ec26
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/4b00ec26
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/4b00ec26
Branch: refs/heads/master
Commit: 4b00ec26de1e1cc9bf9b2e55a1b162d123114df8
Parents: 20c3294
Author: kangkaisen
Authored: Tue Nov 29 19:51:39 2016 +0800
Committer: kangkaisen
Committed: Sat Jan 14 19:00:09 2017 +0800
--
.../storage/gtrecord/SequentialCubeTupleIterator.java | 14 +++---
1 file changed, 11 insertions(+), 3 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/4b00ec26/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SequentialCubeTupleIterator.java
--
diff --git
a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SequentialCubeTupleIterator.java
b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SequentialCubeTupleIterator.java
index ee868c7..c621215 100644
---
a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SequentialCubeTupleIterator.java
+++
b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SequentialCubeTupleIterator.java
@@ -26,6 +26,7 @@ import java.util.Set;
import javax.annotation.Nullable;
+import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.cube.cuboid.Cuboid;
import org.apache.kylin.metadata.model.FunctionDesc;
import org.apache.kylin.metadata.model.TblColRef;
@@ -33,6 +34,7 @@ import org.apache.kylin.metadata.tuple.ITuple;
import org.apache.kylin.metadata.tuple.ITupleIterator;
import org.apache.kylin.metadata.tuple.TupleInfo;
import org.apache.kylin.storage.StorageContext;
+import org.apache.kylin.storage.exception.ScanOutOfLimitException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -45,6 +47,8 @@ public class SequentialCubeTupleIterator implements
ITupleIterator {
private static final Logger logger =
LoggerFactory.getLogger(SequentialCubeTupleIterator.class);
+private final int SCAN_THRESHOLD =
KylinConfig.getInstanceFromEnv().getScanThreshold();
+
protected List scanners;
protected List segmentCubeTupleIterators;
protected Iterator tupleIterator;
@@ -78,7 +82,7 @@ public class SequentialCubeTupleIterator implements
ITupleIterator {
tupleIterator = new
SortedIteratorMergerWithLimit(transformed,
context.getFinalPushDownLimit(), getTupleDimensionComparator(cuboid,
returnTupleInfo)).getIterator();
}
}
-
+
public Comparator getTupleDimensionComparator(Cuboid cuboid,
TupleInfo returnTupleInfo) {
// dimensionIndexOnTuple is for SQL with limit
List temp = Lists.newArrayList();
@@ -92,7 +96,7 @@ public class SequentialCubeTupleIterator implements
ITupleIterator {
for (int i = 0; i < temp.size(); i++) {
dimensionIndexOnTuple[i] = temp.get(i);
}
-
+
return new Comparator() {
@Override
public int compare(ITuple o1, ITuple o2) {
@@ -137,7 +141,11 @@ public class SequentialCubeTupleIterator implements
ITupleIterator {
@Override
public ITuple next() {
-scanCount++;
+// prevent the big query to make the Query Server OOM
+if (scanCount++ > SCAN_THRESHOLD) {
+throw new ScanOutOfLimitException("Scan count exceed the scan
threshold: " + SCAN_THRESHOLD);
+}
+
if (++scanCountDelta >= 1000)
flushScanCountDelta();
kylin git commit: KYLIN-2304 Only copy latest version dict for global dict
Repository: kylin
Updated Branches:
refs/heads/master 4b00ec26d -> eb18ac59f
KYLIN-2304 Only copy latest version dict for global dict
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/eb18ac59
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/eb18ac59
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/eb18ac59
Branch: refs/heads/master
Commit: eb18ac59fa74b3f2f1f0163f7728739a1027a233
Parents: 4b00ec2
Author: kangkaisen
Authored: Sun Dec 18 20:09:03 2016 +0800
Committer: kangkaisen
Committed: Sat Jan 14 19:07:59 2017 +0800
--
.../apache/kylin/dict/AppendTrieDictionary.java | 19 +--
.../org/apache/kylin/dict/CachedTreeMap.java | 2 +-
2 files changed, 14 insertions(+), 7 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/eb18ac59/core-dictionary/src/main/java/org/apache/kylin/dict/AppendTrieDictionary.java
--
diff --git
a/core-dictionary/src/main/java/org/apache/kylin/dict/AppendTrieDictionary.java
b/core-dictionary/src/main/java/org/apache/kylin/dict/AppendTrieDictionary.java
index 5029bc4..80403cc 100644
---
a/core-dictionary/src/main/java/org/apache/kylin/dict/AppendTrieDictionary.java
+++
b/core-dictionary/src/main/java/org/apache/kylin/dict/AppendTrieDictionary.java
@@ -1173,13 +1173,15 @@ public class AppendTrieDictionary extends
CacheDictionary {
@Override
public AppendTrieDictionary copyToAnotherMeta(KylinConfig srcConfig,
KylinConfig dstConfig) throws IOException {
+//copy appendDict
Configuration conf = new Configuration();
-AppendTrieDictionary newDict = new AppendTrieDictionary();
-
newDict.initParams(baseDir.replaceFirst(srcConfig.getHdfsWorkingDirectory(),
dstConfig.getHdfsWorkingDirectory()), baseId, maxId, maxValueLength, nValues,
bytesConvert);
-newDict.initDictSliceMap((CachedTreeMap)dictSliceMap);
-logger.info("Copy AppendDict from {} to {}", this.baseDir,
newDict.baseDir);
-Path srcPath = new Path(this.baseDir);
-Path dstPath = new Path(newDict.baseDir);
+
+Path base = new Path(baseDir);
+FileSystem srcFs = FileSystem.get(base.toUri(), conf);
+Path srcPath = CachedTreeMap.getLatestVersion(conf, srcFs, base);
+Path dstPath = new
Path(srcPath.toString().replaceFirst(srcConfig.getHdfsWorkingDirectory(),
dstConfig.getHdfsWorkingDirectory()));
+logger.info("Copy appendDict from {} to {}", srcPath, dstPath);
+
FileSystem dstFs = FileSystem.get(dstPath.toUri(), conf);
if (dstFs.exists(dstPath)) {
logger.info("Delete existing AppendDict {}", dstPath);
@@ -1187,6 +1189,11 @@ public class AppendTrieDictionary extends
CacheDictionary {
}
FileUtil.copy(FileSystem.get(srcPath.toUri(), conf), srcPath,
FileSystem.get(dstPath.toUri(), conf), dstPath, false, true, conf);
+// init new AppendTrieDictionary
+AppendTrieDictionary newDict = new AppendTrieDictionary();
+
newDict.initParams(baseDir.replaceFirst(srcConfig.getHdfsWorkingDirectory(),
dstConfig.getHdfsWorkingDirectory()), baseId, maxId, maxValueLength, nValues,
bytesConverter);
+newDict.initDictSliceMap((CachedTreeMap) dictSliceMap);
+
return newDict;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/eb18ac59/core-dictionary/src/main/java/org/apache/kylin/dict/CachedTreeMap.java
--
diff --git
a/core-dictionary/src/main/java/org/apache/kylin/dict/CachedTreeMap.java
b/core-dictionary/src/main/java/org/apache/kylin/dict/CachedTreeMap.java
index 3be4fd7..cc23261 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/CachedTreeMap.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/CachedTreeMap.java
@@ -225,7 +225,7 @@ public class CachedTreeMap ext
return getLatestVersion(conf, fs, baseDir).toUri().getPath();
}
-private static Path getLatestVersion(Configuration conf, FileSystem fs,
Path baseDir) throws IOException {
+public static Path getLatestVersion(Configuration conf, FileSystem fs,
Path baseDir) throws IOException {
String[] versions = listAllVersions(fs, baseDir);
if (versions.length > 0) {
return new Path(versions[versions.length - 1]);
kylin git commit: KYLIN-2357 Make ERROR_RECORD_LOG_THRESHOLD configurable
Repository: kylin
Updated Branches:
refs/heads/master eb18ac59f -> 8d374f843
KYLIN-2357 Make ERROR_RECORD_LOG_THRESHOLD configurable
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/8d374f84
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/8d374f84
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/8d374f84
Branch: refs/heads/master
Commit: 8d374f84368194d78208ea3b7683a697e8ec4314
Parents: eb18ac5
Author: kangkaisen
Authored: Sat Jan 7 19:16:30 2017 +0800
Committer: kangkaisen
Committed: Sat Jan 14 19:14:42 2017 +0800
--
.../src/main/java/org/apache/kylin/common/KylinConfigBase.java | 4
.../java/org/apache/kylin/engine/mr/common/BatchConstants.java | 1 -
.../org/apache/kylin/engine/mr/steps/BaseCuboidMapperBase.java | 2 +-
.../kylin/engine/mr/steps/FactDistinctColumnsMapperBase.java | 2 +-
4 files changed, 6 insertions(+), 3 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/8d374f84/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
--
diff --git
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 494c641..d6774ff 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -423,6 +423,10 @@ abstract public class KylinConfigBase implements
Serializable {
return Integer.parseInt(getOptional("kylin.job.scheduler.default",
"0"));
}
+public Integer getErrorRecordThreshold() {
+return
Integer.parseInt(getOptional("kylin.job.error-record-threshold", "0"));
+}
+
//
// SOURCE.HIVE
//
http://git-wip-us.apache.org/repos/asf/kylin/blob/8d374f84/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/BatchConstants.java
--
diff --git
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/BatchConstants.java
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/BatchConstants.java
index 078d80f..0281539 100644
---
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/BatchConstants.java
+++
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/BatchConstants.java
@@ -80,7 +80,6 @@ public interface BatchConstants {
*/
String MAPREDUCE_COUNTER_GROUP_NAME = "Cube Builder";
int NORMAL_RECORD_LOG_THRESHOLD = 10;
-int ERROR_RECORD_LOG_THRESHOLD = 100;
/**
* dictionaries builder class
http://git-wip-us.apache.org/repos/asf/kylin/blob/8d374f84/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/BaseCuboidMapperBase.java
--
diff --git
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/BaseCuboidMapperBase.java
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/BaseCuboidMapperBase.java
index d08e29a..93e413b 100644
---
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/BaseCuboidMapperBase.java
+++
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/BaseCuboidMapperBase.java
@@ -87,7 +87,7 @@ abstract public class BaseCuboidMapperBase
extends KylinMapper BatchConstants.ERROR_RECORD_LOG_THRESHOLD) {
+if (errorRecordCounter >
cubeSegment.getConfig().getErrorRecordThreshold()) {
if (ex instanceof IOException)
throw (IOException) ex;
else if (ex instanceof RuntimeException)
http://git-wip-us.apache.org/repos/asf/kylin/blob/8d374f84/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsMapperBase.java
--
diff --git
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsMapperBase.java
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsMapperBase.java
index 74aebb0..34624e3 100644
---
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsMapperBase.java
+++
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsMapperBase.java
@@ -105,7 +105,7 @@ abstract public class FactDistinctColumnsMapperBase extends Kyli
ex.printStackTrace(System.err);
errorRecordCounter++;
-if (errorRecordCounter > BatchConstants.ERROR_RECORD_LOG_THRESHOLD) {
+if (errorRecordCounter >
cubeSeg.getConfig().getErrorR
kylin git commit: KYLIN-2304 minor, fix variable name error
Repository: kylin
Updated Branches:
refs/heads/master 8d374f843 -> e79a55d94
KYLIN-2304 minor, fix variable name error
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/e79a55d9
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/e79a55d9
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/e79a55d9
Branch: refs/heads/master
Commit: e79a55d94c6c45c315f991b8a8e6f606a6b2c25e
Parents: 8d374f8
Author: kangkaisen
Authored: Sat Jan 14 19:31:55 2017 +0800
Committer: kangkaisen
Committed: Sat Jan 14 19:31:55 2017 +0800
--
.../src/main/java/org/apache/kylin/dict/AppendTrieDictionary.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/e79a55d9/core-dictionary/src/main/java/org/apache/kylin/dict/AppendTrieDictionary.java
--
diff --git
a/core-dictionary/src/main/java/org/apache/kylin/dict/AppendTrieDictionary.java
b/core-dictionary/src/main/java/org/apache/kylin/dict/AppendTrieDictionary.java
index 80403cc..b797167 100644
---
a/core-dictionary/src/main/java/org/apache/kylin/dict/AppendTrieDictionary.java
+++
b/core-dictionary/src/main/java/org/apache/kylin/dict/AppendTrieDictionary.java
@@ -1191,7 +1191,7 @@ public class AppendTrieDictionary extends
CacheDictionary {
// init new AppendTrieDictionary
AppendTrieDictionary newDict = new AppendTrieDictionary();
-
newDict.initParams(baseDir.replaceFirst(srcConfig.getHdfsWorkingDirectory(),
dstConfig.getHdfsWorkingDirectory()), baseId, maxId, maxValueLength, nValues,
bytesConverter);
+
newDict.initParams(baseDir.replaceFirst(srcConfig.getHdfsWorkingDirectory(),
dstConfig.getHdfsWorkingDirectory()), baseId, maxId, maxValueLength, nValues,
bytesConvert);
newDict.initDictSliceMap((CachedTreeMap) dictSliceMap);
return newDict;
kylin git commit: KYLIN-2389 Improve resource utilization for DistributedScheduler
Repository: kylin
Updated Branches:
refs/heads/master e89446500 -> 837bd8200
KYLIN-2389 Improve resource utilization for DistributedScheduler
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/837bd820
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/837bd820
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/837bd820
Branch: refs/heads/master
Commit: 837bd8200b250f38fcfb2d221764d5aca0c66403
Parents: e894465
Author: kangkaisen
Authored: Fri Jan 13 19:58:41 2017 +0800
Committer: kangkaisen
Committed: Wed Jan 18 16:14:24 2017 +0800
--
.../impl/threadpool/DistributedScheduler.java | 8 +--
.../kylin/job/lock/DistributedJobLock.java | 2 +
.../apache/kylin/rest/service/JobService.java | 45 --
.../hbase/util/ZookeeperDistributedJobLock.java | 63
4 files changed, 58 insertions(+), 60 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/837bd820/core-job/src/main/java/org/apache/kylin/job/impl/threadpool/DistributedScheduler.java
--
diff --git
a/core-job/src/main/java/org/apache/kylin/job/impl/threadpool/DistributedScheduler.java
b/core-job/src/main/java/org/apache/kylin/job/impl/threadpool/DistributedScheduler.java
index 3436529..84e62d5 100644
---
a/core-job/src/main/java/org/apache/kylin/job/impl/threadpool/DistributedScheduler.java
+++
b/core-job/src/main/java/org/apache/kylin/job/impl/threadpool/DistributedScheduler.java
@@ -195,13 +195,13 @@ public class DistributedScheduler implements
Scheduler, Conn
}
}
-//release job lock only when the all tasks of the job finish and the
job server keep the cube lock.
+//release job lock when job state is ready or running and the job
server keep the cube lock.
private void releaseJobLock(AbstractExecutable executable) {
if (executable instanceof DefaultChainedExecutable) {
String segmentId = executable.getParam(SEGMENT_ID);
ExecutableState state = executable.getStatus();
-if (state == ExecutableState.SUCCEED || state ==
ExecutableState.ERROR || state == ExecutableState.DISCARDED) {
+if (state != ExecutableState.READY && state !=
ExecutableState.RUNNING) {
if (segmentWithLocks.contains(segmentId)) {
logger.info(executable.toString() + " will release the
lock for the segment: " + segmentId);
jobLock.unlockWithName(segmentId);
@@ -232,7 +232,7 @@ public class DistributedScheduler implements
Scheduler, Conn
if (executable instanceof DefaultChainedExecutable &&
executable.getParams().get(SEGMENT_ID).equalsIgnoreCase(segmentId) &&
!nodeData.equalsIgnoreCase(serverName)) {
try {
logger.warn(nodeData + " has released the lock
for: " + segmentId + " but the job still running. so " + serverName + " resume
the job");
-if (jobLock.lockWithName(segmentId, serverName)) {
+if (!jobLock.isHasLocked(segmentId)) {
executableManager.resumeRunningJobForce(executable.getId());
fetcherPool.schedule(fetcher, 0,
TimeUnit.SECONDS);
break;
@@ -302,7 +302,7 @@ public class DistributedScheduler implements
Scheduler, Conn
AbstractExecutable executable = executableManager.getJob(id);
if (output.getState() == ExecutableState.RUNNING && executable
instanceof DefaultChainedExecutable) {
try {
-if (jobLock.lockWithName(executable.getParam(SEGMENT_ID),
serverName)) {
+if (!jobLock.isHasLocked(executable.getParam(SEGMENT_ID)))
{
executableManager.resumeRunningJobForce(executable.getId());
fetcherPool.schedule(fetcher, 0, TimeUnit.SECONDS);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/837bd820/core-job/src/main/java/org/apache/kylin/job/lock/DistributedJobLock.java
--
diff --git
a/core-job/src/main/java/org/apache/kylin/job/lock/DistributedJobLock.java
b/core-job/src/main/java/org/apache/kylin/job/lock/DistributedJobLock.java
index 9335e56..1c173ec 100644
--- a/core-job/src/main/java/org/apache/kylin/job/lock/DistributedJobLock.java
+++ b/core-job/src/main/java/org/apache/kylin/job/lock/DistributedJobLock.java
@@ -24,6 +24,8 @@ public interface DistributedJobLock extends JobLock {
boolean lock
kylin git commit: KYLIN-2242 write multiple files in FactDistinctColumnsReducer with MultipleOutputs
Repository: kylin
Updated Branches:
refs/heads/master c2229c9c7 -> 7de8aa120
KYLIN-2242 write multiple files in FactDistinctColumnsReducer with
MultipleOutputs
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/7de8aa12
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/7de8aa12
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/7de8aa12
Branch: refs/heads/master
Commit: 7de8aa1203a72bad105ed692f7100535939b03af
Parents: c2229c9
Author: kangkaisen
Authored: Sat Dec 17 14:12:48 2016 +0800
Committer: kangkaisen
Committed: Sat Jan 21 23:19:50 2017 +0800
--
.../apache/kylin/common/util/HadoopUtil.java| 16 ++
.../kylin/engine/mr/JobBuilderSupport.java | 2 +-
.../kylin/engine/mr/common/BatchConstants.java | 9 +-
.../engine/mr/steps/CreateDictionaryJob.java| 43 ++--
.../engine/mr/steps/FactDistinctColumnsJob.java | 32 ++-
.../mr/steps/FactDistinctColumnsReducer.java| 240 +++
.../engine/mr/steps/SaveStatisticsStep.java | 10 +-
.../mr/steps/UpdateCubeInfoAfterBuildStep.java | 10 +-
8 files changed, 175 insertions(+), 187 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/7de8aa12/core-common/src/main/java/org/apache/kylin/common/util/HadoopUtil.java
--
diff --git
a/core-common/src/main/java/org/apache/kylin/common/util/HadoopUtil.java
b/core-common/src/main/java/org/apache/kylin/common/util/HadoopUtil.java
index bdc4c3e..b9ffe38 100644
--- a/core-common/src/main/java/org/apache/kylin/common/util/HadoopUtil.java
+++ b/core-common/src/main/java/org/apache/kylin/common/util/HadoopUtil.java
@@ -26,8 +26,10 @@ import java.net.URISyntaxException;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.PathFilter;
import org.apache.hadoop.io.Writable;
import org.apache.kylin.common.KylinConfig;
import org.slf4j.Logger;
@@ -140,4 +142,18 @@ public class HadoopUtil {
}
}
+public static Path getFilterOnlyPath(FileSystem fs, Path baseDir, final
String filter) throws IOException {
+FileStatus[] fileStatus = fs.listStatus(baseDir, new PathFilter() {
+@Override
+public boolean accept(Path path) {
+return path.getName().startsWith(filter);
+}
+});
+
+if (fileStatus.length == 1) {
+return fileStatus[0].getPath();
+} else {
+return null;
+}
+}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/7de8aa12/engine-mr/src/main/java/org/apache/kylin/engine/mr/JobBuilderSupport.java
--
diff --git
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/JobBuilderSupport.java
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/JobBuilderSupport.java
index 696b22a..c34a904 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/JobBuilderSupport.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/JobBuilderSupport.java
@@ -171,7 +171,7 @@ public class JobBuilderSupport {
}
public String getStatisticsPath(String jobId) {
-return getRealizationRootPath(jobId) + "/statistics";
+return getRealizationRootPath(jobId) + "/fact_distinct_columns/" +
BatchConstants.CFG_OUTPUT_STATISTICS;
}
//
http://git-wip-us.apache.org/repos/asf/kylin/blob/7de8aa12/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/BatchConstants.java
--
diff --git
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/BatchConstants.java
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/BatchConstants.java
index 0281539..602b4bb 100644
---
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/BatchConstants.java
+++
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/BatchConstants.java
@@ -53,9 +53,16 @@ public interface BatchConstants {
String CFG_STATISTICS_ENABLED = "statistics.enabled";
String CFG_STATISTICS_OUTPUT = "statistics.ouput";//spell error, for
compatibility issue better not change it
String CFG_STATISTICS_SAMPLING_PERCENT = "statistics.sampling.percent";
-String CFG_STATISTICS_CUBE_ESTIMATION_FILENAME = "cube_statistics.txt";
String CFG_STATISTICS_CUBOID_ESTIMATION_FILENAME = "cuboid_statistics.seq";
+String CFG_MAPRED_OUTPUT_COMPRESS = "mapred.output.compress&quo
kylin git commit: KYLIN-2433 Fix NPE in MergeCuboidMapper
Repository: kylin
Updated Branches:
refs/heads/master 6d11dd1d2 -> b32cc9545
KYLIN-2433 Fix NPE in MergeCuboidMapper
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/b32cc954
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/b32cc954
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/b32cc954
Branch: refs/heads/master
Commit: b32cc9545a76657570c4a3353469ded43892c772
Parents: 6d11dd1
Author: kangkaisen
Authored: Fri Feb 3 14:00:50 2017 +0800
Committer: kangkaisen
Committed: Tue Feb 7 17:20:30 2017 +0800
--
.../apache/kylin/engine/mr/steps/MergeCuboidMapper.java | 10 ++
1 file changed, 10 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/b32cc954/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidMapper.java
--
diff --git
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidMapper.java
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidMapper.java
index 047e2b1..acf1403 100644
---
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidMapper.java
+++
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidMapper.java
@@ -129,6 +129,11 @@ public class MergeCuboidMapper extends KylinMapper {
List columns =
measureType.getColumnsNeedDictionary(measureDesc.getFunction());
boolean needReEncode = false;
for (TblColRef col : columns) {
+//handle the column that all records is null
+if (sourceCubeSegment.getDictionary(col) == null) {
+continue;
+}
+
if
(!sourceCubeSegment.getDictionary(col).equals(mergedCubeSegment.getDictionary(col)))
{
oldDicts.put(col, sourceCubeSegment.getDictionary(col));
newDicts.put(col, mergedCubeSegment.getDictionary(col));
@@ -249,6 +254,11 @@ public class MergeCuboidMapper extends KylinMapper {
}
private Boolean checkNeedMerging(TblColRef col) throws IOException {
+//handle the column that all records is null
+if (sourceCubeSegment.getDictionary(col) == null) {
+return false;
+}
+
Boolean ret = dimensionsNeedDict.get(col);
if (ret != null)
return ret;
kylin git commit: KYLIN-2430 Get exitCode form FsShell.run in BulkLoadJob
Repository: kylin
Updated Branches:
refs/heads/master eee9ecbcb -> 7b860adb0
KYLIN-2430 Get exitCode form FsShell.run in BulkLoadJob
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/7b860adb
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/7b860adb
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/7b860adb
Branch: refs/heads/master
Commit: 7b860adb0a23b53d6e2e40337f4317a3a2b067ba
Parents: eee9ecb
Author: kangkaisen
Authored: Tue Feb 7 17:35:48 2017 +0800
Committer: kangkaisen
Committed: Tue Feb 7 17:40:39 2017 +0800
--
.../kylin/storage/hbase/steps/BulkLoadJob.java | 17 -
1 file changed, 12 insertions(+), 5 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/7b860adb/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/BulkLoadJob.java
--
diff --git
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/BulkLoadJob.java
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/BulkLoadJob.java
index 1c05767..30616c5 100644
---
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/BulkLoadJob.java
+++
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/BulkLoadJob.java
@@ -55,11 +55,18 @@ public class BulkLoadJob extends AbstractHadoopJob {
Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
FsShell shell = new FsShell(conf);
-try {
-shell.run(new String[] { "-chmod", "-R", "777", input });
-} catch (Exception e) {
-logger.error("Couldn't change the file permissions ", e);
-throw new IOException(e);
+
+int exitCode = -1;
+int retryCount = 10;
+while (exitCode != 0 && retryCount >= 1) {
+exitCode = shell.run(new String[] { "-chmod", "-R", "777", input
});
+retryCount--;
+Thread.sleep(5000);
+}
+
+if (exitCode != 0) {
+logger.error("Failed to change the file permissions: " + input);
+throw new IOException("Failed to change the file permissions: " +
input);
}
String[] newArgs = new String[2];
kylin git commit: KYLIN-2388 Hot load kylin config from web
Repository: kylin
Updated Branches:
refs/heads/master 7b860adb0 -> 39afa5197
KYLIN-2388 Hot load kylin config from web
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/39afa519
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/39afa519
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/39afa519
Branch: refs/heads/master
Commit: 39afa5197e197d5143843a3bb62c528749b03418
Parents: 7b860ad
Author: kangkaisen
Authored: Thu Jan 12 14:35:59 2017 +0800
Committer: kangkaisen
Committed: Tue Feb 7 19:57:29 2017 +0800
--
.../org/apache/kylin/common/KylinConfig.java| 6 +-
.../apache/kylin/common/KylinConfigTest.java| 28 -
.../util/HotLoadKylinPropertiesTestCase.java| 64
.../kylin/cube/CubeSpecificConfigTest.java | 37 ++-
.../apache/kylin/job/JobEngineConfigTest.java | 47 ++
.../kylin/rest/controller/CacheController.java | 9 ++-
webapp/app/js/controllers/admin.js | 26
webapp/app/js/services/cache.js | 3 +-
webapp/app/partials/admin/admin.html| 3 +
9 files changed, 191 insertions(+), 32 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/39afa519/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java
--
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java
b/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java
index 0f40654..c7a18c6 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java
@@ -201,7 +201,7 @@ public class KylinConfig extends KylinConfigBase {
return kylinHome + File.separator + "conf";
}
-static File getKylinPropertiesFile() {
+public static File getKylinPropertiesFile() {
String kylinConfHome = System.getProperty(KYLIN_CONF);
if (!StringUtils.isEmpty(kylinConfHome)) {
logger.info("Use KYLIN_CONF=" + kylinConfHome);
@@ -385,4 +385,8 @@ public class KylinConfig extends KylinConfigBase {
//}
//logger.info(buf.toString());
}
+
+public synchronized void hotLoadKylinProperties() {
+reloadKylinConfig(getKylinProperties());
+}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/39afa519/core-common/src/test/java/org/apache/kylin/common/KylinConfigTest.java
--
diff --git
a/core-common/src/test/java/org/apache/kylin/common/KylinConfigTest.java
b/core-common/src/test/java/org/apache/kylin/common/KylinConfigTest.java
index a426fc6..4d5f130 100644
--- a/core-common/src/test/java/org/apache/kylin/common/KylinConfigTest.java
+++ b/core-common/src/test/java/org/apache/kylin/common/KylinConfigTest.java
@@ -26,24 +26,12 @@ import static org.junit.Assert.assertTrue;
import java.util.Map;
-import org.apache.kylin.common.util.LocalFileMetadataTestCase;
-import org.junit.After;
-import org.junit.Before;
+import org.apache.kylin.common.util.HotLoadKylinPropertiesTestCase;
import org.junit.Test;
import com.google.common.collect.Maps;
-public class KylinConfigTest extends LocalFileMetadataTestCase {
-@Before
-public void setUp() throws Exception {
-this.createTestMetadata();
-}
-
-@After
-public void after() throws Exception {
-this.cleanupTestMetadata();
-}
-
+public class KylinConfigTest extends HotLoadKylinPropertiesTestCase{
@Test
public void testMRConfigOverride() {
KylinConfig config = KylinConfig.getInstanceFromEnv();
@@ -78,9 +66,19 @@ public class KylinConfigTest extends
LocalFileMetadataTestCase {
KylinConfig config = KylinConfig.getInstanceFromEnv();
Map override = Maps.newHashMap();
KylinConfig configExt = KylinConfigExt.createInstance(config,
override);
-
assertTrue(config.properties == configExt.properties);
config.setProperty("1234", "1234");
assertEquals("1234", configExt.getOptional("1234"));
}
+
+@Test
+public void testPropertiesHotLoad() {
+KylinConfig config = KylinConfig.getInstanceFromEnv();
+assertEquals("who...@kylin.apache.org", config.getKylinOwner());
+
+updateProperty("kylin.storage.hbase.owner-tag",
"ky...@kylin.apache.org");
+KylinConfig.getInstanceFromEnv().hotLoadKylinProperties();
+
+assertEquals("ky...@kylin.apache.org", config.getKylinOwner());
+}
}
http://git-wip-us.apache.org/repos/asf/kylin/b
kylin git commit: KYLIN-2377 Add kylin client query timeout
Repository: kylin
Updated Branches:
refs/heads/master 0dcce15f7 -> 8263752a4
KYLIN-2377 Add kylin client query timeout
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/8263752a
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/8263752a
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/8263752a
Branch: refs/heads/master
Commit: 8263752a499158342e0588fda851a4006e8b1669
Parents: 0dcce15
Author: kangkaisen
Authored: Tue Jan 10 14:02:18 2017 +0800
Committer: kangkaisen
Committed: Wed Feb 8 15:51:30 2017 +0800
--
.../org/apache/kylin/common/KylinConfigBase.java| 4
.../java/org/apache/kylin/cube/CubeInstance.java| 1 +
.../kylin/metadata/realization/IRealization.java| 5 -
.../org/apache/kylin/storage/StorageContext.java| 16 +++-
.../storage/gtrecord/GTCubeStorageQueryBase.java| 2 ++
.../gtrecord/SequentialCubeTupleIterator.java | 5 +
.../apache/kylin/storage/hybrid/HybridInstance.java | 1 +
7 files changed, 32 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/8263752a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
--
diff --git
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index fe15b1e..ebd9dfc 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -877,6 +877,10 @@ abstract public class KylinConfigBase implements
Serializable {
return udfMap;
}
+public int getQueryTimeoutSeconds() {
+return
Integer.parseInt(this.getOptional("kylin.query.timeout-seconds", "0"));
+}
+
//
// SERVER
//
http://git-wip-us.apache.org/repos/asf/kylin/blob/8263752a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
--
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
index 1d60575..fb9a7a7 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
@@ -178,6 +178,7 @@ public class CubeInstance extends RootPersistentEntity
implements IRealization,
return sizeRecordSize;
}
+@Override
public KylinConfig getConfig() {
return config;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/8263752a/core-metadata/src/main/java/org/apache/kylin/metadata/realization/IRealization.java
--
diff --git
a/core-metadata/src/main/java/org/apache/kylin/metadata/realization/IRealization.java
b/core-metadata/src/main/java/org/apache/kylin/metadata/realization/IRealization.java
index a0243f4..aafc0f0 100644
---
a/core-metadata/src/main/java/org/apache/kylin/metadata/realization/IRealization.java
+++
b/core-metadata/src/main/java/org/apache/kylin/metadata/realization/IRealization.java
@@ -21,6 +21,7 @@ package org.apache.kylin.metadata.realization;
import java.util.List;
import java.util.Set;
+import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.metadata.model.ColumnDesc;
import org.apache.kylin.metadata.model.DataModelDesc;
import org.apache.kylin.metadata.model.IStorageAware;
@@ -42,7 +43,7 @@ public interface IRealization extends IStorageAware {
public DataModelDesc getModel();
public Set getAllColumns();
-
+
public Set getAllColumnDescs();
public List getAllDimensions();
@@ -60,4 +61,6 @@ public interface IRealization extends IStorageAware {
public long getDateRangeEnd();
public boolean supportsLimitPushDown();
+
+public KylinConfig getConfig();
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/8263752a/core-storage/src/main/java/org/apache/kylin/storage/StorageContext.java
--
diff --git
a/core-storage/src/main/java/org/apache/kylin/storage/StorageContext.java
b/core-storage/src/main/java/org/apache/kylin/storage/StorageContext.java
index ec46f83..ab0ea73 100644
--- a/core-storage/src/main/java/org/apache/kylin/storage/StorageContext.java
+++ b/core-storage/src/main/java/org/apache/kylin/storage/StorageContext.java
@@ -41,6 +41,7 @@ public class StorageContext {
private int finalPushDownLimit = Integer.
kylin git commit: Fix bug in static init of DataType to pass ExtendedColumnSerializerTest UT
Repository: kylin
Updated Branches:
refs/heads/master df3ecd324 -> 570ab42a9
Fix bug in static init of DataType to pass ExtendedColumnSerializerTest UT
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/570ab42a
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/570ab42a
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/570ab42a
Branch: refs/heads/master
Commit: 570ab42a9b7597ce85b17202b67cd0ea5403cba4
Parents: df3ecd3
Author: kangkaisen
Authored: Wed Feb 8 21:25:37 2017 +0800
Committer: kangkaisen
Committed: Wed Feb 8 21:38:00 2017 +0800
--
.../main/java/org/apache/kylin/metadata/datatype/DataType.java| 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/570ab42a/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/DataType.java
--
diff --git
a/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/DataType.java
b/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/DataType.java
index b726c5f..d3756b8 100644
---
a/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/DataType.java
+++
b/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/DataType.java
@@ -114,7 +114,8 @@ public class DataType implements Serializable {
public static final DataType ANY = DataType.getType("any");
static {
-MeasureTypeFactory.init();
+//to ensure the MeasureTypeFactory class has initialized
+MeasureTypeFactory.getUDAFs();
}
public static DataType getType(String type) {
kylin git commit: KYLIN-2308 bug fix and improve user experience
Repository: kylin
Updated Branches:
refs/heads/master 7742b31d8 -> 1e9f0908b
KYLIN-2308 bug fix and improve user experience
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/1e9f0908
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/1e9f0908
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/1e9f0908
Branch: refs/heads/master
Commit: 1e9f0908b9942113bbd251dc5eeba71973cc77e1
Parents: 7742b31
Author: kangkaisen
Authored: Thu Feb 9 12:06:56 2017 +0800
Committer: kangkaisen
Committed: Thu Feb 9 15:54:01 2017 +0800
--
webapp/app/js/controllers/cubeAdvanceSetting.js | 36 ++--
webapp/app/js/controllers/cubeSchema.js | 8 +
.../cubeDesigner/advanced_settings.html | 2 +-
3 files changed, 43 insertions(+), 3 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/1e9f0908/webapp/app/js/controllers/cubeAdvanceSetting.js
--
diff --git a/webapp/app/js/controllers/cubeAdvanceSetting.js
b/webapp/app/js/controllers/cubeAdvanceSetting.js
index 39d36b0..7cdb1cd 100644
--- a/webapp/app/js/controllers/cubeAdvanceSetting.js
+++ b/webapp/app/js/controllers/cubeAdvanceSetting.js
@@ -342,12 +342,44 @@ KylinApp.controller('CubeAdvanceSettingCtrl', function
($scope, $modal,cubeConfi
return assignedMeasures;
};
- if ($scope.getAllMeasureNames().length !=
$scope.getAssignedMeasureNames().length) {
+ $scope.rmDeprecatedMeasureNames = function () {
+var allMeasureNames = $scope.getAllMeasureNames();
+var tmpColumnFamily = $scope.cubeMetaFrame.hbase_mapping.column_family;
+
+angular.forEach($scope.cubeMetaFrame.hbase_mapping.column_family, function
(colFamily,index1) {
+ angular.forEach(colFamily.columns[0].measure_refs, function
(measureName, index2) {
+var allIndex = allMeasureNames.indexOf(measureName);
+if (allIndex == -1) {
+ tmpColumnFamily[index1].columns[0].measure_refs.splice(index2, 1);
+}
+
+if (tmpColumnFamily[index1].columns[0].measure_refs == 0) {
+ tmpColumnFamily.splice(index1, 1);
+}
+ });
+});
+
+$scope.cubeMetaFrame.hbase_mapping.column_family = tmpColumnFamily;
+ };
+
+ if ($scope.getAssignedMeasureNames().length == 0) {
$scope.initColumnFamily();
+ } else {
+$scope.rmDeprecatedMeasureNames();
+if ($scope.getAllMeasureNames().length >
$scope.getAssignedMeasureNames().length) {
+ $scope.initColumnFamily();
+}
}
-
$scope.addColumnFamily = function () {
+var isCFEmpty = _.some($scope.cubeMetaFrame.hbase_mapping.column_family,
function(colFamily) {
+ return colFamily.columns[0].measure_refs.length == 0;
+});
+
+if (isCFEmpty === true) {
+ return;
+}
+
var colFamily =
$scope.newColFamily($scope.cubeMetaFrame.hbase_mapping.column_family.length +
1);
$scope.cubeMetaFrame.hbase_mapping.column_family.push(colFamily);
};
http://git-wip-us.apache.org/repos/asf/kylin/blob/1e9f0908/webapp/app/js/controllers/cubeSchema.js
--
diff --git a/webapp/app/js/controllers/cubeSchema.js
b/webapp/app/js/controllers/cubeSchema.js
index 17371f2..4beeebe 100755
--- a/webapp/app/js/controllers/cubeSchema.js
+++ b/webapp/app/js/controllers/cubeSchema.js
@@ -287,6 +287,14 @@ KylinApp.controller('CubeSchemaCtrl', function ($scope,
QueryService, UserServic
errors.push("All measures need to be assigned to column family");
}
+var isCFEmpty =
_.some($scope.cubeMetaFrame.hbase_mapping.column_family, function(colFamily) {
+ return colFamily.columns[0].measure_refs.length == 0;
+});
+
+if (isCFEmpty == true) {
+ errors.push("Each column family can't not be empty");
+}
+
var errorInfo = "";
angular.forEach(errors,function(item){
http://git-wip-us.apache.org/repos/asf/kylin/blob/1e9f0908/webapp/app/partials/cubeDesigner/advanced_settings.html
--
diff --git a/webapp/app/partials/cubeDesigner/advanced_settings.html
b/webapp/app/partials/cubeDesigner/advanced_settings.html
index bf95256..f26722b 100755
--- a/webapp/app/partials/cubeDesigner/advanced_settings.html
+++ b/webapp/app/partials/cubeDesigner/advanced_settings.html
@@ -453,7 +453,7 @@
-
+
kylin git commit: KYLIN-2316 Global dict couldn't be used for dimension column
Repository: kylin
Updated Branches:
refs/heads/master 7fa25e0fa -> 3ee49467c
KYLIN-2316 Global dict couldn't be used for dimension column
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/3ee49467
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/3ee49467
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/3ee49467
Branch: refs/heads/master
Commit: 3ee49467ce686d11afcecbc636bb5213b0927427
Parents: 7fa25e0
Author: kangkaisen
Authored: Thu Feb 9 19:54:26 2017 +0800
Committer: kangkaisen
Committed: Thu Feb 9 20:28:17 2017 +0800
--
.../kylin/cube/model/validation/rule/DictionaryRule.java | 9 +
.../cube/model/validation/rule/DictionaryRuleTest.java | 11 +--
2 files changed, 18 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/3ee49467/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/DictionaryRule.java
--
diff --git
a/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/DictionaryRule.java
b/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/DictionaryRule.java
index 37889c2..8da3ca0 100644
---
a/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/DictionaryRule.java
+++
b/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/DictionaryRule.java
@@ -29,6 +29,7 @@ import org.apache.kylin.cube.model.DictionaryDesc;
import org.apache.kylin.cube.model.validation.IValidatorRule;
import org.apache.kylin.cube.model.validation.ResultLevel;
import org.apache.kylin.cube.model.validation.ValidateContext;
+import org.apache.kylin.dict.GlobalDictionaryBuilder;
import org.apache.kylin.metadata.model.TblColRef;
/**
@@ -45,10 +46,13 @@ public class DictionaryRule implements
IValidatorRule {
static final String ERROR_REUSE_BUILDER_BOTH_SET = "REUSE and BUILDER both
set on dictionary for column: ";
static final String ERROR_REUSE_BUILDER_BOTH_EMPTY = "REUSE and BUILDER
both empty on dictionary for column: ";
static final String ERROR_TRANSITIVE_REUSE = "Transitive REUSE is not
allowed for dictionary: ";
+static final String ERROR_GLOBAL_DICTIONNARY_ONLY_MEASURE = "Global
dictionary couldn't be used for dimension column: ";
@Override
public void validate(CubeDesc cubeDesc, ValidateContext context) {
List dictDescs = cubeDesc.getDictionaries();
+Set dimensionColumns =
cubeDesc.listDimensionColumnsIncludingDerived();
+
if (dictDescs == null || dictDescs.isEmpty()) {
return;
}
@@ -78,6 +82,11 @@ public class DictionaryRule implements
IValidatorRule {
return;
}
+if (StringUtils.isNotEmpty(builderClass) &&
builderClass.equalsIgnoreCase(GlobalDictionaryBuilder.class.getName()) &&
dimensionColumns.contains(dictCol)) {
+context.addResult(ResultLevel.ERROR,
ERROR_GLOBAL_DICTIONNARY_ONLY_MEASURE + dictCol);
+return;
+}
+
if (reuseCol != null) {
reuseDictionaries.add(dictDesc);
} else {
http://git-wip-us.apache.org/repos/asf/kylin/blob/3ee49467/core-cube/src/test/java/org/apache/kylin/cube/model/validation/rule/DictionaryRuleTest.java
--
diff --git
a/core-cube/src/test/java/org/apache/kylin/cube/model/validation/rule/DictionaryRuleTest.java
b/core-cube/src/test/java/org/apache/kylin/cube/model/validation/rule/DictionaryRuleTest.java
index 452768f..fcb723e 100644
---
a/core-cube/src/test/java/org/apache/kylin/cube/model/validation/rule/DictionaryRuleTest.java
+++
b/core-cube/src/test/java/org/apache/kylin/cube/model/validation/rule/DictionaryRuleTest.java
@@ -19,6 +19,7 @@
package org.apache.kylin.cube.model.validation.rule;
import static
org.apache.kylin.cube.model.validation.rule.DictionaryRule.ERROR_DUPLICATE_DICTIONARY_COLUMN;
+import static
org.apache.kylin.cube.model.validation.rule.DictionaryRule.ERROR_GLOBAL_DICTIONNARY_ONLY_MEASURE;
import static
org.apache.kylin.cube.model.validation.rule.DictionaryRule.ERROR_REUSE_BUILDER_BOTH_EMPTY;
import static
org.apache.kylin.cube.model.validation.rule.DictionaryRule.ERROR_REUSE_BUILDER_BOTH_SET;
import static
org.apache.kylin.cube.model.validation.rule.DictionaryRule.ERROR_TRANSITIVE_REUSE;
@@ -91,10 +92,16 @@ public class DictionaryRuleTest extends
LocalFileMetadataTestCase {
DictionaryDesc.create("lstg_site_id", "SELLER_ID", null),
DictionaryDesc.create("price", "lstg_site_id", null)
kylin git commit: minor, UHCReducerCount change back to 3 by default to make IT cover KYLIN-2135
Repository: kylin
Updated Branches:
refs/heads/master e09338b34 -> edf6cef15
minor, UHCReducerCount change back to 3 by default to make IT cover KYLIN-2135
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/edf6cef1
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/edf6cef1
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/edf6cef1
Branch: refs/heads/master
Commit: edf6cef15753eae8675e9f2f4cf4ba5c7fb4e513
Parents: e09338b
Author: kangkaisen
Authored: Fri Feb 10 17:36:51 2017 +0800
Committer: kangkaisen
Committed: Fri Feb 10 17:36:51 2017 +0800
--
.../src/main/java/org/apache/kylin/common/KylinConfigBase.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/edf6cef1/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
--
diff --git
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 81f0187..894e28a 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -746,7 +746,7 @@ abstract public class KylinConfigBase implements
Serializable {
//UHC: ultra high cardinality columns, contain the ShardByColumns and the
GlobalDictionaryColumns
public int getUHCReducerCount() {
-return
Integer.parseInt(getOptional("kylin.engine.mr.uhc-reducer-count", "1"));
+return
Integer.parseInt(getOptional("kylin.engine.mr.uhc-reducer-count", "3"));
}
public boolean isBuildDictInReducerEnabled() {
kylin git commit: KYLIN-2446 Support project names filter in DeployCoprocessorCLI
Repository: kylin
Updated Branches:
refs/heads/master e2ff7842e -> 69f898482
KYLIN-2446 Support project names filter in DeployCoprocessorCLI
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/69f89848
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/69f89848
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/69f89848
Branch: refs/heads/master
Commit: 69f898482499d530dddeb35b3327f65a50899dd5
Parents: e2ff784
Author: kangkaisen
Authored: Tue Feb 14 17:25:44 2017 +0800
Committer: kangkaisen
Committed: Tue Feb 14 17:43:14 2017 +0800
--
.../hbase/util/DeployCoprocessorCLI.java| 40 +++-
1 file changed, 39 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/69f89848/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java
--
diff --git
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java
index e72859d..29c738e 100644
---
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java
+++
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java
@@ -54,7 +54,11 @@ import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeManager;
import org.apache.kylin.cube.CubeSegment;
import org.apache.kylin.metadata.model.SegmentStatusEnum;
+import org.apache.kylin.metadata.project.ProjectInstance;
+import org.apache.kylin.metadata.project.ProjectManager;
+import org.apache.kylin.metadata.project.RealizationEntry;
import org.apache.kylin.metadata.realization.IRealizationConstants;
+import org.apache.kylin.metadata.realization.RealizationType;
import org.apache.kylin.storage.hbase.HBaseConnection;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -102,11 +106,14 @@ public class DeployCoprocessorCLI {
tableNames = filterByTables(tableNames,
Arrays.asList(args).subList(2, args.length));
} else if (filterType.equals("-cube")) {
tableNames = filterByCubes(tableNames,
Arrays.asList(args).subList(2, args.length));
+} else if (filterType.equals("-project")) {
+tableNames = filterByProjects(tableNames,
Arrays.asList(args).subList(2, args.length));
} else if (!filterType.equals("all")) {
printUsageAndExit();
}
logger.info("Will execute tables " + tableNames);
+long start = System.currentTimeMillis();
Set oldJarPaths = getCoprocessorJarPaths(hbaseAdmin,
tableNames);
logger.info("Old coprocessor jar: " + oldJarPaths);
@@ -121,16 +128,47 @@ public class DeployCoprocessorCLI {
hbaseAdmin.close();
+logger.info("Processed time: " + (System.currentTimeMillis() - start));
logger.info("Processed tables count: " + processedTables.size());
logger.info("Processed tables: " + processedTables);
logger.info("Active coprocessor jar: " + hdfsCoprocessorJar);
}
private static void printUsageAndExit() {
-logger.info("Probe run, exiting. Append argument 'all' or specific
tables/cubes to execute.");
+logger.info("Usage: ");
+logger.info("$KYLIN_HOME/bin/kylin.sh
org.apache.kylin.storage.hbase.util.DeployCoprocessorCLI
$KYLIN_HOME/lib/kylin-coprocessor-*.jar all");
+logger.info("$KYLIN_HOME/bin/kylin.sh
org.apache.kylin.storage.hbase.util.DeployCoprocessorCLI
$KYLIN_HOME/lib/kylin-coprocessor-*.jar -table tableName1 tableName2 ...");
+logger.info("$KYLIN_HOME/bin/kylin.sh
org.apache.kylin.storage.hbase.util.DeployCoprocessorCLI
$KYLIN_HOME/lib/kylin-coprocessor-*.jar -cube cubeName1 cubeName2 ... ");
+logger.info("$KYLIN_HOME/bin/kylin.sh
org.apache.kylin.storage.hbase.util.DeployCoprocessorCLI
$KYLIN_HOME/lib/kylin-coprocessor-*.jar -project projectName1 projectName2
...");
System.exit(0);
}
+private static List filterByProjects(List allTableNames,
List projectNames) {
+ProjectManager projectManager =
ProjectManager.getInstance(KylinConfig.getInstanceFromEnv());
+CubeManager cubeManager =
CubeManager.getInstance(KylinConfig.getInstanceFromEnv());
+
+List result = Lists.newArrayList();
+for (String p : projectNames) {
+p = p.trim();
+if (p.endsWith(",")) {
+p = p.substring(0, p.length() - 1);
+}
+
+ProjectInstance
kylin git commit: KYLIN-2433 Handle the column that all records is null in MergeCuboidMapper
Repository: kylin
Updated Branches:
refs/heads/master 69f898482 -> a5326fcb5
KYLIN-2433 Handle the column that all records is null in MergeCuboidMapper
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/a5326fcb
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/a5326fcb
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/a5326fcb
Branch: refs/heads/master
Commit: a5326fcb5a670970ee20b08486a77ec02dcf5c66
Parents: 69f8984
Author: kangkaisen
Authored: Tue Feb 14 11:29:23 2017 +0800
Committer: kangkaisen
Committed: Tue Feb 14 17:50:26 2017 +0800
--
.../kylin/engine/mr/steps/MergeCuboidMapper.java| 16 ++--
1 file changed, 10 insertions(+), 6 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/a5326fcb/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidMapper.java
--
diff --git
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidMapper.java
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidMapper.java
index acf1403..fccd48a 100644
---
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidMapper.java
+++
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidMapper.java
@@ -191,9 +191,18 @@ public class MergeCuboidMapper extends KylinMapper {
if (this.checkNeedMerging(col)) {
// if dictionary on fact table column, needs rewrite
DictionaryManager dictMgr =
DictionaryManager.getInstance(config);
-Dictionary sourceDict =
dictMgr.getDictionary(sourceCubeSegment.getDictResPath(col));
Dictionary mergedDict =
dictMgr.getDictionary(mergedCubeSegment.getDictResPath(col));
+Dictionary sourceDict;
+// handle the column that all records is null
+if (sourceCubeSegment.getDictionary(col) == null) {
+BytesUtil.writeUnsigned(mergedDict.nullId(),
newKeyBodyBuf, bufOffset, mergedDict.getSizeOfId());
+bufOffset += mergedDict.getSizeOfId();
+continue;
+} else {
+sourceDict =
dictMgr.getDictionary(sourceCubeSegment.getDictResPath(col));
+}
+
while (sourceDict.getSizeOfValue() > newKeyBodyBuf.length -
bufOffset || //
mergedDict.getSizeOfValue() > newKeyBodyBuf.length -
bufOffset || //
mergedDict.getSizeOfId() > newKeyBodyBuf.length -
bufOffset) {
@@ -254,11 +263,6 @@ public class MergeCuboidMapper extends KylinMapper {
}
private Boolean checkNeedMerging(TblColRef col) throws IOException {
-//handle the column that all records is null
-if (sourceCubeSegment.getDictionary(col) == null) {
-return false;
-}
-
Boolean ret = dimensionsNeedDict.get(col);
if (ret != null)
return ret;
[1/3] kylin git commit: KYLIN-2506 Refactor Global Dictionary
Repository: kylin
Updated Branches:
refs/heads/KYLIN-2506 [created] ee3f8e6e0
http://git-wip-us.apache.org/repos/asf/kylin/blob/ee3f8e6e/core-dictionary/src/test/java/org/apache/kylin/dict/CachedTreeMapTest.java
--
diff --git
a/core-dictionary/src/test/java/org/apache/kylin/dict/CachedTreeMapTest.java
b/core-dictionary/src/test/java/org/apache/kylin/dict/CachedTreeMapTest.java
deleted file mode 100644
index 3c29d9c..000
--- a/core-dictionary/src/test/java/org/apache/kylin/dict/CachedTreeMapTest.java
+++ /dev/null
@@ -1,378 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-package org.apache.kylin.dict;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertNull;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
-
-import java.io.DataInput;
-import java.io.DataInputStream;
-import java.io.DataOutput;
-import java.io.DataOutputStream;
-import java.io.File;
-import java.io.FileFilter;
-import java.io.IOException;
-import java.util.UUID;
-
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.Writable;
-import org.apache.hadoop.io.WritableComparable;
-import org.apache.kylin.common.util.HadoopUtil;
-import org.junit.After;
-import org.junit.AfterClass;
-import org.junit.Test;
-
-/**
- * Created by sunyerui on 16/7/12.
- */
-public class CachedTreeMapTest {
-
-public static class Key implements WritableComparable {
-int keyInt;
-
-public static Key of(int keyInt) {
-Key newKey = new Key();
-newKey.keyInt = keyInt;
-return newKey;
-}
-
-@Override
-public int compareTo(Object o) {
-return keyInt - ((Key)o).keyInt;
-}
-
-@Override
-public void write(DataOutput out) throws IOException {
-out.writeInt(keyInt);
-}
-
-@Override
-public void readFields(DataInput in) throws IOException {
-keyInt = in.readInt();
-}
-
-@Override
-public String toString() {
-return String.valueOf(keyInt);
-}
-}
-
-public static boolean VALUE_WRITE_ERROR_TOGGLE = false;
-public static class Value implements Writable {
-String valueStr;
-
-public static Value of(String valueStr) {
-Value newValue = new Value();
-newValue.valueStr = valueStr;
-return newValue;
-}
-
-@Override
-public void write(DataOutput out) throws IOException {
-if (VALUE_WRITE_ERROR_TOGGLE) {
-out.write(new byte[0]);
-return;
-}
-out.writeUTF(valueStr);
-}
-
-@Override
-public void readFields(DataInput in) throws IOException {
-valueStr = in.readUTF();
-}
-}
-
-public static class CachedFileFilter implements FileFilter {
-@Override
-public boolean accept(File pathname) {
-return pathname.getName().startsWith(CachedTreeMap.CACHED_PREFIX);
-}
-}
-
-public static class VersionFilter implements FileFilter {
-@Override
-public boolean accept(File pathname) {
-return pathname.getName().startsWith(CachedTreeMap.VERSION_PREFIX);
-}
-}
-
-
-static final UUID uuid = UUID.randomUUID();
-static final String baseDir = "/tmp/kylin_cachedtreemap_test/" + uuid;
-static final String workingDir = baseDir + "/working";
-
-private static void cleanup() {
-Path basePath = new Path(baseDir);
-try {
-HadoopUtil.getFileSystem(basePath).delete(basePath, true);
-} catch (IOException e) {}
-VALUE_WRITE_ERROR_TOGGLE = false;
-}
-
-@After
-public void afterTest() {
-cleanup();
-}
-
-@AfterClass
-public static void tearDown() {
-cleanup();
-}
-
-@Test
-public void testCachedTreeMap() throws IOException {
-CachedTreeMap map = createMutableMap();
-map.put(Key.of(1), Value.of("a"));
-
[3/3] kylin git commit: KYLIN-2506 Refactor Global Dictionary
KYLIN-2506 Refactor Global Dictionary
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/ee3f8e6e
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/ee3f8e6e
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/ee3f8e6e
Branch: refs/heads/KYLIN-2506
Commit: ee3f8e6e0d202ea3aacf24caf75762f1e6048ebb
Parents: 3a3e967
Author: kangkaisen
Authored: Mon Feb 20 21:06:44 2017 +0800
Committer: kangkaisen
Committed: Mon Mar 13 21:27:46 2017 +0800
--
.../apache/kylin/dict/AppendTrieDictionary.java | 1197 +-
.../kylin/dict/AppendTrieDictionaryBuilder.java | 289 +
.../kylin/dict/AppendTrieDictionaryChecker.java |9 +-
.../org/apache/kylin/dict/CachedTreeMap.java| 481 ---
.../java/org/apache/kylin/dict/DictNode.java| 376 ++
.../java/org/apache/kylin/dict/DictSlice.java | 283 +
.../org/apache/kylin/dict/DictSliceKey.java | 75 ++
.../apache/kylin/dict/GlobalDictHDFSStore.java | 419 ++
.../apache/kylin/dict/GlobalDictMetadata.java | 50 +
.../org/apache/kylin/dict/GlobalDictStore.java | 102 ++
.../kylin/dict/GlobalDictionaryBuilder.java | 12 +-
.../kylin/dict/AppendTrieDictionaryTest.java| 324 -
.../apache/kylin/dict/CachedTreeMapTest.java| 378 --
.../engine/spark/KylinKryoRegistrator.java |3 -
14 files changed, 1943 insertions(+), 2055 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/ee3f8e6e/core-dictionary/src/main/java/org/apache/kylin/dict/AppendTrieDictionary.java
--
diff --git
a/core-dictionary/src/main/java/org/apache/kylin/dict/AppendTrieDictionary.java
b/core-dictionary/src/main/java/org/apache/kylin/dict/AppendTrieDictionary.java
index 962686d..1205c92 100644
---
a/core-dictionary/src/main/java/org/apache/kylin/dict/AppendTrieDictionary.java
+++
b/core-dictionary/src/main/java/org/apache/kylin/dict/AppendTrieDictionary.java
@@ -15,1173 +15,125 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
package org.apache.kylin.dict;
-import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
+import com.google.common.cache.CacheBuilder;
+import com.google.common.cache.CacheLoader;
+import com.google.common.cache.LoadingCache;
+import com.google.common.cache.RemovalListener;
+import com.google.common.cache.RemovalNotification;
+import org.apache.hadoop.fs.Path;
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.util.Dictionary;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import java.io.DataInput;
-import java.io.DataInputStream;
import java.io.DataOutput;
-import java.io.DataOutputStream;
import java.io.IOException;
import java.io.PrintStream;
-import java.io.UnsupportedEncodingException;
-import java.util.ArrayList;
import java.util.Arrays;
-import java.util.HashSet;
-import java.util.IdentityHashMap;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.NavigableSet;
import java.util.Objects;
-import java.util.TreeMap;
-import java.util.concurrent.ConcurrentHashMap;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FSDataInputStream;
-import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.FileUtil;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.Writable;
-import org.apache.hadoop.io.WritableComparable;
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.Bytes;
-import org.apache.kylin.common.util.BytesUtil;
-import org.apache.kylin.common.util.ClassUtil;
-import org.apache.kylin.common.util.Dictionary;
-import org.apache.kylin.common.util.HadoopUtil;
-import org.apache.kylin.common.util.Pair;
-import org.apache.kylin.metadata.MetadataManager;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
+import java.util.concurrent.ExecutionException;
+import static com.google.common.base.Preconditions.checkState;
/**
* A dictionary based on Trie data structure that maps enumerations of byte[]
to
* int IDs, used for global dictionary.
- *
- * Trie data is split into sub trees, called {@link DictSlice}, and stored in
a {@link CachedTreeMap} with a configurable cache size.
- *
+ *
+ * Trie data is split into sub trees, called {@link DictSlice}.
+ *
* With Trie the memory footprint of the mapping is kinda minimized at the cost
* CPU, if compared to HashMap of ID Arrays. Performance test shows Trie is
* roughly 10 times slower, so there's a cache layer overlays on top of Trie
and
* gracefully fall back to Trie using a weak reference.
- *
+ *
* The implementation is NOT thread-safe for now.
- *
+ *
*
[2/3] kylin git commit: KYLIN-2506 Refactor Global Dictionary
http://git-wip-us.apache.org/repos/asf/kylin/blob/ee3f8e6e/core-dictionary/src/main/java/org/apache/kylin/dict/DictNode.java
--
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/DictNode.java
b/core-dictionary/src/main/java/org/apache/kylin/dict/DictNode.java
new file mode 100644
index 000..dd9593a
--- /dev/null
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/DictNode.java
@@ -0,0 +1,376 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.dict;
+
+import org.apache.kylin.common.util.Bytes;
+import org.apache.kylin.common.util.BytesUtil;
+
+import java.io.ByteArrayOutputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.io.PrintStream;
+import java.io.UnsupportedEncodingException;
+import java.util.ArrayList;
+import java.util.IdentityHashMap;
+import java.util.LinkedList;
+
+public class DictNode {
+public byte[] part;
+public int id = -1;
+public boolean isEndOfValue;
+public ArrayList children = new ArrayList<>();
+
+public int nValuesBeneath;
+public DictNode parent;
+public int childrenCount = 1;
+
+DictNode(byte[] value, boolean isEndOfValue) {
+reset(value, isEndOfValue);
+}
+
+DictNode(byte[] value, boolean isEndOfValue, ArrayList children)
{
+reset(value, isEndOfValue, children);
+}
+
+void reset(byte[] value, boolean isEndOfValue) {
+reset(value, isEndOfValue, new ArrayList());
+}
+
+void reset(byte[] value, boolean isEndOfValue, ArrayList
children) {
+this.part = value;
+this.isEndOfValue = isEndOfValue;
+clearChild();
+for (DictNode child : children) {
+addChild(child);
+}
+this.id = -1;
+}
+
+void clearChild() {
+this.children.clear();
+int childrenCountDelta = this.childrenCount - 1;
+for (DictNode p = this; p != null; p = p.parent) {
+p.childrenCount -= childrenCountDelta;
+}
+}
+
+void addChild(DictNode child) {
+addChild(-1, child);
+}
+
+void addChild(int index, DictNode child) {
+child.parent = this;
+if (index < 0) {
+this.children.add(child);
+} else {
+this.children.add(index, child);
+}
+for (DictNode p = this; p != null; p = p.parent) {
+p.childrenCount += child.childrenCount;
+}
+}
+
+private DictNode removeChild(int index) {
+DictNode child = children.remove(index);
+child.parent = null;
+for (DictNode p = this; p != null; p = p.parent) {
+p.childrenCount -= child.childrenCount;
+}
+return child;
+}
+
+private DictNode duplicateNode() {
+DictNode newChild = new DictNode(part, false);
+newChild.parent = parent;
+if (parent != null) {
+int index = parent.children.indexOf(this);
+parent.addChild(index + 1, newChild);
+}
+return newChild;
+}
+
+public byte[] firstValue() {
+ByteArrayOutputStream bytes = new ByteArrayOutputStream();
+DictNode p = this;
+while (true) {
+bytes.write(p.part, 0, p.part.length);
+if (p.isEndOfValue || p.children.size() == 0) {
+break;
+}
+p = p.children.get(0);
+}
+return bytes.toByteArray();
+}
+
+public static DictNode splitNodeTree(final DictNode splitNode) {
+if (splitNode == null) {
+return null;
+}
+DictNode current = splitNode;
+DictNode p = current.parent;
+while (p != null) {
+int index = p.children.indexOf(current);
+assert index != -1;
+DictNode newParent = p.duplicateNode();
+for (int i = p.children.size() - 1; i >= index; i--) {
+DictNode child = p.removeChild(i);
+newParent.addChild(0, child);
+}
+current = newParent;
+p = p.parent;
+}
+return current;
+}
+
+public byte[] buildTrieBytes() {
+
[20/50] [abbrv] kylin git commit: minor, allow DebugTomcat on sandbox in different LAN
minor, allow DebugTomcat on sandbox in different LAN Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/551f5ed2 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/551f5ed2 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/551f5ed2 Branch: refs/heads/KYLIN-2506 Commit: 551f5ed2e7922f63e260b5adf1e04c8a86a77298 Parents: 7af5529 Author: lidongsjtu Authored: Sat Apr 1 20:55:58 2017 +0800 Committer: lidongsjtu Committed: Tue Apr 4 10:55:37 2017 +0800 -- examples/test_case_data/sandbox/hdfs-site.xml | 4 1 file changed, 4 insertions(+) -- http://git-wip-us.apache.org/repos/asf/kylin/blob/551f5ed2/examples/test_case_data/sandbox/hdfs-site.xml -- diff --git a/examples/test_case_data/sandbox/hdfs-site.xml b/examples/test_case_data/sandbox/hdfs-site.xml index 1d9040a..c06222e 100644 --- a/examples/test_case_data/sandbox/hdfs-site.xml +++ b/examples/test_case_data/sandbox/hdfs-site.xml @@ -15,6 +15,10 @@ limitations under the License. --> + +dfs.client.use.datanode.hostname +true + dfs.block.access.token.enable
[12/50] [abbrv] kylin git commit: KYLIN-2529 Allow thread-local override of KylinConfig
KYLIN-2529 Allow thread-local override of KylinConfig
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/1e8b6a5e
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/1e8b6a5e
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/1e8b6a5e
Branch: refs/heads/KYLIN-2506
Commit: 1e8b6a5e7b1350930c68e750fb1ce75c27428082
Parents: 782a974
Author: lidongsjtu
Authored: Thu Mar 30 19:54:34 2017 +0800
Committer: lidongsjtu
Committed: Fri Mar 31 19:03:54 2017 +0800
--
.../org/apache/kylin/common/KylinConfig.java| 37 ++--
.../apache/kylin/common/KylinConfigTest.java| 31 +++-
2 files changed, 56 insertions(+), 12 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/1e8b6a5e/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java
--
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java
b/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java
index c6b1511..a9a0c45 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java
@@ -50,30 +50,41 @@ public class KylinConfig extends KylinConfigBase {
public static final String KYLIN_CONF = "KYLIN_CONF";
// static cached instances
-private static KylinConfig ENV_INSTANCE = null;
+private static KylinConfig SYS_ENV_INSTANCE = null;
+
+// thread-local instances, will override SYS_ENV_INSTANCE
+private static final transient ThreadLocal
THREAD_ENV_INSTANCE = new ThreadLocal<>();
public static KylinConfig getInstanceFromEnv() {
synchronized (KylinConfig.class) {
-if (ENV_INSTANCE == null) {
+KylinConfig config = THREAD_ENV_INSTANCE.get();
+if (config != null) {
+return config;
+}
+
+if (SYS_ENV_INSTANCE == null) {
try {
-KylinConfig config = new KylinConfig();
+config = new KylinConfig();
config.reloadKylinConfig(getKylinProperties());
logger.info("Initialized a new KylinConfig from
getInstanceFromEnv : " + System.identityHashCode(config));
-ENV_INSTANCE = config;
+SYS_ENV_INSTANCE = config;
} catch (IllegalArgumentException e) {
throw new IllegalStateException("Failed to find
KylinConfig ", e);
}
}
-return ENV_INSTANCE;
+return SYS_ENV_INSTANCE;
}
}
//Only used in test cases!!!
public static void destroyInstance() {
-logger.info("Destory KylinConfig");
-dumpStackTrace();
-ENV_INSTANCE = null;
+synchronized (KylinConfig.class) {
+logger.info("Destroy KylinConfig");
+dumpStackTrace();
+SYS_ENV_INSTANCE = null;
+THREAD_ENV_INSTANCE.remove();
+}
}
public enum UriType {
@@ -158,12 +169,12 @@ public class KylinConfig extends KylinConfigBase {
public static void setKylinConfigInEnvIfMissing(Properties prop) {
synchronized (KylinConfig.class) {
-if (ENV_INSTANCE == null) {
+if (SYS_ENV_INSTANCE == null) {
try {
KylinConfig config = new KylinConfig();
config.reloadKylinConfig(prop);
-logger.info("Resetting ENV_INSTANCE by a input stream: " +
System.identityHashCode(config));
-ENV_INSTANCE = config;
+logger.info("Resetting SYS_ENV_INSTANCE by a input stream:
" + System.identityHashCode(config));
+SYS_ENV_INSTANCE = config;
} catch (IllegalArgumentException e) {
throw new IllegalStateException("Failed to find
KylinConfig ", e);
}
@@ -177,6 +188,10 @@ public class KylinConfig extends KylinConfigBase {
setKylinConfigInEnvIfMissing(props);
}
+public static void setKylinConfigThreadLocal(KylinConfig config) {
+THREAD_ENV_INSTANCE.set(config);
+}
+
public static KylinConfig createKylinConfig(String propsInStr) throws
IOException {
Properties props = new Properties();
props.load(new StringReader(propsInStr));
http://git-wip-us.apache.org/repos/asf/kylin/blob/1e8b6a5e/core-common/src/test/java/org/apache/kylin/common/KylinConfigTest.java
--
diff --git
a/core-common/src/test/java/org/apache/kylin/common/KylinConfigTest.java
b/core-common/src/test/java/org/apache/kylin/common/KylinConfigTest.java
ind
[32/50] [abbrv] kylin git commit: minor, remove method fillbackOptimizedColumn
minor, remove method fillbackOptimizedColumn
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/6b4f70d2
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/6b4f70d2
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/6b4f70d2
Branch: refs/heads/KYLIN-2506
Commit: 6b4f70d257e1eb363a7b792cde8f6f59821094a6
Parents: 75630b8
Author: Roger Shi
Authored: Sun Apr 9 21:26:09 2017 +0800
Committer: Roger Shi
Committed: Sun Apr 9 21:26:09 2017 +0800
--
.../kylin/query/relnode/OLAPAggregateRel.java | 18 --
1 file changed, 18 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b4f70d2/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java
--
diff --git
a/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java
b/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java
index 2c75a14..1378681 100644
--- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java
+++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java
@@ -345,8 +345,6 @@ public class OLAPAggregateRel extends Aggregate implements
OLAPRel {
}
private void buildRewriteFieldsAndMetricsColumns() {
-fillbackOptimizedColumn();
-
ColumnRowType inputColumnRowType = ((OLAPRel)
getInput()).getColumnRowType();
RelDataTypeFactory typeFactory = getCluster().getTypeFactory();
for (int i = 0; i < this.aggregations.size(); i++) {
@@ -385,22 +383,6 @@ public class OLAPAggregateRel extends Aggregate implements
OLAPRel {
}
}
-private void fillbackOptimizedColumn() {
-// some aggcall will be optimized out in sub-query (e.g. tableau
generated sql), we need to fill them back
-RelDataType inputAggRow = getInput().getRowType();
-RelDataType outputAggRow = getRowType();
-if (inputAggRow.getFieldCount() != outputAggRow.getFieldCount()) {
-for (RelDataTypeField inputField : inputAggRow.getFieldList()) {
-String inputFieldName = inputField.getName();
-// constant columns(starts with $) should not be added to
context.
-if (!inputFieldName.startsWith("$") &&
outputAggRow.getField(inputFieldName, true, false) == null) {
-TblColRef column =
this.columnRowType.getColumnByIndex(inputField.getIndex());
-this.context.metricsColumns.add(column);
-}
-}
-}
-}
-
private boolean noPrecaculatedFieldsAvailable() {
return !this.context.hasPrecalculatedFields() ||
!RewriteImplementor.needRewrite(this.context);
}
[05/50] [abbrv] kylin git commit: KYLIN-2501 Stream Aggregate GTRecords at Query Server
KYLIN-2501 Stream Aggregate GTRecords at Query Server
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/0fa57248
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/0fa57248
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/0fa57248
Branch: refs/heads/KYLIN-2506
Commit: 0fa572482f209d25cca6968812d293c08d077210
Parents: fa3ee3f
Author: gaodayue
Authored: Wed Mar 15 22:45:02 2017 +0800
Committer: Hongbin Ma
Committed: Fri Mar 31 16:39:35 2017 +0800
--
.../apache/kylin/common/KylinConfigBase.java| 4 +
.../kylin/common/util/ImmutableBitSet.java | 29 ++-
.../org/apache/kylin/GTForwardingScanner.java | 56 +
.../kylin/cube/gridtable/CubeGridTable.java | 18 --
.../gridtable/CuboidToGridTableMapping.java | 18 ++
.../cube/inmemcubing/InMemCubeBuilder.java | 6 +-
.../kylin/gridtable/GTAggregateScanner.java | 16 +-
.../apache/kylin/gridtable/GTFilterScanner.java | 22 +-
.../org/apache/kylin/gridtable/GTRecord.java| 80 +++
.../apache/kylin/gridtable/GTScanRequest.java | 13 ++
.../gridtable/GTStreamAggregateScanner.java | 211 +++
.../kylin/gridtable/GTScanReqSerDerTest.java| 4 +-
.../apache/kylin/storage/StorageContext.java| 20 ++
.../storage/gtrecord/CubeScanRangePlanner.java | 3 +-
.../storage/gtrecord/CubeSegmentScanner.java| 7 +-
.../storage/gtrecord/CubeTupleConverter.java| 31 +--
.../gtrecord/GTCubeStorageQueryBase.java| 38 +++-
.../kylin/storage/gtrecord/ITupleConverter.java | 3 +-
.../gtrecord/PartitionResultIterator.java | 59 ++
.../storage/gtrecord/PartitionResultMerger.java | 100 +
.../kylin/storage/gtrecord/ScannerWorker.java | 5 +-
.../gtrecord/SegmentCubeTupleIterator.java | 72 ++-
.../gtrecord/StorageResponseGTScatter.java | 82 +++
.../hbase/cube/v2/CubeHBaseEndpointRPC.java | 7 +-
.../storage/hbase/cube/v2/CubeHBaseRPC.java | 5 +-
.../storage/hbase/cube/v2/CubeHBaseScanRPC.java | 5 +-
26 files changed, 704 insertions(+), 210 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/0fa57248/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
--
diff --git
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 02349ad..9cd35c8 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -802,6 +802,10 @@ abstract public class KylinConfigBase implements
Serializable {
return Boolean.valueOf(getOptional("kylin.query.skip-empty-segments",
"true"));
}
+public boolean isStreamAggregateEnabled() {
+return
Boolean.parseBoolean(getOptional("kylin.query.stream-aggregate-enabled",
"true"));
+}
+
@Deprecated //Limit is good even it's large. This config is meaning less
since we already have scan threshold
public int getStoragePushDownLimitMax() {
return Integer.parseInt(getOptional("kylin.query.max-limit-pushdown",
"1"));
http://git-wip-us.apache.org/repos/asf/kylin/blob/0fa57248/core-common/src/main/java/org/apache/kylin/common/util/ImmutableBitSet.java
--
diff --git
a/core-common/src/main/java/org/apache/kylin/common/util/ImmutableBitSet.java
b/core-common/src/main/java/org/apache/kylin/common/util/ImmutableBitSet.java
index b417877..5cdf08c 100644
---
a/core-common/src/main/java/org/apache/kylin/common/util/ImmutableBitSet.java
+++
b/core-common/src/main/java/org/apache/kylin/common/util/ImmutableBitSet.java
@@ -19,8 +19,9 @@ package org.apache.kylin.common.util;
import java.nio.ByteBuffer;
import java.util.BitSet;
+import java.util.Iterator;
-public class ImmutableBitSet {
+public class ImmutableBitSet implements Iterable {
public static final ImmutableBitSet EMPTY = new ImmutableBitSet(new
BitSet());
@@ -168,4 +169,30 @@ public class ImmutableBitSet {
return new ImmutableBitSet(bitSet);
}
};
+
+/**
+ * Iterate over the positions of true value.
+ * @return the iterator
+ */
+@Override
+public Iterator iterator() {
+return new Iterator() {
+int index = 0;
+
+@Override
+public boolean hasNext() {
+return index < arr.length;
+}
+
+@Override
+public Integer next() {
+return arr[index++];
+}
+
+@Override
+public void remove() {
+throw new UnsupportedOperation
[39/50] [abbrv] kylin git commit: KYLIN-2540 remove unused test
KYLIN-2540 remove unused test
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/9532bf2b
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/9532bf2b
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/9532bf2b
Branch: refs/heads/KYLIN-2506
Commit: 9532bf2b089b0b77b7ca49ec73f9304fcbe3
Parents: 86645af
Author: lidongsjtu
Authored: Mon Apr 10 22:22:19 2017 +0800
Committer: lidongsjtu
Committed: Tue Apr 11 11:13:25 2017 +0800
--
.../test/java/org/apache/kylin/rest/util/QueryUtilTest.java| 6 --
1 file changed, 6 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/9532bf2b/server-base/src/test/java/org/apache/kylin/rest/util/QueryUtilTest.java
--
diff --git
a/server-base/src/test/java/org/apache/kylin/rest/util/QueryUtilTest.java
b/server-base/src/test/java/org/apache/kylin/rest/util/QueryUtilTest.java
index c00cd3f..dcd8c98 100644
--- a/server-base/src/test/java/org/apache/kylin/rest/util/QueryUtilTest.java
+++ b/server-base/src/test/java/org/apache/kylin/rest/util/QueryUtilTest.java
@@ -51,12 +51,6 @@ public class QueryUtilTest extends LocalFileMetadataTestCase
{
String s = QueryUtil.massageSql(sqlRequest);
Assert.assertEquals("select ( date '2001-09-28' + interval '2'
month) from test_kylin_fact group by ( date '2001-09-28' + interval '2'
month)", s);
}
-{
-SQLRequest sqlRequest = new SQLRequest();
-sqlRequest.setSql("select
concat(\"TEST_KYLIN_FACT\".\"LSTG_FORMAT_NAME\",\"TEST_KYLIN_FACT\".\"LSTG_FORMAT_NAME\")
concat(\"TEST_KYLIN_FACT\".\"LSTG_FORMAT_NAME\",\"TEST_KYLIN_FACT\".\"LSTG_FORMAT_NAME\")
()");
-String s = QueryUtil.massageSql(sqlRequest);
-Assert.assertEquals("select {fn
concat(\"TEST_KYLIN_FACT\".\"LSTG_FORMAT_NAME\",\"TEST_KYLIN_FACT\".\"LSTG_FORMAT_NAME\")
} {fn
concat(\"TEST_KYLIN_FACT\".\"LSTG_FORMAT_NAME\",\"TEST_KYLIN_FACT\".\"LSTG_FORMAT_NAME\")
} ()", s);
-}
}
@Test
[44/50] [abbrv] kylin git commit: Merge commit 'a36288142f30cad931881107872adb4ba0a574a3'
Merge commit 'a36288142f30cad931881107872adb4ba0a574a3'
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/e9ec116c
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/e9ec116c
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/e9ec116c
Branch: refs/heads/KYLIN-2506
Commit: e9ec116c247072603a5405ddba425fd349ba670b
Parents: f44f828 a362881
Author: Hongbin Ma
Authored: Tue Apr 11 17:12:46 2017 +0800
Committer: Hongbin Ma
Committed: Tue Apr 11 17:12:46 2017 +0800
--
build/conf/kylin.properties | 1 +
.../kylin/common/util/HiveCmdBuilder.java | 16 +++-
.../kylin/common/util/HiveCmdBuilderTest.java | 1 +
.../cube/model/CubeJoinedFlatTableDesc.java | 4 +-
.../org/apache/kylin/job/JoinedFlatTable.java | 34 +++
.../kylin/job/engine/JobEngineConfig.java | 2 +-
.../apache/kylin/job/JoinedFlatTableTest.java | 4 +-
.../metadata/filter/CompareTupleFilter.java | 14 +++
.../filter/FilterOptimizeTransformer.java | 89 ++
.../kylin/metadata/model/FunctionDesc.java | 8 +-
.../kylin/metadata/model/ParameterDesc.java | 58 ++--
.../apache/kylin/metadata/model/TblColRef.java | 10 ++
.../gtrecord/GTCubeStorageQueryBase.java| 2 +-
.../gtrecord/StorageResponseGTScatter.java | 22 +++--
.../engine/mr/common/HadoopStatusGetter.java| 15 +--
examples/test_case_data/sandbox/hdfs-site.xml | 4 +
.../test_case_data/sandbox/kylin.properties | 1 +
.../src/test/resources/query/sql/query104.sql | 31 +++
.../resources/query/sql_casewhen/query55.sql| 22 +
.../query/sql_casewhen/query55.sql.disable | 22 -
.../resources/query/sql_subquery/query15.sql| 29 ++
.../resources/query/sql_subquery/query16.sql| 38
.../kylin/query/relnode/OLAPAggregateRel.java | 13 ++-
.../kylin/query/relnode/OLAPFilterRel.java | 25 ++---
.../kylin/query/relnode/OLAPProjectRel.java | 12 +--
.../kylin/query/relnode/OLAPTableScan.java | 2 +-
.../org/apache/kylin/query/udf/ConcatUDF.java | 28 ++
.../org/apache/kylin/rest/util/QueryUtil.java | 15 ---
.../apache/kylin/rest/util/QueryUtilTest.java | 6 --
.../apache/kylin/source/hive/HiveMRInput.java | 97 +---
.../kylin/storage/hbase/HBaseConnection.java| 7 +-
.../hbase/util/DeployCoprocessorCLI.java| 1 +
32 files changed, 425 insertions(+), 208 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/e9ec116c/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
--
diff --cc core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
index 7313630,5553d34..54c1e61
--- a/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
+++ b/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
@@@ -19,8 -19,8 +19,9 @@@
package org.apache.kylin.job;
import java.io.File;
+import java.lang.reflect.Method;
import java.util.HashSet;
+ import java.util.Map;
import java.util.Set;
import javax.xml.parsers.DocumentBuilder;
@@@ -49,18 -46,18 +50,17 @@@ public class JoinedFlatTable
return storageDfsDir + "/" + flatDesc.getTableName();
}
- public static String generateHiveSetStatements(JobEngineConfig
engineConfig) {
-public static String generateHiveInitStatements(
-String flatTableDatabase, String kylinHiveFile, Map cubeOverrides) {
++public static String generateHiveInitStatements(String flatTableDatabase,
String kylinHiveFile, Map cubeOverrides) {
+
StringBuilder buffer = new StringBuilder();
+ buffer.append("USE ").append(flatTableDatabase).append(";\n");
try {
- File hadoopPropertiesFile = new
File(engineConfig.getHiveConfFilePath());
-
- if (hadoopPropertiesFile.exists()) {
+ File file = new File(kylinHiveFile);
+ if (file.exists()) {
DocumentBuilderFactory factory =
DocumentBuilderFactory.newInstance();
- DocumentBuilder builder;
- Document doc;
- builder = factory.newDocumentBuilder();
- doc = builder.parse(hadoopPropertiesFile);
+ DocumentBuilder builder = factory.newDocumentBuilder();
+ Document doc = builder.parse(file);
NodeList nl = doc.getElementsByTagName("property");
for (int i = 0; i < nl.getLength(); i++) {
String name =
doc.getElementsByTagName("name").item(i).getFirstChild().getNodeValue();
@@@ -101,24 -103,8 +106,19 @@@
return ddl.toString();
}
- public static String generateInsertDataStatement(IJoinedFlatTableDe
[48/50] [abbrv] kylin git commit: KYLIN-2506 Refactor Global Dictionary
KYLIN-2506 Refactor Global Dictionary
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/ce8b24f6
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/ce8b24f6
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/ce8b24f6
Branch: refs/heads/KYLIN-2506
Commit: ce8b24f60a01d03eb57a223dbae095e7ceb98c7f
Parents: 1caf19a
Author: kangkaisen
Authored: Mon Feb 20 21:06:44 2017 +0800
Committer: kangkaisen
Committed: Thu Apr 13 16:20:57 2017 +0800
--
.../apache/kylin/dict/AppendTrieDictionary.java | 1197 +-
.../kylin/dict/AppendTrieDictionaryBuilder.java | 289 +
.../kylin/dict/AppendTrieDictionaryChecker.java |9 +-
.../org/apache/kylin/dict/CachedTreeMap.java| 481 ---
.../java/org/apache/kylin/dict/DictNode.java| 376 ++
.../java/org/apache/kylin/dict/DictSlice.java | 283 +
.../org/apache/kylin/dict/DictSliceKey.java | 75 ++
.../apache/kylin/dict/GlobalDictHDFSStore.java | 420 ++
.../apache/kylin/dict/GlobalDictMetadata.java | 50 +
.../org/apache/kylin/dict/GlobalDictStore.java | 102 ++
.../kylin/dict/GlobalDictionaryBuilder.java | 12 +-
.../kylin/dict/AppendTrieDictionaryTest.java| 329 -
.../apache/kylin/dict/CachedTreeMapTest.java| 378 --
.../engine/spark/KylinKryoRegistrator.java |3 -
14 files changed, 1946 insertions(+), 2058 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/ce8b24f6/core-dictionary/src/main/java/org/apache/kylin/dict/AppendTrieDictionary.java
--
diff --git
a/core-dictionary/src/main/java/org/apache/kylin/dict/AppendTrieDictionary.java
b/core-dictionary/src/main/java/org/apache/kylin/dict/AppendTrieDictionary.java
index 962686d..ea216ba 100644
---
a/core-dictionary/src/main/java/org/apache/kylin/dict/AppendTrieDictionary.java
+++
b/core-dictionary/src/main/java/org/apache/kylin/dict/AppendTrieDictionary.java
@@ -15,1173 +15,125 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
package org.apache.kylin.dict;
-import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
+import com.google.common.cache.CacheBuilder;
+import com.google.common.cache.CacheLoader;
+import com.google.common.cache.LoadingCache;
+import com.google.common.cache.RemovalListener;
+import com.google.common.cache.RemovalNotification;
+import org.apache.hadoop.fs.Path;
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.util.Dictionary;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import java.io.DataInput;
-import java.io.DataInputStream;
import java.io.DataOutput;
-import java.io.DataOutputStream;
import java.io.IOException;
import java.io.PrintStream;
-import java.io.UnsupportedEncodingException;
-import java.util.ArrayList;
import java.util.Arrays;
-import java.util.HashSet;
-import java.util.IdentityHashMap;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.NavigableSet;
import java.util.Objects;
-import java.util.TreeMap;
-import java.util.concurrent.ConcurrentHashMap;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FSDataInputStream;
-import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.FileUtil;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.Writable;
-import org.apache.hadoop.io.WritableComparable;
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.Bytes;
-import org.apache.kylin.common.util.BytesUtil;
-import org.apache.kylin.common.util.ClassUtil;
-import org.apache.kylin.common.util.Dictionary;
-import org.apache.kylin.common.util.HadoopUtil;
-import org.apache.kylin.common.util.Pair;
-import org.apache.kylin.metadata.MetadataManager;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
+import java.util.concurrent.ExecutionException;
+import static com.google.common.base.Preconditions.checkState;
/**
* A dictionary based on Trie data structure that maps enumerations of byte[]
to
* int IDs, used for global dictionary.
- *
- * Trie data is split into sub trees, called {@link DictSlice}, and stored in
a {@link CachedTreeMap} with a configurable cache size.
- *
+ *
+ * Trie data is split into sub trees, called {@link DictSlice}.
+ *
* With Trie the memory footprint of the mapping is kinda minimized at the cost
* CPU, if compared to HashMap of ID Arrays. Performance test shows Trie is
* roughly 10 times slower, so there's a cache layer overlays on top of Trie
and
* gracefully fall back to Trie using a weak reference.
- *
+ *
* The implementation is NOT thread-safe for now.
- *
+ *
*
[10/50] [abbrv] kylin git commit: KYLIN-2528 refine job email notification to support starttls and customized port
KYLIN-2528 refine job email notification to support starttls and customized port
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/f6ee76c8
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/f6ee76c8
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/f6ee76c8
Branch: refs/heads/KYLIN-2506
Commit: f6ee76c85a86971e71f998b2dd096b035bc7613c
Parents: 19d5b3d
Author: Hongbin Ma
Authored: Fri Mar 31 18:12:22 2017 +0800
Committer: Hongbin Ma
Committed: Fri Mar 31 18:12:22 2017 +0800
--
build/conf/kylin.properties | 12 +++-
.../org/apache/kylin/common/KylinConfigBase.java | 8
.../org/apache/kylin/common/util/MailService.java| 15 +--
examples/test_case_data/localmeta/kylin.properties | 15 +--
examples/test_case_data/sandbox/kylin.properties | 8 +++-
5 files changed, 44 insertions(+), 14 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/f6ee76c8/build/conf/kylin.properties
--
diff --git a/build/conf/kylin.properties b/build/conf/kylin.properties
index 1a55c94..0a77093 100644
--- a/build/conf/kylin.properties
+++ b/build/conf/kylin.properties
@@ -117,11 +117,13 @@ kylin.job.status.with.kerberos=false
kylin.job.step.timeout=7200
# If true, will send email notification on job complete
-kylin.job.notification-enabled=false
-kylin.job.notification-mail-host=
-kylin.job.notification-mail-username=
-kylin.job.notification-mail-password=
-kylin.job.notification-mail-sender=
+#kylin.job.notification-enabled=true
+#kylin.job.notification-mail-enable-starttls=true
+#kylin.job.notification-mail-host=smtp.office365.com
+#kylin.job.notification-mail-port=587
+#kylin.job.notification-mail-username=ky...@example.com
+#kylin.job.notification-mail-password=mypassword
+#kylin.job.notification-mail-sender=ky...@example.com
### ENGINE ###
http://git-wip-us.apache.org/repos/asf/kylin/blob/f6ee76c8/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
--
diff --git
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 02349ad..36c2b16 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -392,6 +392,14 @@ abstract public class KylinConfigBase implements
Serializable {
return
Boolean.parseBoolean(getOptional("kylin.job.notification-enabled", "false"));
}
+public boolean isStarttlsEnabled() {
+return
Boolean.parseBoolean(getOptional("kylin.job.notification-mail-enable-starttls",
"false"));
+}
+
+public String getSmtpPort() {
+return getOptional("kylin.job.notification-mail-port", "25");
+}
+
public String getMailHost() {
return getOptional("kylin.job.notification-mail-host", "");
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/f6ee76c8/core-common/src/main/java/org/apache/kylin/common/util/MailService.java
--
diff --git
a/core-common/src/main/java/org/apache/kylin/common/util/MailService.java
b/core-common/src/main/java/org/apache/kylin/common/util/MailService.java
index 5793967..25bc03c 100644
--- a/core-common/src/main/java/org/apache/kylin/common/util/MailService.java
+++ b/core-common/src/main/java/org/apache/kylin/common/util/MailService.java
@@ -34,18 +34,22 @@ public class MailService {
private static final org.slf4j.Logger logger =
LoggerFactory.getLogger(MailService.class);
private Boolean enabled = Boolean.TRUE;
+private Boolean starttlsEnabled = Boolean.FALSE;
private String host;
+private String port;
private String username;
private String password;
private String sender;
public MailService(KylinConfig config) {
-this(config.isMailEnabled(), config.getMailHost(),
config.getMailUsername(), config.getMailPassword(), config.getMailSender());
+this(config.isMailEnabled(), config.isStarttlsEnabled(),
config.getMailHost(), config.getSmtpPort(), config.getMailUsername(),
config.getMailPassword(), config.getMailSender());
}
-private MailService(boolean enabled, String host, String username, String
password, String sender) {
+private MailService(boolean enabled, boolean starttlsEnabled, String host,
String port, String username, String password, String sender) {
this.enabled = enabled;
+this.starttlsEnabled = starttlsEnabled;
this.host = host;
+this.port = port;
this.username = u
[27/50] [abbrv] kylin git commit: KYLIN 2534 column family select mistake
KYLIN 2534 column family select mistake
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/75630b8d
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/75630b8d
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/75630b8d
Branch: refs/heads/KYLIN-2506
Commit: 75630b8d90a900550a55f4a931eedee4b9eb3d21
Parents: 023fa23
Author: chenzhx <346839...@qq.com>
Authored: Thu Apr 6 15:42:52 2017 +0800
Committer: chenzhx <346839...@qq.com>
Committed: Thu Apr 6 15:42:52 2017 +0800
--
webapp/app/js/controllers/cubeAdvanceSetting.js | 21 ++--
1 file changed, 11 insertions(+), 10 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/75630b8d/webapp/app/js/controllers/cubeAdvanceSetting.js
--
diff --git a/webapp/app/js/controllers/cubeAdvanceSetting.js
b/webapp/app/js/controllers/cubeAdvanceSetting.js
index 5e7de33..d3e2922 100644
--- a/webapp/app/js/controllers/cubeAdvanceSetting.js
+++ b/webapp/app/js/controllers/cubeAdvanceSetting.js
@@ -339,18 +339,19 @@ KylinApp.controller('CubeAdvanceSettingCtrl', function
($scope, $modal,cubeConfi
var allMeasureNames = $scope.getAllMeasureNames();
var tmpColumnFamily = $scope.cubeMetaFrame.hbase_mapping.column_family;
-angular.forEach($scope.cubeMetaFrame.hbase_mapping.column_family, function
(colFamily,index1) {
- angular.forEach(colFamily.columns[0].measure_refs, function
(measureName, index2) {
-var allIndex = allMeasureNames.indexOf(measureName);
+for(var j=0;j<$scope.cubeMetaFrame.hbase_mapping.column_family.length;
j++) {
+ for (var
i=0;i<$scope.cubeMetaFrame.hbase_mapping.column_family[j].columns[0].measure_refs.length;
i++){
+var allIndex =
allMeasureNames.indexOf($scope.cubeMetaFrame.hbase_mapping.column_family[j].columns[0].measure_refs[i]);
if (allIndex == -1) {
- tmpColumnFamily[index1].columns[0].measure_refs.splice(index2, 1);
+ tmpColumnFamily[j].columns[0].measure_refs.splice(i, 1);
+ i--
}
-
-if (tmpColumnFamily[index1].columns[0].measure_refs == 0) {
- tmpColumnFamily.splice(index1, 1);
-}
- });
-});
+ }
+ if (tmpColumnFamily[j].columns[0].measure_refs.length == 0) {
+tmpColumnFamily.splice(j, 1);
+j--
+ }
+}
$scope.cubeMetaFrame.hbase_mapping.column_family = tmpColumnFamily;
};
[33/50] [abbrv] kylin git commit: KYLIN-2536 replace codehaus jackson with fasterxml
KYLIN-2536 replace codehaus jackson with fasterxml
Signed-off-by: lidongsjtu
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/b641032b
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/b641032b
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/b641032b
Branch: refs/heads/KYLIN-2506
Commit: b641032bfe2d79ae162c88ac7faf082250ca3a73
Parents: 2e87fb4
Author: etherge
Authored: Fri Apr 7 18:28:07 2017 +0800
Committer: lidongsjtu
Committed: Sun Apr 9 23:54:44 2017 +0800
--
.../kylin/engine/mr/common/HadoopStatusGetter.java | 15 ---
1 file changed, 8 insertions(+), 7 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/b641032b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/HadoopStatusGetter.java
--
diff --git
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/HadoopStatusGetter.java
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/HadoopStatusGetter.java
index 1107aa5..f31369b 100644
---
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/HadoopStatusGetter.java
+++
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/HadoopStatusGetter.java
@@ -43,11 +43,12 @@ import org.apache.http.client.params.AuthPolicy;
import org.apache.http.impl.auth.SPNegoSchemeFactory;
import org.apache.http.impl.client.BasicCredentialsProvider;
import org.apache.http.impl.client.DefaultHttpClient;
-import org.codehaus.jackson.JsonNode;
-import org.codehaus.jackson.map.ObjectMapper;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+
/**
*/
public class HadoopStatusGetter {
@@ -68,8 +69,8 @@ public class HadoopStatusGetter {
String response = useKerberosAuth ?
getHttpResponseWithKerberosAuth(url) : getHttpResponse(url);
logger.debug("Hadoop job " + mrJobId + " status : " + response);
JsonNode root = new ObjectMapper().readTree(response);
-RMAppState state =
RMAppState.valueOf(root.findValue("state").getTextValue());
-FinalApplicationStatus finalStatus =
FinalApplicationStatus.valueOf(root.findValue("finalStatus").getTextValue());
+RMAppState state =
RMAppState.valueOf(root.findValue("state").textValue());
+FinalApplicationStatus finalStatus =
FinalApplicationStatus.valueOf(root.findValue("finalStatus").textValue());
return Pair.of(state, finalStatus);
}
@@ -84,12 +85,12 @@ public class HadoopStatusGetter {
System.setProperty("java.security.krb5.conf", krb5ConfigPath);
System.setProperty("sun.security.krb5.debug", "true");
System.setProperty("javax.security.auth.useSubjectCredsOnly", "false");
-
+
DefaultHttpClient client = new DefaultHttpClient();
AuthSchemeRegistry authSchemeRegistry = new AuthSchemeRegistry();
authSchemeRegistry.register(AuthPolicy.SPNEGO, new
SPNegoSchemeFactory(skipPortAtKerberosDatabaseLookup));
client.setAuthSchemes(authSchemeRegistry);
-
+
BasicCredentialsProvider credentialsProvider = new
BasicCredentialsProvider();
Credentials useJaasCreds = new Credentials() {
public String getPassword() {
@@ -102,7 +103,7 @@ public class HadoopStatusGetter {
};
credentialsProvider.setCredentials(new AuthScope(null, -1, null),
useJaasCreds);
client.setCredentialsProvider(credentialsProvider);
-
+
String response = null;
while (response == null) {
if (url.startsWith("https://";)) {
[13/50] [abbrv] kylin git commit: Merge commit '782a97482ca1e92cf24e04badd8ee48c9b829f46'
Merge commit '782a97482ca1e92cf24e04badd8ee48c9b829f46'
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/ee5397bc
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/ee5397bc
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/ee5397bc
Branch: refs/heads/KYLIN-2506
Commit: ee5397bc6207c9cdf57cdce93bcda6bcac58a5a9
Parents: d4cce76 782a974
Author: Hongbin Ma
Authored: Fri Mar 31 20:00:16 2017 +0800
Committer: Hongbin Ma
Committed: Fri Mar 31 20:00:59 2017 +0800
--
build/conf/kylin.properties | 2 +-
.../apache/kylin/common/KylinConfigBase.java| 4 +
.../kylin/common/util/ImmutableBitSet.java | 29 +-
.../org/apache/kylin/GTForwardingScanner.java | 56
.../java/org/apache/kylin/cube/CubeManager.java | 8 +-
.../kylin/cube/gridtable/CubeGridTable.java | 18 --
.../gridtable/CuboidToGridTableMapping.java | 18 ++
.../cube/inmemcubing/InMemCubeBuilder.java | 6 +-
.../org/apache/kylin/cube/model/CubeDesc.java | 11 +-
.../kylin/gridtable/GTAggregateScanner.java | 16 +-
.../apache/kylin/gridtable/GTFilterScanner.java | 22 +-
.../org/apache/kylin/gridtable/GTRecord.java| 80 ++---
.../apache/kylin/gridtable/GTScanRequest.java | 13 +
.../gridtable/GTStreamAggregateScanner.java | 219 ++
.../kylin/gridtable/GTScanReqSerDerTest.java| 4 +-
.../apache/kylin/measure/hllc/HLLCounter.java | 54 ++--
.../apache/kylin/storage/StorageContext.java| 8 +
.../storage/gtrecord/CubeScanRangePlanner.java | 3 +-
.../storage/gtrecord/CubeSegmentScanner.java| 7 +-
.../storage/gtrecord/CubeTupleConverter.java| 26 +-
.../gtrecord/GTCubeStorageQueryBase.java| 35 ++-
.../kylin/storage/gtrecord/ITupleConverter.java | 3 +-
.../gtrecord/PartitionResultIterator.java | 59
.../kylin/storage/gtrecord/ScannerWorker.java | 5 +-
.../gtrecord/SegmentCubeTupleIterator.java | 71 -
.../SortMergedPartitionResultIterator.java | 81 +
.../gtrecord/StorageResponseGTScatter.java | 83 ++---
.../mr/steps/FactDistinctColumnsMapper.java | 31 +-
.../mr/steps/NewCubeSamplingMethodTest.java | 299 +++
.../hbase/cube/v2/CubeHBaseEndpointRPC.java | 7 +-
.../storage/hbase/cube/v2/CubeHBaseRPC.java | 5 +-
.../storage/hbase/cube/v2/CubeHBaseScanRPC.java | 5 +-
.../apache/kylin/tool/StorageCleanupJob.java| 13 +-
33 files changed, 1047 insertions(+), 254 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/ee5397bc/build/conf/kylin.properties
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/ee5397bc/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/ee5397bc/core-cube/src/main/java/org/apache/kylin/gridtable/GTRecord.java
--
diff --cc core-cube/src/main/java/org/apache/kylin/gridtable/GTRecord.java
index f4480c8,3397adc..3e62ea7
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTRecord.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTRecord.java
@@@ -118,20 -114,14 +114,24 @@@ public class GTRecord implements Compar
return result;
}
++
+/** decode and return the values of this record */
+public Object[] getValues(int[] selectedColumns, Object[] result) {
+assert selectedColumns.length <= result.length;
+for (int i = 0; i < selectedColumns.length; i++) {
- int c = selectedColumns[i];
- if (cols[c].array() == null) {
- result[i] = null;
- } else {
- result[i] = info.codeSystem.decodeColumnValue(c,
cols[c].asBuffer());
- }
++result[i] = decodeValue(selectedColumns[i]);
+}
+return result;
+}
+
+ public Object decodeValue(int c) {
+ ByteArray col = cols[c];
+ if (col != null && col.array() != null) {
+ return info.codeSystem.decodeColumnValue(c, col.asBuffer());
+ }
+ return null;
+ }
+
public int sizeOf(ImmutableBitSet selectedCols) {
int size = 0;
for (int i = 0; i < selectedCols.trueBitCount(); i++) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/ee5397bc/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeTupleConverter.java
--
diff --cc
core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeTupleConverter.java
index 526a554,b762e5c..7df80ca
---
a/core-storage/src/main/ja
[47/50] [abbrv] kylin git commit: KYLIN-2506 Refactor Global Dictionary
http://git-wip-us.apache.org/repos/asf/kylin/blob/ce8b24f6/core-dictionary/src/main/java/org/apache/kylin/dict/DictNode.java
--
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/DictNode.java
b/core-dictionary/src/main/java/org/apache/kylin/dict/DictNode.java
new file mode 100644
index 000..dd9593a
--- /dev/null
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/DictNode.java
@@ -0,0 +1,376 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.dict;
+
+import org.apache.kylin.common.util.Bytes;
+import org.apache.kylin.common.util.BytesUtil;
+
+import java.io.ByteArrayOutputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.io.PrintStream;
+import java.io.UnsupportedEncodingException;
+import java.util.ArrayList;
+import java.util.IdentityHashMap;
+import java.util.LinkedList;
+
+public class DictNode {
+public byte[] part;
+public int id = -1;
+public boolean isEndOfValue;
+public ArrayList children = new ArrayList<>();
+
+public int nValuesBeneath;
+public DictNode parent;
+public int childrenCount = 1;
+
+DictNode(byte[] value, boolean isEndOfValue) {
+reset(value, isEndOfValue);
+}
+
+DictNode(byte[] value, boolean isEndOfValue, ArrayList children)
{
+reset(value, isEndOfValue, children);
+}
+
+void reset(byte[] value, boolean isEndOfValue) {
+reset(value, isEndOfValue, new ArrayList());
+}
+
+void reset(byte[] value, boolean isEndOfValue, ArrayList
children) {
+this.part = value;
+this.isEndOfValue = isEndOfValue;
+clearChild();
+for (DictNode child : children) {
+addChild(child);
+}
+this.id = -1;
+}
+
+void clearChild() {
+this.children.clear();
+int childrenCountDelta = this.childrenCount - 1;
+for (DictNode p = this; p != null; p = p.parent) {
+p.childrenCount -= childrenCountDelta;
+}
+}
+
+void addChild(DictNode child) {
+addChild(-1, child);
+}
+
+void addChild(int index, DictNode child) {
+child.parent = this;
+if (index < 0) {
+this.children.add(child);
+} else {
+this.children.add(index, child);
+}
+for (DictNode p = this; p != null; p = p.parent) {
+p.childrenCount += child.childrenCount;
+}
+}
+
+private DictNode removeChild(int index) {
+DictNode child = children.remove(index);
+child.parent = null;
+for (DictNode p = this; p != null; p = p.parent) {
+p.childrenCount -= child.childrenCount;
+}
+return child;
+}
+
+private DictNode duplicateNode() {
+DictNode newChild = new DictNode(part, false);
+newChild.parent = parent;
+if (parent != null) {
+int index = parent.children.indexOf(this);
+parent.addChild(index + 1, newChild);
+}
+return newChild;
+}
+
+public byte[] firstValue() {
+ByteArrayOutputStream bytes = new ByteArrayOutputStream();
+DictNode p = this;
+while (true) {
+bytes.write(p.part, 0, p.part.length);
+if (p.isEndOfValue || p.children.size() == 0) {
+break;
+}
+p = p.children.get(0);
+}
+return bytes.toByteArray();
+}
+
+public static DictNode splitNodeTree(final DictNode splitNode) {
+if (splitNode == null) {
+return null;
+}
+DictNode current = splitNode;
+DictNode p = current.parent;
+while (p != null) {
+int index = p.children.indexOf(current);
+assert index != -1;
+DictNode newParent = p.duplicateNode();
+for (int i = p.children.size() - 1; i >= index; i--) {
+DictNode child = p.removeChild(i);
+newParent.addChild(0, child);
+}
+current = newParent;
+p = p.parent;
+}
+return current;
+}
+
+public byte[] buildTrieBytes() {
+
[08/50] [abbrv] kylin git commit: KYLIN-2521 upgrade calcite to 1.12
http://git-wip-us.apache.org/repos/asf/kylin/blob/19d5b3de/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java -- diff --git a/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java b/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java index 2c308f5..c199c31 100644 --- a/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java +++ b/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java @@ -1,12 +1,13 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to you under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -16,25 +17,6 @@ */ package org.apache.calcite.sql2rel; -import static org.apache.calcite.sql.SqlUtil.stripAs; -import static org.apache.calcite.util.Static.RESOURCE; - -import java.lang.reflect.Type; -import java.math.BigDecimal; -import java.util.AbstractList; -import java.util.ArrayDeque; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.Deque; -import java.util.EnumSet; -import java.util.HashMap; -import java.util.LinkedHashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.TreeSet; - import org.apache.calcite.avatica.util.Spaces; import org.apache.calcite.linq4j.Ord; import org.apache.calcite.plan.Convention; @@ -106,6 +88,7 @@ import org.apache.calcite.schema.ModifiableTable; import org.apache.calcite.schema.ModifiableView; import org.apache.calcite.schema.Table; import org.apache.calcite.schema.TranslatableTable; +import org.apache.calcite.schema.Wrapper; import org.apache.calcite.sql.JoinConditionType; import org.apache.calcite.sql.JoinType; import org.apache.calcite.sql.SemiJoinType; @@ -125,6 +108,7 @@ import org.apache.calcite.sql.SqlIntervalQualifier; import org.apache.calcite.sql.SqlJoin; import org.apache.calcite.sql.SqlKind; import org.apache.calcite.sql.SqlLiteral; +import org.apache.calcite.sql.SqlMatchRecognize; import org.apache.calcite.sql.SqlMerge; import org.apache.calcite.sql.SqlNode; import org.apache.calcite.sql.SqlNodeList; @@ -161,6 +145,7 @@ import org.apache.calcite.sql.validate.ListScope; import org.apache.calcite.sql.validate.ParameterScope; import org.apache.calcite.sql.validate.SelectScope; import org.apache.calcite.sql.validate.SqlMonotonicity; +import org.apache.calcite.sql.validate.SqlNameMatcher; import org.apache.calcite.sql.validate.SqlQualified; import org.apache.calcite.sql.validate.SqlUserDefinedTableFunction; import org.apache.calcite.sql.validate.SqlUserDefinedTableMacro; @@ -168,6 +153,7 @@ import org.apache.calcite.sql.validate.SqlValidator; import org.apache.calcite.sql.validate.SqlValidatorImpl; import org.apache.calcite.sql.validate.SqlValidatorNamespace; import org.apache.calcite.sql.validate.SqlValidatorScope; +import org.apache.calcite.sql.validate.SqlValidatorTable; import org.apache.calcite.sql.validate.SqlValidatorUtil; import org.apache.calcite.tools.RelBuilder; import org.apache.calcite.util.ImmutableBitSet; @@ -178,7 +164,6 @@ import org.apache.calcite.util.NumberUtil; import org.apache.calcite.util.Pair; import org.apache.calcite.util.Util; import org.apache.calcite.util.trace.CalciteTrace; -import org.slf4j.Logger; import com.google.common.base.Function; import com.google.common.base.Preconditions; @@ -190,6 +175,28 @@ import com.google.common.collect.Iterables; import com.google.common.collect.Lists; import com.google.common.collect.Maps; +import org.slf4j.Logger; + +import java.lang.reflect.Type; +import java.math.BigDecimal; +import java.util.AbstractList; +import java.util.ArrayDeque; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.Deque; +import java.util.EnumSet; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashSet; +import java.util.List; +import
[03/50] [abbrv] kylin git commit: KYLIN-2525 tolerate job metadata exception in StorageCleanupJob
KYLIN-2525 tolerate job metadata exception in StorageCleanupJob
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/fa3ee3ff
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/fa3ee3ff
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/fa3ee3ff
Branch: refs/heads/KYLIN-2506
Commit: fa3ee3ffbbb11ac3d4dc79dbb719f56a7e913857
Parents: c6b4592
Author: Li Yang
Authored: Wed Mar 29 18:03:59 2017 +0800
Committer: Yang Li
Committed: Wed Mar 29 21:02:23 2017 +0800
--
.../java/org/apache/kylin/tool/StorageCleanupJob.java | 13 +
1 file changed, 9 insertions(+), 4 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/fa3ee3ff/tool/src/main/java/org/apache/kylin/tool/StorageCleanupJob.java
--
diff --git a/tool/src/main/java/org/apache/kylin/tool/StorageCleanupJob.java
b/tool/src/main/java/org/apache/kylin/tool/StorageCleanupJob.java
index 477c58a..f1a3ebe 100644
--- a/tool/src/main/java/org/apache/kylin/tool/StorageCleanupJob.java
+++ b/tool/src/main/java/org/apache/kylin/tool/StorageCleanupJob.java
@@ -46,6 +46,7 @@ import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.AbstractApplication;
import org.apache.kylin.common.util.CliCommandExecutor;
import org.apache.kylin.common.util.HadoopUtil;
+import org.apache.kylin.common.util.HiveCmdBuilder;
import org.apache.kylin.common.util.OptionsHelper;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeManager;
@@ -57,7 +58,6 @@ import org.apache.kylin.job.execution.ExecutableManager;
import org.apache.kylin.job.execution.ExecutableState;
import org.apache.kylin.metadata.realization.IRealizationConstants;
import org.apache.kylin.source.hive.HiveClientFactory;
-import org.apache.kylin.common.util.HiveCmdBuilder;
import org.apache.kylin.source.hive.IHiveClient;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -276,9 +276,14 @@ public class StorageCleanupJob extends AbstractApplication
{
sb.append(jobId).append("(").append(state).append("), ");
}
-String segmentId = getSegmentIdFromJobId(jobId);
-if (segmentId != null) {//some jobs are not cubing jobs
-segmentId2JobId.put(segmentId, jobId);
+try {
+String segmentId = getSegmentIdFromJobId(jobId);
+if (segmentId != null) {//some jobs are not cubing jobs
+segmentId2JobId.put(segmentId, jobId);
+}
+} catch (Exception ex) {
+logger.warn("Failed to find segment ID from job ID " + jobId +
", ignore it");
+// some older version job metadata may fail to read, ignore it
}
}
logger.info("Working jobIDs: " + workingJobList);
[16/50] [abbrv] kylin git commit: KYLIN-2449 skip rewriting if no realiztion in OLAPAggregateRel
KYLIN-2449 skip rewriting if no realiztion in OLAPAggregateRel
Signed-off-by: Yang Li
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/4ddcc699
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/4ddcc699
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/4ddcc699
Branch: refs/heads/KYLIN-2506
Commit: 4ddcc699ec807aea73cf4c158550678803f48eef
Parents: 9250d9b
Author: etherge
Authored: Wed Feb 15 15:18:18 2017 +0800
Committer: Yang Li
Committed: Sun Apr 2 08:56:34 2017 +0800
--
.../kylin/metadata/model/ParameterDesc.java | 1 +
.../resources/query/sql_subquery/query16.sql| 38
.../kylin/query/relnode/OLAPAggregateRel.java | 5 +--
3 files changed, 42 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/4ddcc699/core-metadata/src/main/java/org/apache/kylin/metadata/model/ParameterDesc.java
--
diff --git
a/core-metadata/src/main/java/org/apache/kylin/metadata/model/ParameterDesc.java
b/core-metadata/src/main/java/org/apache/kylin/metadata/model/ParameterDesc.java
index 5ba2f14..272c4ee 100644
---
a/core-metadata/src/main/java/org/apache/kylin/metadata/model/ParameterDesc.java
+++
b/core-metadata/src/main/java/org/apache/kylin/metadata/model/ParameterDesc.java
@@ -34,6 +34,7 @@ import com.google.common.collect.Sets;
/**
*/
+@SuppressWarnings("serial")
@JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility =
Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility =
Visibility.NONE)
public class ParameterDesc implements Serializable {
http://git-wip-us.apache.org/repos/asf/kylin/blob/4ddcc699/kylin-it/src/test/resources/query/sql_subquery/query16.sql
--
diff --git a/kylin-it/src/test/resources/query/sql_subquery/query16.sql
b/kylin-it/src/test/resources/query/sql_subquery/query16.sql
new file mode 100644
index 000..5477e54
--- /dev/null
+++ b/kylin-it/src/test/resources/query/sql_subquery/query16.sql
@@ -0,0 +1,38 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+SELECT
+t1.leaf_categ_id, COUNT(*) AS nums
+FROM
+(SELECT
+f.leaf_categ_id
+FROM
+test_kylin_fact f inner join TEST_CATEGORY_GROUPINGS o on
f.leaf_categ_id = o.leaf_categ_id and f.LSTG_SITE_ID = o.site_id
+WHERE
+f.lstg_format_name = 'ABIN') t1
+INNER JOIN
+(SELECT
+leaf_categ_id
+FROM
+test_kylin_fact f
+INNER JOIN test_order o ON f.order_id = o.order_id
+WHERE
+buyer_id > 100) t2 ON t1.leaf_categ_id = t2.leaf_categ_id
+GROUP BY t1.leaf_categ_id
+ORDER BY nums, leaf_categ_id
+limit 100
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/4ddcc699/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java
--
diff --git
a/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java
b/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java
index 2c75a14..d9385e9 100644
--- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java
+++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java
@@ -279,7 +279,8 @@ public class OLAPAggregateRel extends Aggregate implements
OLAPRel {
@Override
public void implementRewrite(RewriteImplementor implementor) {
// only rewrite the innermost aggregation
-if (!this.afterAggregate) {
+boolean hasRealization = (null != this.context.realization);
+if (hasRealization && !this.afterAggregate) {
translateAggregation();
buildRewriteFieldsAndMetricsColumns();
}
@@ -287,7 +288,7 @@ public class OLAPAggregateRel extends Aggregate implements
OLAPRel {
implementor.visitChild(this, getInput());
// only rewrite the innermost aggregation
-if (!th
[17/50] [abbrv] kylin git commit: KYLIN-2407 fix OLAPTableScan choose exec function issue
KYLIN-2407 fix OLAPTableScan choose exec function issue
Signed-off-by: Yang Li
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/c85a0e73
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/c85a0e73
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/c85a0e73
Branch: refs/heads/KYLIN-2506
Commit: c85a0e73192ea65d5bf12d116a0f791261365753
Parents: 4ddcc69
Author: etherge
Authored: Wed Feb 15 13:30:44 2017 +0800
Committer: Yang Li
Committed: Sun Apr 2 09:15:10 2017 +0800
--
.../resources/query/sql_subquery/query15.sql| 29
.../kylin/query/relnode/OLAPTableScan.java | 2 +-
2 files changed, 30 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/c85a0e73/kylin-it/src/test/resources/query/sql_subquery/query15.sql
--
diff --git a/kylin-it/src/test/resources/query/sql_subquery/query15.sql
b/kylin-it/src/test/resources/query/sql_subquery/query15.sql
new file mode 100644
index 000..330dc27
--- /dev/null
+++ b/kylin-it/src/test/resources/query/sql_subquery/query15.sql
@@ -0,0 +1,29 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+SELECT
+week_beg_dt
+FROM
+edw.test_cal_dt t1
+WHERE
+t1.cal_dt IN (SELECT
+cal_dt
+FROM
+test_kylin_fact
+WHERE
+lstg_format_name = 'ABIN')
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/c85a0e73/query/src/main/java/org/apache/kylin/query/relnode/OLAPTableScan.java
--
diff --git
a/query/src/main/java/org/apache/kylin/query/relnode/OLAPTableScan.java
b/query/src/main/java/org/apache/kylin/query/relnode/OLAPTableScan.java
index 35dd73f..75c9c3e 100644
--- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPTableScan.java
+++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPTableScan.java
@@ -283,7 +283,7 @@ public class OLAPTableScan extends TableScan implements
OLAPRel, EnumerableRel {
private String genExecFunc() {
// if the table to scan is not the fact table of cube, then it's a
lookup table
-if (context.hasJoin == false &&
context.realization.getModel().isLookupTable(tableName)) {
+if (context.realization.getModel().isLookupTable(tableName)) {
return "executeLookupTableQuery";
} else {
return "executeOLAPQuery";
[23/50] [abbrv] kylin git commit: KYLIN-2501 pipeline partition results if possible
KYLIN-2501 pipeline partition results if possible
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/6f9bce27
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/6f9bce27
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/6f9bce27
Branch: refs/heads/KYLIN-2506
Commit: 6f9bce27545ef185a64c24aff67790150fb469b9
Parents: 7cb88f5
Author: gaodayue
Authored: Wed Apr 5 13:47:34 2017 +0800
Committer: gaodayue
Committed: Wed Apr 5 13:56:03 2017 +0800
--
.../gtrecord/StorageResponseGTScatter.java | 22 +++-
1 file changed, 12 insertions(+), 10 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/6f9bce27/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/StorageResponseGTScatter.java
--
diff --git
a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/StorageResponseGTScatter.java
b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/StorageResponseGTScatter.java
index f1ab20c..ef12ff0 100644
---
a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/StorageResponseGTScatter.java
+++
b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/StorageResponseGTScatter.java
@@ -18,6 +18,7 @@
package org.apache.kylin.storage.gtrecord;
+import com.google.common.base.Function;
import com.google.common.collect.Iterators;
import com.google.common.collect.Lists;
import org.apache.kylin.common.util.ImmutableBitSet;
@@ -69,21 +70,22 @@ public class StorageResponseGTScatter implements IGTScanner
{
@Override
public Iterator iterator() {
-List partitionResults = Lists.newArrayList();
-while (blocks.hasNext()) {
-partitionResults.add(new PartitionResultIterator(blocks.next(),
info, columns));
+Iterator iterators =
Iterators.transform(blocks, new Function() {
+public PartitionResultIterator apply(byte[] input) {
+return new PartitionResultIterator(input, info, columns);
+}
+});
+
+if (!needSorted) {
+logger.debug("Using Iterators.concat to pipeline partition
results");
+return Iterators.concat(iterators);
}
+List partitionResults =
Lists.newArrayList(iterators);
if (partitionResults.size() == 1) {
return partitionResults.get(0);
}
-
-if (!needSorted) {
-logger.debug("Using Iterators.concat to merge partition results");
-return Iterators.concat(partitionResults.iterator());
-}
-
-logger.debug("Using SortMergedPartitionResultIterator to merge
partition results");
+logger.debug("Using SortMergedPartitionResultIterator to merge {}
partition results", partitionResults.size());
return new SortMergedPartitionResultIterator(partitionResults, info,
GTRecord.getComparator(groupByDims));
}
}
[19/50] [abbrv] kylin git commit: KYLIN-2341 report error on unsupported aggr func
KYLIN-2341 report error on unsupported aggr func
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/7af5529e
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/7af5529e
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/7af5529e
Branch: refs/heads/KYLIN-2506
Commit: 7af5529ef566434b34131a6b77c0f32e738bb318
Parents: 2057831
Author: Yang Li
Authored: Mon Apr 3 22:35:21 2017 +0800
Committer: Yang Li
Committed: Mon Apr 3 22:35:21 2017 +0800
--
.../kylin/metadata/model/FunctionDesc.java | 8 +--
.../kylin/metadata/model/ParameterDesc.java | 57 ++--
.../apache/kylin/metadata/model/TblColRef.java | 10
.../resources/query/sql_casewhen/query04.sql| 30 ---
.../kylin/query/relnode/OLAPAggregateRel.java | 4 +-
.../kylin/query/relnode/OLAPProjectRel.java | 2 +-
6 files changed, 44 insertions(+), 67 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/7af5529e/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java
--
diff --git
a/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java
b/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java
index 61c5fac..e969be2 100644
---
a/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java
+++
b/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java
@@ -19,7 +19,6 @@
package org.apache.kylin.metadata.model;
import java.io.Serializable;
-import java.util.ArrayList;
import java.util.Collections;
import java.util.LinkedHashMap;
import java.util.List;
@@ -41,6 +40,7 @@ import com.google.common.collect.Sets;
/**
*/
+@SuppressWarnings("serial")
@JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility =
Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility =
Visibility.NONE)
public class FunctionDesc implements Serializable {
@@ -90,17 +90,13 @@ public class FunctionDesc implements Serializable {
expression = expression.toUpperCase();
returnDataType = DataType.getType(returnType);
-ArrayList colRefs = Lists.newArrayList();
for (ParameterDesc p = parameter; p != null; p = p.getNextParameter())
{
if (p.isColumnType()) {
TblColRef colRef = model.findColumn(p.getValue());
p.setValue(colRef.getIdentity());
-colRefs.add(colRef);
+p.setColRef(colRef);
}
}
-
-if (parameter != null)
-parameter.setColRefs(colRefs);
}
private void reInitMeasureType() {
http://git-wip-us.apache.org/repos/asf/kylin/blob/7af5529e/core-metadata/src/main/java/org/apache/kylin/metadata/model/ParameterDesc.java
--
diff --git
a/core-metadata/src/main/java/org/apache/kylin/metadata/model/ParameterDesc.java
b/core-metadata/src/main/java/org/apache/kylin/metadata/model/ParameterDesc.java
index 272c4ee..0cf31b1 100644
---
a/core-metadata/src/main/java/org/apache/kylin/metadata/model/ParameterDesc.java
+++
b/core-metadata/src/main/java/org/apache/kylin/metadata/model/ParameterDesc.java
@@ -20,6 +20,7 @@ package org.apache.kylin.metadata.model;
import java.io.Serializable;
import java.io.UnsupportedEncodingException;
+import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Set;
@@ -28,8 +29,6 @@ import com.fasterxml.jackson.annotation.JsonAutoDetect;
import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonProperty;
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.Iterables;
import com.google.common.collect.Sets;
/**
@@ -49,7 +48,7 @@ public class ParameterDesc implements Serializable {
TblColRef col = (TblColRef) obj;
r.type = FunctionDesc.PARAMETER_TYPE_COLUMN;
r.value = col.getIdentity();
-r.colRefs = ImmutableList.of(col);
+r.colRef = col;
} else {
r.type = FunctionDesc.PARAMETER_TYPE_CONSTANT;
r.value = (String) obj;
@@ -57,12 +56,6 @@ public class ParameterDesc implements Serializable {
if (objs.length >= 2) {
r.nextParameter = newInstance(Arrays.copyOfRange(objs, 1,
objs.length));
-if (r.nextParameter.colRefs.size() > 0) {
-if (r.colRefs.isEmpty())
-r.colRefs = r.nextParameter.colRefs;
-else
-r.colRefs =
ImmutableList.copyOf(Iterables.concat(r.colRefs, r.nextParameter.colR
