date:20160226

[2/4] kylin git commit: KYLIN-1366 simply metadata version binding

2016-02-26 Thread mahongbin

KYLIN-1366 simply metadata version binding

kylin 1366


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/ab9d5791
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/ab9d5791
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/ab9d5791

Branch: refs/heads/2.x-staging
Commit: ab9d57914a119b5b426d2bfae712ca55ff35c92a
Parents: 4c08ded
Author: honma 
Authored: Wed Feb 24 15:58:07 2016 +0800
Committer: honma 
Committed: Fri Feb 26 17:54:37 2016 +0800

--
 .../org/apache/kylin/common/KylinConfigBase.java   |  8 
 .../java/org/apache/kylin/common/KylinVersion.java |  2 +-
 .../common/persistence/RootPersistentEntity.java   | 17 +
 .../java/org/apache/kylin/cube/CubeInstance.java   |  5 ++---
 .../java/org/apache/kylin/cube/model/CubeDesc.java |  2 +-
 .../java/org/apache/kylin/dict/DictionaryInfo.java |  4 ++--
 .../apache/kylin/dict/lookup/SnapshotManager.java  |  2 +-
 .../apache/kylin/metadata/model/DataModelDesc.java |  2 +-
 .../kylin/metadata/project/ProjectInstance.java|  2 +-
 .../kylin/metadata/project/ProjectManager.java |  2 +-
 .../kylin/storage/hybrid/HybridInstance.java   |  2 +-
 .../org/apache/kylin/invertedindex/IIInstance.java |  2 +-
 .../kylin/rest/controller/CubeController.java  |  4 ++--
 .../storage/hbase/util/ExtendCubeToHybridCLI.java  |  4 ++--
 14 files changed, 21 insertions(+), 37 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/ab9d5791/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java 
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 5f9983a..7707684 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -61,14 +61,6 @@ public class KylinConfigBase implements Serializable {
 return kylinHome;
 }
 
-/**
- * @see KylinVersion
- * @return current kylin version
- */
-public static String getKylinVersion(){
-return KylinVersion.getCurrentVersion();
-}
-
 // 

 
 private volatile Properties properties = new Properties();

http://git-wip-us.apache.org/repos/asf/kylin/blob/ab9d5791/core-common/src/main/java/org/apache/kylin/common/KylinVersion.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/KylinVersion.java 
b/core-common/src/main/java/org/apache/kylin/common/KylinVersion.java
index d711b38..42cf237 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinVersion.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinVersion.java
@@ -21,7 +21,7 @@ package org.apache.kylin.common;
  *
  * @since 2.1
  */
-class KylinVersion {
+public class KylinVersion {
 /**
  * Require MANUAL updating kylin version per ANY upgrading.
  */

http://git-wip-us.apache.org/repos/asf/kylin/blob/ab9d5791/core-common/src/main/java/org/apache/kylin/common/persistence/RootPersistentEntity.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/persistence/RootPersistentEntity.java
 
b/core-common/src/main/java/org/apache/kylin/common/persistence/RootPersistentEntity.java
index 327ddcc..c46abe7 100644
--- 
a/core-common/src/main/java/org/apache/kylin/common/persistence/RootPersistentEntity.java
+++ 
b/core-common/src/main/java/org/apache/kylin/common/persistence/RootPersistentEntity.java
@@ -26,11 +26,11 @@ import java.util.Date;
 import java.util.UUID;
 
 import org.apache.commons.lang.time.FastDateFormat;
+import org.apache.kylin.common.KylinVersion;
 
 import com.fasterxml.jackson.annotation.JsonAutoDetect;
 import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
 import com.fasterxml.jackson.annotation.JsonProperty;
-import org.apache.kylin.common.KylinConfig;
 
 /**
  * Marks the root entity of JSON persistence. Unit of read, write, cache, and
@@ -80,14 +80,14 @@ abstract public class RootPersistentEntity implements 
AclEntity, Serializable {
  * For example: 2.1
  */
 @JsonProperty("version")
-protected String version;
+protected String version = KylinVersion.getCurrentVersion();
 
 public String getVersion() {
-  return version;
+return version;
 }
 
 public void setVersion(String version) {
-  this.version = version;
+this.version = version;
 }
 
 public String getUuid() {
@@ -110,14 +110,7

[1/4] kylin git commit: KYLIN-1453 cuboid sharding based on specific column

2016-02-26 Thread mahongbin

Repository: kylin
Updated Branches:
  refs/heads/2.x-staging 2e1d2f6b6 -> 294fc7078


KYLIN-1453 cuboid sharding based on specific column


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/294fc707
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/294fc707
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/294fc707

Branch: refs/heads/2.x-staging
Commit: 294fc70785ab009e6b4d8a12cbeb609d46f89a93
Parents: 3f5074e
Author: honma 
Authored: Thu Feb 25 19:02:28 2016 +0800
Committer: honma 
Committed: Fri Feb 26 17:54:37 2016 +0800

--
 .../org/apache/kylin/common/util/BytesUtil.java |  18 +-
 .../dict/TupleFilterFunctionTransformer.java| 170 +++
 .../dict/TupleFilterFunctionTranslator.java | 166 --
 .../kylin/metadata/filter/CaseTupleFilter.java  |   7 +-
 .../metadata/filter/ColumnTupleFilter.java  |  11 +-
 .../metadata/filter/CompareTupleFilter.java |  10 +-
 .../metadata/filter/ConstantTupleFilter.java|  16 +-
 .../metadata/filter/DynamicTupleFilter.java |   9 +-
 .../metadata/filter/ExtractTupleFilter.java |   7 +-
 .../metadata/filter/FunctionTupleFilter.java|  16 +-
 .../filter/ITupleFilterTransformer.java |  23 +++
 .../metadata/filter/ITupleFilterTranslator.java |  26 ---
 .../metadata/filter/LogicalTupleFilter.java |  10 +-
 .../kylin/metadata/filter/TupleFilter.java  |   5 +-
 .../metadata/filter/TupleFilterSerializer.java  |  25 ++-
 .../common/coprocessor/FilterDecorator.java |   8 +-
 .../hbase/cube/v2/CubeSegmentScanner.java   |  10 +-
 .../common/coprocessor/FilterBaseTest.java  |  46 -
 .../common/coprocessor/FilterEvaluateTest.java  |   4 +-
 .../common/coprocessor/FilterSerializeTest.java |  26 ++-
 20 files changed, 325 insertions(+), 288 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/294fc707/core-common/src/main/java/org/apache/kylin/common/util/BytesUtil.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/util/BytesUtil.java 
b/core-common/src/main/java/org/apache/kylin/common/util/BytesUtil.java
index e01ce4f..4e0701c 100644
--- a/core-common/src/main/java/org/apache/kylin/common/util/BytesUtil.java
+++ b/core-common/src/main/java/org/apache/kylin/common/util/BytesUtil.java
@@ -57,7 +57,6 @@ public class BytesUtil {
 return integer;
 }
 
-
 public static long readLong(ByteBuffer buffer, int size) {
 long integer = 0;
 for (int i = 0; i < size; i++) {
@@ -133,11 +132,15 @@ public class BytesUtil {
 // from WritableUtils
 // 

 
+
 public static void writeVInt(int i, ByteBuffer out) {
+
 writeVLong(i, out);
+
 }
 
 public static void writeVLong(long i, ByteBuffer out) {
+
 if (i >= -112 && i <= 127) {
 out.put((byte) i);
 return;
@@ -203,6 +206,8 @@ public class BytesUtil {
 }
 
 public static void writeUnsigned(int num, int size, ByteBuffer out) {
+
+
 int mask = 0xff << ((size - 1) * 8);
 for (int i = size; i > 0; i--) {
 int v = (num & mask) >> (i - 1) * 8;
@@ -222,6 +227,7 @@ public class BytesUtil {
 }
 
 public static void writeLong(long num, ByteBuffer out) {
+
 for (int i = 0; i < 8; i++) {
 out.put((byte) num);
 num >>>= 8;
@@ -257,6 +263,8 @@ public class BytesUtil {
 }
 int len = str.length();
 BytesUtil.writeVInt(len, out);
+
+
 for (int i = 0; i < len; i++) {
 out.put((byte) str.charAt(i));
 }
@@ -335,7 +343,7 @@ public class BytesUtil {
 writeVInt(-1, out);
 return;
 }
-writeVInt(array.length, out);
+writeVInt(length, out);
 out.put(array, offset, length);
 }
 
@@ -348,7 +356,7 @@ public class BytesUtil {
 in.get(array);
 return array;
 }
-
+
 public static int peekByteArrayLength(ByteBuffer in) {
 int start = in.position();
 int arrayLen = readVInt(in);
@@ -369,6 +377,7 @@ public class BytesUtil {
 writeVInt(array.length, out);
 byte b_true = (byte) 1;
 byte b_false = (byte) 0;
+
 for (int i = 0; i < array.length; i++) {
 if (array[i])
 out.put(b_true);
@@ -428,7 +437,4 @@ public class BytesUtil {
 return sb.toString();
 }
 
-public static void main(String[] args) throws Exception {
-}
-
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/294fc707/core-dictionary/src/main/java/org/apache/kylin/dict/TupleFilterFunctionTransformer.java

[3/4] kylin git commit: minor, fix CI occasionally fail issue

2016-02-26 Thread mahongbin

minor, fix CI occasionally fail issue


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/3f5074ee
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/3f5074ee
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/3f5074ee

Branch: refs/heads/2.x-staging
Commit: 3f5074ee1568d5b0ba50d70d5c35319cd8223cc9
Parents: ab9d579
Author: honma 
Authored: Thu Feb 25 14:13:48 2016 +0800
Committer: honma 
Committed: Fri Feb 26 17:54:37 2016 +0800

--
 .../dict/TupleFilterDictionaryTranslater.java   | 165 --
 .../dict/TupleFilterFunctionTranslator.java | 166 +++
 .../metadata/filter/FunctionTupleFilter.java|   5 +-
 .../metadata/filter/function/BuiltInMethod.java |  33 ++--
 .../cache/AbstractCacheFledgedQuery.java|  32 +---
 .../kylin/storage/cache/DynamicCacheTest.java   |  15 +-
 .../kylin/storage/cache/StaticCacheTest.java|  19 ++-
 .../kylin/storage/hbase/ITStorageTest.java  |  11 +-
 .../common/coprocessor/FilterDecorator.java |   4 +-
 .../hbase/cube/v2/CubeSegmentScanner.java   |   4 +-
 10 files changed, 226 insertions(+), 228 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/3f5074ee/core-dictionary/src/main/java/org/apache/kylin/dict/TupleFilterDictionaryTranslater.java
--
diff --git 
a/core-dictionary/src/main/java/org/apache/kylin/dict/TupleFilterDictionaryTranslater.java
 
b/core-dictionary/src/main/java/org/apache/kylin/dict/TupleFilterDictionaryTranslater.java
deleted file mode 100644
index 9ef360d..000
--- 
a/core-dictionary/src/main/java/org/apache/kylin/dict/TupleFilterDictionaryTranslater.java
+++ /dev/null
@@ -1,165 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.dict;
-
-import com.google.common.primitives.Primitives;
-import org.apache.kylin.common.util.Dictionary;
-import org.apache.kylin.metadata.filter.ColumnTupleFilter;
-import org.apache.kylin.metadata.filter.CompareTupleFilter;
-import org.apache.kylin.metadata.filter.ConstantTupleFilter;
-import org.apache.kylin.metadata.filter.FunctionTupleFilter;
-import org.apache.kylin.metadata.filter.ITupleFilterTranslator;
-import org.apache.kylin.metadata.filter.LogicalTupleFilter;
-import org.apache.kylin.metadata.filter.TupleFilter;
-import org.apache.kylin.metadata.filter.TupleFilter.FilterOperatorEnum;
-import org.apache.kylin.metadata.model.TblColRef;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.ListIterator;
-
-/**
- * Created by dongli on 1/7/16.
- */
-public class TupleFilterDictionaryTranslater implements ITupleFilterTranslator 
{
-public static final Logger logger = 
LoggerFactory.getLogger(TupleFilterDictionaryTranslater.class);
-
-private IDictionaryAware dictionaryAware;
-
-public TupleFilterDictionaryTranslater(IDictionaryAware dictionaryAware) {
-this.dictionaryAware = dictionaryAware;
-}
-
-@Override
-public TupleFilter translate(TupleFilter tupleFilter) {
-TupleFilter translated = null;
-if (tupleFilter instanceof CompareTupleFilter) {
-translated = translateCompareTupleFilter((CompareTupleFilter) 
tupleFilter);
-if (translated != null) {
-logger.info("Translated {" + tupleFilter + "} to IN clause: {" 
+ translated + "}");
-}
-} else if (tupleFilter instanceof FunctionTupleFilter) {
-translated = translateFunctionTupleFilter((FunctionTupleFilter) 
tupleFilter);
-if (translated != null) {
-logger.info("Translated {" + tupleFilter + "} to IN clause: {" 
+ translated + "}");
-}
-} else if (tupleFilter instanceof LogicalTupleFilter) {
-ListIterator childIterator = 
(ListIterator) tupleFilter.getChildren().listIterator();
-while (childIterator.hasNext()) {
-TupleFilter tempTranslated =

kylin git commit: KYLIN-579 Unload Hive table from kylin

2016-02-26 Thread shaofengshi

Repository: kylin
Updated Branches:
  refs/heads/2.x-staging 6ee409ccc -> 2e1d2f6b6


KYLIN-579 Unload Hive table from kylin

Signed-off-by: wangxianbin1987 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/2e1d2f6b
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/2e1d2f6b
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/2e1d2f6b

Branch: refs/heads/2.x-staging
Commit: 2e1d2f6b62903b16b17bd2442e3456107dc0aa6a
Parents: 6ee409c
Author: wangxianbin1987 
Authored: Thu Feb 25 19:22:34 2016 +0800
Committer: shaofengshi 
Committed: Fri Feb 26 16:18:50 2016 +0800

--
 .../java/org/apache/kylin/cube/CubeManager.java |  2 +
 .../apache/kylin/metadata/MetadataManager.java  | 30 +
 .../kylin/metadata/project/ProjectManager.java  | 12 
 .../kylin/rest/controller/TableController.java  | 59 -
 .../apache/kylin/rest/service/CubeService.java  | 14 +
 .../apache/kylin/rest/service/ModelService.java | 15 +
 .../kylin/rest/service/ProjectService.java  | 20 ++
 .../source/hive/HiveSourceTableLoader.java  |  6 ++
 webapp/app/js/controllers/sourceMeta.js | 66 
 webapp/app/js/services/tables.js|  1 +
 .../app/partials/tables/source_table_tree.html  |  6 +-
 webapp/app/partials/tables/table_unload.html| 33 ++
 12 files changed, 259 insertions(+), 5 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/2e1d2f6b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
--
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java 
b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
index 84dd30a..4951ce6 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
@@ -46,7 +46,9 @@ import org.apache.kylin.metadata.MetadataManager;
 import org.apache.kylin.metadata.model.SegmentStatusEnum;
 import org.apache.kylin.metadata.model.TableDesc;
 import org.apache.kylin.metadata.model.TblColRef;
+import org.apache.kylin.metadata.project.ProjectInstance;
 import org.apache.kylin.metadata.project.ProjectManager;
+import org.apache.kylin.metadata.project.RealizationEntry;
 import org.apache.kylin.metadata.realization.IRealization;
 import org.apache.kylin.metadata.realization.IRealizationConstants;
 import org.apache.kylin.metadata.realization.IRealizationProvider;

http://git-wip-us.apache.org/repos/asf/kylin/blob/2e1d2f6b/core-metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java 
b/core-metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java
index 80ee8b3..9f2a934 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java
@@ -197,6 +197,12 @@ public class MetadataManager {
 srcTableMap.put(srcTable.getIdentity(), srcTable);
 }
 
+public void removeSourceTable(String tableIdentity) throws IOException {
+String path = TableDesc.concatResourcePath(tableIdentity);
+getStore().deleteResource(path);
+srcTableMap.remove(tableIdentity);
+}
+
 private void init(KylinConfig config) throws IOException {
 this.config = config;
 this.srcTableMap = new CaseInsensitiveStringCache(config, 
Broadcaster.TYPE.TABLE);
@@ -336,6 +342,24 @@ public class MetadataManager {
 return new ArrayList<>(ret);
 }
 
+public boolean isTableInModel(String tableName, String projectName) throws 
IOException {
+for(DataModelDesc modelDesc : getModels(projectName)) {
+if(modelDesc.getAllTables().contains(tableName.toUpperCase())) {
+return true;
+}
+}
+return false;
+}
+
+public boolean isTableInAnyModel(String tableName) {
+for(DataModelDesc modelDesc : getModels()) {
+if(modelDesc.getAllTables().contains(tableName.toUpperCase())){
+return true;
+}
+}
+return false;
+}
+
 private void reloadAllDataModel() throws IOException {
 ResourceStore store = getStore();
 logger.debug("Reloading DataModel from folder " + 
store.getReadableResourcePath(ResourceStore.DATA_MODEL_DESC_RESOURCE_ROOT));
@@ -441,6 +465,12 @@ public class MetadataManager {
 srcTableExdMap.put(tableId, tableExdProperties);
 }
 
+public void removeTableExd(String

[2/4] kylin git commit: KYLIN-1366 simply metadata version binding

[1/4] kylin git commit: KYLIN-1453 cuboid sharding based on specific column

[3/4] kylin git commit: minor, fix CI occasionally fail issue

kylin git commit: KYLIN-579 Unload Hive table from kylin

4 matches

Site Navigation

Mail list logo

Footer information