http://git-wip-us.apache.org/repos/asf/kylin/blob/826f23f1/core-cube/src/main/java/org/apache/kylin/cube/kv/RowKeyColumnIO.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/kv/RowKeyColumnIO.java b/core-cube/src/main/java/org/apache/kylin/cube/kv/RowKeyColumnIO.java index b3facd2..fbb93db 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/kv/RowKeyColumnIO.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/kv/RowKeyColumnIO.java @@ -1,63 +1,63 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -package org.apache.kylin.cube.kv; - -import org.apache.kylin.common.util.Dictionary; -import org.apache.kylin.dimension.DictionaryDimEnc; -import org.apache.kylin.dimension.DimensionEncoding; -import org.apache.kylin.dimension.IDimensionEncodingMap; -import org.apache.kylin.metadata.model.TblColRef; - -/** - * Read/Write column values from/into bytes - * - * @author yangli9 - */ -public class RowKeyColumnIO { - - //private static final Logger logger = LoggerFactory.getLogger(RowKeyColumnIO.class); - - private final IDimensionEncodingMap dimEncMap; - - public RowKeyColumnIO(IDimensionEncodingMap dimEncMap) { - this.dimEncMap = dimEncMap; - } - - public int getColumnLength(TblColRef col) { - return dimEncMap.get(col).getLengthOfEncoding(); - } - - public Dictionary<String> getDictionary(TblColRef col) { - return dimEncMap.getDictionary(col); - } - - public void writeColumn(TblColRef col, String value, int roundingFlag, byte defaultValue, byte[] output, int outputOffset) { - DimensionEncoding dimEnc = dimEncMap.get(col); - if (dimEnc instanceof DictionaryDimEnc) - dimEnc = ((DictionaryDimEnc) dimEnc).copy(roundingFlag, defaultValue); - - dimEnc.encode(value, output, outputOffset); - } - - public String readColumnString(TblColRef col, byte[] bytes, int offset, int length) { - DimensionEncoding dimEnc = dimEncMap.get(col); - return dimEnc.decode(bytes, offset, length); - } - -} +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +package org.apache.kylin.cube.kv; + +import org.apache.kylin.common.util.Dictionary; +import org.apache.kylin.dimension.DictionaryDimEnc; +import org.apache.kylin.dimension.DimensionEncoding; +import org.apache.kylin.dimension.IDimensionEncodingMap; +import org.apache.kylin.metadata.model.TblColRef; + +/** + * Read/Write column values from/into bytes + * + * @author yangli9 + */ +public class RowKeyColumnIO { + + //private static final Logger logger = LoggerFactory.getLogger(RowKeyColumnIO.class); + + private final IDimensionEncodingMap dimEncMap; + + public RowKeyColumnIO(IDimensionEncodingMap dimEncMap) { + this.dimEncMap = dimEncMap; + } + + public int getColumnLength(TblColRef col) { + return dimEncMap.get(col).getLengthOfEncoding(); + } + + public Dictionary<String> getDictionary(TblColRef col) { + return dimEncMap.getDictionary(col); + } + + public void writeColumn(TblColRef col, String value, int roundingFlag, byte defaultValue, byte[] output, int outputOffset) { + DimensionEncoding dimEnc = dimEncMap.get(col); + if (dimEnc instanceof DictionaryDimEnc) + dimEnc = ((DictionaryDimEnc) dimEnc).copy(roundingFlag, defaultValue); + + dimEnc.encode(value, output, outputOffset); + } + + public String readColumnString(TblColRef col, byte[] bytes, int offset, int length) { + DimensionEncoding dimEnc = dimEncMap.get(col); + return dimEnc.decode(bytes, offset, length); + } + +}
http://git-wip-us.apache.org/repos/asf/kylin/blob/826f23f1/core-cube/src/main/java/org/apache/kylin/cube/kv/RowKeyColumnOrder.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/kv/RowKeyColumnOrder.java b/core-cube/src/main/java/org/apache/kylin/cube/kv/RowKeyColumnOrder.java index 23c8f6a..3f4f6f4 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/kv/RowKeyColumnOrder.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/kv/RowKeyColumnOrder.java @@ -1,108 +1,108 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -package org.apache.kylin.cube.kv; - -import java.util.Collection; -import java.util.Comparator; - -import org.apache.kylin.metadata.datatype.DataType; - -/** - * @author yangli9 - */ -abstract public class RowKeyColumnOrder implements Comparator<String> { - - public static final NumberOrder NUMBER_ORDER = new NumberOrder(); - public static final StringOrder STRING_ORDER = new StringOrder(); - - public static RowKeyColumnOrder getInstance(DataType type) { - if (type.isNumberFamily() || type.isDateTimeFamily()) - return NUMBER_ORDER; - else - return STRING_ORDER; - } - - public String max(Collection<String> values) { - String max = null; - for (String v : values) { - if (max == null || compare(max, v) < 0) - max = v; - } - return max; - } - - public String min(Collection<String> values) { - String min = null; - for (String v : values) { - if (min == null || compare(min, v) > 0) - min = v; - } - return min; - } - - public String min(String v1, String v2) { - if (v1 == null) - return v2; - else if (v2 == null) - return v1; - else - return compare(v1, v2) <= 0 ? v1 : v2; - } - - public String max(String v1, String v2) { - if (v1 == null) - return v2; - else if (v2 == null) - return v1; - else - return compare(v1, v2) >= 0 ? v1 : v2; - } - - @Override - public int compare(String o1, String o2) { - // consider null - if (o1 == o2) - return 0; - if (o1 == null) - return -1; - if (o2 == null) - return 1; - - return compareNonNull(o1, o2); - } - - abstract int compareNonNull(String o1, String o2); - - private static class StringOrder extends RowKeyColumnOrder { - @Override - public int compareNonNull(String o1, String o2) { - return o1.compareTo(o2); - } - } - - private static class NumberOrder extends RowKeyColumnOrder { - @Override - public int compareNonNull(String o1, String o2) { - double d1 = Double.parseDouble(o1); - double d2 = Double.parseDouble(o2); - return Double.compare(d1, d2); - } - } - -} +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +package org.apache.kylin.cube.kv; + +import java.util.Collection; +import java.util.Comparator; + +import org.apache.kylin.metadata.datatype.DataType; + +/** + * @author yangli9 + */ +abstract public class RowKeyColumnOrder implements Comparator<String> { + + public static final NumberOrder NUMBER_ORDER = new NumberOrder(); + public static final StringOrder STRING_ORDER = new StringOrder(); + + public static RowKeyColumnOrder getInstance(DataType type) { + if (type.isNumberFamily() || type.isDateTimeFamily()) + return NUMBER_ORDER; + else + return STRING_ORDER; + } + + public String max(Collection<String> values) { + String max = null; + for (String v : values) { + if (max == null || compare(max, v) < 0) + max = v; + } + return max; + } + + public String min(Collection<String> values) { + String min = null; + for (String v : values) { + if (min == null || compare(min, v) > 0) + min = v; + } + return min; + } + + public String min(String v1, String v2) { + if (v1 == null) + return v2; + else if (v2 == null) + return v1; + else + return compare(v1, v2) <= 0 ? v1 : v2; + } + + public String max(String v1, String v2) { + if (v1 == null) + return v2; + else if (v2 == null) + return v1; + else + return compare(v1, v2) >= 0 ? v1 : v2; + } + + @Override + public int compare(String o1, String o2) { + // consider null + if (o1 == o2) + return 0; + if (o1 == null) + return -1; + if (o2 == null) + return 1; + + return compareNonNull(o1, o2); + } + + abstract int compareNonNull(String o1, String o2); + + private static class StringOrder extends RowKeyColumnOrder { + @Override + public int compareNonNull(String o1, String o2) { + return o1.compareTo(o2); + } + } + + private static class NumberOrder extends RowKeyColumnOrder { + @Override + public int compareNonNull(String o1, String o2) { + double d1 = Double.parseDouble(o1); + double d2 = Double.parseDouble(o2); + return Double.compare(d1, d2); + } + } + +} http://git-wip-us.apache.org/repos/asf/kylin/blob/826f23f1/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java index b62c1ec..47063d0 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java @@ -609,7 +609,7 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware { combination = combination * (1L << normalDims.size()); if (combination > config.getCubeAggrGroupMaxCombination()) { - String msg = "Aggregation group " + index + " has too many combinations, use 'mandatory'/'hierarchy'/'joint' to optimize; or update 'kylin.cube.aggrgroup.max.combination' to a bigger value."; + String msg = "Aggregation group " + index + " has too many combinations, use 'mandatory'/'hierarchy'/'joint' to optimize; or update 'kylin.cube.aggrgroup.max-combination' to a bigger value."; logger.error("Aggregation group " + index + " has " + combination + " combinations;"); logger.error(msg); throw new IllegalStateException(msg); http://git-wip-us.apache.org/repos/asf/kylin/blob/826f23f1/core-cube/src/main/java/org/apache/kylin/cube/model/RowKeyColDesc.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/RowKeyColDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/RowKeyColDesc.java index cc264d5..326052e 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/model/RowKeyColDesc.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/model/RowKeyColDesc.java @@ -1,162 +1,162 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -package org.apache.kylin.cube.model; - -import org.apache.commons.lang.StringUtils; -import org.apache.kylin.dimension.DateDimEnc; -import org.apache.kylin.dimension.DictionaryDimEnc; -import org.apache.kylin.dimension.DimensionEncoding; -import org.apache.kylin.dimension.DimensionEncodingFactory; -import org.apache.kylin.dimension.FixedLenDimEnc; -import org.apache.kylin.dimension.TimeDimEnc; -import org.apache.kylin.metadata.datatype.DataType; -import org.apache.kylin.metadata.model.TblColRef; - -import com.fasterxml.jackson.annotation.JsonAutoDetect; -import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility; -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.google.common.base.Objects; -import com.google.common.base.Preconditions; - -/** - * @author yangli9 - * - */ -@JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE) -public class RowKeyColDesc { - - @JsonProperty("column") - private String column; - @JsonProperty("encoding") - private String encoding; - @JsonProperty("encoding_version") - @JsonInclude(JsonInclude.Include.NON_DEFAULT) - private int encodingVersion = 1; - @JsonProperty("isShardBy") - private boolean isShardBy;//usually it is ultra high cardinality column, shard by such column can reduce the agg cache for each shard - @JsonProperty("index") - @JsonInclude(JsonInclude.Include.NON_NULL) - private String index; - - // computed - private String encodingName; - private String[] encodingArgs; - private int bitIndex; - private TblColRef colRef; - - public void init(int index, CubeDesc cubeDesc) { - column = column.toUpperCase(); - bitIndex = index; - colRef = cubeDesc.getModel().findColumn(column); - Preconditions.checkArgument(colRef != null, "Cannot find rowkey column %s in cube %s", column, cubeDesc); - - Preconditions.checkState(StringUtils.isNotEmpty(this.encoding)); - Object[] encodingConf = DimensionEncoding.parseEncodingConf(this.encoding); - encodingName = (String) encodingConf[0]; - encodingArgs = (String[]) encodingConf[1]; - - if (!DimensionEncodingFactory.isValidEncoding(this.encodingName)) - throw new IllegalArgumentException("Not supported row key col encoding: '" + this.encoding + "'"); - - // convert date/time dictionary on date/time column to DimensionEncoding implicitly - // however date/time dictionary on varchar column is still required - DataType type = colRef.getType(); - if (DictionaryDimEnc.ENCODING_NAME.equals(encodingName)) { - if (type.isDate()) { - encoding = encodingName = DateDimEnc.ENCODING_NAME; - } - if (type.isTimeFamily()) { - encoding = encodingName = TimeDimEnc.ENCODING_NAME; - } - } - if (DateDimEnc.ENCODING_NAME.equals(encodingName) && type.isDate() == false) - throw new IllegalArgumentException(colRef + " type is " + type + " and cannot apply date encoding"); - if (TimeDimEnc.ENCODING_NAME.equals(encodingName) && type.isTimeFamily() == false) - throw new IllegalArgumentException(colRef + " type is " + type + " and cannot apply time encoding"); - if (encodingName.startsWith(FixedLenDimEnc.ENCODING_NAME) && (type.isIntegerFamily() || type.isNumberFamily())) - throw new IllegalArgumentException(colRef + " type is " + type + " and cannot apply fixed_length encoding"); - } - - public String getEncoding() { - return encoding; - } - - public void setEncoding(String encoding) { - this.encoding = encoding; - } - - public String getColumn() { - return column; - } - - public void setColumn(String column) { - this.column = column; - } - - public boolean isShardBy() { - return isShardBy; - } - - public void setShardBy(boolean shardBy) { - isShardBy = shardBy; - } - - public String getEncodingName() { - return encodingName; - } - - public String[] getEncodingArgs() { - return encodingArgs; - } - - public boolean isUsingDictionary() { - return DictionaryDimEnc.ENCODING_NAME.equals(encodingName); - } - - public int getBitIndex() { - return bitIndex; - } - - public TblColRef getColRef() { - return colRef; - } - - public String getIndex() { - return index; - } - - public void setIndex(String index) { - this.index = index; - } - - public int getEncodingVersion() { - return encodingVersion; - } - - public void setEncodingVersion(int encodingVersion) { - this.encodingVersion = encodingVersion; - } - - @Override - public String toString() { - return Objects.toStringHelper(this).add("column", column).add("encoding", encoding).toString(); - } - +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +package org.apache.kylin.cube.model; + +import org.apache.commons.lang.StringUtils; +import org.apache.kylin.dimension.DateDimEnc; +import org.apache.kylin.dimension.DictionaryDimEnc; +import org.apache.kylin.dimension.DimensionEncoding; +import org.apache.kylin.dimension.DimensionEncodingFactory; +import org.apache.kylin.dimension.FixedLenDimEnc; +import org.apache.kylin.dimension.TimeDimEnc; +import org.apache.kylin.metadata.datatype.DataType; +import org.apache.kylin.metadata.model.TblColRef; + +import com.fasterxml.jackson.annotation.JsonAutoDetect; +import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.base.Objects; +import com.google.common.base.Preconditions; + +/** + * @author yangli9 + * + */ +@JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE) +public class RowKeyColDesc { + + @JsonProperty("column") + private String column; + @JsonProperty("encoding") + private String encoding; + @JsonProperty("encoding_version") + @JsonInclude(JsonInclude.Include.NON_DEFAULT) + private int encodingVersion = 1; + @JsonProperty("isShardBy") + private boolean isShardBy;//usually it is ultra high cardinality column, shard by such column can reduce the agg cache for each shard + @JsonProperty("index") + @JsonInclude(JsonInclude.Include.NON_NULL) + private String index; + + // computed + private String encodingName; + private String[] encodingArgs; + private int bitIndex; + private TblColRef colRef; + + public void init(int index, CubeDesc cubeDesc) { + column = column.toUpperCase(); + bitIndex = index; + colRef = cubeDesc.getModel().findColumn(column); + Preconditions.checkArgument(colRef != null, "Cannot find rowkey column %s in cube %s", column, cubeDesc); + + Preconditions.checkState(StringUtils.isNotEmpty(this.encoding)); + Object[] encodingConf = DimensionEncoding.parseEncodingConf(this.encoding); + encodingName = (String) encodingConf[0]; + encodingArgs = (String[]) encodingConf[1]; + + if (!DimensionEncodingFactory.isValidEncoding(this.encodingName)) + throw new IllegalArgumentException("Not supported row key col encoding: '" + this.encoding + "'"); + + // convert date/time dictionary on date/time column to DimensionEncoding implicitly + // however date/time dictionary on varchar column is still required + DataType type = colRef.getType(); + if (DictionaryDimEnc.ENCODING_NAME.equals(encodingName)) { + if (type.isDate()) { + encoding = encodingName = DateDimEnc.ENCODING_NAME; + } + if (type.isTimeFamily()) { + encoding = encodingName = TimeDimEnc.ENCODING_NAME; + } + } + if (DateDimEnc.ENCODING_NAME.equals(encodingName) && type.isDate() == false) + throw new IllegalArgumentException(colRef + " type is " + type + " and cannot apply date encoding"); + if (TimeDimEnc.ENCODING_NAME.equals(encodingName) && type.isTimeFamily() == false) + throw new IllegalArgumentException(colRef + " type is " + type + " and cannot apply time encoding"); + if (encodingName.startsWith(FixedLenDimEnc.ENCODING_NAME) && (type.isIntegerFamily() || type.isNumberFamily())) + throw new IllegalArgumentException(colRef + " type is " + type + " and cannot apply fixed_length encoding"); + } + + public String getEncoding() { + return encoding; + } + + public void setEncoding(String encoding) { + this.encoding = encoding; + } + + public String getColumn() { + return column; + } + + public void setColumn(String column) { + this.column = column; + } + + public boolean isShardBy() { + return isShardBy; + } + + public void setShardBy(boolean shardBy) { + isShardBy = shardBy; + } + + public String getEncodingName() { + return encodingName; + } + + public String[] getEncodingArgs() { + return encodingArgs; + } + + public boolean isUsingDictionary() { + return DictionaryDimEnc.ENCODING_NAME.equals(encodingName); + } + + public int getBitIndex() { + return bitIndex; + } + + public TblColRef getColRef() { + return colRef; + } + + public String getIndex() { + return index; + } + + public void setIndex(String index) { + this.index = index; + } + + public int getEncodingVersion() { + return encodingVersion; + } + + public void setEncodingVersion(int encodingVersion) { + this.encodingVersion = encodingVersion; + } + + @Override + public String toString() { + return Objects.toStringHelper(this).add("column", column).add("encoding", encoding).toString(); + } + } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/kylin/blob/826f23f1/core-cube/src/main/java/org/apache/kylin/cube/model/v1_4_0/RowKeyColDesc.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/v1_4_0/RowKeyColDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/v1_4_0/RowKeyColDesc.java index 20245f4..8ef3698 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/model/v1_4_0/RowKeyColDesc.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/model/v1_4_0/RowKeyColDesc.java @@ -1,92 +1,92 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.kylin.cube.model.v1_4_0; - -import org.apache.kylin.metadata.model.TblColRef; - -import com.fasterxml.jackson.annotation.JsonAutoDetect; -import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility; -import com.fasterxml.jackson.annotation.JsonProperty; - -/** - * @author yangli9 - * - */ -@JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE) -public class RowKeyColDesc { - - @JsonProperty("column") - private String column; - @JsonProperty("length") - private int length; - @JsonProperty("dictionary") - private String dictionary; - @JsonProperty("mandatory") - private boolean mandatory = false; - - // computed - private int bitIndex; - private TblColRef colRef; - - public String getDictionary() { - return dictionary; - } - - public String getColumn() { - return column; - } - - void setColumn(String column) { - this.column = column; - } - - public int getLength() { - return length; - } - - public boolean isMandatory() { - return mandatory; - } - - public int getBitIndex() { - return bitIndex; - } - - void setBitIndex(int index) { - this.bitIndex = index; - } - - public TblColRef getColRef() { - return colRef; - } - - void setColRef(TblColRef colRef) { - this.colRef = colRef; - } - - public void setDictionary(String dictionary) { - this.dictionary = dictionary; - } - - @Override - public String toString() { - return "RowKeyColDesc [column=" + column + ", length=" + length + ", dictionary=" + dictionary + ", mandatory=" + mandatory + "]"; - } - -} +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kylin.cube.model.v1_4_0; + +import org.apache.kylin.metadata.model.TblColRef; + +import com.fasterxml.jackson.annotation.JsonAutoDetect; +import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility; +import com.fasterxml.jackson.annotation.JsonProperty; + +/** + * @author yangli9 + * + */ +@JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE) +public class RowKeyColDesc { + + @JsonProperty("column") + private String column; + @JsonProperty("length") + private int length; + @JsonProperty("dictionary") + private String dictionary; + @JsonProperty("mandatory") + private boolean mandatory = false; + + // computed + private int bitIndex; + private TblColRef colRef; + + public String getDictionary() { + return dictionary; + } + + public String getColumn() { + return column; + } + + void setColumn(String column) { + this.column = column; + } + + public int getLength() { + return length; + } + + public boolean isMandatory() { + return mandatory; + } + + public int getBitIndex() { + return bitIndex; + } + + void setBitIndex(int index) { + this.bitIndex = index; + } + + public TblColRef getColRef() { + return colRef; + } + + void setColRef(TblColRef colRef) { + this.colRef = colRef; + } + + public void setDictionary(String dictionary) { + this.dictionary = dictionary; + } + + @Override + public String toString() { + return "RowKeyColDesc [column=" + column + ", length=" + length + ", dictionary=" + dictionary + ", mandatory=" + mandatory + "]"; + } + +} http://git-wip-us.apache.org/repos/asf/kylin/blob/826f23f1/core-cube/src/main/java/org/apache/kylin/cube/model/validation/IValidatorRule.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/validation/IValidatorRule.java b/core-cube/src/main/java/org/apache/kylin/cube/model/validation/IValidatorRule.java index 4787e61..b805b62 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/model/validation/IValidatorRule.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/model/validation/IValidatorRule.java @@ -1,27 +1,27 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -package org.apache.kylin.cube.model.validation; - -/** - * @author jianliu - * - */ -public interface IValidatorRule<T> { - public void validate(T element, ValidateContext context); -} +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +package org.apache.kylin.cube.model.validation; + +/** + * @author jianliu + * + */ +public interface IValidatorRule<T> { + public void validate(T element, ValidateContext context); +} http://git-wip-us.apache.org/repos/asf/kylin/blob/826f23f1/core-cube/src/main/java/org/apache/kylin/cube/model/validation/ResultLevel.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/validation/ResultLevel.java b/core-cube/src/main/java/org/apache/kylin/cube/model/validation/ResultLevel.java index 020e2a3..c562914 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/model/validation/ResultLevel.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/model/validation/ResultLevel.java @@ -1,38 +1,38 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -package org.apache.kylin.cube.model.validation; - -/** - * Validation result level - * - * @author jianliu - * - */ -public enum ResultLevel { - ERROR("ERROR"), WARN("WARN"); - private String level; - - ResultLevel(String level) { - this.level = level; - } - - public String toString() { - return level; - } -} +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +package org.apache.kylin.cube.model.validation; + +/** + * Validation result level + * + * @author jianliu + * + */ +public enum ResultLevel { + ERROR("ERROR"), WARN("WARN"); + private String level; + + ResultLevel(String level) { + this.level = level; + } + + public String toString() { + return level; + } +} http://git-wip-us.apache.org/repos/asf/kylin/blob/826f23f1/core-cube/src/main/java/org/apache/kylin/cube/model/validation/ValidateContext.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/validation/ValidateContext.java b/core-cube/src/main/java/org/apache/kylin/cube/model/validation/ValidateContext.java index 632cbff..459e38d 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/model/validation/ValidateContext.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/model/validation/ValidateContext.java @@ -1,100 +1,100 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -package org.apache.kylin.cube.model.validation; - -import java.io.PrintStream; -import java.util.ArrayList; -import java.util.Iterator; -import java.util.List; - -/** - * Context. Supply all dependent objects for validator - */ -public class ValidateContext { - private List<Result> results = new ArrayList<ValidateContext.Result>(); - - public void addResult(ResultLevel level, String message) { - results.add(new Result(level, message)); - } - - public void addResult(Result result) { - results.add(result); - } - - public class Result { - private ResultLevel level; - private String message; - - /** - * @param level - * @param message - */ - public Result(ResultLevel level, String message) { - this.level = level; - this.message = message; - } - - /** - * @return the level - */ - public ResultLevel getLevel() { - return level; - } - - /** - * @return the message - */ - public String getMessage() { - return message; - } - } - - /** - * Get validation result - * - * @return - */ - public Result[] getResults() { - Result[] rs = new Result[0]; - rs = results.toArray(rs); - return rs; - } - - /** - * - */ - public void print(PrintStream out) { - if (results.isEmpty()) { - out.println("The element is perfect."); - } - Iterator<Result> it = results.iterator(); - while (it.hasNext()) { - Result result = it.next(); - out.println(result.level + " : " + result.message); - } - } - - /** - * @return if there is not validation errors - */ - public boolean ifPass() { - return results.isEmpty(); - } - -} +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +package org.apache.kylin.cube.model.validation; + +import java.io.PrintStream; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; + +/** + * Context. Supply all dependent objects for validator + */ +public class ValidateContext { + private List<Result> results = new ArrayList<ValidateContext.Result>(); + + public void addResult(ResultLevel level, String message) { + results.add(new Result(level, message)); + } + + public void addResult(Result result) { + results.add(result); + } + + public class Result { + private ResultLevel level; + private String message; + + /** + * @param level + * @param message + */ + public Result(ResultLevel level, String message) { + this.level = level; + this.message = message; + } + + /** + * @return the level + */ + public ResultLevel getLevel() { + return level; + } + + /** + * @return the message + */ + public String getMessage() { + return message; + } + } + + /** + * Get validation result + * + * @return + */ + public Result[] getResults() { + Result[] rs = new Result[0]; + rs = results.toArray(rs); + return rs; + } + + /** + * + */ + public void print(PrintStream out) { + if (results.isEmpty()) { + out.println("The element is perfect."); + } + Iterator<Result> it = results.iterator(); + while (it.hasNext()) { + Result result = it.next(); + out.println(result.level + " : " + result.message); + } + } + + /** + * @return if there is not validation errors + */ + public boolean ifPass() { + return results.isEmpty(); + } + +} http://git-wip-us.apache.org/repos/asf/kylin/blob/826f23f1/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/AggregationGroupRule.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/AggregationGroupRule.java b/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/AggregationGroupRule.java index a5cf407..c73f5ca 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/AggregationGroupRule.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/AggregationGroupRule.java @@ -1,202 +1,202 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -package org.apache.kylin.cube.model.validation.rule; - -import java.util.Arrays; -import java.util.HashSet; -import java.util.List; -import java.util.Set; -import java.util.TreeSet; - -import org.apache.commons.collections.CollectionUtils; -import org.apache.kylin.cube.model.AggregationGroup; -import org.apache.kylin.cube.model.CubeDesc; -import org.apache.kylin.cube.model.validation.IValidatorRule; -import org.apache.kylin.cube.model.validation.ResultLevel; -import org.apache.kylin.cube.model.validation.ValidateContext; - -import com.google.common.collect.Iterables; -import com.google.common.collect.Lists; - -/** - * find forbid overlaps in each AggregationGroup - * the include dims in AggregationGroup must contain all mandatory, hierarchy and joint - */ -public class AggregationGroupRule implements IValidatorRule<CubeDesc> { - - @Override - public void validate(CubeDesc cube, ValidateContext context) { - inner(cube, context); - } - - public AggregationGroupRule() { - } - - private void inner(CubeDesc cube, ValidateContext context) { - - int index = 0; - for (AggregationGroup agg : cube.getAggregationGroups()) { - if (agg.getIncludes() == null) { - context.addResult(ResultLevel.ERROR, "Aggregation group " + index + " 'includes' field not set"); - continue; - } - - if (agg.getSelectRule() == null) { - context.addResult(ResultLevel.ERROR, "Aggregation group " + index + " 'select rule' field not set"); - continue; - } - +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +package org.apache.kylin.cube.model.validation.rule; + +import java.util.Arrays; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.TreeSet; + +import org.apache.commons.collections.CollectionUtils; +import org.apache.kylin.cube.model.AggregationGroup; +import org.apache.kylin.cube.model.CubeDesc; +import org.apache.kylin.cube.model.validation.IValidatorRule; +import org.apache.kylin.cube.model.validation.ResultLevel; +import org.apache.kylin.cube.model.validation.ValidateContext; + +import com.google.common.collect.Iterables; +import com.google.common.collect.Lists; + +/** + * find forbid overlaps in each AggregationGroup + * the include dims in AggregationGroup must contain all mandatory, hierarchy and joint + */ +public class AggregationGroupRule implements IValidatorRule<CubeDesc> { + + @Override + public void validate(CubeDesc cube, ValidateContext context) { + inner(cube, context); + } + + public AggregationGroupRule() { + } + + private void inner(CubeDesc cube, ValidateContext context) { + + int index = 0; + for (AggregationGroup agg : cube.getAggregationGroups()) { + if (agg.getIncludes() == null) { + context.addResult(ResultLevel.ERROR, "Aggregation group " + index + " 'includes' field not set"); + continue; + } + + if (agg.getSelectRule() == null) { + context.addResult(ResultLevel.ERROR, "Aggregation group " + index + " 'select rule' field not set"); + continue; + } + long combination = 1; - Set<String> includeDims = new TreeSet<>(String.CASE_INSENSITIVE_ORDER); - if (agg.getIncludes() != null) { - for (String include : agg.getIncludes()) { - includeDims.add(include); - } - } - - Set<String> mandatoryDims = new TreeSet<>(String.CASE_INSENSITIVE_ORDER); - if (agg.getSelectRule().mandatory_dims != null) { - for (String m : agg.getSelectRule().mandatory_dims) { - mandatoryDims.add(m); - } - } - - Set<String> hierarchyDims = new TreeSet<>(String.CASE_INSENSITIVE_ORDER); - if (agg.getSelectRule().hierarchy_dims != null) { - for (String[] ss : agg.getSelectRule().hierarchy_dims) { - for (String s : ss) { - hierarchyDims.add(s); - } - combination = combination * (ss.length + 1); - } - } - - Set<String> jointDims = new TreeSet<>(String.CASE_INSENSITIVE_ORDER); - if (agg.getSelectRule().joint_dims != null) { - for (String[] ss : agg.getSelectRule().joint_dims) { - for (String s : ss) { - jointDims.add(s); - } - combination = combination * 2; - } - } - - if (!includeDims.containsAll(mandatoryDims) || !includeDims.containsAll(hierarchyDims) || !includeDims.containsAll(jointDims)) { - List<String> notIncluded = Lists.newArrayList(); - final Iterable<String> all = Iterables.unmodifiableIterable(Iterables.concat(mandatoryDims, hierarchyDims, jointDims)); - for (String dim : all) { - if (includeDims.contains(dim) == false) { - notIncluded.add(dim); - } - } - context.addResult(ResultLevel.ERROR, "Aggregation group " + index + " 'includes' dimensions not include all the dimensions:" + notIncluded.toString()); - continue; - } - - Set<String> normalDims = new TreeSet<>(String.CASE_INSENSITIVE_ORDER); - normalDims.addAll(includeDims); - normalDims.removeAll(mandatoryDims); - normalDims.removeAll(hierarchyDims); - normalDims.removeAll(jointDims); - + Set<String> includeDims = new TreeSet<>(String.CASE_INSENSITIVE_ORDER); + if (agg.getIncludes() != null) { + for (String include : agg.getIncludes()) { + includeDims.add(include); + } + } + + Set<String> mandatoryDims = new TreeSet<>(String.CASE_INSENSITIVE_ORDER); + if (agg.getSelectRule().mandatory_dims != null) { + for (String m : agg.getSelectRule().mandatory_dims) { + mandatoryDims.add(m); + } + } + + Set<String> hierarchyDims = new TreeSet<>(String.CASE_INSENSITIVE_ORDER); + if (agg.getSelectRule().hierarchy_dims != null) { + for (String[] ss : agg.getSelectRule().hierarchy_dims) { + for (String s : ss) { + hierarchyDims.add(s); + } + combination = combination * (ss.length + 1); + } + } + + Set<String> jointDims = new TreeSet<>(String.CASE_INSENSITIVE_ORDER); + if (agg.getSelectRule().joint_dims != null) { + for (String[] ss : agg.getSelectRule().joint_dims) { + for (String s : ss) { + jointDims.add(s); + } + combination = combination * 2; + } + } + + if (!includeDims.containsAll(mandatoryDims) || !includeDims.containsAll(hierarchyDims) || !includeDims.containsAll(jointDims)) { + List<String> notIncluded = Lists.newArrayList(); + final Iterable<String> all = Iterables.unmodifiableIterable(Iterables.concat(mandatoryDims, hierarchyDims, jointDims)); + for (String dim : all) { + if (includeDims.contains(dim) == false) { + notIncluded.add(dim); + } + } + context.addResult(ResultLevel.ERROR, "Aggregation group " + index + " 'includes' dimensions not include all the dimensions:" + notIncluded.toString()); + continue; + } + + Set<String> normalDims = new TreeSet<>(String.CASE_INSENSITIVE_ORDER); + normalDims.addAll(includeDims); + normalDims.removeAll(mandatoryDims); + normalDims.removeAll(hierarchyDims); + normalDims.removeAll(jointDims); + combination = combination * (1L << normalDims.size()); - - if (CollectionUtils.containsAny(mandatoryDims, hierarchyDims)) { - Set<String> intersection = new HashSet<>(mandatoryDims); - intersection.retainAll(hierarchyDims); - context.addResult(ResultLevel.ERROR, "Aggregation group " + index + " mandatory dimension has overlap with hierarchy dimension: " + intersection.toString()); - continue; - } - if (CollectionUtils.containsAny(mandatoryDims, jointDims)) { - Set<String> intersection = new HashSet<>(mandatoryDims); - intersection.retainAll(jointDims); - context.addResult(ResultLevel.ERROR, "Aggregation group " + index + " mandatory dimension has overlap with joint dimension: " + intersection.toString()); - continue; - } - - int jointDimNum = 0; - if (agg.getSelectRule().joint_dims != null) { - for (String[] joints : agg.getSelectRule().joint_dims) { - - Set<String> oneJoint = new TreeSet<>(String.CASE_INSENSITIVE_ORDER); - for (String s : joints) { - oneJoint.add(s); - } - - if (oneJoint.size() < 2) { - context.addResult(ResultLevel.ERROR, "Aggregation group " + index + " require at least 2 dimensions in a joint: " + oneJoint.toString()); - continue; - } - jointDimNum += oneJoint.size(); - - int overlapHierarchies = 0; - if (agg.getSelectRule().hierarchy_dims != null) { - for (String[] oneHierarchy : agg.getSelectRule().hierarchy_dims) { - Set<String> share = new TreeSet<>(String.CASE_INSENSITIVE_ORDER); - share.addAll(CollectionUtils.intersection(oneJoint, Arrays.asList(oneHierarchy))); - - if (!share.isEmpty()) { - overlapHierarchies++; - } - if (share.size() > 1) { - context.addResult(ResultLevel.ERROR, "Aggregation group " + index + " joint dimensions has overlap with more than 1 dimensions in same hierarchy: " + share.toString()); - continue; - } - } - - if (overlapHierarchies > 1) { - context.addResult(ResultLevel.ERROR, "Aggregation group " + index + " joint dimensions has overlap with more than 1 hierarchies"); - continue; - } - } - } - - if (jointDimNum != jointDims.size()) { - - Set<String> existing = new TreeSet<>(String.CASE_INSENSITIVE_ORDER); - Set<String> overlap = new TreeSet<>(String.CASE_INSENSITIVE_ORDER); - for (String[] joints : agg.getSelectRule().joint_dims) { - Set<String> oneJoint = new TreeSet<>(String.CASE_INSENSITIVE_ORDER); - for (String s : joints) { - oneJoint.add(s); - } - if (CollectionUtils.containsAny(existing, oneJoint)) { - overlap.addAll(CollectionUtils.intersection(existing, oneJoint)); - } - existing.addAll(oneJoint); - } - context.addResult(ResultLevel.ERROR, "Aggregation group " + index + " a dimension exists in more than one joint: " + overlap.toString()); - continue; - } - } - - if (combination > getMaxCombinations(cube)) { - String msg = "Aggregation group " + index + " has too many combinations, current combination is " + combination + ", max allowed combination is " + getMaxCombinations(cube) + "; use 'mandatory'/'hierarchy'/'joint' to optimize; or update 'kylin.cube.aggrgroup.max.combination' to a bigger value."; - context.addResult(ResultLevel.ERROR, msg); - continue; - } - - index++; - } - } - - protected int getMaxCombinations(CubeDesc cubeDesc) { - return cubeDesc.getConfig().getCubeAggrGroupMaxCombination(); - } -} + + if (CollectionUtils.containsAny(mandatoryDims, hierarchyDims)) { + Set<String> intersection = new HashSet<>(mandatoryDims); + intersection.retainAll(hierarchyDims); + context.addResult(ResultLevel.ERROR, "Aggregation group " + index + " mandatory dimension has overlap with hierarchy dimension: " + intersection.toString()); + continue; + } + if (CollectionUtils.containsAny(mandatoryDims, jointDims)) { + Set<String> intersection = new HashSet<>(mandatoryDims); + intersection.retainAll(jointDims); + context.addResult(ResultLevel.ERROR, "Aggregation group " + index + " mandatory dimension has overlap with joint dimension: " + intersection.toString()); + continue; + } + + int jointDimNum = 0; + if (agg.getSelectRule().joint_dims != null) { + for (String[] joints : agg.getSelectRule().joint_dims) { + + Set<String> oneJoint = new TreeSet<>(String.CASE_INSENSITIVE_ORDER); + for (String s : joints) { + oneJoint.add(s); + } + + if (oneJoint.size() < 2) { + context.addResult(ResultLevel.ERROR, "Aggregation group " + index + " require at least 2 dimensions in a joint: " + oneJoint.toString()); + continue; + } + jointDimNum += oneJoint.size(); + + int overlapHierarchies = 0; + if (agg.getSelectRule().hierarchy_dims != null) { + for (String[] oneHierarchy : agg.getSelectRule().hierarchy_dims) { + Set<String> share = new TreeSet<>(String.CASE_INSENSITIVE_ORDER); + share.addAll(CollectionUtils.intersection(oneJoint, Arrays.asList(oneHierarchy))); + + if (!share.isEmpty()) { + overlapHierarchies++; + } + if (share.size() > 1) { + context.addResult(ResultLevel.ERROR, "Aggregation group " + index + " joint dimensions has overlap with more than 1 dimensions in same hierarchy: " + share.toString()); + continue; + } + } + + if (overlapHierarchies > 1) { + context.addResult(ResultLevel.ERROR, "Aggregation group " + index + " joint dimensions has overlap with more than 1 hierarchies"); + continue; + } + } + } + + if (jointDimNum != jointDims.size()) { + + Set<String> existing = new TreeSet<>(String.CASE_INSENSITIVE_ORDER); + Set<String> overlap = new TreeSet<>(String.CASE_INSENSITIVE_ORDER); + for (String[] joints : agg.getSelectRule().joint_dims) { + Set<String> oneJoint = new TreeSet<>(String.CASE_INSENSITIVE_ORDER); + for (String s : joints) { + oneJoint.add(s); + } + if (CollectionUtils.containsAny(existing, oneJoint)) { + overlap.addAll(CollectionUtils.intersection(existing, oneJoint)); + } + existing.addAll(oneJoint); + } + context.addResult(ResultLevel.ERROR, "Aggregation group " + index + " a dimension exists in more than one joint: " + overlap.toString()); + continue; + } + } + + if (combination > getMaxCombinations(cube)) { + String msg = "Aggregation group " + index + " has too many combinations, current combination is " + combination + ", max allowed combination is " + getMaxCombinations(cube) + "; use 'mandatory'/'hierarchy'/'joint' to optimize; or update 'kylin.cube.aggrgroup.max-combination' to a bigger value."; + context.addResult(ResultLevel.ERROR, msg); + continue; + } + + index++; + } + } + + protected int getMaxCombinations(CubeDesc cubeDesc) { + return cubeDesc.getConfig().getCubeAggrGroupMaxCombination(); + } +} http://git-wip-us.apache.org/repos/asf/kylin/blob/826f23f1/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/FunctionRule.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/FunctionRule.java b/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/FunctionRule.java index ee93f72..1ed7325 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/FunctionRule.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/FunctionRule.java @@ -1,202 +1,202 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -package org.apache.kylin.cube.model.validation.rule; - -import java.util.ArrayList; -import java.util.HashSet; -import java.util.Iterator; -import java.util.List; -import java.util.Set; - -import org.apache.commons.lang.StringUtils; -import org.apache.kylin.cube.model.CubeDesc; -import org.apache.kylin.cube.model.DimensionDesc; -import org.apache.kylin.cube.model.validation.IValidatorRule; -import org.apache.kylin.cube.model.validation.ResultLevel; -import org.apache.kylin.cube.model.validation.ValidateContext; -import org.apache.kylin.measure.topn.TopNMeasureType; -import org.apache.kylin.metadata.MetadataManager; -import org.apache.kylin.metadata.model.ColumnDesc; -import org.apache.kylin.metadata.model.FunctionDesc; -import org.apache.kylin.metadata.model.MeasureDesc; -import org.apache.kylin.metadata.model.ParameterDesc; -import org.apache.kylin.metadata.model.TableDesc; - -import com.google.common.collect.Lists; - -/** - * Validate function parameter. - * <p/> - * if type is column, check values are valid fact table columns if type is - * constant, the value only can be numberic - * <p/> - * the return type only can be int/bigint/long/double/decimal - * - */ -public class FunctionRule implements IValidatorRule<CubeDesc> { - - /* - * (non-Javadoc) - * - * @see - * org.apache.kylin.metadata.validation.IValidatorRule#validate(java.lang.Object - * , org.apache.kylin.metadata.validation.ValidateContext) - */ - @Override - public void validate(CubeDesc cube, ValidateContext context) { - List<MeasureDesc> measures = cube.getMeasures(); - - if (validateMeasureNamesDuplicated(measures, context)) { - return; - } - - List<FunctionDesc> countFuncs = new ArrayList<FunctionDesc>(); - - Iterator<MeasureDesc> it = measures.iterator(); - while (it.hasNext()) { - MeasureDesc measure = it.next(); - FunctionDesc func = measure.getFunction(); - ParameterDesc parameter = func.getParameter(); - if (parameter == null) { - context.addResult(ResultLevel.ERROR, "Must define parameter for function " + func.getExpression() + " in " + measure.getName()); - return; - } - - String type = func.getParameter().getType(); - String value = func.getParameter().getValue(); - if (StringUtils.isEmpty(type)) { - context.addResult(ResultLevel.ERROR, "Must define type for parameter type " + func.getExpression() + " in " + measure.getName()); - return; - } - if (StringUtils.isEmpty(value)) { - context.addResult(ResultLevel.ERROR, "Must define type for parameter value " + func.getExpression() + " in " + measure.getName()); - return; - } - if (StringUtils.isEmpty(func.getReturnType())) { - context.addResult(ResultLevel.ERROR, "Must define return type for function " + func.getExpression() + " in " + measure.getName()); - return; - } - - if (StringUtils.equalsIgnoreCase(FunctionDesc.PARAMETER_TYPE_COLUMN, type)) { - validateColumnParameter(context, cube, value); - } else if (StringUtils.equals(FunctionDesc.PARAMETER_TYPE_CONSTANT, type)) { - validateCostantParameter(context, cube, value); - } - - try { - func.getMeasureType().validate(func); - } catch (IllegalArgumentException ex) { - context.addResult(ResultLevel.ERROR, ex.getMessage()); - } - - if (func.isCount()) - countFuncs.add(func); - - if (TopNMeasureType.FUNC_TOP_N.equalsIgnoreCase(func.getExpression())) { - if (parameter.getNextParameter() == null) { - context.addResult(ResultLevel.ERROR, "Must define at least 2 parameters for function " + func.getExpression() + " in " + measure.getName()); - return; - } - - ParameterDesc groupByCol = parameter.getNextParameter(); - List<String> duplicatedCol = Lists.newArrayList(); - while (groupByCol != null) { - String embeded_groupby = groupByCol.getValue(); - for (DimensionDesc dimensionDesc : cube.getDimensions()) { - if (dimensionDesc.getColumn() != null && dimensionDesc.getColumn().equalsIgnoreCase(embeded_groupby)) { - duplicatedCol.add(embeded_groupby); - } - } - groupByCol = groupByCol.getNextParameter(); - } - - } - } - - if (countFuncs.size() != 1) { - context.addResult(ResultLevel.ERROR, "Must define one and only one count(1) function, but there are " + countFuncs.size() + " -- " + countFuncs); - } - } - - /** - * @param context - * @param cube - * @param value - */ - private void validateCostantParameter(ValidateContext context, CubeDesc cube, String value) { - try { - Integer.parseInt(value); - } catch (Exception e) { - context.addResult(ResultLevel.ERROR, "Parameter value must be number, but it is " + value); - } - } - - /** - * @param context - * @param cube - * @param value - */ - private void validateColumnParameter(ValidateContext context, CubeDesc cube, String value) { - String factTable = cube.getModel().getRootFactTable().getTableIdentity(); - if (StringUtils.isEmpty(factTable)) { - context.addResult(ResultLevel.ERROR, "Fact table can not be null."); - return; - } - TableDesc table = MetadataManager.getInstance(cube.getConfig()).getTableDesc(factTable); - if (table == null) { - context.addResult(ResultLevel.ERROR, "Fact table can not be found: " + cube); - return; - } - // Prepare column set - Set<String> set = new HashSet<String>(); - ColumnDesc[] cdesc = table.getColumns(); - for (int i = 0; i < cdesc.length; i++) { - ColumnDesc columnDesc = cdesc[i]; - set.add(columnDesc.getName()); - } - - String[] items = value.split(","); - for (int i = 0; i < items.length; i++) { - String item = items[i].trim(); - if (StringUtils.isEmpty(item)) { - continue; - } - if (!set.contains(item)) { - context.addResult(ResultLevel.ERROR, "Column [" + item + "] does not exist in factable table" + factTable); - } - } - - } - - /** - * @param measures - */ - private boolean validateMeasureNamesDuplicated(List<MeasureDesc> measures, ValidateContext context) { - Set<String> nameSet = new HashSet<>(); - for (MeasureDesc measure: measures){ - if (nameSet.contains(measure.getName())){ - context.addResult(ResultLevel.ERROR, "There is duplicated measure's name: " + measure.getName()); - return true; - } else { - nameSet.add(measure.getName()); - } - } - return false; - } -} +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +package org.apache.kylin.cube.model.validation.rule; + +import java.util.ArrayList; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Set; + +import org.apache.commons.lang.StringUtils; +import org.apache.kylin.cube.model.CubeDesc; +import org.apache.kylin.cube.model.DimensionDesc; +import org.apache.kylin.cube.model.validation.IValidatorRule; +import org.apache.kylin.cube.model.validation.ResultLevel; +import org.apache.kylin.cube.model.validation.ValidateContext; +import org.apache.kylin.measure.topn.TopNMeasureType; +import org.apache.kylin.metadata.MetadataManager; +import org.apache.kylin.metadata.model.ColumnDesc; +import org.apache.kylin.metadata.model.FunctionDesc; +import org.apache.kylin.metadata.model.MeasureDesc; +import org.apache.kylin.metadata.model.ParameterDesc; +import org.apache.kylin.metadata.model.TableDesc; + +import com.google.common.collect.Lists; + +/** + * Validate function parameter. + * <p/> + * if type is column, check values are valid fact table columns if type is + * constant, the value only can be numberic + * <p/> + * the return type only can be int/bigint/long/double/decimal + * + */ +public class FunctionRule implements IValidatorRule<CubeDesc> { + + /* + * (non-Javadoc) + * + * @see + * org.apache.kylin.metadata.validation.IValidatorRule#validate(java.lang.Object + * , org.apache.kylin.metadata.validation.ValidateContext) + */ + @Override + public void validate(CubeDesc cube, ValidateContext context) { + List<MeasureDesc> measures = cube.getMeasures(); + + if (validateMeasureNamesDuplicated(measures, context)) { + return; + } + + List<FunctionDesc> countFuncs = new ArrayList<FunctionDesc>(); + + Iterator<MeasureDesc> it = measures.iterator(); + while (it.hasNext()) { + MeasureDesc measure = it.next(); + FunctionDesc func = measure.getFunction(); + ParameterDesc parameter = func.getParameter(); + if (parameter == null) { + context.addResult(ResultLevel.ERROR, "Must define parameter for function " + func.getExpression() + " in " + measure.getName()); + return; + } + + String type = func.getParameter().getType(); + String value = func.getParameter().getValue(); + if (StringUtils.isEmpty(type)) { + context.addResult(ResultLevel.ERROR, "Must define type for parameter type " + func.getExpression() + " in " + measure.getName()); + return; + } + if (StringUtils.isEmpty(value)) { + context.addResult(ResultLevel.ERROR, "Must define type for parameter value " + func.getExpression() + " in " + measure.getName()); + return; + } + if (StringUtils.isEmpty(func.getReturnType())) { + context.addResult(ResultLevel.ERROR, "Must define return type for function " + func.getExpression() + " in " + measure.getName()); + return; + } + + if (StringUtils.equalsIgnoreCase(FunctionDesc.PARAMETER_TYPE_COLUMN, type)) { + validateColumnParameter(context, cube, value); + } else if (StringUtils.equals(FunctionDesc.PARAMETER_TYPE_CONSTANT, type)) { + validateCostantParameter(context, cube, value); + } + + try { + func.getMeasureType().validate(func); + } catch (IllegalArgumentException ex) { + context.addResult(ResultLevel.ERROR, ex.getMessage()); + } + + if (func.isCount()) + countFuncs.add(func); + + if (TopNMeasureType.FUNC_TOP_N.equalsIgnoreCase(func.getExpression())) { + if (parameter.getNextParameter() == null) { + context.addResult(ResultLevel.ERROR, "Must define at least 2 parameters for function " + func.getExpression() + " in " + measure.getName()); + return; + } + + ParameterDesc groupByCol = parameter.getNextParameter(); + List<String> duplicatedCol = Lists.newArrayList(); + while (groupByCol != null) { + String embeded_groupby = groupByCol.getValue(); + for (DimensionDesc dimensionDesc : cube.getDimensions()) { + if (dimensionDesc.getColumn() != null && dimensionDesc.getColumn().equalsIgnoreCase(embeded_groupby)) { + duplicatedCol.add(embeded_groupby); + } + } + groupByCol = groupByCol.getNextParameter(); + } + + } + } + + if (countFuncs.size() != 1) { + context.addResult(ResultLevel.ERROR, "Must define one and only one count(1) function, but there are " + countFuncs.size() + " -- " + countFuncs); + } + } + + /** + * @param context + * @param cube + * @param value + */ + private void validateCostantParameter(ValidateContext context, CubeDesc cube, String value) { + try { + Integer.parseInt(value); + } catch (Exception e) { + context.addResult(ResultLevel.ERROR, "Parameter value must be number, but it is " + value); + } + } + + /** + * @param context + * @param cube + * @param value + */ + private void validateColumnParameter(ValidateContext context, CubeDesc cube, String value) { + String factTable = cube.getModel().getRootFactTable().getTableIdentity(); + if (StringUtils.isEmpty(factTable)) { + context.addResult(ResultLevel.ERROR, "Fact table can not be null."); + return; + } + TableDesc table = MetadataManager.getInstance(cube.getConfig()).getTableDesc(factTable); + if (table == null) { + context.addResult(ResultLevel.ERROR, "Fact table can not be found: " + cube); + return; + } + // Prepare column set + Set<String> set = new HashSet<String>(); + ColumnDesc[] cdesc = table.getColumns(); + for (int i = 0; i < cdesc.length; i++) { + ColumnDesc columnDesc = cdesc[i]; + set.add(columnDesc.getName()); + } + + String[] items = value.split(","); + for (int i = 0; i < items.length; i++) { + String item = items[i].trim(); + if (StringUtils.isEmpty(item)) { + continue; + } + if (!set.contains(item)) { + context.addResult(ResultLevel.ERROR, "Column [" + item + "] does not exist in factable table" + factTable); + } + } + + } + + /** + * @param measures + */ + private boolean validateMeasureNamesDuplicated(List<MeasureDesc> measures, ValidateContext context) { + Set<String> nameSet = new HashSet<>(); + for (MeasureDesc measure: measures){ + if (nameSet.contains(measure.getName())){ + context.addResult(ResultLevel.ERROR, "There is duplicated measure's name: " + measure.getName()); + return true; + } else { + nameSet.add(measure.getName()); + } + } + return false; + } +} http://git-wip-us.apache.org/repos/asf/kylin/blob/826f23f1/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/RowKeyAttrRule.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/RowKeyAttrRule.java b/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/RowKeyAttrRule.java index df3cd40..390dfb9 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/RowKeyAttrRule.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/RowKeyAttrRule.java @@ -1,55 +1,55 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -package org.apache.kylin.cube.model.validation.rule; - -import org.apache.kylin.cube.model.CubeDesc; -import org.apache.kylin.cube.model.RowKeyColDesc; -import org.apache.kylin.cube.model.RowKeyDesc; -import org.apache.kylin.cube.model.validation.IValidatorRule; -import org.apache.kylin.cube.model.validation.ResultLevel; -import org.apache.kylin.cube.model.validation.ValidateContext; - -/** - */ -public class RowKeyAttrRule implements IValidatorRule<CubeDesc> { - - @Override - public void validate(CubeDesc cube, ValidateContext context) { - RowKeyDesc row = cube.getRowkey(); - if (row == null) { - context.addResult(ResultLevel.ERROR, "Rowkey does not exist"); - return; - } - - RowKeyColDesc[] rcd = row.getRowKeyColumns(); - if (rcd == null || rcd.length == 0) { - context.addResult(ResultLevel.ERROR, "Rowkey columns do not exist"); - return; - } - - for (int i = 0; i < rcd.length; i++) { - RowKeyColDesc rd = rcd[i]; - if (rd.getColumn() == null || rd.getColumn().length() == 0) { - context.addResult(ResultLevel.ERROR, "Rowkey column empty"); - } - - } - - } -} +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +package org.apache.kylin.cube.model.validation.rule; + +import org.apache.kylin.cube.model.CubeDesc; +import org.apache.kylin.cube.model.RowKeyColDesc; +import org.apache.kylin.cube.model.RowKeyDesc; +import org.apache.kylin.cube.model.validation.IValidatorRule; +import org.apache.kylin.cube.model.validation.ResultLevel; +import org.apache.kylin.cube.model.validation.ValidateContext; + +/** + */ +public class RowKeyAttrRule implements IValidatorRule<CubeDesc> { + + @Override + public void validate(CubeDesc cube, ValidateContext context) { + RowKeyDesc row = cube.getRowkey(); + if (row == null) { + context.addResult(ResultLevel.ERROR, "Rowkey does not exist"); + return; + } + + RowKeyColDesc[] rcd = row.getRowKeyColumns(); + if (rcd == null || rcd.length == 0) { + context.addResult(ResultLevel.ERROR, "Rowkey columns do not exist"); + return; + } + + for (int i = 0; i < rcd.length; i++) { + RowKeyColDesc rd = rcd[i]; + if (rd.getColumn() == null || rd.getColumn().length() == 0) { + context.addResult(ResultLevel.ERROR, "Rowkey column empty"); + } + + } + + } +}