minor refactor
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/6c23313e Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/6c23313e Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/6c23313e Branch: refs/heads/1.5.x-CDH5.7 Commit: 6c23313e9d14f8eabf165566ae0a8156b25a47e8 Parents: 27426ae Author: Hongbin Ma <mahong...@apache.org> Authored: Tue Aug 30 16:15:28 2016 +0800 Committer: Hongbin Ma <mahong...@apache.org> Committed: Tue Aug 30 19:08:32 2016 +0800 ---------------------------------------------------------------------- .../org/apache/kylin/common/util/ByteArray.java | 4 + .../kylin/dict/BuildInFunctionTransformer.java | 178 ------------------- .../kylin/dict/BuiltInFunctionTransformer.java | 178 +++++++++++++++++++ .../filter/BuildInFunctionTupleFilter.java | 165 ----------------- .../filter/BuiltInFunctionTupleFilter.java | 157 ++++++++++++++++ .../kylin/metadata/filter/TupleFilter.java | 1 - .../metadata/filter/TupleFilterSerializer.java | 2 +- .../metadata/filter/function/Functions.java | 4 +- .../storage/gtrecord/CubeSegmentScanner.java | 4 +- .../test/resources/query/sql_like/query03.sql | 33 ++++ .../common/coprocessor/FilterDecorator.java | 4 +- 11 files changed, 379 insertions(+), 351 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/6c23313e/core-common/src/main/java/org/apache/kylin/common/util/ByteArray.java ---------------------------------------------------------------------- diff --git a/core-common/src/main/java/org/apache/kylin/common/util/ByteArray.java b/core-common/src/main/java/org/apache/kylin/common/util/ByteArray.java index a72425b..db02f34 100644 --- a/core-common/src/main/java/org/apache/kylin/common/util/ByteArray.java +++ b/core-common/src/main/java/org/apache/kylin/common/util/ByteArray.java @@ -136,6 +136,10 @@ public class ByteArray implements Comparable<ByteArray>, Serializable { return ByteBuffer.wrap(data, offset, length).slice(); } + public byte[] toBytes() { + return Bytes.copy(this.array(), this.offset(), this.length()); + } + @Override public int hashCode() { if (data == null) { http://git-wip-us.apache.org/repos/asf/kylin/blob/6c23313e/core-dictionary/src/main/java/org/apache/kylin/dict/BuildInFunctionTransformer.java ---------------------------------------------------------------------- diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/BuildInFunctionTransformer.java b/core-dictionary/src/main/java/org/apache/kylin/dict/BuildInFunctionTransformer.java deleted file mode 100644 index 665a7e6..0000000 --- a/core-dictionary/src/main/java/org/apache/kylin/dict/BuildInFunctionTransformer.java +++ /dev/null @@ -1,178 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.kylin.dict; - -import java.util.Collection; -import java.util.ListIterator; - -import org.apache.kylin.common.util.Dictionary; -import org.apache.kylin.dimension.IDimensionEncodingMap; -import org.apache.kylin.metadata.filter.BuildInFunctionTupleFilter; -import org.apache.kylin.metadata.filter.ColumnTupleFilter; -import org.apache.kylin.metadata.filter.CompareTupleFilter; -import org.apache.kylin.metadata.filter.ConstantTupleFilter; -import org.apache.kylin.metadata.filter.ITupleFilterTransformer; -import org.apache.kylin.metadata.filter.LogicalTupleFilter; -import org.apache.kylin.metadata.filter.TupleFilter; -import org.apache.kylin.metadata.filter.TupleFilter.FilterOperatorEnum; -import org.apache.kylin.metadata.model.TblColRef; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import com.google.common.collect.Lists; -import com.google.common.primitives.Primitives; - -/** - * only take effect when the compare filter has function - */ -public class BuildInFunctionTransformer implements ITupleFilterTransformer { - public static final Logger logger = LoggerFactory.getLogger(BuildInFunctionTransformer.class); - - private IDimensionEncodingMap dimEncMap; - - public BuildInFunctionTransformer(IDimensionEncodingMap dimEncMap) { - this.dimEncMap = dimEncMap; - } - - @Override - public TupleFilter transform(TupleFilter tupleFilter) { - TupleFilter translated = null; - if (tupleFilter instanceof CompareTupleFilter) { - //normal case - translated = translateCompareTupleFilter((CompareTupleFilter) tupleFilter); - if (translated != null) { - logger.info("Translated {" + tupleFilter + "} to IN clause: {" + translated + "}"); - } - } else if (tupleFilter instanceof BuildInFunctionTupleFilter) { - //like case - translated = translateFunctionTupleFilter((BuildInFunctionTupleFilter) tupleFilter); - if (translated != null) { - logger.info("Translated {" + tupleFilter + "} to IN clause: {" + translated + "}"); - } - } else if (tupleFilter instanceof LogicalTupleFilter) { - @SuppressWarnings("unchecked") - ListIterator<TupleFilter> childIterator = (ListIterator<TupleFilter>) tupleFilter.getChildren().listIterator(); - while (childIterator.hasNext()) { - TupleFilter transformed = transform(childIterator.next()); - if (transformed != null) { - childIterator.set(transformed); - } else { - throw new IllegalStateException("Should not be null"); - } - } - } - return translated == null ? tupleFilter : translated; - } - - private TupleFilter translateFunctionTupleFilter(BuildInFunctionTupleFilter buildInFunctionTupleFilter) { - if (!buildInFunctionTupleFilter.isValid()) - return null; - - TblColRef columnRef = buildInFunctionTupleFilter.getColumn(); - Dictionary<?> dict = dimEncMap.getDictionary(columnRef); - if (dict == null) - return null; - - CompareTupleFilter translated = new CompareTupleFilter(FilterOperatorEnum.IN); - translated.addChild(new ColumnTupleFilter(columnRef)); - - try { - for (int i = dict.getMinId(); i <= dict.getMaxId(); i++) { - Object dictVal = dict.getValueFromId(i); - if ((Boolean) buildInFunctionTupleFilter.invokeFunction(dictVal)) { - translated.addChild(new ConstantTupleFilter(dictVal)); - } - } - } catch (Exception e) { - logger.debug(e.getMessage()); - return null; - } - return translated; - } - - @SuppressWarnings({ "unchecked", "rawtypes" }) - private TupleFilter translateCompareTupleFilter(CompareTupleFilter compTupleFilter) { - if (compTupleFilter.getFunction() == null || (!(compTupleFilter.getFunction() instanceof BuildInFunctionTupleFilter))) - return null; - - BuildInFunctionTupleFilter buildInFunctionTupleFilter = (BuildInFunctionTupleFilter) compTupleFilter.getFunction(); - - if (!buildInFunctionTupleFilter.isValid()) - return null; - - TblColRef columnRef = buildInFunctionTupleFilter.getColumn(); - Dictionary<?> dict = dimEncMap.getDictionary(columnRef); - if (dict == null) - return null; - - CompareTupleFilter translated = new CompareTupleFilter(FilterOperatorEnum.IN); - translated.addChild(new ColumnTupleFilter(columnRef)); - - try { - Collection<Object> inValues = Lists.newArrayList(); - for (int i = dict.getMinId(); i <= dict.getMaxId(); i++) { - Object dictVal = dict.getValueFromId(i); - Object computedVal = buildInFunctionTupleFilter.invokeFunction(dictVal); - Class clazz = Primitives.wrap(computedVal.getClass()); - Object targetVal = compTupleFilter.getFirstValue(); - if (Primitives.isWrapperType(clazz)) - targetVal = clazz.cast(clazz.getDeclaredMethod("valueOf", String.class).invoke(null, compTupleFilter.getFirstValue())); - - int comp = ((Comparable) computedVal).compareTo(targetVal); - boolean compResult = false; - switch (compTupleFilter.getOperator()) { - case EQ: - compResult = comp == 0; - break; - case NEQ: - compResult = comp != 0; - break; - case LT: - compResult = comp < 0; - break; - case LTE: - compResult = comp <= 0; - break; - case GT: - compResult = comp > 0; - break; - case GTE: - compResult = comp >= 0; - break; - case IN: - compResult = compTupleFilter.getValues().contains(computedVal.toString()); - break; - case NOTIN: - compResult = !compTupleFilter.getValues().contains(computedVal.toString()); - break; - default: - break; - } - if (compResult) { - inValues.add(dictVal); - } - } - translated.addChild(new ConstantTupleFilter(inValues)); - } catch (Exception e) { - logger.debug(e.getMessage()); - return null; - } - return translated; - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/kylin/blob/6c23313e/core-dictionary/src/main/java/org/apache/kylin/dict/BuiltInFunctionTransformer.java ---------------------------------------------------------------------- diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/BuiltInFunctionTransformer.java b/core-dictionary/src/main/java/org/apache/kylin/dict/BuiltInFunctionTransformer.java new file mode 100644 index 0000000..9707f8f --- /dev/null +++ b/core-dictionary/src/main/java/org/apache/kylin/dict/BuiltInFunctionTransformer.java @@ -0,0 +1,178 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kylin.dict; + +import java.util.Collection; +import java.util.ListIterator; + +import org.apache.kylin.common.util.Dictionary; +import org.apache.kylin.dimension.IDimensionEncodingMap; +import org.apache.kylin.metadata.filter.BuiltInFunctionTupleFilter; +import org.apache.kylin.metadata.filter.ColumnTupleFilter; +import org.apache.kylin.metadata.filter.CompareTupleFilter; +import org.apache.kylin.metadata.filter.ConstantTupleFilter; +import org.apache.kylin.metadata.filter.ITupleFilterTransformer; +import org.apache.kylin.metadata.filter.LogicalTupleFilter; +import org.apache.kylin.metadata.filter.TupleFilter; +import org.apache.kylin.metadata.filter.TupleFilter.FilterOperatorEnum; +import org.apache.kylin.metadata.model.TblColRef; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.common.collect.Lists; +import com.google.common.primitives.Primitives; + +/** + * only take effect when the compare filter has function + */ +public class BuiltInFunctionTransformer implements ITupleFilterTransformer { + public static final Logger logger = LoggerFactory.getLogger(BuiltInFunctionTransformer.class); + + private IDimensionEncodingMap dimEncMap; + + public BuiltInFunctionTransformer(IDimensionEncodingMap dimEncMap) { + this.dimEncMap = dimEncMap; + } + + @Override + public TupleFilter transform(TupleFilter tupleFilter) { + TupleFilter translated = null; + if (tupleFilter instanceof CompareTupleFilter) { + //normal case + translated = translateCompareTupleFilter((CompareTupleFilter) tupleFilter); + if (translated != null) { + logger.info("Translated {" + tupleFilter + "} to IN clause: {" + translated + "}"); + } + } else if (tupleFilter instanceof BuiltInFunctionTupleFilter) { + //like case + translated = translateFunctionTupleFilter((BuiltInFunctionTupleFilter) tupleFilter); + if (translated != null) { + logger.info("Translated {" + tupleFilter + "} to IN clause: {" + translated + "}"); + } + } else if (tupleFilter instanceof LogicalTupleFilter) { + @SuppressWarnings("unchecked") + ListIterator<TupleFilter> childIterator = (ListIterator<TupleFilter>) tupleFilter.getChildren().listIterator(); + while (childIterator.hasNext()) { + TupleFilter transformed = transform(childIterator.next()); + if (transformed != null) { + childIterator.set(transformed); + } else { + throw new IllegalStateException("Should not be null"); + } + } + } + return translated == null ? tupleFilter : translated; + } + + private TupleFilter translateFunctionTupleFilter(BuiltInFunctionTupleFilter builtInFunctionTupleFilter) { + if (!builtInFunctionTupleFilter.isValid()) + return null; + + TblColRef columnRef = builtInFunctionTupleFilter.getColumn(); + Dictionary<?> dict = dimEncMap.getDictionary(columnRef); + if (dict == null) + return null; + + CompareTupleFilter translated = new CompareTupleFilter(FilterOperatorEnum.IN); + translated.addChild(new ColumnTupleFilter(columnRef)); + + try { + for (int i = dict.getMinId(); i <= dict.getMaxId(); i++) { + Object dictVal = dict.getValueFromId(i); + if ((Boolean) builtInFunctionTupleFilter.invokeFunction(dictVal)) { + translated.addChild(new ConstantTupleFilter(dictVal)); + } + } + } catch (Exception e) { + logger.debug(e.getMessage()); + return null; + } + return translated; + } + + @SuppressWarnings({ "unchecked", "rawtypes" }) + private TupleFilter translateCompareTupleFilter(CompareTupleFilter compTupleFilter) { + if (compTupleFilter.getFunction() == null || (!(compTupleFilter.getFunction() instanceof BuiltInFunctionTupleFilter))) + return null; + + BuiltInFunctionTupleFilter builtInFunctionTupleFilter = (BuiltInFunctionTupleFilter) compTupleFilter.getFunction(); + + if (!builtInFunctionTupleFilter.isValid()) + return null; + + TblColRef columnRef = builtInFunctionTupleFilter.getColumn(); + Dictionary<?> dict = dimEncMap.getDictionary(columnRef); + if (dict == null) + return null; + + CompareTupleFilter translated = new CompareTupleFilter(FilterOperatorEnum.IN); + translated.addChild(new ColumnTupleFilter(columnRef)); + + try { + Collection<Object> inValues = Lists.newArrayList(); + for (int i = dict.getMinId(); i <= dict.getMaxId(); i++) { + Object dictVal = dict.getValueFromId(i); + Object computedVal = builtInFunctionTupleFilter.invokeFunction(dictVal); + Class clazz = Primitives.wrap(computedVal.getClass()); + Object targetVal = compTupleFilter.getFirstValue(); + if (Primitives.isWrapperType(clazz)) + targetVal = clazz.cast(clazz.getDeclaredMethod("valueOf", String.class).invoke(null, compTupleFilter.getFirstValue())); + + int comp = ((Comparable) computedVal).compareTo(targetVal); + boolean compResult = false; + switch (compTupleFilter.getOperator()) { + case EQ: + compResult = comp == 0; + break; + case NEQ: + compResult = comp != 0; + break; + case LT: + compResult = comp < 0; + break; + case LTE: + compResult = comp <= 0; + break; + case GT: + compResult = comp > 0; + break; + case GTE: + compResult = comp >= 0; + break; + case IN: + compResult = compTupleFilter.getValues().contains(computedVal.toString()); + break; + case NOTIN: + compResult = !compTupleFilter.getValues().contains(computedVal.toString()); + break; + default: + break; + } + if (compResult) { + inValues.add(dictVal); + } + } + translated.addChild(new ConstantTupleFilter(inValues)); + } catch (Exception e) { + logger.debug(e.getMessage()); + return null; + } + return translated; + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/kylin/blob/6c23313e/core-metadata/src/main/java/org/apache/kylin/metadata/filter/BuildInFunctionTupleFilter.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/BuildInFunctionTupleFilter.java b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/BuildInFunctionTupleFilter.java deleted file mode 100644 index c15b083..0000000 --- a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/BuildInFunctionTupleFilter.java +++ /dev/null @@ -1,165 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.kylin.metadata.filter; - -import java.io.Serializable; -import java.lang.reflect.InvocationTargetException; -import java.lang.reflect.Method; -import java.nio.ByteBuffer; -import java.util.Collection; -import java.util.List; - -import org.apache.kylin.common.util.BytesUtil; -import org.apache.kylin.metadata.filter.function.BuiltInMethod; -import org.apache.kylin.metadata.model.TblColRef; -import org.apache.kylin.metadata.tuple.IEvaluatableTuple; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import com.google.common.collect.Lists; -import com.google.common.primitives.Primitives; - -public class BuildInFunctionTupleFilter extends FunctionTupleFilter { - public static final Logger logger = LoggerFactory.getLogger(BuildInFunctionTupleFilter.class); - - private String name; - // FIXME Only supports single parameter functions currently - private TupleFilter columnContainerFilter;//might be a ColumnTupleFilter(simple case) or FunctionTupleFilter(complex case like substr(lower())) - private int colPosition; - private Method method; - private List<Serializable> methodParams; - private boolean isValid = false; - - public BuildInFunctionTupleFilter(String name) { - super(Lists.<TupleFilter> newArrayList(), FilterOperatorEnum.FUNCTION); - this.methodParams = Lists.newArrayList(); - - if (name != null) { - this.name = name.toUpperCase(); - initMethod(); - } - } - - public String getName() { - return name; - } - - public TblColRef getColumn() { - if (columnContainerFilter == null) - return null; - - if (columnContainerFilter instanceof ColumnTupleFilter) - return ((ColumnTupleFilter) columnContainerFilter).getColumn(); - else if (columnContainerFilter instanceof BuildInFunctionTupleFilter) - return ((BuildInFunctionTupleFilter) columnContainerFilter).getColumn(); - - throw new UnsupportedOperationException("Wrong type TupleFilter in FunctionTupleFilter."); - } - - public Object invokeFunction(Object input) throws InvocationTargetException, IllegalAccessException { - if (columnContainerFilter instanceof ColumnTupleFilter) - methodParams.set(colPosition, (Serializable) input); - else if (columnContainerFilter instanceof BuildInFunctionTupleFilter) - methodParams.set(colPosition, (Serializable) ((BuildInFunctionTupleFilter) columnContainerFilter).invokeFunction(input)); - return method.invoke(null, (Object[]) (methodParams.toArray())); - } - - public boolean isValid() { - return isValid && method != null && methodParams.size() == children.size(); - } - - @Override - public void addChild(TupleFilter child) { - if (child instanceof ColumnTupleFilter || child instanceof BuildInFunctionTupleFilter) { - columnContainerFilter = child; - colPosition = methodParams.size(); - methodParams.add(null); - } else if (child instanceof ConstantTupleFilter) { - Serializable constVal = (Serializable) child.getValues().iterator().next(); - try { - Class<?> clazz = Primitives.wrap(method.getParameterTypes()[methodParams.size()]); - if (!Primitives.isWrapperType(clazz)) - methodParams.add(constVal); - else - methodParams.add((Serializable) clazz.cast(clazz.getDeclaredMethod("valueOf", String.class).invoke(null, constVal))); - } catch (Exception e) { - logger.warn(e.getMessage()); - isValid = false; - } - } - super.addChild(child); - } - - @Override - public boolean isEvaluable() { - return false; - } - - @Override - public boolean evaluate(IEvaluatableTuple tuple, IFilterCodeSystem<?> cs) { - throw new UnsupportedOperationException("Function filter cannot be evaluated immediately"); - } - - @Override - public Collection<String> getValues() { - throw new UnsupportedOperationException("Function filter cannot be evaluated immediately"); - } - - @Override - public void serialize(IFilterCodeSystem<?> cs, ByteBuffer buffer) { - BytesUtil.writeUTFString(name, buffer); - BytesUtil.writeVInt(colPosition, buffer); - BytesUtil.writeVInt(isValid ? 1 : 0, buffer); - } - - @Override - public void deserialize(IFilterCodeSystem<?> cs, ByteBuffer buffer) { - - this.name = BytesUtil.readUTFString(buffer); - this.initMethod(); - - this.colPosition = BytesUtil.readVInt(buffer); - this.isValid = BytesUtil.readVInt(buffer) == 1; - } - - @Override - public String toString() { - StringBuilder sb = new StringBuilder(); - sb.append(name); - sb.append("("); - for (int i = 0; i < methodParams.size(); i++) { - if (colPosition == i) { - sb.append(columnContainerFilter); - } else { - sb.append(methodParams.get(i)); - } - if (i < methodParams.size() - 1) - sb.append(","); - } - sb.append(")"); - return sb.toString(); - } - - private void initMethod() { - if (BuiltInMethod.MAP.containsKey(name)) { - this.method = BuiltInMethod.MAP.get(name).method; - isValid = true; - } - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/kylin/blob/6c23313e/core-metadata/src/main/java/org/apache/kylin/metadata/filter/BuiltInFunctionTupleFilter.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/BuiltInFunctionTupleFilter.java b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/BuiltInFunctionTupleFilter.java new file mode 100644 index 0000000..5a10371 --- /dev/null +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/BuiltInFunctionTupleFilter.java @@ -0,0 +1,157 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kylin.metadata.filter; + +import java.io.Serializable; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.nio.ByteBuffer; +import java.util.Collection; +import java.util.List; + +import org.apache.kylin.metadata.filter.function.BuiltInMethod; +import org.apache.kylin.metadata.model.TblColRef; +import org.apache.kylin.metadata.tuple.IEvaluatableTuple; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.common.collect.Lists; +import com.google.common.primitives.Primitives; + +public class BuiltInFunctionTupleFilter extends FunctionTupleFilter { + public static final Logger logger = LoggerFactory.getLogger(BuiltInFunctionTupleFilter.class); + + protected String name; + // FIXME Only supports single parameter functions currently + protected TupleFilter columnContainerFilter;//might be a ColumnTupleFilter(simple case) or FunctionTupleFilter(complex case like substr(lower())) + protected int colPosition; + protected Method method; + protected List<Serializable> methodParams; + protected boolean isValidFunc = false; + + public BuiltInFunctionTupleFilter(String name) { + super(Lists.<TupleFilter> newArrayList(), FilterOperatorEnum.FUNCTION); + this.methodParams = Lists.newArrayList(); + + if (name != null) { + this.name = name.toUpperCase(); + initMethod(); + } + } + + public String getName() { + return name; + } + + public TblColRef getColumn() { + if (columnContainerFilter == null) + return null; + + if (columnContainerFilter instanceof ColumnTupleFilter) + return ((ColumnTupleFilter) columnContainerFilter).getColumn(); + else if (columnContainerFilter instanceof BuiltInFunctionTupleFilter) + return ((BuiltInFunctionTupleFilter) columnContainerFilter).getColumn(); + + throw new UnsupportedOperationException("Wrong type TupleFilter in FunctionTupleFilter."); + } + + public Object invokeFunction(Object input) throws InvocationTargetException, IllegalAccessException { + if (columnContainerFilter instanceof ColumnTupleFilter) + methodParams.set(colPosition, (Serializable) input); + else if (columnContainerFilter instanceof BuiltInFunctionTupleFilter) + methodParams.set(colPosition, (Serializable) ((BuiltInFunctionTupleFilter) columnContainerFilter).invokeFunction(input)); + return method.invoke(null, (Object[]) (methodParams.toArray())); + } + + public boolean isValid() { + return isValidFunc && method != null && methodParams.size() == children.size(); + } + + @Override + public void addChild(TupleFilter child) { + if (child instanceof ColumnTupleFilter || child instanceof BuiltInFunctionTupleFilter) { + columnContainerFilter = child; + colPosition = methodParams.size(); + methodParams.add(null); + } else if (child instanceof ConstantTupleFilter) { + Serializable constVal = (Serializable) child.getValues().iterator().next(); + try { + Class<?> clazz = Primitives.wrap(method.getParameterTypes()[methodParams.size()]); + if (!Primitives.isWrapperType(clazz)) + methodParams.add(constVal); + else + methodParams.add((Serializable) clazz.cast(clazz.getDeclaredMethod("valueOf", String.class).invoke(null, constVal))); + } catch (Exception e) { + logger.warn(e.getMessage()); + isValidFunc = false; + } + } + super.addChild(child); + } + + @Override + public boolean isEvaluable() { + return false; + } + + @Override + public boolean evaluate(IEvaluatableTuple tuple, IFilterCodeSystem<?> cs) { + throw new UnsupportedOperationException("Function filter cannot be evaluated immediately"); + } + + @Override + public Collection<String> getValues() { + throw new UnsupportedOperationException("Function filter cannot be evaluated immediately"); + } + + @Override + public void serialize(IFilterCodeSystem<?> cs, ByteBuffer buffer) { + throw new UnsupportedOperationException("Function filter cannot serialized"); + } + + @Override + public void deserialize(IFilterCodeSystem<?> cs, ByteBuffer buffer) { + throw new UnsupportedOperationException("Function filter cannot serialized"); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append(name); + sb.append("("); + for (int i = 0; i < methodParams.size(); i++) { + if (colPosition == i) { + sb.append(columnContainerFilter); + } else { + sb.append(methodParams.get(i)); + } + if (i < methodParams.size() - 1) + sb.append(","); + } + sb.append(")"); + return sb.toString(); + } + + protected void initMethod() { + if (BuiltInMethod.MAP.containsKey(name)) { + this.method = BuiltInMethod.MAP.get(name).method; + isValidFunc = true; + } + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/kylin/blob/6c23313e/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilter.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilter.java b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilter.java index 4647c51..3250640 100644 --- a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilter.java +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilter.java @@ -80,7 +80,6 @@ public abstract class TupleFilter { protected final List<TupleFilter> children; protected FilterOperatorEnum operator; - protected boolean hasChildren; protected TupleFilter(List<TupleFilter> filters, FilterOperatorEnum op) { this.children = filters; http://git-wip-us.apache.org/repos/asf/kylin/blob/6c23313e/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilterSerializer.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilterSerializer.java b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilterSerializer.java index bcb005f..f6c8bc4 100644 --- a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilterSerializer.java +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilterSerializer.java @@ -186,7 +186,7 @@ public class TupleFilterSerializer { filter = new DynamicTupleFilter(null); break; case FUNCTION: - filter = new BuildInFunctionTupleFilter(null); + filter = new BuiltInFunctionTupleFilter(null); break; case MASSIN: filter = new MassInTupleFilter(); http://git-wip-us.apache.org/repos/asf/kylin/blob/6c23313e/core-metadata/src/main/java/org/apache/kylin/metadata/filter/function/Functions.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/function/Functions.java b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/function/Functions.java index 401c90b..861e530 100644 --- a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/function/Functions.java +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/function/Functions.java @@ -21,7 +21,7 @@ package org.apache.kylin.metadata.filter.function; import java.lang.reflect.InvocationTargetException; import java.util.Map; -import org.apache.kylin.metadata.filter.BuildInFunctionTupleFilter; +import org.apache.kylin.metadata.filter.BuiltInFunctionTupleFilter; import org.apache.kylin.metadata.filter.TupleFilter; import org.apache.kylin.metadata.filter.UDF.MassInTupleFilter; @@ -54,7 +54,7 @@ public class Functions { } } - return new BuildInFunctionTupleFilter(name); + return new BuiltInFunctionTupleFilter(name); } } http://git-wip-us.apache.org/repos/asf/kylin/blob/6c23313e/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeSegmentScanner.java ---------------------------------------------------------------------- diff --git a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeSegmentScanner.java b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeSegmentScanner.java index 6ed7d3b..3b9d9c6 100644 --- a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeSegmentScanner.java +++ b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeSegmentScanner.java @@ -27,7 +27,7 @@ import org.apache.kylin.common.KylinConfig; import org.apache.kylin.cube.CubeSegment; import org.apache.kylin.cube.cuboid.Cuboid; import org.apache.kylin.cube.gridtable.CubeScanRangePlanner; -import org.apache.kylin.dict.BuildInFunctionTransformer; +import org.apache.kylin.dict.BuiltInFunctionTransformer; import org.apache.kylin.gridtable.GTInfo; import org.apache.kylin.gridtable.GTRecord; import org.apache.kylin.gridtable.GTScanRequest; @@ -64,7 +64,7 @@ public class CubeSegmentScanner implements IGTScanner { byte[] serialize = TupleFilterSerializer.serialize(originalfilter, StringCodeSystem.INSTANCE); TupleFilter filter = TupleFilterSerializer.deserialize(serialize, StringCodeSystem.INSTANCE); // translate FunctionTupleFilter to IN clause - ITupleFilterTransformer translator = new BuildInFunctionTransformer(cubeSeg.getDimensionEncodingMap()); + ITupleFilterTransformer translator = new BuiltInFunctionTransformer(cubeSeg.getDimensionEncodingMap()); filter = translator.transform(filter); String plannerName = KylinConfig.getInstanceFromEnv().getQueryStorageVisitPlanner(); http://git-wip-us.apache.org/repos/asf/kylin/blob/6c23313e/kylin-it/src/test/resources/query/sql_like/query03.sql ---------------------------------------------------------------------- diff --git a/kylin-it/src/test/resources/query/sql_like/query03.sql b/kylin-it/src/test/resources/query/sql_like/query03.sql new file mode 100644 index 0000000..ae095fc --- /dev/null +++ b/kylin-it/src/test/resources/query/sql_like/query03.sql @@ -0,0 +1,33 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- + +-- test case for KYLIN-1954 + +select lstg_format_name as lstg_format_name, count(*) as cnt + + from test_kylin_fact +inner JOIN edw.test_cal_dt as test_cal_dt + ON test_kylin_fact.cal_dt = test_cal_dt.cal_dt + inner JOIN test_category_groupings + ON test_kylin_fact.leaf_categ_id = test_category_groupings.leaf_categ_id AND test_kylin_fact.lstg_site_id = test_category_groupings.site_id + inner JOIN edw.test_sites as test_sites + ON test_kylin_fact.lstg_site_id = test_sites.site_id + + +where lstg_format_name like '%BIN%' and lstg_format_name > 'A' +group by lstg_format_name \ No newline at end of file http://git-wip-us.apache.org/repos/asf/kylin/blob/6c23313e/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/common/coprocessor/FilterDecorator.java ---------------------------------------------------------------------- diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/common/coprocessor/FilterDecorator.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/common/coprocessor/FilterDecorator.java index 01bd4e6..ea4b504 100644 --- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/common/coprocessor/FilterDecorator.java +++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/common/coprocessor/FilterDecorator.java @@ -24,7 +24,7 @@ import java.util.Set; import org.apache.kylin.common.util.Bytes; import org.apache.kylin.common.util.Dictionary; import org.apache.kylin.cube.kv.RowKeyColumnIO; -import org.apache.kylin.dict.BuildInFunctionTransformer; +import org.apache.kylin.dict.BuiltInFunctionTransformer; import org.apache.kylin.dict.DictCodeSystem; import org.apache.kylin.dimension.DimensionEncoding; import org.apache.kylin.dimension.IDimensionEncodingMap; @@ -151,7 +151,7 @@ public class FilterDecorator implements TupleFilterSerializer.Decorator { if (filter == null) return null; - BuildInFunctionTransformer translator = new BuildInFunctionTransformer(dimEncMap); + BuiltInFunctionTransformer translator = new BuiltInFunctionTransformer(dimEncMap); filter = translator.transform(filter); // un-evaluatable filter is replaced with TRUE