Minor: do not hardcode DefaultQueryTransformer sequence. (cherry picked from commit 447c9c27aacbc1524a70da195096e8954c03accf)
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/48a08e5f Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/48a08e5f Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/48a08e5f Branch: refs/heads/2.2.x Commit: 48a08e5f5f112ed2dc0a9408601d265815a42f37 Parents: 5095f1f Author: Yifan Zhang <event.dim...@gmail.com> Authored: Tue Oct 10 14:25:20 2017 +0800 Committer: Yifan Zhang <event.dim...@gmail.com> Committed: Wed Oct 18 13:14:14 2017 +0800 ---------------------------------------------------------------------- .../main/resources/kylin-defaults.properties | 2 +- .../query/util/DefaultQueryTransformer.java | 97 ++++++++++++++++++++ .../org/apache/kylin/query/util/QueryUtil.java | 77 +--------------- 3 files changed, 99 insertions(+), 77 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/48a08e5f/core-common/src/main/resources/kylin-defaults.properties ---------------------------------------------------------------------- diff --git a/core-common/src/main/resources/kylin-defaults.properties b/core-common/src/main/resources/kylin-defaults.properties index e3632ae..75436d6 100644 --- a/core-common/src/main/resources/kylin-defaults.properties +++ b/core-common/src/main/resources/kylin-defaults.properties @@ -200,7 +200,7 @@ kylin.query.interceptors=org.apache.kylin.rest.security.TableInterceptor kylin.query.escape-default-keyword=false # Usually should not modify this -kylin.query.transformers=org.apache.kylin.query.util.KeywordDefaultDirtyHack +kylin.query.transformers=org.apache.kylin.query.util.DefaultQueryTransformer,org.apache.kylin.query.util.KeywordDefaultDirtyHack ### SECURITY ### http://git-wip-us.apache.org/repos/asf/kylin/blob/48a08e5f/query/src/main/java/org/apache/kylin/query/util/DefaultQueryTransformer.java ---------------------------------------------------------------------- diff --git a/query/src/main/java/org/apache/kylin/query/util/DefaultQueryTransformer.java b/query/src/main/java/org/apache/kylin/query/util/DefaultQueryTransformer.java new file mode 100644 index 0000000..0afe1ed --- /dev/null +++ b/query/src/main/java/org/apache/kylin/query/util/DefaultQueryTransformer.java @@ -0,0 +1,97 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +package org.apache.kylin.query.util; + +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import org.apache.kylin.query.util.QueryUtil.IQueryTransformer; + +public class DefaultQueryTransformer implements IQueryTransformer { + + private static final String S0 = "\\s*"; + private static final String S1 = "\\s"; + private static final String SM = "\\s+"; + private static final Pattern PTN_GROUP_BY = Pattern.compile(S1 + "GROUP" + SM + "BY" + S1, + Pattern.CASE_INSENSITIVE); + private static final Pattern PTN_HAVING_COUNT_GREATER_THAN_ZERO = Pattern.compile(S1 + "HAVING" + SM + "[(]?" + S0 + + "COUNT" + S0 + "[(]" + S0 + "1" + S0 + "[)]" + S0 + ">" + S0 + "0" + S0 + "[)]?", + Pattern.CASE_INSENSITIVE); + private static final Pattern PTN_SUM_1 = Pattern.compile(S0 + "SUM" + S0 + "[(]" + S0 + "[1]" + S0 + "[)]" + S0, + Pattern.CASE_INSENSITIVE); + private static final Pattern PTN_NOT_EQ = Pattern.compile(S0 + "!=" + S0, Pattern.CASE_INSENSITIVE); + private static final Pattern PTN_INTERVAL = Pattern.compile( + "interval" + SM + "(floor\\()([\\d\\.]+)(\\))" + SM + "(second|minute|hour|day|month|year)", + Pattern.CASE_INSENSITIVE); + private static final Pattern PTN_HAVING_ESCAPE_FUNCTION = Pattern.compile("\\{fn" + "(.*?)" + "\\}", + Pattern.CASE_INSENSITIVE); + + @Override + public String transform(String sql, String project, String defaultSchema) { + Matcher m; + + // Case {fn EXTRACT(...) } + // Use non-greedy regrex matching to remove escape functions + while (true) { + m = PTN_HAVING_ESCAPE_FUNCTION.matcher(sql); + if (!m.find()) + break; + sql = sql.substring(0, m.start()) + m.group(1) + sql.substring(m.end()); + } + + // Case: HAVING COUNT(1)>0 without Group By + // Tableau generates: SELECT SUM(1) AS "COL" FROM "VAC_SW" HAVING + // COUNT(1)>0 + m = PTN_HAVING_COUNT_GREATER_THAN_ZERO.matcher(sql); + if (m.find() && PTN_GROUP_BY.matcher(sql).find() == false) { + sql = sql.substring(0, m.start()) + " " + sql.substring(m.end()); + } + + // Case: SUM(1) + // Replace it with COUNT(1) + while (true) { + m = PTN_SUM_1.matcher(sql); + if (!m.find()) + break; + sql = sql.substring(0, m.start()) + " COUNT(1) " + sql.substring(m.end()); + } + + // Case: != + // Replace it with <> + while (true) { + m = PTN_NOT_EQ.matcher(sql); + if (!m.find()) + break; + sql = sql.substring(0, m.start()) + " <> " + sql.substring(m.end()); + } + + // ( date '2001-09-28' + interval floor(1) day ) generated by cognos + // calcite only recognizes date '2001-09-28' + interval '1' day + while (true) { + m = PTN_INTERVAL.matcher(sql); + if (!m.find()) + break; + + int value = (int) Math.floor(Double.valueOf(m.group(2))); + sql = sql.substring(0, m.start(1)) + "'" + value + "'" + sql.substring(m.end(3)); + } + + return sql; + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/kylin/blob/48a08e5f/query/src/main/java/org/apache/kylin/query/util/QueryUtil.java ---------------------------------------------------------------------- diff --git a/query/src/main/java/org/apache/kylin/query/util/QueryUtil.java b/query/src/main/java/org/apache/kylin/query/util/QueryUtil.java index 377ca89..ab9d490 100644 --- a/query/src/main/java/org/apache/kylin/query/util/QueryUtil.java +++ b/query/src/main/java/org/apache/kylin/query/util/QueryUtil.java @@ -75,7 +75,6 @@ public class QueryUtil { private static void initQueryTransformers() { List<IQueryTransformer> transformers = Lists.newArrayList(); - transformers.add(new DefaultQueryTransformer()); String[] classes = KylinConfig.getInstanceFromEnv().getQueryTransformers(); for (String clz : classes) { @@ -86,82 +85,8 @@ public class QueryUtil { throw new RuntimeException("Failed to init query transformer", e); } } - queryTransformers = transformers; - } - - // correct sick / invalid SQL - private static class DefaultQueryTransformer implements IQueryTransformer { - - private static final String S0 = "\\s*"; - private static final String S1 = "\\s"; - private static final String SM = "\\s+"; - private static final Pattern PTN_GROUP_BY = Pattern.compile(S1 + "GROUP" + SM + "BY" + S1, - Pattern.CASE_INSENSITIVE); - private static final Pattern PTN_HAVING_COUNT_GREATER_THAN_ZERO = Pattern.compile(S1 + "HAVING" + SM + "[(]?" - + S0 + "COUNT" + S0 + "[(]" + S0 + "1" + S0 + "[)]" + S0 + ">" + S0 + "0" + S0 + "[)]?", - Pattern.CASE_INSENSITIVE); - private static final Pattern PTN_SUM_1 = Pattern.compile(S0 + "SUM" + S0 + "[(]" + S0 + "[1]" + S0 + "[)]" + S0, - Pattern.CASE_INSENSITIVE); - private static final Pattern PTN_NOT_EQ = Pattern.compile(S0 + "!=" + S0, Pattern.CASE_INSENSITIVE); - private static final Pattern PTN_INTERVAL = Pattern.compile( - "interval" + SM + "(floor\\()([\\d\\.]+)(\\))" + SM + "(second|minute|hour|day|month|year)", - Pattern.CASE_INSENSITIVE); - private static final Pattern PTN_HAVING_ESCAPE_FUNCTION = Pattern.compile("\\{fn" + "(.*?)" + "\\}", - Pattern.CASE_INSENSITIVE); - - @Override - public String transform(String sql, String project, String defaultSchema) { - Matcher m; - - // Case fn{ EXTRACT(...) } - // Use non-greedy regrex matching to remove escape functions - while (true) { - m = PTN_HAVING_ESCAPE_FUNCTION.matcher(sql); - if (!m.find()) - break; - sql = sql.substring(0, m.start()) + m.group(1) + sql.substring(m.end()); - } - - // Case: HAVING COUNT(1)>0 without Group By - // Tableau generates: SELECT SUM(1) AS "COL" FROM "VAC_SW" HAVING - // COUNT(1)>0 - m = PTN_HAVING_COUNT_GREATER_THAN_ZERO.matcher(sql); - if (m.find() && PTN_GROUP_BY.matcher(sql).find() == false) { - sql = sql.substring(0, m.start()) + " " + sql.substring(m.end()); - } - - // Case: SUM(1) - // Replace it with COUNT(1) - while (true) { - m = PTN_SUM_1.matcher(sql); - if (!m.find()) - break; - sql = sql.substring(0, m.start()) + " COUNT(1) " + sql.substring(m.end()); - } - - // Case: != - // Replace it with <> - while (true) { - m = PTN_NOT_EQ.matcher(sql); - if (!m.find()) - break; - sql = sql.substring(0, m.start()) + " <> " + sql.substring(m.end()); - } - - // ( date '2001-09-28' + interval floor(1) day ) generated by cognos - // calcite only recognizes date '2001-09-28' + interval '1' day - while (true) { - m = PTN_INTERVAL.matcher(sql); - if (!m.find()) - break; - - int value = (int) Math.floor(Double.valueOf(m.group(2))); - sql = sql.substring(0, m.start(1)) + "'" + value + "'" + sql.substring(m.end(3)); - } - - return sql; - } + queryTransformers = transformers; } public static String makeErrorMsgUserFriendly(Throwable e) {