Repository: kylin Updated Branches: refs/heads/2.x-staging 9021f17d8 -> f73abf6c2
http://git-wip-us.apache.org/repos/asf/kylin/blob/f73abf6c/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MergeCuboidJobTest.java ---------------------------------------------------------------------- diff --git a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MergeCuboidJobTest.java b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MergeCuboidJobTest.java index 7e530db..97b1ef2 100644 --- a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MergeCuboidJobTest.java +++ b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MergeCuboidJobTest.java @@ -44,7 +44,6 @@ public class MergeCuboidJobTest extends LocalFileMetadataTestCase { conf.set("fs.default.name", "file:///"); conf.set("mapreduce.framework.name", "local"); conf.set("mapreduce.application.framework.path", ""); - // for local runner out-of-memory issue conf.set("mapreduce.task.io.sort.mb", "10"); http://git-wip-us.apache.org/repos/asf/kylin/blob/f73abf6c/jdbc/src/main/java/org/apache/kylin/jdbc/KylinConnection.java ---------------------------------------------------------------------- diff --git a/jdbc/src/main/java/org/apache/kylin/jdbc/KylinConnection.java b/jdbc/src/main/java/org/apache/kylin/jdbc/KylinConnection.java index 86c8d1d..6852998 100644 --- a/jdbc/src/main/java/org/apache/kylin/jdbc/KylinConnection.java +++ b/jdbc/src/main/java/org/apache/kylin/jdbc/KylinConnection.java @@ -118,7 +118,7 @@ public class KylinConnection extends AvaticaConnection { ArrayList<ColumnMetaData> columns = new ArrayList<ColumnMetaData>(); Map<String, Object> internalParams = Collections.<String, Object> emptyMap(); - return new Meta.Signature(columns, sql, params, internalParams, CursorFactory.ARRAY); + return new Meta.Signature(columns, sql, params, internalParams, CursorFactory.ARRAY, Meta.StatementType.SELECT); } private KylinJdbcFactory factory() { http://git-wip-us.apache.org/repos/asf/kylin/blob/f73abf6c/jdbc/src/main/java/org/apache/kylin/jdbc/KylinJdbcFactory.java ---------------------------------------------------------------------- diff --git a/jdbc/src/main/java/org/apache/kylin/jdbc/KylinJdbcFactory.java b/jdbc/src/main/java/org/apache/kylin/jdbc/KylinJdbcFactory.java index f1a4939..6aae983 100644 --- a/jdbc/src/main/java/org/apache/kylin/jdbc/KylinJdbcFactory.java +++ b/jdbc/src/main/java/org/apache/kylin/jdbc/KylinJdbcFactory.java @@ -33,6 +33,7 @@ import org.apache.calcite.avatica.AvaticaStatement; import org.apache.calcite.avatica.Meta.Frame; import org.apache.calcite.avatica.Meta.Signature; import org.apache.calcite.avatica.Meta.StatementHandle; +import org.apache.calcite.avatica.QueryState; import org.apache.calcite.avatica.UnregisteredDriver; /** @@ -93,9 +94,9 @@ public class KylinJdbcFactory implements AvaticaFactory { } @Override - public AvaticaResultSet newResultSet(AvaticaStatement statement, Signature signature, TimeZone timeZone, Frame firstFrame) throws SQLException { + public AvaticaResultSet newResultSet(AvaticaStatement statement, QueryState state, Signature signature, TimeZone timeZone, Frame firstFrame) throws SQLException { AvaticaResultSetMetaData resultSetMetaData = new AvaticaResultSetMetaData(statement, null, signature); - return new KylinResultSet(statement, signature, resultSetMetaData, timeZone, firstFrame); + return new KylinResultSet(statement, state, signature, resultSetMetaData, timeZone, firstFrame); } @Override http://git-wip-us.apache.org/repos/asf/kylin/blob/f73abf6c/jdbc/src/main/java/org/apache/kylin/jdbc/KylinMeta.java ---------------------------------------------------------------------- diff --git a/jdbc/src/main/java/org/apache/kylin/jdbc/KylinMeta.java b/jdbc/src/main/java/org/apache/kylin/jdbc/KylinMeta.java index cae39ad..05f7983 100644 --- a/jdbc/src/main/java/org/apache/kylin/jdbc/KylinMeta.java +++ b/jdbc/src/main/java/org/apache/kylin/jdbc/KylinMeta.java @@ -30,6 +30,10 @@ import java.util.regex.Pattern; import org.apache.calcite.avatica.AvaticaUtils; import org.apache.calcite.avatica.ColumnMetaData; import org.apache.calcite.avatica.MetaImpl; +import org.apache.calcite.avatica.MissingResultsException; +import org.apache.calcite.avatica.NoSuchStatementException; +import org.apache.calcite.avatica.QueryState; +import org.apache.calcite.avatica.remote.TypedValue; import com.google.common.collect.ImmutableList; @@ -55,6 +59,13 @@ public class KylinMeta extends MetaImpl { result.signature = connection().mockPreparedSignature(sql); return result; } + + // real execution happens in KylinResultSet.execute() + @Override + public ExecuteResult execute(StatementHandle sh, List<TypedValue> parameterValues, long maxRowCount) throws NoSuchStatementException { + final MetaResultSet metaResultSet = MetaResultSet.create(sh.connectionId, sh.id, false, sh.signature, null); + return new ExecuteResult(ImmutableList.of(metaResultSet)); + } // mimic from CalciteMetaImpl, real execution happens via callback in KylinResultSet.execute() @Override @@ -91,24 +102,24 @@ public class KylinMeta extends MetaImpl { } @Override - public MetaResultSet getTableTypes() { + public MetaResultSet getTableTypes(ConnectionHandle ch) { return createResultSet(metaTableTypes, MetaTableType.class, "TABLE_TYPE"); } @Override - public MetaResultSet getCatalogs() { + public MetaResultSet getCatalogs(ConnectionHandle ch) { List<KMetaCatalog> catalogs = getMetaProject().catalogs; return createResultSet(catalogs, KMetaCatalog.class, "TABLE_CAT"); } @Override - public MetaResultSet getSchemas(String catalog, Pat schemaPattern) { + public MetaResultSet getSchemas(ConnectionHandle ch, String catalog, Pat schemaPattern) { List<KMetaSchema> schemas = getMetaProject().getSchemas(catalog, schemaPattern); return createResultSet(schemas, KMetaSchema.class, "TABLE_SCHEM", "TABLE_CATALOG"); } @Override - public MetaResultSet getTables(String catalog, Pat schemaPattern, Pat tableNamePattern, List<String> typeList) { + public MetaResultSet getTables(ConnectionHandle ch, String catalog, Pat schemaPattern, Pat tableNamePattern, List<String> typeList) { List<KMetaTable> tables = getMetaProject().getTables(catalog, schemaPattern, tableNamePattern, typeList); return createResultSet(tables, KMetaTable.class, // "TABLE_CAT", // @@ -124,7 +135,7 @@ public class KylinMeta extends MetaImpl { } @Override - public MetaResultSet getColumns(String catalog, Pat schemaPattern, Pat tableNamePattern, Pat columnNamePattern) { + public MetaResultSet getColumns(ConnectionHandle ch, String catalog, Pat schemaPattern, Pat tableNamePattern, Pat columnNamePattern) { List<KMetaColumn> columns = getMetaProject().getColumns(catalog, schemaPattern, tableNamePattern, columnNamePattern); return createResultSet(columns, KMetaColumn.class, // "TABLE_CAT", // @@ -172,7 +183,7 @@ public class KylinMeta extends MetaImpl { } CursorFactory cursorFactory = CursorFactory.record(clazz, fields, fieldNames); - Signature signature = new Signature(columns, "", null, Collections.<String, Object> emptyMap(), cursorFactory); + Signature signature = new Signature(columns, "", null, Collections.<String, Object> emptyMap(), cursorFactory, StatementType.SELECT); StatementHandle sh = this.createStatement(connection().handle); Frame frame = new Frame(0, true, iterable); @@ -355,4 +366,27 @@ public class KylinMeta extends MetaImpl { } } + @Override + public Frame fetch(StatementHandle h, long offset, int fetchMaxRowCount) throws NoSuchStatementException, MissingResultsException { + // TODO Auto-generated method stub + return null; + } + + @Override + public boolean syncResults(StatementHandle sh, QueryState state, long offset) throws NoSuchStatementException { + // TODO Auto-generated method stub + return false; + } + + @Override + public void commit(ConnectionHandle ch) { + // TODO Auto-generated method stub + + } + + @Override + public void rollback(ConnectionHandle ch) { + // TODO Auto-generated method stub + + } } http://git-wip-us.apache.org/repos/asf/kylin/blob/f73abf6c/jdbc/src/main/java/org/apache/kylin/jdbc/KylinPreparedStatement.java ---------------------------------------------------------------------- diff --git a/jdbc/src/main/java/org/apache/kylin/jdbc/KylinPreparedStatement.java b/jdbc/src/main/java/org/apache/kylin/jdbc/KylinPreparedStatement.java index b14865b..04567cb 100644 --- a/jdbc/src/main/java/org/apache/kylin/jdbc/KylinPreparedStatement.java +++ b/jdbc/src/main/java/org/apache/kylin/jdbc/KylinPreparedStatement.java @@ -18,6 +18,8 @@ public class KylinPreparedStatement extends AvaticaPreparedStatement { protected KylinPreparedStatement(AvaticaConnection connection, StatementHandle h, Signature signature, int resultSetType, int resultSetConcurrency, int resultSetHoldability) throws SQLException { super(connection, h, signature, resultSetType, resultSetConcurrency, resultSetHoldability); + if (this.handle.signature == null) + this.handle.signature = signature; } protected List<Object> getParameterValues2() { http://git-wip-us.apache.org/repos/asf/kylin/blob/f73abf6c/jdbc/src/main/java/org/apache/kylin/jdbc/KylinResultSet.java ---------------------------------------------------------------------- diff --git a/jdbc/src/main/java/org/apache/kylin/jdbc/KylinResultSet.java b/jdbc/src/main/java/org/apache/kylin/jdbc/KylinResultSet.java index b6ac261..2b186bf 100644 --- a/jdbc/src/main/java/org/apache/kylin/jdbc/KylinResultSet.java +++ b/jdbc/src/main/java/org/apache/kylin/jdbc/KylinResultSet.java @@ -30,12 +30,13 @@ import org.apache.calcite.avatica.AvaticaStatement; import org.apache.calcite.avatica.Meta.Frame; import org.apache.calcite.avatica.Meta.Signature; import org.apache.calcite.avatica.MetaImpl; +import org.apache.calcite.avatica.QueryState; import org.apache.kylin.jdbc.IRemoteClient.QueryResult; public class KylinResultSet extends AvaticaResultSet { - public KylinResultSet(AvaticaStatement statement, Signature signature, ResultSetMetaData resultSetMetaData, TimeZone timeZone, Frame firstFrame) { - super(statement, signature, resultSetMetaData, timeZone, firstFrame); + public KylinResultSet(AvaticaStatement statement, QueryState state, Signature signature, ResultSetMetaData resultSetMetaData, TimeZone timeZone, Frame firstFrame) { + super(statement, state, signature, resultSetMetaData, timeZone, firstFrame); } @Override http://git-wip-us.apache.org/repos/asf/kylin/blob/f73abf6c/kylin-it/src/test/resources/logging.properties ---------------------------------------------------------------------- diff --git a/kylin-it/src/test/resources/logging.properties b/kylin-it/src/test/resources/logging.properties index a925478..5d2afe2 100644 --- a/kylin-it/src/test/resources/logging.properties +++ b/kylin-it/src/test/resources/logging.properties @@ -1,5 +1,5 @@ handlers=java.util.logging.ConsoleHandler .level=INFO -#org.eigenbase.relopt.RelOptPlanner.level=FINEST +#org.apache.calcite.plan.RelOptPlanner.level=FINE java.util.logging.ConsoleHandler.level=ALL java.util.logging.ConsoleHandler.formatter=org.apache.kylin.common.util.MyLogFormatter \ No newline at end of file http://git-wip-us.apache.org/repos/asf/kylin/blob/f73abf6c/kylin-it/src/test/resources/query/sql/query92.sql ---------------------------------------------------------------------- diff --git a/kylin-it/src/test/resources/query/sql/query92.sql b/kylin-it/src/test/resources/query/sql/query92.sql new file mode 100644 index 0000000..e551a45 --- /dev/null +++ b/kylin-it/src/test/resources/query/sql/query92.sql @@ -0,0 +1,30 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- + +select meta_categ_name, count(1) as cnt, sum(price) as GMV + + from test_kylin_fact + left JOIN edw.test_cal_dt as test_cal_dt + ON test_kylin_fact.cal_dt = test_cal_dt.cal_dt + left JOIN test_category_groupings + ON test_kylin_fact.leaf_categ_id = test_category_groupings.leaf_categ_id AND test_kylin_fact.lstg_site_id = test_category_groupings.site_id + left JOIN edw.test_sites as test_sites + ON test_kylin_fact.lstg_site_id = test_sites.site_id + + where meta_categ_name not in ('', 'a') + group by meta_categ_name http://git-wip-us.apache.org/repos/asf/kylin/blob/f73abf6c/kylin-it/src/test/resources/query/sql/query92.sql.disabled ---------------------------------------------------------------------- diff --git a/kylin-it/src/test/resources/query/sql/query92.sql.disabled b/kylin-it/src/test/resources/query/sql/query92.sql.disabled deleted file mode 100644 index e551a45..0000000 --- a/kylin-it/src/test/resources/query/sql/query92.sql.disabled +++ /dev/null @@ -1,30 +0,0 @@ --- --- Licensed to the Apache Software Foundation (ASF) under one --- or more contributor license agreements. See the NOTICE file --- distributed with this work for additional information --- regarding copyright ownership. The ASF licenses this file --- to you under the Apache License, Version 2.0 (the --- "License"); you may not use this file except in compliance --- with the License. You may obtain a copy of the License at --- --- http://www.apache.org/licenses/LICENSE-2.0 --- --- Unless required by applicable law or agreed to in writing, software --- distributed under the License is distributed on an "AS IS" BASIS, --- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. --- See the License for the specific language governing permissions and --- limitations under the License. --- - -select meta_categ_name, count(1) as cnt, sum(price) as GMV - - from test_kylin_fact - left JOIN edw.test_cal_dt as test_cal_dt - ON test_kylin_fact.cal_dt = test_cal_dt.cal_dt - left JOIN test_category_groupings - ON test_kylin_fact.leaf_categ_id = test_category_groupings.leaf_categ_id AND test_kylin_fact.lstg_site_id = test_category_groupings.site_id - left JOIN edw.test_sites as test_sites - ON test_kylin_fact.lstg_site_id = test_sites.site_id - - where meta_categ_name not in ('', 'a') - group by meta_categ_name http://git-wip-us.apache.org/repos/asf/kylin/blob/f73abf6c/kylin-it/src/test/resources/query/sql/query93.sql ---------------------------------------------------------------------- diff --git a/kylin-it/src/test/resources/query/sql/query93.sql b/kylin-it/src/test/resources/query/sql/query93.sql new file mode 100644 index 0000000..cc6dca5 --- /dev/null +++ b/kylin-it/src/test/resources/query/sql/query93.sql @@ -0,0 +1,30 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- + +select meta_categ_name, count(1) as cnt, sum(price) as GMV + + from test_kylin_fact + left JOIN edw.test_cal_dt as test_cal_dt + ON test_kylin_fact.cal_dt = test_cal_dt.cal_dt + left JOIN test_category_groupings + ON test_kylin_fact.leaf_categ_id = test_category_groupings.leaf_categ_id AND test_kylin_fact.lstg_site_id = test_category_groupings.site_id + left JOIN edw.test_sites as test_sites + ON test_kylin_fact.lstg_site_id = test_sites.site_id + + where meta_categ_name is not null + group by meta_categ_name http://git-wip-us.apache.org/repos/asf/kylin/blob/f73abf6c/kylin-it/src/test/resources/query/sql/query93.sql.disabled ---------------------------------------------------------------------- diff --git a/kylin-it/src/test/resources/query/sql/query93.sql.disabled b/kylin-it/src/test/resources/query/sql/query93.sql.disabled deleted file mode 100644 index cc6dca5..0000000 --- a/kylin-it/src/test/resources/query/sql/query93.sql.disabled +++ /dev/null @@ -1,30 +0,0 @@ --- --- Licensed to the Apache Software Foundation (ASF) under one --- or more contributor license agreements. See the NOTICE file --- distributed with this work for additional information --- regarding copyright ownership. The ASF licenses this file --- to you under the Apache License, Version 2.0 (the --- "License"); you may not use this file except in compliance --- with the License. You may obtain a copy of the License at --- --- http://www.apache.org/licenses/LICENSE-2.0 --- --- Unless required by applicable law or agreed to in writing, software --- distributed under the License is distributed on an "AS IS" BASIS, --- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. --- See the License for the specific language governing permissions and --- limitations under the License. --- - -select meta_categ_name, count(1) as cnt, sum(price) as GMV - - from test_kylin_fact - left JOIN edw.test_cal_dt as test_cal_dt - ON test_kylin_fact.cal_dt = test_cal_dt.cal_dt - left JOIN test_category_groupings - ON test_kylin_fact.leaf_categ_id = test_category_groupings.leaf_categ_id AND test_kylin_fact.lstg_site_id = test_category_groupings.site_id - left JOIN edw.test_sites as test_sites - ON test_kylin_fact.lstg_site_id = test_sites.site_id - - where meta_categ_name is not null - group by meta_categ_name http://git-wip-us.apache.org/repos/asf/kylin/blob/f73abf6c/kylin-it/src/test/resources/query/sql/query94.sql ---------------------------------------------------------------------- diff --git a/kylin-it/src/test/resources/query/sql/query94.sql b/kylin-it/src/test/resources/query/sql/query94.sql new file mode 100644 index 0000000..c7899fd --- /dev/null +++ b/kylin-it/src/test/resources/query/sql/query94.sql @@ -0,0 +1,30 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- + +select meta_categ_name, count(1) as cnt, sum(price) as GMV + + from test_kylin_fact + left JOIN edw.test_cal_dt as test_cal_dt + ON test_kylin_fact.cal_dt = test_cal_dt.cal_dt + left JOIN test_category_groupings + ON test_kylin_fact.leaf_categ_id = test_category_groupings.leaf_categ_id AND test_kylin_fact.lstg_site_id = test_category_groupings.site_id + left JOIN edw.test_sites as test_sites + ON test_kylin_fact.lstg_site_id = test_sites.site_id + + where meta_categ_name not in ('Unknown') + group by meta_categ_name http://git-wip-us.apache.org/repos/asf/kylin/blob/f73abf6c/kylin-it/src/test/resources/query/sql/query94.sql.disabled ---------------------------------------------------------------------- diff --git a/kylin-it/src/test/resources/query/sql/query94.sql.disabled b/kylin-it/src/test/resources/query/sql/query94.sql.disabled deleted file mode 100644 index c7899fd..0000000 --- a/kylin-it/src/test/resources/query/sql/query94.sql.disabled +++ /dev/null @@ -1,30 +0,0 @@ --- --- Licensed to the Apache Software Foundation (ASF) under one --- or more contributor license agreements. See the NOTICE file --- distributed with this work for additional information --- regarding copyright ownership. The ASF licenses this file --- to you under the Apache License, Version 2.0 (the --- "License"); you may not use this file except in compliance --- with the License. You may obtain a copy of the License at --- --- http://www.apache.org/licenses/LICENSE-2.0 --- --- Unless required by applicable law or agreed to in writing, software --- distributed under the License is distributed on an "AS IS" BASIS, --- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. --- See the License for the specific language governing permissions and --- limitations under the License. --- - -select meta_categ_name, count(1) as cnt, sum(price) as GMV - - from test_kylin_fact - left JOIN edw.test_cal_dt as test_cal_dt - ON test_kylin_fact.cal_dt = test_cal_dt.cal_dt - left JOIN test_category_groupings - ON test_kylin_fact.leaf_categ_id = test_category_groupings.leaf_categ_id AND test_kylin_fact.lstg_site_id = test_category_groupings.site_id - left JOIN edw.test_sites as test_sites - ON test_kylin_fact.lstg_site_id = test_sites.site_id - - where meta_categ_name not in ('Unknown') - group by meta_categ_name http://git-wip-us.apache.org/repos/asf/kylin/blob/f73abf6c/kylin-it/src/test/resources/query/sql/query95.sql ---------------------------------------------------------------------- diff --git a/kylin-it/src/test/resources/query/sql/query95.sql b/kylin-it/src/test/resources/query/sql/query95.sql new file mode 100644 index 0000000..578b93f --- /dev/null +++ b/kylin-it/src/test/resources/query/sql/query95.sql @@ -0,0 +1,30 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- + +select meta_categ_name, count(1) as cnt, sum(price) as GMV + + from test_kylin_fact + left JOIN edw.test_cal_dt as test_cal_dt + ON test_kylin_fact.cal_dt = test_cal_dt.cal_dt + left JOIN test_category_groupings + ON test_kylin_fact.leaf_categ_id = test_category_groupings.leaf_categ_id AND test_kylin_fact.lstg_site_id = test_category_groupings.site_id + left JOIN edw.test_sites as test_sites + ON test_kylin_fact.lstg_site_id = test_sites.site_id + + where meta_categ_name not in ('Unknown', 'ToyHobbies', '', 'a', 'BookMagazines') + group by meta_categ_name http://git-wip-us.apache.org/repos/asf/kylin/blob/f73abf6c/kylin-it/src/test/resources/query/sql/query95.sql.disabled ---------------------------------------------------------------------- diff --git a/kylin-it/src/test/resources/query/sql/query95.sql.disabled b/kylin-it/src/test/resources/query/sql/query95.sql.disabled deleted file mode 100644 index 578b93f..0000000 --- a/kylin-it/src/test/resources/query/sql/query95.sql.disabled +++ /dev/null @@ -1,30 +0,0 @@ --- --- Licensed to the Apache Software Foundation (ASF) under one --- or more contributor license agreements. See the NOTICE file --- distributed with this work for additional information --- regarding copyright ownership. The ASF licenses this file --- to you under the Apache License, Version 2.0 (the --- "License"); you may not use this file except in compliance --- with the License. You may obtain a copy of the License at --- --- http://www.apache.org/licenses/LICENSE-2.0 --- --- Unless required by applicable law or agreed to in writing, software --- distributed under the License is distributed on an "AS IS" BASIS, --- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. --- See the License for the specific language governing permissions and --- limitations under the License. --- - -select meta_categ_name, count(1) as cnt, sum(price) as GMV - - from test_kylin_fact - left JOIN edw.test_cal_dt as test_cal_dt - ON test_kylin_fact.cal_dt = test_cal_dt.cal_dt - left JOIN test_category_groupings - ON test_kylin_fact.leaf_categ_id = test_category_groupings.leaf_categ_id AND test_kylin_fact.lstg_site_id = test_category_groupings.site_id - left JOIN edw.test_sites as test_sites - ON test_kylin_fact.lstg_site_id = test_sites.site_id - - where meta_categ_name not in ('Unknown', 'ToyHobbies', '', 'a', 'BookMagazines') - group by meta_categ_name http://git-wip-us.apache.org/repos/asf/kylin/blob/f73abf6c/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index 2e42841..501bf77 100644 --- a/pom.xml +++ b/pom.xml @@ -99,7 +99,7 @@ <spring.boot.version>1.2.7.RELEASE</spring.boot.version> <!-- Calcite Version --> - <calcite.version>1.4.0-incubating</calcite.version> + <calcite.version>1.6.0</calcite.version> <!-- Curator.version Version --> <curator.version>2.6.0</curator.version> http://git-wip-us.apache.org/repos/asf/kylin/blob/f73abf6c/query/src/main/java/org/apache/kylin/query/optrule/OLAPJoinRule.java ---------------------------------------------------------------------- diff --git a/query/src/main/java/org/apache/kylin/query/optrule/OLAPJoinRule.java b/query/src/main/java/org/apache/kylin/query/optrule/OLAPJoinRule.java index 10047e6..c31d1d0 100644 --- a/query/src/main/java/org/apache/kylin/query/optrule/OLAPJoinRule.java +++ b/query/src/main/java/org/apache/kylin/query/optrule/OLAPJoinRule.java @@ -63,7 +63,7 @@ public class OLAPJoinRule extends ConverterRule { try { newRel = new OLAPJoinRel(cluster, traitSet, left, right, // info.getEquiCondition(left, right, cluster.getRexBuilder()), // - info.leftKeys, info.rightKeys, join.getJoinType(), join.getVariablesStopped()); + info.leftKeys, info.rightKeys, join.getVariablesSet(), join.getJoinType()); } catch (InvalidRelException e) { // Semantic error not possible. Must be a bug. Convert to internal error. throw new AssertionError(e); http://git-wip-us.apache.org/repos/asf/kylin/blob/f73abf6c/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java ---------------------------------------------------------------------- diff --git a/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java b/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java index eed5636..9414757 100644 --- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java +++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java @@ -37,6 +37,7 @@ import org.apache.calcite.rel.InvalidRelException; import org.apache.calcite.rel.RelNode; import org.apache.calcite.rel.core.Aggregate; import org.apache.calcite.rel.core.AggregateCall; +import org.apache.calcite.rel.metadata.RelMetadataQuery; import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rel.type.RelDataTypeFactory; import org.apache.calcite.rel.type.RelDataTypeField; @@ -60,6 +61,7 @@ import org.apache.kylin.metadata.model.MeasureDesc; import org.apache.kylin.metadata.model.ParameterDesc; import org.apache.kylin.metadata.model.TableDesc; import org.apache.kylin.metadata.model.TblColRef; +import org.apache.kylin.query.schema.OLAPTable; import com.google.common.base.Preconditions; import com.google.common.collect.Lists; @@ -122,8 +124,8 @@ public class OLAPAggregateRel extends Aggregate implements OLAPRel { } @Override - public RelOptCost computeSelfCost(RelOptPlanner planner) { - return super.computeSelfCost(planner).multiplyBy(.05); + public RelOptCost computeSelfCost(RelOptPlanner planner, RelMetadataQuery mq) { + return super.computeSelfCost(planner, mq).multiplyBy(.05); } @Override @@ -280,6 +282,7 @@ public class OLAPAggregateRel extends Aggregate implements OLAPRel { fillbackOptimizedColumn(); ColumnRowType inputColumnRowType = ((OLAPRel) getInput()).getColumnRowType(); + RelDataTypeFactory typeFactory = getCluster().getTypeFactory(); for (int i = 0; i < this.aggregations.size(); i++) { FunctionDesc aggFunc = this.aggregations.get(i); @@ -290,7 +293,8 @@ public class OLAPAggregateRel extends Aggregate implements OLAPRel { if (aggFunc.needRewrite()) { String rewriteFieldName = aggFunc.getRewriteFieldName(); - this.context.rewriteFields.put(rewriteFieldName, null); + RelDataType rewriteFieldType = OLAPTable.createSqlType(typeFactory, aggFunc.getRewriteFieldType(), true); + this.context.rewriteFields.put(rewriteFieldName, rewriteFieldType); TblColRef column = buildRewriteColumn(aggFunc); this.context.metricsColumns.add(column); http://git-wip-us.apache.org/repos/asf/kylin/blob/f73abf6c/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java ---------------------------------------------------------------------- diff --git a/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java b/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java index a847890..5ea138f 100644 --- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java +++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java @@ -36,6 +36,7 @@ import org.apache.calcite.plan.RelTrait; import org.apache.calcite.plan.RelTraitSet; import org.apache.calcite.rel.RelNode; import org.apache.calcite.rel.core.Filter; +import org.apache.calcite.rel.metadata.RelMetadataQuery; import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rex.RexBuilder; import org.apache.calcite.rex.RexCall; @@ -252,8 +253,8 @@ public class OLAPFilterRel extends Filter implements OLAPRel { } @Override - public RelOptCost computeSelfCost(RelOptPlanner planner) { - return super.computeSelfCost(planner).multiplyBy(.05); + public RelOptCost computeSelfCost(RelOptPlanner planner, RelMetadataQuery mq) { + return super.computeSelfCost(planner, mq).multiplyBy(.05); } @Override http://git-wip-us.apache.org/repos/asf/kylin/blob/f73abf6c/query/src/main/java/org/apache/kylin/query/relnode/OLAPJoinRel.java ---------------------------------------------------------------------- diff --git a/query/src/main/java/org/apache/kylin/query/relnode/OLAPJoinRel.java b/query/src/main/java/org/apache/kylin/query/relnode/OLAPJoinRel.java index 09a0b17..efe404b 100644 --- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPJoinRel.java +++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPJoinRel.java @@ -43,8 +43,10 @@ import org.apache.calcite.plan.RelTrait; import org.apache.calcite.plan.RelTraitSet; import org.apache.calcite.rel.InvalidRelException; import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.core.CorrelationId; import org.apache.calcite.rel.core.JoinInfo; import org.apache.calcite.rel.core.JoinRelType; +import org.apache.calcite.rel.metadata.RelMetadataQuery; import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rel.type.RelDataTypeFactory.FieldInfoBuilder; import org.apache.calcite.rel.type.RelDataTypeField; @@ -73,8 +75,8 @@ public class OLAPJoinRel extends EnumerableJoin implements OLAPRel { public OLAPJoinRel(RelOptCluster cluster, RelTraitSet traits, RelNode left, RelNode right, // RexNode condition, ImmutableIntList leftKeys, ImmutableIntList rightKeys, // - JoinRelType joinType, Set<String> variablesStopped) throws InvalidRelException { - super(cluster, traits, left, right, condition, leftKeys, rightKeys, joinType, variablesStopped); + Set<CorrelationId> variablesSet, JoinRelType joinType) throws InvalidRelException { + super(cluster, traits, left, right, condition, leftKeys, rightKeys, variablesSet, joinType); Preconditions.checkArgument(getConvention() == OLAPRel.CONVENTION); this.rowType = getRowType(); this.isTopJoin = false; @@ -87,7 +89,7 @@ public class OLAPJoinRel extends EnumerableJoin implements OLAPRel { final JoinInfo joinInfo = JoinInfo.of(left, right, condition); assert joinInfo.isEqui(); try { - return new OLAPJoinRel(getCluster(), traitSet, left, right, condition, joinInfo.leftKeys, joinInfo.rightKeys, joinType, variablesStopped); + return new OLAPJoinRel(getCluster(), traitSet, left, right, condition, joinInfo.leftKeys, joinInfo.rightKeys, variablesSet, joinType); } catch (InvalidRelException e) { // Semantic error not possible. Must be a bug. Convert to internal error. throw new AssertionError(e); @@ -95,13 +97,13 @@ public class OLAPJoinRel extends EnumerableJoin implements OLAPRel { } @Override - public RelOptCost computeSelfCost(RelOptPlanner planner) { - return super.computeSelfCost(planner).multiplyBy(.05); + public RelOptCost computeSelfCost(RelOptPlanner planner, RelMetadataQuery mq) { + return super.computeSelfCost(planner, mq).multiplyBy(.05); } @Override - public double getRows() { - return super.getRows() * 0.1; + public double estimateRowCount(RelMetadataQuery mq) { + return super.estimateRowCount(mq) * 0.1; } @Override @@ -225,8 +227,8 @@ public class OLAPJoinRel extends EnumerableJoin implements OLAPRel { RexNode.class, // ImmutableIntList.class, // ImmutableIntList.class, // - JoinRelType.class, // - Set.class); + Set.class, // + JoinRelType.class); constr.setAccessible(true); } catch (Exception e) { throw new RuntimeException(e); @@ -238,7 +240,7 @@ public class OLAPJoinRel extends EnumerableJoin implements OLAPRel { if (this.hasSubQuery) { try { return constr.newInstance(getCluster(), getCluster().traitSetOf(EnumerableConvention.INSTANCE), // - inputs.get(0), inputs.get(1), condition, leftKeys, rightKeys, joinType, variablesStopped); + inputs.get(0), inputs.get(1), condition, leftKeys, rightKeys, variablesSet, joinType); } catch (Exception e) { throw new IllegalStateException("Can't create EnumerableJoin!", e); } http://git-wip-us.apache.org/repos/asf/kylin/blob/f73abf6c/query/src/main/java/org/apache/kylin/query/relnode/OLAPLimitRel.java ---------------------------------------------------------------------- diff --git a/query/src/main/java/org/apache/kylin/query/relnode/OLAPLimitRel.java b/query/src/main/java/org/apache/kylin/query/relnode/OLAPLimitRel.java index 82aa9de..988d1fa 100644 --- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPLimitRel.java +++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPLimitRel.java @@ -31,6 +31,7 @@ import org.apache.calcite.plan.RelTraitSet; import org.apache.calcite.rel.RelNode; import org.apache.calcite.rel.RelWriter; import org.apache.calcite.rel.SingleRel; +import org.apache.calcite.rel.metadata.RelMetadataQuery; import org.apache.calcite.rex.RexLiteral; import org.apache.calcite.rex.RexNode; @@ -59,8 +60,8 @@ public class OLAPLimitRel extends SingleRel implements OLAPRel { } @Override - public RelOptCost computeSelfCost(RelOptPlanner planner) { - return super.computeSelfCost(planner).multiplyBy(.05); + public RelOptCost computeSelfCost(RelOptPlanner planner, RelMetadataQuery mq) { + return super.computeSelfCost(planner, mq).multiplyBy(.05); } @Override http://git-wip-us.apache.org/repos/asf/kylin/blob/f73abf6c/query/src/main/java/org/apache/kylin/query/relnode/OLAPProjectRel.java ---------------------------------------------------------------------- diff --git a/query/src/main/java/org/apache/kylin/query/relnode/OLAPProjectRel.java b/query/src/main/java/org/apache/kylin/query/relnode/OLAPProjectRel.java index b0436e2..8e454c9 100644 --- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPProjectRel.java +++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPProjectRel.java @@ -35,6 +35,7 @@ import org.apache.calcite.plan.RelTrait; import org.apache.calcite.plan.RelTraitSet; import org.apache.calcite.rel.RelNode; import org.apache.calcite.rel.core.Project; +import org.apache.calcite.rel.metadata.RelMetadataQuery; import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rel.type.RelDataTypeFactory.FieldInfoBuilder; import org.apache.calcite.rel.type.RelDataTypeField; @@ -86,8 +87,8 @@ public class OLAPProjectRel extends Project implements OLAPRel { } @Override - public RelOptCost computeSelfCost(RelOptPlanner planner) { - return super.computeSelfCost(planner).multiplyBy(.05); + public RelOptCost computeSelfCost(RelOptPlanner planner, RelMetadataQuery mq) { + return super.computeSelfCost(planner, mq).multiplyBy(.05); } @Override http://git-wip-us.apache.org/repos/asf/kylin/blob/f73abf6c/query/src/main/java/org/apache/kylin/query/relnode/OLAPSortRel.java ---------------------------------------------------------------------- diff --git a/query/src/main/java/org/apache/kylin/query/relnode/OLAPSortRel.java b/query/src/main/java/org/apache/kylin/query/relnode/OLAPSortRel.java index c3e0595..64a6c3c 100644 --- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPSortRel.java +++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPSortRel.java @@ -32,6 +32,7 @@ import org.apache.calcite.rel.RelCollation; import org.apache.calcite.rel.RelFieldCollation; import org.apache.calcite.rel.RelNode; import org.apache.calcite.rel.core.Sort; +import org.apache.calcite.rel.metadata.RelMetadataQuery; import org.apache.calcite.rex.RexNode; import org.apache.kylin.metadata.model.MeasureDesc; import org.apache.kylin.metadata.model.TblColRef; @@ -58,8 +59,8 @@ public class OLAPSortRel extends Sort implements OLAPRel { } @Override - public RelOptCost computeSelfCost(RelOptPlanner planner) { - return super.computeSelfCost(planner).multiplyBy(.05); + public RelOptCost computeSelfCost(RelOptPlanner planner, RelMetadataQuery mq) { + return super.computeSelfCost(planner, mq).multiplyBy(.05); } @Override http://git-wip-us.apache.org/repos/asf/kylin/blob/f73abf6c/query/src/main/java/org/apache/kylin/query/relnode/OLAPTableScan.java ---------------------------------------------------------------------- diff --git a/query/src/main/java/org/apache/kylin/query/relnode/OLAPTableScan.java b/query/src/main/java/org/apache/kylin/query/relnode/OLAPTableScan.java index 675fd99..bcef413 100644 --- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPTableScan.java +++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPTableScan.java @@ -40,14 +40,17 @@ import org.apache.calcite.plan.volcano.AbstractConverter.ExpandConversionRule; import org.apache.calcite.rel.RelNode; import org.apache.calcite.rel.RelWriter; import org.apache.calcite.rel.core.TableScan; +import org.apache.calcite.rel.metadata.RelMetadataQuery; import org.apache.calcite.rel.rules.AggregateExpandDistinctAggregatesRule; import org.apache.calcite.rel.rules.AggregateJoinTransposeRule; import org.apache.calcite.rel.rules.AggregateProjectMergeRule; import org.apache.calcite.rel.rules.FilterJoinRule; import org.apache.calcite.rel.rules.FilterProjectTransposeRule; import org.apache.calcite.rel.rules.JoinCommuteRule; +import org.apache.calcite.rel.rules.JoinPushExpressionsRule; import org.apache.calcite.rel.rules.JoinPushThroughJoinRule; import org.apache.calcite.rel.rules.ReduceExpressionsRule; +import org.apache.calcite.rel.rules.SortJoinTransposeRule; import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rel.type.RelDataTypeFactory; import org.apache.calcite.rel.type.RelDataTypeField; @@ -147,6 +150,8 @@ public class OLAPTableScan extends TableScan implements OLAPRel, EnumerableRel { planner.removeRule(AggregateJoinTransposeRule.INSTANCE); planner.removeRule(AggregateProjectMergeRule.INSTANCE); planner.removeRule(FilterProjectTransposeRule.INSTANCE); + planner.removeRule(SortJoinTransposeRule.INSTANCE); + planner.removeRule(JoinPushExpressionsRule.INSTANCE); // distinct count will be split into a separated query that is joined with the left query planner.removeRule(AggregateExpandDistinctAggregatesRule.INSTANCE); @@ -165,8 +170,8 @@ public class OLAPTableScan extends TableScan implements OLAPRel, EnumerableRel { } @Override - public RelOptCost computeSelfCost(RelOptPlanner planner) { - return super.computeSelfCost(planner).multiplyBy(.05); + public RelOptCost computeSelfCost(RelOptPlanner planner, RelMetadataQuery mq) { + return super.computeSelfCost(planner, mq).multiplyBy(.05); } @Override http://git-wip-us.apache.org/repos/asf/kylin/blob/f73abf6c/query/src/main/java/org/apache/kylin/query/relnode/OLAPToEnumerableConverter.java ---------------------------------------------------------------------- diff --git a/query/src/main/java/org/apache/kylin/query/relnode/OLAPToEnumerableConverter.java b/query/src/main/java/org/apache/kylin/query/relnode/OLAPToEnumerableConverter.java index 7053694..de7e7e2 100644 --- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPToEnumerableConverter.java +++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPToEnumerableConverter.java @@ -35,6 +35,7 @@ import org.apache.calcite.plan.RelOptUtil; import org.apache.calcite.plan.RelTraitSet; import org.apache.calcite.rel.RelNode; import org.apache.calcite.rel.convert.ConverterImpl; +import org.apache.calcite.rel.metadata.RelMetadataQuery; import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.sql.SqlExplainLevel; import org.apache.kylin.metadata.realization.IRealization; @@ -56,8 +57,9 @@ public class OLAPToEnumerableConverter extends ConverterImpl implements Enumerab } @Override - public RelOptCost computeSelfCost(RelOptPlanner planner) { - return super.computeSelfCost(planner).multiplyBy(.05); + public RelOptCost computeSelfCost(RelOptPlanner planner, RelMetadataQuery mq) { + // huge cost to ensure OLAPToEnumerableConverter only appears once in rel tree + return planner.getCostFactory().makeCost(1E100, 0, 0); } @Override http://git-wip-us.apache.org/repos/asf/kylin/blob/f73abf6c/query/src/main/java/org/apache/kylin/query/schema/OLAPTable.java ---------------------------------------------------------------------- diff --git a/query/src/main/java/org/apache/kylin/query/schema/OLAPTable.java b/query/src/main/java/org/apache/kylin/query/schema/OLAPTable.java index a8789ea..12ed6e5 100644 --- a/query/src/main/java/org/apache/kylin/query/schema/OLAPTable.java +++ b/query/src/main/java/org/apache/kylin/query/schema/OLAPTable.java @@ -43,6 +43,7 @@ import org.apache.calcite.schema.impl.AbstractTableQueryable; import org.apache.calcite.sql.type.SqlTypeName; import org.apache.calcite.sql.type.SqlTypeUtil; import org.apache.calcite.util.ImmutableBitSet; +import org.apache.kylin.metadata.datatype.DataType; import org.apache.kylin.metadata.model.ColumnDesc; import org.apache.kylin.metadata.model.FunctionDesc; import org.apache.kylin.metadata.model.MeasureDesc; @@ -128,21 +129,21 @@ public class OLAPTable extends AbstractQueryableTable implements TranslatableTab private RelDataType deriveRowType(RelDataTypeFactory typeFactory) { RelDataTypeFactory.FieldInfoBuilder fieldInfo = typeFactory.builder(); for (ColumnDesc column : exposedColumns) { - RelDataType sqlType = createSqlType(typeFactory, column); + RelDataType sqlType = createSqlType(typeFactory, column.getType(), column.isNullable()); sqlType = SqlTypeUtil.addCharsetAndCollation(sqlType, typeFactory); fieldInfo.add(column.getName(), sqlType); } return typeFactory.createStructType(fieldInfo); } - private RelDataType createSqlType(RelDataTypeFactory typeFactory, ColumnDesc column) { - SqlTypeName sqlTypeName = SQLTYPE_MAPPING.get(column.getTypeName()); + public static RelDataType createSqlType(RelDataTypeFactory typeFactory, DataType dataType, boolean isNullable) { + SqlTypeName sqlTypeName = SQLTYPE_MAPPING.get(dataType.getName()); if (sqlTypeName == null) - throw new IllegalArgumentException("Unrecognized column type " + column.getTypeName() + " from " + column); - - int precision = column.getTypePrecision(); - int scale = column.getTypeScale(); + throw new IllegalArgumentException("Unrecognized data type " + dataType); + int precision = dataType.getPrecision(); + int scale = dataType.getScale(); + RelDataType result; if (precision >= 0 && scale >= 0) result = typeFactory.createSqlType(sqlTypeName, precision, scale); @@ -152,7 +153,7 @@ public class OLAPTable extends AbstractQueryableTable implements TranslatableTab result = typeFactory.createSqlType(sqlTypeName); // due to left join and uncertain data quality, dimension value can be null - if (column.isNullable()) { + if (isNullable) { result = typeFactory.createTypeWithNullability(result, true); } else { result = typeFactory.createTypeWithNullability(result, false);
