Copilot commented on code in PR #3155: URL: https://github.com/apache/groovy/pull/3155#discussion_r4431681028
########## subprojects/groovy-ginq-sql/src/main/groovy/org/apache/groovy/ginq/provider/sql/GinqSqlWalker.groovy: ########## @@ -0,0 +1,441 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.groovy.ginq.provider.sql + +import groovy.transform.CompileStatic +import org.apache.groovy.ginq.GinqGroovyMethods +import org.apache.groovy.ginq.dsl.GinqAstVisitor +import org.apache.groovy.ginq.dsl.GinqSyntaxError +import org.apache.groovy.ginq.dsl.SyntaxErrorReportable +import org.apache.groovy.ginq.dsl.expression.AbstractGinqExpression +import org.apache.groovy.ginq.dsl.expression.DataSourceExpression +import org.apache.groovy.ginq.dsl.expression.FilterExpression +import org.apache.groovy.ginq.dsl.expression.FromExpression +import org.apache.groovy.ginq.dsl.expression.GinqExpression +import org.apache.groovy.ginq.dsl.expression.GroupExpression +import org.apache.groovy.ginq.dsl.expression.HavingExpression +import org.apache.groovy.ginq.dsl.expression.JoinExpression +import org.apache.groovy.ginq.dsl.expression.LimitExpression +import org.apache.groovy.ginq.dsl.expression.OnExpression +import org.apache.groovy.ginq.dsl.expression.OrderExpression +import org.apache.groovy.ginq.dsl.expression.SelectExpression +import org.apache.groovy.ginq.dsl.expression.SetOperationExpression +import org.apache.groovy.ginq.dsl.expression.ShutdownExpression +import org.apache.groovy.ginq.dsl.expression.WhereExpression +import org.apache.groovy.ginq.provider.collection.GinqAstWalker +import org.apache.groovy.ginq.provider.sql.ir.SqlQuery +import org.apache.groovy.ginq.provider.sql.ir.SqlQueryNode +import org.apache.groovy.ginq.provider.sql.render.AnsiDialect +import org.apache.groovy.ginq.provider.sql.render.Dialect +import org.apache.groovy.ginq.provider.sql.render.Dialects +import org.apache.groovy.ginq.provider.sql.render.RenderedSql +import org.apache.groovy.ginq.provider.sql.render.SqlRenderer +import org.apache.groovy.ginq.provider.sql.runtime.SqlGinqRuntime +import org.apache.groovy.lang.annotation.Incubating +import org.codehaus.groovy.GroovyBugError +import org.codehaus.groovy.ast.ASTNode +import org.codehaus.groovy.ast.ClassHelper +import org.codehaus.groovy.ast.ClassNode +import org.codehaus.groovy.ast.expr.ClosureExpression +import org.codehaus.groovy.ast.expr.ConstantExpression +import org.codehaus.groovy.ast.expr.Expression +import org.codehaus.groovy.ast.expr.ExpressionTransformer +import org.codehaus.groovy.ast.expr.ListExpression +import org.codehaus.groovy.ast.expr.MethodCallExpression +import org.codehaus.groovy.ast.expr.TupleExpression +import org.codehaus.groovy.ast.expr.VariableExpression +import org.codehaus.groovy.ast.stmt.BlockStatement +import org.codehaus.groovy.ast.stmt.ExpressionStatement +import org.codehaus.groovy.ast.stmt.ReturnStatement +import org.codehaus.groovy.ast.stmt.Statement +import org.codehaus.groovy.control.SourceUnit + +import static org.codehaus.groovy.ast.tools.GeneralUtils.args +import static org.codehaus.groovy.ast.tools.GeneralUtils.block +import static org.codehaus.groovy.ast.tools.GeneralUtils.callX +import static org.codehaus.groovy.ast.tools.GeneralUtils.classX +import static org.codehaus.groovy.ast.tools.GeneralUtils.constX +import static org.codehaus.groovy.ast.tools.GeneralUtils.declS +import static org.codehaus.groovy.ast.tools.GeneralUtils.lambdaX +import static org.codehaus.groovy.ast.tools.GeneralUtils.localVarX +import static org.codehaus.groovy.ast.tools.GeneralUtils.params +import static org.codehaus.groovy.ast.tools.GeneralUtils.returnS +import static org.codehaus.groovy.ast.tools.GeneralUtils.varX + +/** + * The native SQL provider: transforms a GINQ query into a call executing the + * equivalent SQL, rendered at compile time with all user values bound as + * positional parameters, against the configured {@code dataSource}. + * + * @since 6.0.0 + */ +@Incubating +@CompileStatic +class GinqSqlWalker implements GinqAstVisitor<Expression>, SyntaxErrorReportable { + private final SourceUnit sourceUnit + private Map<String, String> configuration = [:] + private Map<String, Expression> configurationExpressions = [:] + private Dialect dialect = new AnsiDialect() + + GinqSqlWalker(SourceUnit sourceUnit) { + this.sourceUnit = sourceUnit + } + + @Override + SourceUnit getSourceUnit() { + return sourceUnit + } + + @Override + Expression visit(AbstractGinqExpression expression) { + if ('true' == configuration.get(GinqGroovyMethods.CONF_PARALLEL)) { + error("`parallel` is not supported by the ${providerName} provider", expression) + } + if (expression instanceof ShutdownExpression) { + error("`shutdown` is not supported by the ${providerName} provider", expression) + } + Expression dataSourceExpr = configurationExpressions.get(GinqGroovyMethods.CONF_DATA_SOURCE) + if (dataSourceExpr == null) { + error("the ${providerName} provider requires a `dataSource`, e.g. GQ(provider: '${providerName}', dataSource: db) {...}; " + + 'note that `@GQ` does not support `dataSource`', expression) + } + validateOptions(expression) + if (sourceUnit.errorCollector.hasErrors()) { + return constX(null) + } + + if (isMixedMode()) { + return visitMixed(expression, dataSourceExpr) + } + + SqlQueryNode queryNode = translator().translate(expression) + if (queryNode == null || sourceUnit.errorCollector.hasErrors()) { + return constX(null) + } + return executeExpression(renderSql(queryNode, expression), dataSourceExpr, expression) + } + + private Expression visitMixed(AbstractGinqExpression expression, Expression dataSourceExpr) { + int before = sourceUnit.errorCollector.errorCount + MixedPlan plan = translator().planMixed(expression) + if (plan == null || sourceUnit.errorCollector.errorCount != before) { + return constX(null) + } + if (plan.fullQuery != null) { + return executeExpression(renderSql(plan.fullQuery, expression), dataSourceExpr, expression) + } + return emitMixed(plan, expression, dataSourceExpr) + } + + private Expression emitMixed(MixedPlan plan, AbstractGinqExpression expression, Expression dataSourceExpr) { + GinqExpression tail = plan.tail + if (plan.columnCountQueries.isEmpty()) { + tail.fromExpression.dataSourceExpr = executeExpression(renderSql(plan.prefix, expression), dataSourceExpr, expression) + materializeQuery(tail) + return runCollection(tail) + } + + String rowsName = "__ginqRows${mixedSeq++}" + VariableExpression rowsVar = varX(rowsName) + List<Expression> counts = [] + for (SqlQuery countQuery : plan.columnCountQueries) { + RenderedSql rendered = renderSql(countQuery, expression) + counts << callX(classX(SQL_GINQ_RUNTIME_TYPE), 'columnCount', + args(dataSourceExpr, constX(rendered.sql), new ListExpression(rendered.parameters))) + } + RenderedSql prefixSql = renderSql(plan.prefix, expression) + Expression zip = callX(classX(SQL_GINQ_RUNTIME_TYPE), 'zipAliases', args( + dataSourceExpr, + constX(prefixSql.sql), + new ListExpression(prefixSql.parameters), + new ListExpression(counts))) + tail.fromExpression.dataSourceExpr = callX(rowsVar, 'get', constX(0)) + for (int i = 0; i < plan.pushedJoins; i++) { + tail.joinExpressionList[i].dataSourceExpr = callX(rowsVar, 'get', constX(i + 1)) + } + materializeQuery(tail) + Expression tailExec = runCollection(tail) + Expression wrapper = callX(lambdaX(params(), block( + declS(localVarX(rowsName), zip), + returnS(tailExec) + )), 'call') + wrapper.setSourcePosition(expression) + return wrapper + } + + private Expression runCollection(GinqExpression tail) { + GinqAstWalker walker = new GinqAstWalker(sourceUnit) + walker.configuration = Collections.<String, String>emptyMap() + walker.configurationExpressions = Collections.<String, Expression>emptyMap() + return (Expression) walker.visit(tail) + } + + private MethodCallExpression executeExpression(RenderedSql renderedSql, Expression dataSourceExpr, AbstractGinqExpression expression) { + MethodCallExpression executeCall = callX( + classX(SQL_GINQ_RUNTIME_TYPE), + 'execute', + args(dataSourceExpr, constX(renderedSql.sql), new ListExpression(renderedSql.parameters))) + executeCall.setSourcePosition(expression) + return executeCall + } + + private boolean isMixedMode() { + return 'true' == configuration.get(NativeSqlGinqProvider.CONF_MIXED_MODE) + } + + /** + * Returns the provider name used in diagnostics. + */ + protected String getProviderName() { + return 'native-sql' + } + + /** + * Validates provider-specific options. + */ + protected void validateOptions(AbstractGinqExpression expression) { + String dialectName = configuration.get(NativeSqlGinqProvider.CONF_DIALECT) + if (dialectName != null) { + Dialect resolved = Dialects.resolve(dialectName) + if (resolved == null) { + error("`${dialectName}` is not a native SQL dialect (supported: ${Dialects.SUPPORTED}). " + + "For other databases use `provider: 'jooq-sql'` with a jOOQ SQLDialect name.", expression) + } else { + dialect = resolved + } + } + validateMixedMode(expression) + } + + /** + * Checks the shared {@code mixedMode} option. Absent means strict mode. + */ + protected void validateMixedMode(AbstractGinqExpression expression) { + String mixedMode = configuration.get(NativeSqlGinqProvider.CONF_MIXED_MODE) + if (mixedMode != null && !('true' == mixedMode || 'false' == mixedMode)) { + error('`mixedMode` must be `true` or `false`', expression) + } + } + + /** + * Renders the translated query to SQL text plus ordered parameters. + */ + protected RenderedSql renderSql(SqlQueryNode queryNode, AbstractGinqExpression expression) { + return new SqlRenderer(dialect).render(queryNode) + } + + private GinqToSqlTranslator translator() { + return new GinqToSqlTranslator(sourceUnit, providerName) + } + + @Override + Expression visitGinqExpression(GinqExpression ginqExpression) { + return visit(ginqExpression) + } + + @Override + Expression visitSetOperationExpression(SetOperationExpression setOperationExpression) { + return visit(setOperationExpression) + } + + @Override + Expression visitShutdownExpression(ShutdownExpression shutdownExpression) { + return visit(shutdownExpression) + } + + @Override + Expression visitFromExpression(FromExpression fromExpression) { + throw new GroovyBugError('clauses are translated by GinqToSqlTranslator') + } + + @Override + Expression visitJoinExpression(JoinExpression joinExpression) { + throw new GroovyBugError('clauses are translated by GinqToSqlTranslator') + } + + @Override + Expression visitOnExpression(OnExpression onExpression) { + throw new GroovyBugError('clauses are translated by GinqToSqlTranslator') + } + + @Override + Expression visitWhereExpression(WhereExpression whereExpression) { + throw new GroovyBugError('clauses are translated by GinqToSqlTranslator') + } + + @Override + Expression visitGroupExpression(GroupExpression groupExpression) { + throw new GroovyBugError('clauses are translated by GinqToSqlTranslator') + } + + @Override + Expression visitHavingExpression(HavingExpression havingExpression) { + throw new GroovyBugError('clauses are translated by GinqToSqlTranslator') + } + + @Override + Expression visitOrderExpression(OrderExpression orderExpression) { + throw new GroovyBugError('clauses are translated by GinqToSqlTranslator') + } + + @Override + Expression visitLimitExpression(LimitExpression limitExpression) { + throw new GroovyBugError('clauses are translated by GinqToSqlTranslator') + } + + @Override + Expression visitSelectExpression(SelectExpression selectExpression) { + throw new GroovyBugError('clauses are translated by GinqToSqlTranslator') + } + + @Override + void setConfiguration(Map<String, String> configuration) { + this.configuration = configuration + } + + @Override + Map<String, String> getConfiguration() { + return configuration + } + + @Override + void setConfigurationExpressions(Map<String, Expression> configurationExpressions) { + this.configurationExpressions = configurationExpressions + } + + @Override + Map<String, Expression> getConfigurationExpressions() { + return configurationExpressions + } + + protected void error(String message, ASTNode node) { + collectSyntaxError(new GinqSyntaxError(message, node.lineNumber, node.columnNumber)) + } + + /** + * Replaces string table names that the in-memory tail would otherwise + * iterate as character sequences. Sources already rewritten to the SQL + * prefix result are left alone. Nested queries stay GINQ so the collection + * provider can correlate them, but their tables are loaded with {@code SELECT *}. + */ + private void materializeQuery(GinqExpression ginqExpression) { + materializeSource(ginqExpression.fromExpression) + for (JoinExpression joinExpression : ginqExpression.joinExpressionList) { + materializeSource(joinExpression) + if (joinExpression.onExpression != null) { + joinExpression.onExpression.filterExpr = materialize(joinExpression.onExpression.filterExpr) + } + } + materializeFilter(ginqExpression.whereExpression) + GroupExpression groupExpression = ginqExpression.groupExpression + if (groupExpression != null) { + materializeTuple(groupExpression.classifierExpr) + materializeFilter(groupExpression.havingExpression) + } + if (ginqExpression.orderExpression != null) { + materializeTuple(ginqExpression.orderExpression.ordersExpr) + } + if (ginqExpression.limitExpression != null) { + materializeTuple(ginqExpression.limitExpression.offsetAndSizeExpr) + } + materializeTuple(ginqExpression.selectExpression.projectionExpr) + } + + private void materializeSet(SetOperationExpression setOperation) { + AbstractGinqExpression left = setOperation.left + if (left instanceof GinqExpression) { + materializeQuery((GinqExpression) left) + } else if (left instanceof SetOperationExpression) { + materializeSet((SetOperationExpression) left) + } + materializeQuery(setOperation.right) + } + + private void materializeFilter(FilterExpression filter) { + if (filter != null) { + filter.filterExpr = materialize(filter.filterExpr) + } + } + + private void materializeSource(DataSourceExpression source) { + Expression dataSource = source.dataSourceExpr + if (dataSource instanceof ConstantExpression && dataSource.value instanceof String) { + source.dataSourceExpr = selectAll((String) dataSource.value) + } else if (dataSource instanceof GinqExpression) { + materializeQuery((GinqExpression) dataSource) + } else if (dataSource instanceof SetOperationExpression) { + materializeSet((SetOperationExpression) dataSource) + } + } + + private void materializeTuple(Expression expr) { + if (expr instanceof TupleExpression) { + List<Expression> expressions = ((TupleExpression) expr).expressions + for (int i = 0; i < expressions.size(); i++) { + expressions.set(i, materialize(expressions.get(i))) + } + } + } + + private Expression materialize(Expression expr) { + if (expr == null) { + return null + } + return ((ListExpression) new ListExpression(Collections.singletonList(expr)).transformExpression(new ExpressionTransformer() { + @Override + Expression transform(Expression expression) { + if (expression instanceof GinqExpression) { + materializeQuery((GinqExpression) expression) + return expression + } + if (expression instanceof SetOperationExpression) { + materializeSet((SetOperationExpression) expression) + return expression + } + if (expression instanceof ClosureExpression) { + materializeStatement(((ClosureExpression) expression).code) + return expression + } + return expression.transformExpression(this) + } + })).getExpression(0) + } + + private void materializeStatement(Statement statement) { + if (statement instanceof BlockStatement) { + for (Statement child : ((BlockStatement) statement).statements) { + materializeStatement(child) + } + } else if (statement instanceof ExpressionStatement) { + ((ExpressionStatement) statement).expression = materialize(((ExpressionStatement) statement).expression) + } else if (statement instanceof ReturnStatement) { + ((ReturnStatement) statement).expression = materialize(((ReturnStatement) statement).expression) + } + } + + private Expression selectAll(String tableName) { + Expression dataSourceExpr = configurationExpressions.get(GinqGroovyMethods.CONF_DATA_SOURCE) + return callX(classX(SQL_GINQ_RUNTIME_TYPE), 'execute', + args(dataSourceExpr, constX('SELECT * FROM ' + tableName), new ListExpression())) Review Comment: Nested queries that are moved into a mixed-mode in-memory tail reach `selectAll` without going through `GinqToSqlTranslator.tableSource`, so this concatenation bypasses the documented `plainTableName` validation. For example, an untranslatable outer predicate can cause `from d in 'departments; ...'` to become executable `SELECT * FROM ...` instead of a compile-time rejection. Validate the table name here (or reuse the same safe renderer) before constructing the SQL text. ########## subprojects/groovy-ginq-sql/src/main/groovy/org/apache/groovy/ginq/provider/sql/render/Dialects.java: ########## @@ -0,0 +1,182 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.groovy.ginq.provider.sql.render; + +import org.apache.groovy.ginq.provider.sql.ir.SqlExpr; +import org.apache.groovy.lang.annotation.Incubating; + +import java.util.Locale; + +/** + * The native dialects that cover the high-variance points called out by + * GEP-26: limit/offset syntax, identifier quoting, and a few function names. + * Comprehensive dialect coverage stays with the jOOQ provider. + * + * @since 6.0.0 + */ +@Incubating +public final class Dialects { + /** + * Names accepted by {@link #resolve(String)}, for diagnostics. + */ + public static final String SUPPORTED = "ansi, h2, postgres, mysql, mariadb, sqlite"; + + private Dialects() { + } + + /** + * Returns the dialect named {@code name}, or ANSI when {@code name} is + * null or empty. The match is case-insensitive. Unknown names return null. + * + * @param name the dialect name + * @return the dialect, or null when the name is not supported + */ + public static Dialect resolve(String name) { + if (name == null || name.isEmpty()) { + return new AnsiDialect(); + } + switch (name.toLowerCase(Locale.ROOT)) { + case "ansi": + case "default": + case "h2": + // H2 folds unquoted identifiers to upper case, so quoting a + // lowercase name would miss tables created without quotes. + // Its limit syntax, null ordering and function names are ANSI. + return new AnsiDialect(); + case "postgres": + case "postgresql": + return new PostgresDialect(); + case "mysql": + case "mariadb": + return new MySqlDialect(); + case "sqlite": + return new SqliteDialect(); + default: + return null; + } + } + + private static String quote(String identifier, char mark) { + String q = String.valueOf(mark); + return q + identifier.replace(q, q + q) + q; + } + + private static void limitOffset(SqlExpr offset, SqlExpr fetch, Dialect.LimitRender out) { + if (fetch == null && offset != null) { + // LIMIT is required before OFFSET. This is the largest unsigned + // BIGINT, the usual stand-in for "no limit". + out.text(" LIMIT 18446744073709551615"); + } else if (fetch != null) { + out.text(" LIMIT "); + out.expr(fetch); + } + if (offset != null) { + out.text(" OFFSET "); + out.expr(offset); + } + } + + private static final class PostgresDialect implements Dialect { + @Override + public String identifier(String identifier) { + return quote(identifier, '"'); + } + + @Override + public String realTypeName() { + return "DOUBLE PRECISION"; + } + } + + private static final class MySqlDialect implements Dialect { + @Override + public String identifier(String identifier) { + return quote(identifier, '`'); + } + + @Override + public boolean explicitNullOrdering() { + return false; + } + + @Override + public Dialect.SearchPosition searchPosition() { + return Dialect.SearchPosition.LOCATE; + } + + @Override + public void renderLimit(SqlExpr offset, SqlExpr fetch, LimitRender out) { + limitOffset(offset, fetch, out); + } + } + + private static final class SqliteDialect implements Dialect { + @Override + public String identifier(String identifier) { + return quote(identifier, '"'); + } + + @Override + public String realTypeName() { + return "REAL"; + } + + @Override + public String functionName(String ansiName) { + switch (ansiName) { + case "CHAR_LENGTH": + return "LENGTH"; + case "SUBSTRING": + return "SUBSTR"; + case "GREATEST": + // scalar max/min; GREATEST/LEAST arrived only in later SQLite + return "MAX"; + case "LEAST": + return "MIN"; + default: + return ansiName; Review Comment: The SQLite dialect leaves `FLOOR` and `CEILING` unchanged in the default branch, even though these functions are not available on all SQLite builds (and the user guide explicitly says they are unsupported for SQLite). A query using `floor()` or `ceil()` with `dialect: 'sqlite'` therefore compiles successfully but can fail at execution instead of being rejected or translated to a supported form; make this dialect reject those mappings or provide an emulation. ########## subprojects/groovy-ginq/src/main/groovy/org/apache/groovy/ginq/GinqGroovyMethods.groovy: ########## @@ -111,27 +112,80 @@ class GinqGroovyMethods { GinqAstBuilder ginqAstBuilder = new GinqAstBuilder(sourceUnit) AbstractGinqExpression ginqExpression = ginqAstBuilder.buildAST(code) Map<String, String> configuration = createConfiguration(sourceUnit, ginqConfigurationMapExpression) + String astWalkerClassName = resolveAstWalkerClassName(sourceUnit, configuration, ginqConfigurationMapExpression) - if (ginqExpression instanceof GinqExpression && TRUE_STR == configuration.get(CONF_OPTIMIZE, TRUE_STR)) { + // the optimizer rewrites queries assuming collection semantics, so only run it for the default provider + if (DEFAULT_AST_WALKER_CLASS_NAME == astWalkerClassName + && ginqExpression instanceof GinqExpression && TRUE_STR == configuration.get(CONF_OPTIMIZE, TRUE_STR)) { GinqAstOptimizer ginqAstOptimizer = new GinqAstOptimizer() ginqAstOptimizer.visitGinqExpression(ginqExpression) } - Class<?> clazz = GinqGroovyMethods.class.classLoader.loadClass(configuration.get(CONF_AST_WALKER, DEFAULT_AST_WALKER_CLASS_NAME)) + Class<?> clazz = GinqGroovyMethods.class.classLoader.loadClass(astWalkerClassName) GinqAstVisitor ginqAstWalker = (GinqAstVisitor) clazz.getDeclaredConstructor(SourceUnit.class).newInstance(sourceUnit) ginqAstWalker.setConfiguration(configuration) + ginqAstWalker.setConfigurationExpressions(createConfigurationExpressions(ginqConfigurationMapExpression)) return (Expression) ginqAstWalker.visit(ginqExpression) } + private static String resolveAstWalkerClassName(SourceUnit sourceUnit, Map<String, String> configuration, MapExpression ginqConfigurationMapExpression) { + String provider = configuration.get(CONF_PROVIDER) + if (!provider || CONF_PROVIDER_COLLECTION == provider) { + return configuration.get(CONF_AST_WALKER, DEFAULT_AST_WALKER_CLASS_NAME) + } + + // `@GQ` always supplies the default astWalker, so only a non-default value conflicts + String astWalker = configuration.get(CONF_AST_WALKER) + if (astWalker && DEFAULT_AST_WALKER_CLASS_NAME != astWalker) { + collectErrors("Options `${CONF_PROVIDER}` and `${CONF_AST_WALKER}` are mutually exclusive", ginqConfigurationMapExpression, sourceUnit) + } + + Map<String, String> providers = [:] + for (GinqProvider ginqProvider : ServiceLoader.load(GinqProvider, GinqGroovyMethods.class.classLoader)) { + providers.put(ginqProvider.name, ginqProvider.astWalkerClassName) + } + + String astWalkerClassName + if (CONF_PROVIDER_AUTO_SQL == provider) { + astWalkerClassName = PREFERRED_SQL_PROVIDERS.findResult { providers.get(it) } Review Comment: The `sql` auto-provider always selects `jooq-sql` whenever `groovy-ginq-sql` is present because the service descriptor registers `JooqSqlGinqProvider` even when the optional jOOQ classes are absent. Thus `provider: 'sql'` does not fall back to `native-sql` on a classpath without jOOQ, contrary to the documented auto-detection behavior, and can fail while loading/rendering the jOOQ walker. Check jOOQ availability before preferring this provider, or make the selection conditional. ########## subprojects/groovy-ginq-sql/src/main/groovy/org/apache/groovy/ginq/provider/sql/GinqToSqlTranslator.groovy: ########## @@ -0,0 +1,1204 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.groovy.ginq.provider.sql + +import groovy.transform.CompileStatic +import groovy.transform.PackageScope +import org.apache.groovy.ginq.dsl.GinqAstBuilder +import org.apache.groovy.ginq.dsl.GinqSyntaxError +import org.apache.groovy.ginq.dsl.SyntaxErrorReportable +import org.apache.groovy.ginq.dsl.expression.AbstractGinqExpression +import org.apache.groovy.ginq.dsl.expression.DataSourceExpression +import org.apache.groovy.ginq.dsl.expression.FromExpression +import org.apache.groovy.ginq.dsl.expression.GinqExpression +import org.apache.groovy.ginq.dsl.expression.GroupExpression +import org.apache.groovy.ginq.dsl.expression.JoinExpression +import org.apache.groovy.ginq.dsl.expression.OnExpression +import org.apache.groovy.ginq.dsl.expression.SelectExpression +import org.apache.groovy.ginq.dsl.expression.SetOperationExpression +import org.apache.groovy.ginq.provider.sql.ir.SqlBinary +import org.apache.groovy.ginq.provider.sql.ir.SqlCaseWhen +import org.apache.groovy.ginq.provider.sql.ir.SqlCast +import org.apache.groovy.ginq.provider.sql.ir.SqlColumn +import org.apache.groovy.ginq.provider.sql.ir.SqlCountStar +import org.apache.groovy.ginq.provider.sql.ir.SqlDerivedTable +import org.apache.groovy.ginq.provider.sql.ir.SqlExists +import org.apache.groovy.ginq.provider.sql.ir.SqlExpr +import org.apache.groovy.ginq.provider.sql.ir.SqlFunction +import org.apache.groovy.ginq.provider.sql.ir.SqlIn +import org.apache.groovy.ginq.provider.sql.ir.SqlInQuery +import org.apache.groovy.ginq.provider.sql.ir.SqlIsNull +import org.apache.groovy.ginq.provider.sql.ir.SqlJoin +import org.apache.groovy.ginq.provider.sql.ir.SqlLike +import org.apache.groovy.ginq.provider.sql.ir.SqlLiteral +import org.apache.groovy.ginq.provider.sql.ir.SqlOrderSpec +import org.apache.groovy.ginq.provider.sql.ir.SqlOrdinal +import org.apache.groovy.ginq.provider.sql.ir.SqlParam +import org.apache.groovy.ginq.provider.sql.ir.SqlProjection +import org.apache.groovy.ginq.provider.sql.ir.SqlQuery +import org.apache.groovy.ginq.provider.sql.ir.SqlQueryNode +import org.apache.groovy.ginq.provider.sql.ir.SqlScalarQuery +import org.apache.groovy.ginq.provider.sql.ir.SqlSetQuery +import org.apache.groovy.ginq.provider.sql.ir.SqlStar +import org.apache.groovy.ginq.provider.sql.ir.SqlTableRef +import org.apache.groovy.ginq.provider.sql.ir.SqlTableSource +import org.apache.groovy.ginq.provider.sql.ir.SqlUnary +import org.apache.groovy.ginq.provider.sql.runtime.SqlGinqRuntime +import org.apache.groovy.lang.annotation.Incubating +import org.codehaus.groovy.ast.ASTNode +import org.codehaus.groovy.ast.ClassHelper +import org.codehaus.groovy.ast.ClassNode +import org.codehaus.groovy.ast.expr.ArgumentListExpression +import org.codehaus.groovy.ast.expr.BinaryExpression +import org.codehaus.groovy.ast.expr.BooleanExpression +import org.codehaus.groovy.ast.expr.CastExpression +import org.codehaus.groovy.ast.expr.ClassExpression +import org.codehaus.groovy.ast.expr.ClosureExpression +import org.codehaus.groovy.ast.expr.ConstantExpression +import org.codehaus.groovy.ast.expr.ElvisOperatorExpression +import org.codehaus.groovy.ast.expr.Expression +import org.codehaus.groovy.ast.expr.ListExpression +import org.codehaus.groovy.ast.expr.MethodCallExpression +import org.codehaus.groovy.ast.expr.NotExpression +import org.codehaus.groovy.ast.expr.PropertyExpression +import org.codehaus.groovy.ast.expr.RangeExpression +import org.codehaus.groovy.ast.expr.TernaryExpression +import org.codehaus.groovy.ast.expr.TupleExpression +import org.codehaus.groovy.ast.expr.UnaryMinusExpression +import org.codehaus.groovy.ast.expr.VariableExpression +import org.codehaus.groovy.ast.stmt.BlockStatement +import org.codehaus.groovy.ast.stmt.ExpressionStatement +import org.codehaus.groovy.ast.stmt.ReturnStatement +import org.codehaus.groovy.ast.stmt.Statement +import org.codehaus.groovy.control.SourceUnit +import org.codehaus.groovy.syntax.Types + +import static org.codehaus.groovy.ast.tools.GeneralUtils.callX +import static org.codehaus.groovy.ast.tools.GeneralUtils.classX +import static org.codehaus.groovy.ast.tools.GeneralUtils.constX +import static org.codehaus.groovy.ast.tools.GeneralUtils.eqX +import static org.codehaus.groovy.ast.tools.GeneralUtils.minusX +import static org.codehaus.groovy.ast.tools.GeneralUtils.plusX +import static org.codehaus.groovy.ast.tools.GeneralUtils.propX +import static org.codehaus.groovy.ast.tools.GeneralUtils.varX + +/** + * Translates the GINQ AST into the SQL IR. In strict mode every construct + * either translates cleanly or is rejected with a compile-time error. With + * {@link #recoverUntranslatable} set, constructs that have no SQL equivalent + * raise {@link Untranslatable} so mixed mode can move that clause in memory. + * + * @since 6.0.0 + */ +@Incubating +@CompileStatic +class GinqToSqlTranslator implements SyntaxErrorReportable { + private final SourceUnit sourceUnit + private final GinqToSqlTranslator outerScope + private final String providerName + private final List<String> tableAliases = [] + private String groupAlias + private final Map<String, SqlExpr> groupKeysByName = [:] + private String aggregateLambdaParam + private boolean recoverUntranslatable + private final SqlFunctionSite calls = new CallSite() + + GinqToSqlTranslator(SourceUnit sourceUnit, String providerName = 'native-sql') { + this.sourceUnit = sourceUnit + this.outerScope = null + this.providerName = providerName + } + + private GinqToSqlTranslator(SourceUnit sourceUnit, GinqToSqlTranslator outerScope) { + this.sourceUnit = sourceUnit + this.outerScope = outerScope + this.providerName = outerScope.providerName + this.recoverUntranslatable = outerScope.recoverUntranslatable + } + + private GinqToSqlTranslator nest(boolean correlated) { + GinqToSqlTranslator child = correlated + ? new GinqToSqlTranslator(sourceUnit, this) + : new GinqToSqlTranslator(sourceUnit, providerName) + child.recoverUntranslatable = recoverUntranslatable + return child + } + + @Override + SourceUnit getSourceUnit() { + return sourceUnit + } + + SqlQueryNode translate(AbstractGinqExpression expression) { + if (expression instanceof SetOperationExpression) { + return translateSetOperation(expression, true) + } + if (expression instanceof GinqExpression) { + return translateQuery(expression, QueryContext.TOP_LEVEL) + } + reject("`${expression.text}` is not supported by the ${providerName} provider", expression) + return null + } + + private SqlSetQuery translateSetOperation(SetOperationExpression setOperationExpression, boolean outermost) { + AbstractGinqExpression leftExpression = setOperationExpression.left + SqlQueryNode left = leftExpression instanceof SetOperationExpression + ? translateSetOperation(leftExpression, false) + : nest(false).translateQuery(requireGinqExpression(leftExpression), QueryContext.INNER_OPERAND) + SqlQuery right = nest(false).translateQuery(setOperationExpression.right, + outermost ? QueryContext.FINAL_OPERAND : QueryContext.INNER_OPERAND) + + SqlSetQuery.SetOp op + switch (setOperationExpression.operation) { + case 'union': op = SqlSetQuery.SetOp.UNION; break + case 'unionall': op = SqlSetQuery.SetOp.UNION_ALL; break + case 'intersect': op = SqlSetQuery.SetOp.INTERSECT; break + case 'minus': op = SqlSetQuery.SetOp.MINUS; break + default: + error("Unknown set operation: ${setOperationExpression.operation}", setOperationExpression) + return null + } + SqlSetQuery setQuery = new SqlSetQuery(left, op, right) + if (outermost) { + hoistOrderAndLimit(setQuery, setOperationExpression.right) + } + return setQuery + } + + /** + * Moves the final operand's {@code orderby}/{@code limit} onto the set operation + * result, matching SQL's trailing {@code ORDER BY} semantics. Order keys must match + * a select-list item and are rendered as ordinals since alias-qualified references + * are not visible after a set operation. + */ + private void hoistOrderAndLimit(SqlSetQuery setQuery, GinqExpression finalOperand) { + SqlQuery right = setQuery.right + if (right.orderBy) { + for (SqlOrderSpec orderSpec : right.orderBy) { + int index = right.projections.findIndexOf { SqlProjection projection -> sameExpr(projection.expr, orderSpec.expr) } + if (index < 0) { + error('`orderby` on a set operation result must reference columns of the final `select` clause', + finalOperand.orderExpression) + } + setQuery.orderBy << new SqlOrderSpec(new SqlOrdinal(index + 1), orderSpec.asc, orderSpec.nullsLast) + } + right.orderBy.clear() + } + setQuery.offset = right.offset + setQuery.fetch = right.fetch + right.offset = null + right.fetch = null + } + + private static boolean sameExpr(SqlExpr a, SqlExpr b) { + if (a.class != b.class) return false + if (a instanceof SqlColumn) { + SqlColumn other = (SqlColumn) b + return a.tableAlias == other.tableAlias && a.column == other.column + } + if (a instanceof SqlCountStar) return true + if (a instanceof SqlStar) return a.tableAlias == ((SqlStar) b).tableAlias + if (a instanceof SqlFunction) { + SqlFunction other = (SqlFunction) b + return a.name == other.name && a.args.size() == other.args.size() && + (0..<a.args.size()).every { int i -> sameExpr(a.args[i], other.args[i]) } + } + if (a instanceof SqlBinary) { + SqlBinary other = (SqlBinary) b + return a.op == other.op && sameExpr(a.left, other.left) && sameExpr(a.right, other.right) + } + if (a instanceof SqlUnary) { + SqlUnary other = (SqlUnary) b + return a.op == other.op && sameExpr(a.operand, other.operand) + } + if (a instanceof SqlIsNull) { + SqlIsNull other = (SqlIsNull) b + return a.negated == other.negated && sameExpr(a.expr, other.expr) + } + if (a instanceof SqlCast) { + SqlCast other = (SqlCast) b + return a.sqlType == other.sqlType && sameExpr(a.expr, other.expr) + } + return false // parameters and parameter-bearing expressions are never comparable + } + + private GinqExpression requireGinqExpression(AbstractGinqExpression expression) { + if (!(expression instanceof GinqExpression)) { + error("`${expression.text}` is not supported by the ${providerName} provider", expression) + } + return (GinqExpression) expression + } + + private SqlQuery translateQuery(GinqExpression ginqExpression, QueryContext context) { + SqlQuery query = new SqlQuery() + + query.from = tableSource(ginqExpression.fromExpression) + tableAliases << ginqExpression.fromExpression.aliasExpr.text + + for (JoinExpression joinExpression : ginqExpression.joinExpressionList) { + query.joins << translateJoin(joinExpression) + } + + if (ginqExpression.whereExpression != null) { + query.where = translateExpr(ginqExpression.whereExpression.filterExpr) + } + + GroupExpression groupExpression = ginqExpression.groupExpression + if (groupExpression != null) { + translateGroup(groupExpression, query) + } + + if (ginqExpression.orderExpression != null) { + if (QueryContext.INNER_OPERAND == context) { + error('`orderby` within set operation operands is only supported on the final operand, ' + + 'where it orders the combined result', ginqExpression.orderExpression) + } + translateOrders(ginqExpression.orderExpression.ordersExpr, query) + } + + if (ginqExpression.limitExpression != null) { + if (QueryContext.INNER_OPERAND == context) { + error('`limit` within set operation operands is only supported on the final operand, ' + + 'where it limits the combined result', ginqExpression.limitExpression) + } + translateLimit(ginqExpression.limitExpression.offsetAndSizeExpr, query) + } + + translateSelect(ginqExpression, query) + + return query + } + + private SqlTableSource tableSource(DataSourceExpression dataSourceExpression) { + Expression dataSourceExpr = dataSourceExpression.dataSourceExpr + String alias = dataSourceExpression.aliasExpr.text + if (dataSourceExpr instanceof GinqExpression) { + // no outer scope: standard SQL derived tables cannot be correlated + SqlQuery query = nest(false).translateQuery(dataSourceExpr, QueryContext.TOP_LEVEL) + return new SqlDerivedTable(query, alias) + } + if (dataSourceExpr instanceof AbstractGinqExpression) { + reject("set operations are not supported as table sources by the ${providerName} provider", dataSourceExpr) + return null + } + if (dataSourceExpr instanceof ConstantExpression && dataSourceExpr.value instanceof String) { + String tableName = (String) dataSourceExpr.value + if (!plainTableName(tableName)) { + reject("table name `${tableName}` must be a simple identifier or a schema-qualified name such as `hr.employees`. " + + 'A quoting `table()` helper is not supported yet', dataSourceExpr, false) + return null + } + return new SqlTableRef(tableName, alias) + } + reject("table references must be string literals or subqueries with the ${providerName} provider, e.g. `from ${alias} in 'employees'`", dataSourceExpr) + return null + } + + private SqlQuery correlatedSubQuery(GinqExpression ginqExpression) { + return nest(true).translateQuery(ginqExpression, QueryContext.TOP_LEVEL) + } + + private boolean isKnownAlias(String name) { + return name in tableAliases || (outerScope != null && outerScope.isKnownAlias(name)) + } + + private SqlJoin translateJoin(JoinExpression joinExpression) { + SqlJoin.Type type + switch (joinExpression.joinName) { + case 'join': + case 'innerjoin': + case 'innerhashjoin': + type = SqlJoin.Type.INNER; break + case 'leftjoin': + case 'lefthashjoin': + type = SqlJoin.Type.LEFT; break + case 'rightjoin': + case 'righthashjoin': + type = SqlJoin.Type.RIGHT; break + case 'fulljoin': + case 'fullhashjoin': + type = SqlJoin.Type.FULL; break + case 'crossjoin': + type = SqlJoin.Type.CROSS; break + default: + error("Unknown join: ${joinExpression.joinName}", joinExpression) + return null + } + + SqlTableSource table = tableSource(joinExpression) + tableAliases << joinExpression.aliasExpr.text + + SqlExpr on = null + if (joinExpression.onExpression != null) { + on = translateExpr(joinExpression.onExpression.filterExpr) + } else if (SqlJoin.Type.CROSS != type) { + error("`on` clause is expected for `${joinExpression.joinName}`", joinExpression) + } + return new SqlJoin(type, table, on) + } + + private void translateGroup(GroupExpression groupExpression, SqlQuery query) { + groupAlias = groupExpression.intoAlias // null for classic (no `into`) groupby + + for (Expression classifier : ((TupleExpression) groupExpression.classifierExpr).expressions) { + String name + Expression valueExpr + if (classifier instanceof CastExpression) { + name = classifier.type.text + valueExpr = classifier.expression + } else if (classifier instanceof PropertyExpression && classifier.property instanceof ConstantExpression) { + name = classifier.propertyAsString + valueExpr = classifier + } else { + name = classifier.text + valueExpr = classifier + } + SqlExpr keyExpr = translateExpr(valueExpr) + query.groupBy << keyExpr + groupKeysByName[name] = keyExpr + } + + if (groupExpression.havingExpression != null) { + query.having = translateExpr(groupExpression.havingExpression.filterExpr) + } + } + + private void translateOrders(Expression ordersExpr, SqlQuery query) { + for (Expression orderExpr : ((TupleExpression) ordersExpr).expressions) { + Expression target = orderExpr + boolean asc = true + boolean nullsLast = true + if (orderExpr instanceof BinaryExpression && orderExpr.operation.type == Types.KEYWORD_IN) { + target = orderExpr.leftExpression + Expression orderOptionExpr = orderExpr.rightExpression + String orderOption = orderOptionExpr instanceof MethodCallExpression + ? orderOptionExpr.methodAsString + : orderOptionExpr.text + if (orderOption !in ORDER_OPTION_LIST) { + error("`${orderOption}` is not an order direction; use `asc` or `desc`", orderOptionExpr) + } + asc = 'asc' == orderOption + if (orderOptionExpr instanceof MethodCallExpression) { + List<Expression> optionArgs = ((TupleExpression) orderOptionExpr.arguments).expressions + String nullsOption = 1 == optionArgs.size() ? optionArgs[0].text : null + if (nullsOption !in NULLS_OPTION_LIST) { + error('`asc`/`desc` takes `nullsfirst` or `nullslast`', orderOptionExpr.arguments) + } + nullsLast = 'nullslast' == nullsOption + } + } + query.orderBy << new SqlOrderSpec(translateExpr(target), asc, nullsLast) + } + } + + private void translateLimit(Expression offsetAndSizeExpr, SqlQuery query) { + List<Expression> expressions = offsetAndSizeExpr instanceof TupleExpression + ? offsetAndSizeExpr.expressions + : [offsetAndSizeExpr] + if (1 == expressions.size()) { + query.fetch = new SqlParam(expressions[0]) + } else { + query.offset = new SqlParam(expressions[0]) + query.fetch = new SqlParam(expressions[1]) + } + } + + private void translateSelect(GinqExpression ginqExpression, SqlQuery query) { + SelectExpression selectExpression = ginqExpression.selectExpression + Boolean distinct = ginqExpression.getNodeMetaData(GinqAstBuilder.GINQ_SELECT_DISTINCT) + query.distinct = Boolean.TRUE == distinct + + for (Expression projection : ((TupleExpression) selectExpression.projectionExpr).expressions) { + String alias = null + Expression expr = projection + if (projection instanceof CastExpression) { + alias = projection.type.text + expr = projection.expression + } + if (expr instanceof VariableExpression && expr.text in tableAliases) { + if (alias != null) { + error("whole-row projection `${expr.text}` cannot be aliased", projection) + } + query.projections << new SqlProjection(new SqlStar(expr.text)) + } else { + if (alias == null) { + alias = groupKeyProjectionAlias(expr) + } + query.projections << new SqlProjection(translateExpr(expr), alias) + } + } + } + + // a projected group key keeps its GINQ name as the column label, since the + // underlying expression's own label, e.g. `e.deptId` for `groupby e.deptId as dept`, + // may differ from the name the query author expects + private String groupKeyProjectionAlias(Expression expr) { + if (expr instanceof VariableExpression && groupKeysByName.containsKey(expr.text)) { + return expr.text + } + if (expr instanceof PropertyExpression && groupAlias != null + && expr.objectExpression instanceof VariableExpression + && groupAlias == expr.objectExpression.text + && expr.property instanceof ConstantExpression) { + return expr.propertyAsString + } + return null + } + + private SqlExpr translateExpr(Expression expr) { + if (expr instanceof GinqExpression) { + return new SqlScalarQuery(correlatedSubQuery(expr)) + } + if (expr instanceof AbstractGinqExpression) { + reject("`${expr.text}` cannot be used as an expression with the ${providerName} provider", expr) + return null + } + // NotExpression extends BooleanExpression; the negation has to be kept + if (expr instanceof NotExpression) return new SqlUnary('NOT', translateExpr(expr.expression)) + if (expr instanceof BooleanExpression) return translateExpr(expr.expression) + if (expr instanceof TernaryExpression) return translateTernary(expr) + if (expr instanceof BinaryExpression) return translateBinary(expr) + if (expr instanceof UnaryMinusExpression) return new SqlUnary('-', translateExpr(expr.expression)) + if (expr instanceof PropertyExpression) return translateProperty(expr) + if (expr instanceof MethodCallExpression) return translateMethodCall(expr) + if (expr instanceof ConstantExpression) return new SqlParam(expr) + if (expr instanceof VariableExpression) return translateVariable(expr) + reject("Expression `${expr.text}` cannot be translated to SQL by the ${providerName} provider", expr) + return null + } + + private SqlExpr translateTernary(TernaryExpression expr) { + if (expr instanceof ElvisOperatorExpression) { + // truthiness is not SQL null, so ?: is not COALESCE + reject("the Elvis operator cannot be translated to SQL by the ${providerName} provider; " + + 'Groovy truthiness is not SQL null. Use `coalesce(...)` or an explicit ternary with a `== null` check', expr) + return null + } + return new SqlCaseWhen( + translateExpr(expr.booleanExpression), + translateCaseBranch(expr.trueExpression), + translateCaseBranch(expr.falseExpression)) + } + + // CASE branch values that are compile-time constants are inlined as literals + // (never runtime values, so parameter safety is preserved) since databases + // cannot infer a type for `THEN ? ELSE ?` + private SqlExpr translateCaseBranch(Expression expr) { + if (expr instanceof ConstantExpression) { + return new SqlLiteral(expr.value) + } + return translateExpr(expr) + } + + private SqlExpr translateBinary(BinaryExpression expr) { + String op = expr.operation.text + Expression left = expr.leftExpression + Expression right = expr.rightExpression + switch (op) { + case 'in': + case '!in': + if (right instanceof GinqExpression) { + return new SqlInQuery(translateExpr(left), correlatedSubQuery(right), '!in' == op) + } + if (right instanceof RangeExpression) { + return translateRangeMember(left, (RangeExpression) right, '!in' == op, expr) + } + if (!(right instanceof ListExpression)) { + reject("only list literals, constant ranges and subqueries are supported on the right-hand side of `in` by the ${providerName} provider", right) + return null + } + List<Expression> elements = ((ListExpression) right).expressions + if (elements.isEmpty()) { + // `x in []` is false and `x !in []` is true, including when x is null + return new SqlBinary('=', new SqlLiteral(1), new SqlLiteral('!in' == op ? 1 : 0)) + } + List<SqlExpr> values = elements.collect { Expression e -> translateExpr(e) } Review Comment: When a list literal contains `null`, emitting an ordinary `IN`/`NOT IN` list is not faithful to collection semantics: `x in [null]` should match a null `x`, but SQL evaluates `NULL IN (NULL)` as unknown, and `x !in [1, null]` becomes unknown for every row because SQL `NOT IN` propagates the null. Split null members into `IS NULL`/`IS NOT NULL` logic (and keep the non-null list) or reject this case instead of generating these predicates. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
