jdaugherty commented on code in PR #16631: URL: https://github.com/apache/grails-core/pull/16631#discussion_r4396907451
########## grails-data-hibernate7/core/src/main/groovy/org/apache/grails/data/hibernate7/core/querytail/QueryTailConnectionProvider.groovy: ########## @@ -0,0 +1,127 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.grails.data.hibernate7.core.querytail + +import java.lang.reflect.InvocationHandler +import java.lang.reflect.InvocationTargetException +import java.lang.reflect.Method +import java.lang.reflect.Proxy +import java.sql.Connection + +import groovy.transform.CompileStatic +import org.hibernate.boot.registry.StandardServiceInitiator +import org.hibernate.boot.registry.StandardServiceRegistryBuilder +import org.hibernate.engine.jdbc.connections.internal.ConnectionProviderInitiator +import org.hibernate.engine.jdbc.connections.internal.MultiTenantConnectionProviderInitiator +import org.hibernate.service.Service +import org.hibernate.service.spi.Configurable +import org.hibernate.service.spi.ServiceBinding +import org.hibernate.service.spi.ServiceRegistryAwareService +import org.hibernate.service.spi.ServiceRegistryImplementor +import org.hibernate.service.spi.Startable +import org.hibernate.service.spi.Stoppable + +import org.apache.grails.datamapping.core.querytail.QueryTailJdbcConnection +import org.apache.grails.datamapping.core.querytail.QueryTailRecorder + +/** + * Decorates the provider Hibernate selects, preserving its configuration and service lifecycle. + * Connections are captured at execution and returned to their provider as the original instances. + * + * @since 9.0 + */ +@CompileStatic +final class QueryTailConnectionProvider<S extends Service> implements StandardServiceInitiator<S> { + + private final StandardServiceInitiator<S> delegate + private final QueryTailRecorder recorder + + private QueryTailConnectionProvider(StandardServiceInitiator<S> delegate, QueryTailRecorder recorder) { + this.delegate = delegate + this.recorder = recorder + } + + static void install(StandardServiceRegistryBuilder builder, QueryTailRecorder recorder) { Review Comment: Fixed in 8976ddec98, for Hibernate 5 and 7. The capture no longer adds initiators to the service registry the session factory is built with. It decorates the connection providers in the session factory's own service registry instead, through a `SessionFactoryServiceContributor`, which is where sessions take their connections from. Whatever provider the main registry has is kept and wrapped: Hibernate's default, one from a custom initiator, or one registered with `addService`. That registry still configures, starts and stops it. `QueryTailSessionFactoryObserverSpec` now covers a configuration class that adds its own initiator: its provider hands out the connections, and their statements are recorded. The spec also checks that a configured provider is stopped exactly once. ########## grails-data-mongodb/core/src/main/groovy/org/apache/grails/data/mongodb/core/querytail/MongoCommandRedactor.groovy: ########## @@ -0,0 +1,548 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.grails.data.mongodb.core.querytail + +import groovy.transform.CompileStatic +import org.bson.BsonDocument +import org.bson.BsonValue + +import org.apache.grails.datamapping.core.querytail.QueryTailRedactor + +/** + * Takes the values out of a MongoDB command before it is captured. A command is a document, so + * there is nothing to parse: every value becomes {@link QueryTailRedactor#PLACEHOLDER}, and the + * keys, which are the options of the command, its operators and the field names of the collection, + * are kept, except where a key may be data. The keys of a document stored in a map property, or in + * a property or a collection the {@linkplain MongoMappedFields mapping} does not know, are recorded + * as the placeholder too, and so is every segment of a dotted path past such a key. So is a key at + * the top of stored data, the documents an insert writes, a replacement document, the fields an + * update writes and the operand of a literal, unless the mapping knows it, as the name of a dynamic + * attribute may be data; the keys at the top of a condition, a pipeline, a sort or a projection + * are the fields the application asks for, and are kept. An update that is a pipeline writes the + * documents its stages build: the keys at the top of what a {@code $set}, {@code $addFields} or + * {@code $project} stage holds, of the new root of a {@code $replaceWith} or {@code $replaceRoot}, and of a + * document {@code $mergeObjects} merges into it are fields written, kept only when the mapping knows + * them, as those of a classic update are. A numeric key + * is kept only where it indexes an array the mapping knows; in a map, in an element of an array, + * which is a map or a document unless the field is a list of lists, or at the top of a document, + * it is data. A {@code $} key is kept as an operator only where operators go, in a condition, a + * pipeline and the operators of an update; in stored data, the documents an insert writes, a + * replacement document and the values an update operator writes, it is a field name, which MongoDB + * allows to start with {@code $}, and is recorded as the placeholder unless the mapping knows it. + * So is a {@code $} key in the operand of a comparison in a condition, such as {@code $eq} or + * {@code $in}, or an implicit document or array equality, which holds the value the field is compared + * with, and in a DBRef, whose {@code $ref}, {@code $id} and {@code $db} a condition reads as fields + * compared with their whole value, in any order; in an aggregation expression, which + * {@code $expr}, the {@code let} of a command and the stages that compute fields hold, the operands + * of a comparison are expressions, and their operators are kept, except in the operand of a + * {@code $literal} or {@code $const}, which is stored data. A comment, and the constants of an + * update, and the scope of a map-reduce command, may be any document, and none of their keys are + * kept. The entries the driver adds for its + * session, the cluster time and the transaction, which say nothing about the command, are left out. + * + * <p>An array is recorded with each of its elements once: an element recorded as one before it in + * the array is left out, and the array then ends with {@code ...}, so that the documents of a batch + * and the values of an {@code $in} are recorded the same whatever their number, and a command is + * counted as the one statement it is: {@code {"$in": [?, ...]}}.</p> + * + * <p>A command is recorded as its name, the database and collection it runs against, and the + * document of its remaining entries: {@code find shop.book {"filter": {"title": ?}, "limit": ?}}. + * The collection is taken only from a command that runs against one; the first value of any other + * command is an argument, such as a user name, and is recorded as a value: + * {@code dropUser admin {"dropUser": ?}}.</p> + * + * @since 9.0 + */ +@CompileStatic +final class MongoCommandRedactor { + + private static final String PLACEHOLDER = QueryTailRedactor.PLACEHOLDER + + /** Ends an array that held an element recorded as one before it, which is left out. */ + private static final String REPEATED = '...' + + private static final String DATABASE = '$db' + + private static final String COLLECTION = 'collection' + + /** + * The commands whose first value is the collection they run against. The first value of any other + * command, such as the user {@code dropUser} drops, is an argument, and is recorded as a value. + */ + private static final Set<String> COLLECTION_COMMANDS = ['find', 'insert', 'update', 'delete', 'aggregate', + 'count', 'distinct', 'findAndModify', 'findandmodify', 'mapReduce', 'mapreduce', 'killCursors', + 'create', 'drop', 'collMod', 'createIndexes', 'dropIndexes', 'deleteIndexes', 'listIndexes', 'reIndex', + 'compact', 'validate', 'collStats', 'convertToCapped', 'cloneCollectionAsCapped', 'analyze', + 'checkMetadataConsistency', 'createSearchIndexes', 'dropSearchIndex', 'updateSearchIndex', + 'planCacheClear', 'planCacheClearFilters', 'planCacheListFilters', 'planCacheSetFilter'] as Set<String> + + /** The command that names its collection in its {@code collection} option, as it continues a cursor. */ + private static final String GET_MORE = 'getMore' + + /** The entries the driver adds, which describe its session and transaction rather than the command. */ + private static final Set<String> LEFT_OUT = ['lsid', '$clusterTime', 'txnNumber', 'autocommit', + 'startTransaction', '$readPreference', 'apiVersion', 'apiStrict', 'apiDeprecationErrors'] as Set<String> + + /** + * The options whose document, or documents, are keyed by the fields of the collection, wherever they appear, + * including those of the operations of a client-level {@code bulkWrite}. + */ + private static final Set<String> FIELD_KEYED = ['filter', 'query', 'q', 'u', 'update', 'document', 'documents', + 'updateMods', 'pipeline', 'sort', 'projection', 'fields', 'key', 'hint', 'min', 'max', 'arrayFilters', + 'partialFilterExpression', 'wildcardProjection', 'validator', 'weights'] as Set<String> + + /** Documents that match fields, whose field values may be implicit equality operands. */ + private static final Set<String> CONDITION_KEYED = ['filter', 'query', 'q', 'arrayFilters', + 'partialFilterExpression', 'validator', 'restrictSearchWithMatch'] as Set<String> + + /** The options whose documents are stored data: what an insert writes. */ + private static final Set<String> DATA_KEYED = ['documents', 'document'] as Set<String> + + /** The options and the operator that may hold any document, none of whose keys say what the command does. */ + private static final Set<String> ANY_VALUE = ['comment', '$comment', 'c', 'scope'] as Set<String> + + /** The option whose values are aggregation expressions. */ + private static final String LET = 'let' + + /** The options that hold an update: update operators, a replacement document or a pipeline. */ + private static final Set<String> UPDATE_KEYED = ['u', 'update', 'updateMods'] as Set<String> + + /** The update operators whose fields take a condition rather than the value to write. */ + private static final Set<String> CONDITION_OPERATORS = ['$pull', '$currentDate'] as Set<String> + + /** The update operators that add to an array, whose value may hold modifiers. */ + private static final Set<String> PUSH_OPERATORS = ['$push', '$addToSet'] as Set<String> + + private static final String EACH = '$each' + + /** The modifiers of a {@code $push} or an {@code $addToSet}. */ + private static final Set<String> PUSH_MODIFIERS = [EACH, '$slice', '$sort', '$position'] as Set<String> + + /** + * The operators of a pipeline whose operand is literal data rather than an expression, including + * {@code $const}, which MongoDB accepts for {@code $literal}. + */ + private static final Set<String> LITERAL_OPERATORS = ['$literal', '$const', '$documents'] as Set<String> + + /** + * The fields of a DBRef, which a condition reads as fields compared with their whole value, never as + * operators, in any order and wherever a condition has fields. + */ + private static final Set<String> DBREF_FIELDS = ['$ref', '$id', '$db'] as Set<String> + + /** + * The query operators whose operand holds values rather than a condition: the comparisons, whose operand + * is the value a field is compared with, a JSON schema, whose {@code enum} lists values, and a + * GeoJSON geometry, which can carry additional data alongside its type and coordinates. + */ + private static final Set<String> VALUE_OPERATORS = ['$eq', '$ne', '$gt', '$gte', '$lt', '$lte', '$in', '$nin', + '$all', '$jsonSchema', '$geometry'] as Set<String> + + /** The operator of a condition, and the stages of a pipeline, whose operand is an aggregation expression. */ + private static final Set<String> EXPRESSION_OPERATORS = ['$expr', '$addFields', '$set', '$project', '$group', + '$replaceRoot', '$replaceWith', '$redact', '$bucket', '$bucketAuto', '$sortByCount', '$setWindowFields', + '$fill'] as Set<String> + + /** The stages of an update pipeline whose operand is the document they write, or the new root that replaces it. */ + private static final Set<String> WRITING_STAGES = ['$set', '$addFields', '$project', '$replaceWith'] as Set<String> + + /** The stage of an update pipeline whose options name the new root that replaces the document. */ + private static final String REPLACE_ROOT = '$replaceRoot' + + /** The option of {@code $replaceRoot} that holds the new root. */ + private static final String NEW_ROOT = 'newRoot' + + /** The operator whose operands are documents merged into one, which in an update pipeline is written. */ + private static final String MERGE_OBJECTS = '$mergeObjects' + + /** The kinds of document whose keys at the top are stored data: what an insert, a replacement or an update writes. */ + private static final Set<Kind> STORED = EnumSet.of(Kind.DATA, Kind.UPDATE, Kind.UPDATE_FIELDS, Kind.WRITTEN) + + private MongoCommandRedactor() { + } + + /** + * Redacts a command. + * + * @param database the database the command runs against + * @param commandName the name of the command, which is its first key + * @param command the command as the driver sends it + * @param fields the fields the mapping of the datastore knows, or null before the datastore is built + * @return the command with its values replaced + */ + static String redact(String database, String commandName, BsonDocument command, MongoMappedFields fields) { + String collection = null + String collectionKey = null + BsonValue first = command.get(commandName) + if (first != null && first.isString() && COLLECTION_COMMANDS.contains(commandName)) { + collection = first.asString().value + collectionKey = commandName + } + else if (commandName == GET_MORE && command.get(COLLECTION)?.isString()) { + collection = command.getString(COLLECTION).value + collectionKey = COLLECTION + } + StringBuilder out = new StringBuilder(commandName) + if (database != null || collection != null) { + out.append(' ') + if (database != null) { + out.append(database) + } + if (collection != null) { + out.append(database != null ? '.' : '').append(collection) + } + } + Scope options = Scope.options(collection, fields) + StringBuilder body = new StringBuilder() + for (Map.Entry<String, BsonValue> entry in command.entrySet()) { + String key = entry.key + if (key == DATABASE || key == collectionKey || LEFT_OUT.contains(key)) { + continue + } + body.append(body.length() == 0 ? '{' : ', ') + appendEntry(body, key, entry.value, options) + } + if (body.length() > 0) { + out.append(' ').append(body).append('}') + } + out.toString() + } + + private static void appendEntry(StringBuilder out, String key, BsonValue value, Scope scope) { + Scope child = scope.of(key, value) + out.append(child.keyText).append(': ') + appendValue(out, value, child) + } + + private static void appendValue(StringBuilder out, BsonValue value, Scope scope) { + if (value.isDocument()) { + out.append('{') + boolean first = true + for (Map.Entry<String, BsonValue> entry in value.asDocument().entrySet()) { + if (!first) { + out.append(', ') + } + first = false + appendEntry(out, entry.key, entry.value, scope) + } + out.append('}') + } + else if (value.isArray()) { + out.append('[') + Scope element = scope.element() + Set<String> written = new HashSet<>() + boolean repeated = false + for (BsonValue item in value.asArray()) { + StringBuilder text = new StringBuilder() + appendValue(text, item, element) + String recorded = text.toString() + if (written.add(recorded)) { + if (written.size() > 1) { + out.append(', ') + } + out.append(recorded) + } + else { + repeated = true + } + } + if (repeated) { + out.append(', ').append(REPEATED) + } + out.append(']') + } + else { + out.append(PLACEHOLDER) + } + } + + private static String quote(String key) { + StringBuilder out = new StringBuilder(key.length() + 2).append('"') + for (int i = 0; i < key.length(); i++) { + char c = key.charAt(i) + if (c == '"' || c == '\\') { + out.append('\\').append(c) + } + else if (c < 0x20) { + out.append(String.format('\\u%04x', (int) c)) + } + else { + out.append(c) + } + } + out.append('"').toString() + } + + /** + * Where a document sits in the command, which decides what its keys are: the options of the + * command, kept; the fields and operators of a condition, a pipeline, a sort or a projection, where + * a {@code $} key is an operator; an aggregation expression, whose {@code $} keys are operators + * too; an update, whose {@code $} keys are update operators; or stored data, the documents an + * insert writes, the values an update writes and the values a condition compares with, where a + * {@code $} key is a field name like any other and is kept only when the mapping knows it. + */ + @CompileStatic + private static final class Scope { + + final String collection + final MongoMappedFields fields + final Kind kind + /** + * The update operator whose fields a document of kind {@link Kind#UPDATE_FIELDS} holds, or the stage + * whose document one of kind {@link Kind#WRITTEN} is. + */ + final String operator + /** The field path of the document; null in a document keyed by fields the mapping does not know. */ + final List<String> path + /** How many indices into the field at the path have been taken: 0 at the field, 1 in one of its elements. */ + final int depth + final String keyText + + private Scope(String collection, MongoMappedFields fields, Kind kind, String operator, List<String> path, + int depth, String keyText) { + this.collection = collection + this.fields = fields + this.kind = kind + this.operator = operator + this.path = path + this.depth = depth + this.keyText = keyText + } + + static Scope options(String collection, MongoMappedFields fields) { + new Scope(collection, fields, Kind.OPTIONS, null, Collections.<String>emptyList(), 0, null) + } + + /** + * @return the scope of an element of an array in this scope: one index deeper into the field + * at the path, so that a numeric key inside the element is data unless the elements are arrays; + * the elements of an update that is an array are the stages of a pipeline, and an element of an + * expression at the top of a document, as the operands of an operator {@code $expr} holds are, is a + * value the expression builds, whose keys are kept only when the mapping knows them, as those of a + * document a stage of an update pipeline writes are + */ + Scope element() { + if (kind == Kind.UPDATE) { + return child(Kind.UPDATE_PIPELINE, null, path, depth, keyText) + } + if (kind == Kind.PUSH) { + return child(Kind.DATA, null, path, depth + 2, keyText) + } + if ((kind == Kind.QUERY || kind == Kind.CONDITION || kind == Kind.EXPRESSION || kind == Kind.DATA) && path != null && !path.isEmpty()) { + return child(kind, null, path, depth + 1, keyText) + } + if (kind == Kind.EXPRESSION && path != null) { + return child(Kind.WRITTEN, null, path, depth, keyText) + } + this + } + + /** + * @return the scope of the value of a key in a document of this scope, with the key as it is recorded + */ + Scope of(String key, BsonValue value) { + if (kind == Kind.OPTIONS) { + if (ANY_VALUE.contains(key)) { + return child(Kind.DATA, null, null, 0, quote(key)) + } + Kind valueKind = DATA_KEYED.contains(key) ? Kind.DATA : + UPDATE_KEYED.contains(key) ? Kind.UPDATE : + CONDITION_KEYED.contains(key) ? Kind.CONDITION : + FIELD_KEYED.contains(key) ? Kind.QUERY : + key == LET ? Kind.EXPRESSION : Kind.OPTIONS + return child(valueKind, null, Collections.<String>emptyList(), 0, quote(key)) + } + if ((kind == Kind.QUERY || kind == Kind.CONDITION) && key.startsWith('$')) { Review Comment: Fixed in 8976ddec98. In a condition, a `$` key is now kept only if it is one of MongoDB's query operators (or a DBRef field). Any other is recorded as `?`, along with everything under it. `MongoCommandRedactorSpec` covers the key directly inside `$near` and `$nearSphere`, next to field operators, at the top of a condition, inside `$elemMatch` and in a `$match` stage. It also checks that the query operators themselves, including the geospatial, `$text` and `$regex` options, are still kept. The guide says the same. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
