jdaugherty commented on code in PR #16631: URL: https://github.com/apache/grails-core/pull/16631#discussion_r4371058328
########## grails-data-mongodb/core/src/main/groovy/org/apache/grails/data/mongo/core/querytail/MongoCommandRedactor.groovy: ########## @@ -0,0 +1,363 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.grails.data.mongo.core.querytail + +import groovy.transform.CompileStatic +import org.bson.BsonDocument +import org.bson.BsonValue + +import org.apache.grails.data.core.querytail.QueryTailRedactor + +/** + * Takes the values out of a MongoDB command before it is captured. A command is a document, so + * there is nothing to parse: every value becomes {@link QueryTailRedactor#PLACEHOLDER}, and the + * keys, which are the options of the command, its operators and the field names of the collection, + * are kept, except where a key may be data. The keys of a document stored in a map property, or in + * a property or a collection the {@linkplain MongoMappedFields mapping} does not know, are recorded + * as the placeholder too, and so is every segment of a dotted path past such a key. A numeric key + * is kept only where it indexes an array the mapping knows; in a map, in an element of an array, + * which is a map or a document unless the field is a list of lists, or at the top of a document, + * it is data. A {@code $} key is kept as an operator only where operators go, in a condition, a + * pipeline and the operators of an update; in stored data, the documents an insert writes, a + * replacement document and the values an update operator writes, it is a field name, which MongoDB + * allows to start with {@code $}, and is recorded as the placeholder unless the mapping knows it. + * The entries the driver adds for its session, the cluster time and the transaction, which say + * nothing about the command, are left out. + * + * <p>A command is recorded as its name, the database and collection it runs against, and the + * document of its remaining entries: {@code find shop.book {"filter": {"title": ?}, "limit": ?}}.</p> + * + * @since 8.1 + */ +@CompileStatic +final class MongoCommandRedactor { + + private static final String PLACEHOLDER = QueryTailRedactor.PLACEHOLDER + + private static final String DATABASE = '$db' + + private static final String COLLECTION = 'collection' + + /** The entries the driver adds, which describe its session and transaction rather than the command. */ + private static final Set<String> LEFT_OUT = ['lsid', '$clusterTime', 'txnNumber', 'autocommit', + 'startTransaction', '$readPreference', 'apiVersion', 'apiStrict', 'apiDeprecationErrors'] as Set<String> + + /** + * The options whose document, or documents, are keyed by the fields of the collection, wherever they appear, + * including those of the operations of a client-level {@code bulkWrite}. + */ + private static final Set<String> FIELD_KEYED = ['filter', 'query', 'q', 'u', 'update', 'document', 'documents', + 'updateMods', 'pipeline', 'sort', 'projection', 'fields', 'key', 'hint', 'min', 'max', 'arrayFilters', + 'partialFilterExpression', 'wildcardProjection'] as Set<String> + + /** The options whose documents are stored data: what an insert writes. */ + private static final Set<String> DATA_KEYED = ['documents', 'document'] as Set<String> + + /** The options that hold an update: update operators, a replacement document or a pipeline. */ + private static final Set<String> UPDATE_KEYED = ['u', 'update', 'updateMods'] as Set<String> + + /** The update operators whose fields take a condition rather than the value to write. */ + private static final Set<String> CONDITION_OPERATORS = ['$pull', '$currentDate'] as Set<String> + + /** The update operators that add to an array, whose value may hold modifiers. */ + private static final Set<String> PUSH_OPERATORS = ['$push', '$addToSet'] as Set<String> + + private static final String EACH = '$each' + + /** The modifiers of a {@code $push} or an {@code $addToSet}. */ + private static final Set<String> PUSH_MODIFIERS = [EACH, '$slice', '$sort', '$position'] as Set<String> + + /** The operators of a pipeline whose operand is literal data rather than an expression. */ + private static final Set<String> LITERAL_OPERATORS = ['$literal', '$documents'] as Set<String> + + private MongoCommandRedactor() { + } + + /** + * Redacts a command. + * + * @param database the database the command runs against + * @param commandName the name of the command, which is its first key + * @param command the command as the driver sends it + * @param fields the fields the mapping of the datastore knows, or null before the datastore is built + * @return the command with its values replaced + */ + static String redact(String database, String commandName, BsonDocument command, MongoMappedFields fields) { + String collection = null + String collectionKey = null + BsonValue first = command.get(commandName) + if (first != null && first.isString()) { + collection = first.asString().value + collectionKey = commandName + } + else if (command.get(COLLECTION)?.isString()) { + collection = command.getString(COLLECTION).value + collectionKey = COLLECTION + } + StringBuilder out = new StringBuilder(commandName) + if (database != null || collection != null) { + out.append(' ') + if (database != null) { + out.append(database) + } + if (collection != null) { + out.append(database != null ? '.' : '').append(collection) + } + } + Scope options = Scope.options(collection, fields) + StringBuilder body = new StringBuilder() + for (Map.Entry<String, BsonValue> entry in command.entrySet()) { + String key = entry.key + if (key == DATABASE || key == collectionKey || LEFT_OUT.contains(key)) { + continue + } + body.append(body.length() == 0 ? '{' : ', ') + appendEntry(body, key, entry.value, options) + } + if (body.length() > 0) { + out.append(' ').append(body).append('}') + } + out.toString() + } + + private static void appendEntry(StringBuilder out, String key, BsonValue value, Scope scope) { + Scope child = scope.of(key) + out.append(child.keyText).append(': ') + appendValue(out, value, child) + } + + private static void appendValue(StringBuilder out, BsonValue value, Scope scope) { + if (value.isDocument()) { + out.append('{') + boolean first = true + for (Map.Entry<String, BsonValue> entry in value.asDocument().entrySet()) { + if (!first) { + out.append(', ') + } + first = false + appendEntry(out, entry.key, entry.value, scope) + } + out.append('}') + } + else if (value.isArray()) { + out.append('[') + boolean first = true + Scope element = scope.element() + for (BsonValue item in value.asArray()) { + if (!first) { + out.append(', ') + } + first = false + appendValue(out, item, element) + } + out.append(']') + } + else { + out.append(PLACEHOLDER) + } + } + + private static String quote(String key) { + StringBuilder out = new StringBuilder(key.length() + 2).append('"') + for (int i = 0; i < key.length(); i++) { + char c = key.charAt(i) + if (c == '"' || c == '\\') { + out.append('\\').append(c) + } + else if (c < 0x20) { + out.append(String.format('\\u%04x', (int) c)) + } + else { + out.append(c) + } + } + out.append('"').toString() + } + + /** + * Where a document sits in the command, which decides what its keys are: the options of the + * command, kept; the fields and operators of a condition, a pipeline, a sort or a projection, where + * a {@code $} key is an operator; an update, whose {@code $} keys are update operators; or stored + * data, the documents an insert writes and the values an update writes, where a {@code $} key is a + * field name like any other and is kept only when the mapping knows it. + */ + @CompileStatic + private static final class Scope { + + final String collection + final MongoMappedFields fields + final Kind kind + /** The update operator whose fields a document of kind {@link Kind#UPDATE_FIELDS} holds. */ + final String operator + /** The field path of the document; null in a document keyed by fields the mapping does not know. */ + final List<String> path + /** How many indices into the field at the path have been taken: 0 at the field, 1 in one of its elements. */ + final int depth + final String keyText + + private Scope(String collection, MongoMappedFields fields, Kind kind, String operator, List<String> path, + int depth, String keyText) { + this.collection = collection + this.fields = fields + this.kind = kind + this.operator = operator + this.path = path + this.depth = depth + this.keyText = keyText + } + + static Scope options(String collection, MongoMappedFields fields) { + new Scope(collection, fields, Kind.OPTIONS, null, Collections.<String>emptyList(), 0, null) + } + + /** + * @return the scope of an element of an array in this scope: one index deeper into the field + * at the path, so that a numeric key inside the element is data unless the elements are arrays; + * the elements of an update that is an array are the stages of a pipeline + */ + Scope element() { + if (kind == Kind.UPDATE) { + return child(Kind.QUERY, null, path, depth, keyText) + } + if (kind == Kind.PUSH) { + return child(Kind.DATA, null, path, depth + 2, keyText) + } + if ((kind == Kind.QUERY || kind == Kind.DATA) && path != null && !path.isEmpty()) { + return child(kind, null, path, depth + 1, keyText) + } + this + } + + /** + * @return the scope of the value of a key in a document of this scope, with the key as it is recorded + */ + Scope of(String key) { + if (kind == Kind.OPTIONS) { + Kind valueKind = DATA_KEYED.contains(key) ? Kind.DATA : + UPDATE_KEYED.contains(key) ? Kind.UPDATE : + FIELD_KEYED.contains(key) ? Kind.QUERY : Kind.OPTIONS + return child(valueKind, null, Collections.<String>emptyList(), 0, quote(key)) + } + if (kind == Kind.QUERY && key.startsWith('$')) { Review Comment: Fixed in f69625477f. In a condition, the operand of a comparison (`$eq`, `$ne`, `$gt`, `$gte`, `$lt`, `$lte`, `$in`, `$nin`, `$all`) is now read as data, so `{"attrs":{"$eq":{"$example-secret":"value"}}}` records `{"attrs": {"$eq": {?: ?}}}`. The same holds under `$not`, `$elemMatch`, `$or`, `$pull`, `arrayFilters` and a `$match` stage. In an aggregation expression (`$expr`, the `let` of a command, and stages such as `$project`, `$group` and `$set`), a `$` key can only be an operator, so `{"$expr": {"$eq": [{"$toLower": "$title"}, "x"]}}` keeps `$toLower`; `$literal` stays data there. `QueryTailCommandListenerSpec` now runs your filter against a real mongod, along with that `$expr` aggregation. It asserts that MongoDB returns the stored document both ways, that the capture keeps `$eq`, `$expr` and `$toLower`, and that `example-secret` does not reach the file. While tracing the scopes I found more places where a key that may be data was kept, and fixed them in the same change. Each has a redactor row: - the keys of a comment document (`comment`, `$comment`) and of the constants of an update (`c`) - a `$jsonSchema`, whose `enum` lists values - the `validator` and text-index `weights` documents, whose keys were kept as option names - a `$literal` in the `let` of a command Where it is not clear whether an operand is an expression, such as a find projection, the `let` of a `$lookup`, or `$all: [{"$elemMatch": ...}]`, a nested operator is now recorded as `?`. That withholds more, never less. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
