jdaugherty commented on code in PR #16631: URL: https://github.com/apache/grails-core/pull/16631#discussion_r4393424575
########## grails-data-mongodb/core/src/main/groovy/org/apache/grails/data/mongodb/core/querytail/MongoCommandRedactor.groovy: ########## @@ -0,0 +1,511 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.grails.data.mongodb.core.querytail + +import groovy.transform.CompileStatic +import org.bson.BsonDocument +import org.bson.BsonValue + +import org.apache.grails.datamapping.core.querytail.QueryTailRedactor + +/** + * Takes the values out of a MongoDB command before it is captured. A command is a document, so + * there is nothing to parse: every value becomes {@link QueryTailRedactor#PLACEHOLDER}, and the + * keys, which are the options of the command, its operators and the field names of the collection, + * are kept, except where a key may be data. The keys of a document stored in a map property, or in + * a property or a collection the {@linkplain MongoMappedFields mapping} does not know, are recorded + * as the placeholder too, and so is every segment of a dotted path past such a key. So is a key at + * the top of stored data, the documents an insert writes, a replacement document, the fields an + * update writes and the operand of a literal, unless the mapping knows it, as the name of a dynamic + * attribute may be data; the keys at the top of a condition, a pipeline, a sort or a projection + * are the fields the application asks for, and are kept. An update that is a pipeline writes the + * documents its stages build: the keys at the top of what a {@code $set}, {@code $addFields} or + * {@code $project} stage holds, of the new root of a {@code $replaceWith} or {@code $replaceRoot}, and of a + * document {@code $mergeObjects} merges into it are fields written, kept only when the mapping knows + * them, as those of a classic update are. A numeric key + * is kept only where it indexes an array the mapping knows; in a map, in an element of an array, + * which is a map or a document unless the field is a list of lists, or at the top of a document, + * it is data. A {@code $} key is kept as an operator only where operators go, in a condition, a + * pipeline and the operators of an update; in stored data, the documents an insert writes, a + * replacement document and the values an update operator writes, it is a field name, which MongoDB + * allows to start with {@code $}, and is recorded as the placeholder unless the mapping knows it. + * So is a {@code $} key in the operand of a comparison in a condition, such as {@code $eq} or + * {@code $in}, or an implicit document or array equality, which holds the value the field is compared + * with, and in a DBRef, whose {@code $ref}, {@code $id} and {@code $db} a condition reads as fields + * compared with their whole value, in any order; in an aggregation expression, which + * {@code $expr}, the {@code let} of a command and the stages that compute fields hold, the operands + * of a comparison are expressions, and their operators are kept, except in the operand of a + * {@code $literal} or {@code $const}, which is stored data. A comment, and the constants of an + * update, may be any document, and none of their keys are kept. The entries the driver adds for its + * session, the cluster time and the transaction, which say nothing about the command, are left out. + * + * <p>A command is recorded as its name, the database and collection it runs against, and the + * document of its remaining entries: {@code find shop.book {"filter": {"title": ?}, "limit": ?}}.</p> + * + * @since 9.0 + */ +@CompileStatic +final class MongoCommandRedactor { + + private static final String PLACEHOLDER = QueryTailRedactor.PLACEHOLDER + + private static final String DATABASE = '$db' + + private static final String COLLECTION = 'collection' + + /** The entries the driver adds, which describe its session and transaction rather than the command. */ + private static final Set<String> LEFT_OUT = ['lsid', '$clusterTime', 'txnNumber', 'autocommit', + 'startTransaction', '$readPreference', 'apiVersion', 'apiStrict', 'apiDeprecationErrors'] as Set<String> + + /** + * The options whose document, or documents, are keyed by the fields of the collection, wherever they appear, + * including those of the operations of a client-level {@code bulkWrite}. + */ + private static final Set<String> FIELD_KEYED = ['filter', 'query', 'q', 'u', 'update', 'document', 'documents', + 'updateMods', 'pipeline', 'sort', 'projection', 'fields', 'key', 'hint', 'min', 'max', 'arrayFilters', + 'partialFilterExpression', 'wildcardProjection', 'validator', 'weights'] as Set<String> + + /** Documents that match fields, whose field values may be implicit equality operands. */ + private static final Set<String> CONDITION_KEYED = ['filter', 'query', 'q', 'arrayFilters', + 'partialFilterExpression', 'validator', 'restrictSearchWithMatch'] as Set<String> + + /** The options whose documents are stored data: what an insert writes. */ + private static final Set<String> DATA_KEYED = ['documents', 'document'] as Set<String> + + /** The options and the operator that may hold any document, none of whose keys say what the command does. */ + private static final Set<String> ANY_VALUE = ['comment', '$comment', 'c'] as Set<String> + + /** The option whose values are aggregation expressions. */ + private static final String LET = 'let' + + /** The options that hold an update: update operators, a replacement document or a pipeline. */ + private static final Set<String> UPDATE_KEYED = ['u', 'update', 'updateMods'] as Set<String> + + /** The update operators whose fields take a condition rather than the value to write. */ + private static final Set<String> CONDITION_OPERATORS = ['$pull', '$currentDate'] as Set<String> + + /** The update operators that add to an array, whose value may hold modifiers. */ + private static final Set<String> PUSH_OPERATORS = ['$push', '$addToSet'] as Set<String> + + private static final String EACH = '$each' + + /** The modifiers of a {@code $push} or an {@code $addToSet}. */ + private static final Set<String> PUSH_MODIFIERS = [EACH, '$slice', '$sort', '$position'] as Set<String> + + /** + * The operators of a pipeline whose operand is literal data rather than an expression, including + * {@code $const}, which MongoDB accepts for {@code $literal}. + */ + private static final Set<String> LITERAL_OPERATORS = ['$literal', '$const', '$documents'] as Set<String> + + /** + * The fields of a DBRef, which a condition reads as fields compared with their whole value, never as + * operators, in any order and wherever a condition has fields. + */ + private static final Set<String> DBREF_FIELDS = ['$ref', '$id', '$db'] as Set<String> + + /** + * The query operators whose operand holds values rather than a condition: the comparisons, whose operand + * is the value a field is compared with, a JSON schema, whose {@code enum} lists values, and a + * GeoJSON geometry, which can carry additional data alongside its type and coordinates. + */ + private static final Set<String> VALUE_OPERATORS = ['$eq', '$ne', '$gt', '$gte', '$lt', '$lte', '$in', '$nin', + '$all', '$jsonSchema', '$geometry'] as Set<String> + + /** The operator of a condition, and the stages of a pipeline, whose operand is an aggregation expression. */ + private static final Set<String> EXPRESSION_OPERATORS = ['$expr', '$addFields', '$set', '$project', '$group', + '$replaceRoot', '$replaceWith', '$redact', '$bucket', '$bucketAuto', '$sortByCount', '$setWindowFields', + '$fill'] as Set<String> + + /** The stages of an update pipeline whose operand is the document they write, or the new root that replaces it. */ + private static final Set<String> WRITING_STAGES = ['$set', '$addFields', '$project', '$replaceWith'] as Set<String> + + /** The stage of an update pipeline whose options name the new root that replaces the document. */ + private static final String REPLACE_ROOT = '$replaceRoot' + + /** The option of {@code $replaceRoot} that holds the new root. */ + private static final String NEW_ROOT = 'newRoot' + + /** The operator whose operands are documents merged into one, which in an update pipeline is written. */ + private static final String MERGE_OBJECTS = '$mergeObjects' + + /** The kinds of document whose keys at the top are stored data: what an insert, a replacement or an update writes. */ + private static final Set<Kind> STORED = EnumSet.of(Kind.DATA, Kind.UPDATE, Kind.UPDATE_FIELDS, Kind.WRITTEN) + + private MongoCommandRedactor() { + } + + /** + * Redacts a command. + * + * @param database the database the command runs against + * @param commandName the name of the command, which is its first key + * @param command the command as the driver sends it + * @param fields the fields the mapping of the datastore knows, or null before the datastore is built + * @return the command with its values replaced + */ + static String redact(String database, String commandName, BsonDocument command, MongoMappedFields fields) { + String collection = null + String collectionKey = null + BsonValue first = command.get(commandName) + if (first != null && first.isString()) { Review Comment: Fixed in fe3a01a69b. A collection is now taken only from a command that runs against one. That means an allowlist of the commands whose first value is their collection: `find`, `insert`, `update`, `delete`, `aggregate`, `count`, `distinct`, `findAndModify`, `mapReduce`, the collection and index commands, `killCursors`, and the search-index and plan-cache commands. The `collection` option is used only for `getMore`. The first value of any other command is recorded as `?` like every other value, so your command is recorded as `dropUser admin {"dropUser": ?}`. A command that is not on the list, including one MongoDB adds later, is redacted rather than named. Tests: - `MongoCommandRedactorSpec`: "only a command that runs against a collection names one..." covers `dropUser`, `usersInfo`, `grantRolesToUser`, `renameCollection` (whose full namespace is redacted too), `enableSharding` and an unknown command with a `collection` option, each asserting the secret is absent. The `findandmodify`, `distinct`, `listIndexes`, `create` and `drop` rows still name their collection. - `QueryTailCommandListenerSpec`: creates `[email protected]` on the test MongoDB and drops it with `dropUser`. It asserts the user is gone, that `dropUser` and `usersInfo` are recorded with `?`, and that neither the account name nor its password is in the file. ########## grails-data-mongodb/core/src/main/groovy/org/apache/grails/data/mongodb/core/querytail/MongoCommandRedactor.groovy: ########## @@ -0,0 +1,511 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.grails.data.mongodb.core.querytail + +import groovy.transform.CompileStatic +import org.bson.BsonDocument +import org.bson.BsonValue + +import org.apache.grails.datamapping.core.querytail.QueryTailRedactor + +/** + * Takes the values out of a MongoDB command before it is captured. A command is a document, so + * there is nothing to parse: every value becomes {@link QueryTailRedactor#PLACEHOLDER}, and the + * keys, which are the options of the command, its operators and the field names of the collection, + * are kept, except where a key may be data. The keys of a document stored in a map property, or in + * a property or a collection the {@linkplain MongoMappedFields mapping} does not know, are recorded + * as the placeholder too, and so is every segment of a dotted path past such a key. So is a key at + * the top of stored data, the documents an insert writes, a replacement document, the fields an + * update writes and the operand of a literal, unless the mapping knows it, as the name of a dynamic + * attribute may be data; the keys at the top of a condition, a pipeline, a sort or a projection + * are the fields the application asks for, and are kept. An update that is a pipeline writes the + * documents its stages build: the keys at the top of what a {@code $set}, {@code $addFields} or + * {@code $project} stage holds, of the new root of a {@code $replaceWith} or {@code $replaceRoot}, and of a + * document {@code $mergeObjects} merges into it are fields written, kept only when the mapping knows + * them, as those of a classic update are. A numeric key + * is kept only where it indexes an array the mapping knows; in a map, in an element of an array, + * which is a map or a document unless the field is a list of lists, or at the top of a document, + * it is data. A {@code $} key is kept as an operator only where operators go, in a condition, a + * pipeline and the operators of an update; in stored data, the documents an insert writes, a + * replacement document and the values an update operator writes, it is a field name, which MongoDB + * allows to start with {@code $}, and is recorded as the placeholder unless the mapping knows it. + * So is a {@code $} key in the operand of a comparison in a condition, such as {@code $eq} or + * {@code $in}, or an implicit document or array equality, which holds the value the field is compared + * with, and in a DBRef, whose {@code $ref}, {@code $id} and {@code $db} a condition reads as fields + * compared with their whole value, in any order; in an aggregation expression, which + * {@code $expr}, the {@code let} of a command and the stages that compute fields hold, the operands + * of a comparison are expressions, and their operators are kept, except in the operand of a + * {@code $literal} or {@code $const}, which is stored data. A comment, and the constants of an + * update, may be any document, and none of their keys are kept. The entries the driver adds for its + * session, the cluster time and the transaction, which say nothing about the command, are left out. + * + * <p>A command is recorded as its name, the database and collection it runs against, and the + * document of its remaining entries: {@code find shop.book {"filter": {"title": ?}, "limit": ?}}.</p> + * + * @since 9.0 + */ +@CompileStatic +final class MongoCommandRedactor { + + private static final String PLACEHOLDER = QueryTailRedactor.PLACEHOLDER + + private static final String DATABASE = '$db' + + private static final String COLLECTION = 'collection' + + /** The entries the driver adds, which describe its session and transaction rather than the command. */ + private static final Set<String> LEFT_OUT = ['lsid', '$clusterTime', 'txnNumber', 'autocommit', + 'startTransaction', '$readPreference', 'apiVersion', 'apiStrict', 'apiDeprecationErrors'] as Set<String> + + /** + * The options whose document, or documents, are keyed by the fields of the collection, wherever they appear, + * including those of the operations of a client-level {@code bulkWrite}. + */ + private static final Set<String> FIELD_KEYED = ['filter', 'query', 'q', 'u', 'update', 'document', 'documents', + 'updateMods', 'pipeline', 'sort', 'projection', 'fields', 'key', 'hint', 'min', 'max', 'arrayFilters', + 'partialFilterExpression', 'wildcardProjection', 'validator', 'weights'] as Set<String> + + /** Documents that match fields, whose field values may be implicit equality operands. */ + private static final Set<String> CONDITION_KEYED = ['filter', 'query', 'q', 'arrayFilters', + 'partialFilterExpression', 'validator', 'restrictSearchWithMatch'] as Set<String> + + /** The options whose documents are stored data: what an insert writes. */ + private static final Set<String> DATA_KEYED = ['documents', 'document'] as Set<String> + + /** The options and the operator that may hold any document, none of whose keys say what the command does. */ + private static final Set<String> ANY_VALUE = ['comment', '$comment', 'c'] as Set<String> + + /** The option whose values are aggregation expressions. */ + private static final String LET = 'let' + + /** The options that hold an update: update operators, a replacement document or a pipeline. */ + private static final Set<String> UPDATE_KEYED = ['u', 'update', 'updateMods'] as Set<String> + + /** The update operators whose fields take a condition rather than the value to write. */ + private static final Set<String> CONDITION_OPERATORS = ['$pull', '$currentDate'] as Set<String> + + /** The update operators that add to an array, whose value may hold modifiers. */ + private static final Set<String> PUSH_OPERATORS = ['$push', '$addToSet'] as Set<String> + + private static final String EACH = '$each' + + /** The modifiers of a {@code $push} or an {@code $addToSet}. */ + private static final Set<String> PUSH_MODIFIERS = [EACH, '$slice', '$sort', '$position'] as Set<String> + + /** + * The operators of a pipeline whose operand is literal data rather than an expression, including + * {@code $const}, which MongoDB accepts for {@code $literal}. + */ + private static final Set<String> LITERAL_OPERATORS = ['$literal', '$const', '$documents'] as Set<String> + + /** + * The fields of a DBRef, which a condition reads as fields compared with their whole value, never as + * operators, in any order and wherever a condition has fields. + */ + private static final Set<String> DBREF_FIELDS = ['$ref', '$id', '$db'] as Set<String> + + /** + * The query operators whose operand holds values rather than a condition: the comparisons, whose operand + * is the value a field is compared with, a JSON schema, whose {@code enum} lists values, and a + * GeoJSON geometry, which can carry additional data alongside its type and coordinates. + */ + private static final Set<String> VALUE_OPERATORS = ['$eq', '$ne', '$gt', '$gte', '$lt', '$lte', '$in', '$nin', + '$all', '$jsonSchema', '$geometry'] as Set<String> + + /** The operator of a condition, and the stages of a pipeline, whose operand is an aggregation expression. */ + private static final Set<String> EXPRESSION_OPERATORS = ['$expr', '$addFields', '$set', '$project', '$group', + '$replaceRoot', '$replaceWith', '$redact', '$bucket', '$bucketAuto', '$sortByCount', '$setWindowFields', + '$fill'] as Set<String> + + /** The stages of an update pipeline whose operand is the document they write, or the new root that replaces it. */ + private static final Set<String> WRITING_STAGES = ['$set', '$addFields', '$project', '$replaceWith'] as Set<String> + + /** The stage of an update pipeline whose options name the new root that replaces the document. */ + private static final String REPLACE_ROOT = '$replaceRoot' + + /** The option of {@code $replaceRoot} that holds the new root. */ + private static final String NEW_ROOT = 'newRoot' + + /** The operator whose operands are documents merged into one, which in an update pipeline is written. */ + private static final String MERGE_OBJECTS = '$mergeObjects' + + /** The kinds of document whose keys at the top are stored data: what an insert, a replacement or an update writes. */ + private static final Set<Kind> STORED = EnumSet.of(Kind.DATA, Kind.UPDATE, Kind.UPDATE_FIELDS, Kind.WRITTEN) + + private MongoCommandRedactor() { + } + + /** + * Redacts a command. + * + * @param database the database the command runs against + * @param commandName the name of the command, which is its first key + * @param command the command as the driver sends it + * @param fields the fields the mapping of the datastore knows, or null before the datastore is built + * @return the command with its values replaced + */ + static String redact(String database, String commandName, BsonDocument command, MongoMappedFields fields) { + String collection = null + String collectionKey = null + BsonValue first = command.get(commandName) + if (first != null && first.isString()) { + collection = first.asString().value + collectionKey = commandName + } + else if (command.get(COLLECTION)?.isString()) { + collection = command.getString(COLLECTION).value + collectionKey = COLLECTION + } + StringBuilder out = new StringBuilder(commandName) + if (database != null || collection != null) { + out.append(' ') + if (database != null) { + out.append(database) + } + if (collection != null) { + out.append(database != null ? '.' : '').append(collection) + } + } + Scope options = Scope.options(collection, fields) + StringBuilder body = new StringBuilder() + for (Map.Entry<String, BsonValue> entry in command.entrySet()) { + String key = entry.key + if (key == DATABASE || key == collectionKey || LEFT_OUT.contains(key)) { + continue + } + body.append(body.length() == 0 ? '{' : ', ') + appendEntry(body, key, entry.value, options) + } + if (body.length() > 0) { + out.append(' ').append(body).append('}') + } + out.toString() + } + + private static void appendEntry(StringBuilder out, String key, BsonValue value, Scope scope) { + Scope child = scope.of(key, value) + out.append(child.keyText).append(': ') + appendValue(out, value, child) + } + + private static void appendValue(StringBuilder out, BsonValue value, Scope scope) { + if (value.isDocument()) { + out.append('{') + boolean first = true + for (Map.Entry<String, BsonValue> entry in value.asDocument().entrySet()) { + if (!first) { + out.append(', ') + } + first = false + appendEntry(out, entry.key, entry.value, scope) + } + out.append('}') + } + else if (value.isArray()) { + out.append('[') + boolean first = true + Scope element = scope.element() + for (BsonValue item in value.asArray()) { + if (!first) { + out.append(', ') + } + first = false + appendValue(out, item, element) + } + out.append(']') + } + else { + out.append(PLACEHOLDER) + } + } + + private static String quote(String key) { + StringBuilder out = new StringBuilder(key.length() + 2).append('"') + for (int i = 0; i < key.length(); i++) { + char c = key.charAt(i) + if (c == '"' || c == '\\') { + out.append('\\').append(c) + } + else if (c < 0x20) { + out.append(String.format('\\u%04x', (int) c)) + } + else { + out.append(c) + } + } + out.append('"').toString() + } + + /** + * Where a document sits in the command, which decides what its keys are: the options of the + * command, kept; the fields and operators of a condition, a pipeline, a sort or a projection, where + * a {@code $} key is an operator; an aggregation expression, whose {@code $} keys are operators + * too; an update, whose {@code $} keys are update operators; or stored data, the documents an + * insert writes, the values an update writes and the values a condition compares with, where a + * {@code $} key is a field name like any other and is kept only when the mapping knows it. + */ + @CompileStatic + private static final class Scope { + + final String collection + final MongoMappedFields fields + final Kind kind + /** + * The update operator whose fields a document of kind {@link Kind#UPDATE_FIELDS} holds, or the stage + * whose document one of kind {@link Kind#WRITTEN} is. + */ + final String operator + /** The field path of the document; null in a document keyed by fields the mapping does not know. */ + final List<String> path + /** How many indices into the field at the path have been taken: 0 at the field, 1 in one of its elements. */ + final int depth + final String keyText + + private Scope(String collection, MongoMappedFields fields, Kind kind, String operator, List<String> path, + int depth, String keyText) { + this.collection = collection + this.fields = fields + this.kind = kind + this.operator = operator + this.path = path + this.depth = depth + this.keyText = keyText + } + + static Scope options(String collection, MongoMappedFields fields) { + new Scope(collection, fields, Kind.OPTIONS, null, Collections.<String>emptyList(), 0, null) + } + + /** + * @return the scope of an element of an array in this scope: one index deeper into the field + * at the path, so that a numeric key inside the element is data unless the elements are arrays; + * the elements of an update that is an array are the stages of a pipeline, and an element of an + * expression at the top of a document, as the operands of an operator {@code $expr} holds are, is a + * value the expression builds, whose keys are kept only when the mapping knows them, as those of a + * document a stage of an update pipeline writes are + */ + Scope element() { + if (kind == Kind.UPDATE) { + return child(Kind.UPDATE_PIPELINE, null, path, depth, keyText) + } + if (kind == Kind.PUSH) { + return child(Kind.DATA, null, path, depth + 2, keyText) + } + if ((kind == Kind.QUERY || kind == Kind.CONDITION || kind == Kind.EXPRESSION || kind == Kind.DATA) && path != null && !path.isEmpty()) { + return child(kind, null, path, depth + 1, keyText) + } + if (kind == Kind.EXPRESSION && path != null) { + return child(Kind.WRITTEN, null, path, depth, keyText) + } + this + } + + /** + * @return the scope of the value of a key in a document of this scope, with the key as it is recorded + */ + Scope of(String key, BsonValue value) { + if (kind == Kind.OPTIONS) { + if (ANY_VALUE.contains(key)) { + return child(Kind.DATA, null, null, 0, quote(key)) + } + Kind valueKind = DATA_KEYED.contains(key) ? Kind.DATA : + UPDATE_KEYED.contains(key) ? Kind.UPDATE : + CONDITION_KEYED.contains(key) ? Kind.CONDITION : + FIELD_KEYED.contains(key) ? Kind.QUERY : + key == LET ? Kind.EXPRESSION : Kind.OPTIONS + return child(valueKind, null, Collections.<String>emptyList(), 0, quote(key)) Review Comment: Fixed in fe3a01a69b. A map-reduce `scope` is now treated like a comment: none of its keys are kept, at any depth, inside documents and arrays, including keys that match a mapped field or start with `$`. Your example is recorded as `"scope": {?: {?: ?}}`. Tests: - `MongoCommandRedactorSpec`: "map-reduce scope keys are data even when they match mapped fields or operators" asserts `"scope": {?: {?: ?}, ?: {?: ?}, ?: [{?: {?: ?}}]}`, both with the mapping bound and before it binds. - `QueryTailCommandListenerSpec`: runs a real `mapReduce` whose map function reads the scope. It checks the result, so the database still received the scope as written, and asserts that `example-secret`, `9876543210` and the queried title are absent from the file. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
