rdblue commented on code in PR #17413: URL: https://github.com/apache/iceberg/pull/17413#discussion_r3779605378
########## core/src/main/java/org/apache/iceberg/expressions/InclusiveStatsEvaluator.java: ########## @@ -0,0 +1,226 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.expressions; + +import static org.apache.iceberg.expressions.Expressions.rewriteNot; + +import java.util.Collections; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; +import org.apache.iceberg.ContentStats; +import org.apache.iceberg.FieldStats; +import org.apache.iceberg.Schema; +import org.apache.iceberg.TrackedFile; +import org.apache.iceberg.types.TypeUtil; +import org.apache.iceberg.types.Types; +import org.apache.iceberg.variants.Variant; +import org.apache.iceberg.variants.VariantObject; + +/** + * Evaluates an {@link Expression} on a {@link TrackedFile} to test whether rows in the file may + * match. + * + * <p>This evaluation is inclusive: it returns true if a file may match and false if it cannot + * match. + * + * <p>Files are passed to {@link #eval(TrackedFile)}, which returns true if the file may contain + * matching rows and false if the file cannot contain matching rows. Files may be skipped if and + * only if the return value of {@code eval} is false. + * + * <p>Due to the comparison implementation of ORC stats, for float/double columns in ORC files, if + * the first value in a file is NaN, metrics of this file will report NaN for both upper and lower + * bound despite that the column could contain non-NaN data. Thus, in some scenarios explicitly + * checks for NaN is necessary in order to not skip files that may contain matching data. + */ +public class InclusiveStatsEvaluator { + private final Expression expr; + private final Set<Integer> alwaysPresentFieldIds; + private final Set<Integer> requiredFieldIds; + + public InclusiveStatsEvaluator(Schema schema, Expression unbound) { + this(schema, unbound, true); + } + + public InclusiveStatsEvaluator(Schema schema, Expression unbound, boolean caseSensitive) { + Types.StructType struct = schema.asStruct(); + this.expr = Binder.bind(struct, rewriteNot(unbound), caseSensitive); + Set<Integer> referencedIds = + Binder.boundReferences(struct, Collections.singletonList(expr), caseSensitive); + this.alwaysPresentFieldIds = alwaysPresentFieldIds(schema, referencedIds); + this.requiredFieldIds = + referencedIds.stream() + .filter(fieldId -> schema.findField(fieldId).isRequired()) + .collect(Collectors.toSet()); + } + + /** + * Returns the IDs of the referenced fields that cannot contain null values. + * + * <p>A field cannot be null if it is required and every field that contains it is required. Such + * a field is never null, even when stats are missing for it. + */ + private static Set<Integer> alwaysPresentFieldIds(Schema schema, Set<Integer> fieldIds) { Review Comment: Style: Can you move private static methods to the end of the file? We usually locate them there. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
