github-advanced-security[bot] commented on code in PR #6758: URL: https://github.com/apache/hive/pull/6758#discussion_r3945207669
########## itests/hive-jmh/src/main/java/org/apache/hive/benchmark/vectorization/parquet/VectorizedParquetReadBench.java: ########## @@ -0,0 +1,388 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hive.benchmark.vectorization.parquet; + +import java.io.File; +import java.io.IOException; +import java.lang.reflect.Field; +import java.lang.reflect.Method; +import java.nio.file.Files; +import java.util.concurrent.TimeUnit; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.ql.exec.Utilities; +import org.apache.hadoop.hive.ql.exec.vector.BytesColumnVector; +import org.apache.hadoop.hive.ql.exec.vector.DoubleColumnVector; +import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector; +import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; +import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatchCtx; +import org.apache.hadoop.hive.ql.io.IOConstants; +import org.apache.hadoop.hive.ql.io.parquet.vector.VectorizedColumnReader; +import org.apache.hadoop.hive.ql.io.parquet.vector.VectorizedParquetRecordReader; +import org.apache.hadoop.hive.ql.io.parquet.vector.probe.ParquetProbeFilter; +import org.apache.hadoop.hive.ql.plan.MapWork; +import org.apache.hadoop.hive.serde2.ColumnProjectionUtils; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; +import org.apache.hadoop.mapred.JobConf; +import org.apache.hadoop.mapreduce.Job; +import org.apache.hadoop.mapreduce.lib.input.FileInputFormat; +import org.apache.parquet.example.data.Group; +import org.apache.parquet.example.data.simple.SimpleGroupFactory; +import org.apache.parquet.hadoop.ParquetInputFormat; +import org.apache.parquet.hadoop.ParquetWriter; +import org.apache.parquet.hadoop.example.GroupWriteSupport; +import org.apache.parquet.hadoop.metadata.CompressionCodecName; +import org.apache.parquet.io.api.Binary; +import org.apache.parquet.schema.MessageType; +import org.apache.parquet.schema.MessageTypeParser; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Level; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Param; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.TearDown; +import org.openjdk.jmh.annotations.Warmup; +import org.openjdk.jmh.infra.Blackhole; +import org.openjdk.jmh.runner.Runner; +import org.openjdk.jmh.runner.options.Options; +import org.openjdk.jmh.runner.options.OptionsBuilder; + +/** + * JMH benchmark that compares the pre-patch decode path (a bare 3-arg + * {@code readBatch(total, col, type)} call — what {@code VectorizedParquetRecordReader.nextBatch} + * would issue before HIVE-30019) against the post-patch filter-aware path + * ({@code readBatch(total, col, type, ParquetProbeFilter)} with a supplied bitmap) across a + * sweep of filter selectivities. + * + * <p>A single {@code @Benchmark} method ({@link #readBatch}) drives every projected column + * through {@link #BATCHES_PER_INVOCATION} batches per invocation; the {@link #filter} @Param + * chooses which shape to call: + * <ul> + * <li><b>filter=none</b> — baseline, unfiltered decode via the 3-arg readBatch. Represents + * the reader's behaviour before HIVE-30019: every row materialises regardless of whether + * a downstream operator will drop it.</li> + * <li><b>filter=P</b> (integer pass-percentage) — filter-aware readBatch with a clumpy bitmap + * that accepts the first {@code P} rows of every 100-row block and rejects the rest. On + * dict-encoded pages the coalesced skip fast-path + * ({@code readDictionaryIDs} → {@code DictionaryValuesReader.skip(int)} + * → {@code RunLengthBitPackingHybridDecoder.skipInts}) drops each reject run in one call; + * on PLAIN pages each filtered row still costs a per-row {@code dataColumn.skip()} but + * never a value-materialise / null-set / setVal copy.</li> + * </ul> + * + * <p>Two encodings are covered: + * <ul> + * <li><b>dict</b> — small distinct-value set, so pages use dictionary encoding.</li> + * <li><b>plain</b> — every value distinct, so pages fall back to PLAIN.</li> + * </ul> + * + * <p>Note on worst case: the current sweep uses a <i>clumpy</i> shape ({@code (i % 100) < P}), + * which matches realistic ProbeDecode join-key hit distributions and is favourable to both the + * dict bulk-skip and to branch prediction on PLAIN. An alternating half-filter + * ({@code i % 2 == 0}) is the branch-predictor worst case but is not the shape ProbeDecode + * produces in practice. + * + * <p>Run: {@code + * java -jar itests/hive-jmh/target/benchmarks.jar + * org.apache.hive.benchmark.vectorization.parquet.VectorizedParquetReadBench + * -wi 5 -i 15 -f 2 -bm avgt -tu us + * } + */ +@BenchmarkMode(Mode.AverageTime) +@OutputTimeUnit(TimeUnit.MICROSECONDS) +@Warmup(iterations = 5, time = 1, timeUnit = TimeUnit.SECONDS) +@Measurement(iterations = 10, time = 1, timeUnit = TimeUnit.SECONDS) +@Fork(1) +@State(Scope.Benchmark) +public class VectorizedParquetReadBench { + + // Enough rows in a single row group that the fixed per-invocation cost -- reader open, split + // init, checkEndOfRowGroup, JobConf lookup -- is amortised across many readBatch calls. With + // 131072 rows (128 batches of 1024) written into one row group, each invocation drives ~128 + // filter-aware calls per column vs one, so the isFilteredOut / skip fast-path signal is no + // longer diluted by the ~5 ms setup floor. + private static final int N_ROWS = 131072; + private static final int BATCHES_PER_INVOCATION = N_ROWS / VectorizedRowBatch.DEFAULT_SIZE; + private static final MessageType WRITE_SCHEMA = MessageTypeParser.parseMessageType( + "message pd_read { " + + "required int32 int_col; " + + "required int64 long_col; " + + "required double dbl_col; " + + "required binary str_col (UTF8); " + + "}"); + + @Param({"dict", "plain"}) + String encoding; + + /** + * Selectivity sweep. {@code none} = call the 3-arg readBatch (baseline / pre-patch shape); + * every other value is an integer pass-percentage in {@code [0, 100]} used to build a bitmap + * where the first {@code p} rows of every 100-row block accept and the remaining + * {@code 100 - p} reject. Clumpy, not alternating: matches real-world join-key hit distributions + * and lets the dict {@code readDictionaryIDs} bulk-skip coalesce the reject runs. {@code 50} + * with a clumpy shape is not the same worst-case as an alternating half-filter (see the + * "worst-case" note in the class javadoc). + */ + @Param({"none", "10", "50", "90"}) + String filter; + + /** + * Toggles {@code hive.optimize.scan.probedecode.parquet.plain.filter.enabled}. When + * {@code off}, the PLAIN-path {@code isFilteredOutPlain} check is constant-folded away by the + * JIT and every row on a PLAIN page is materialised; the dictionary path is unaffected. + * Included as a bench param so the JIT-elimination claim can be verified end-to-end (i.e. the + * {@code plain × <any filter> × off} cells should match {@code plain × none} within noise). + * + * <p>Ignored when {@code filter=none}: the 3-arg readBatch path never calls the check + * regardless of the config, so a {@code none × off} run is not informative and only widens + * the sweep matrix. + */ + @Param({"on", "off"}) + String plainFilter; + + private File dataDir; + private Path dataFile; + private JobConf jobConf; + + private LongColumnVector intVec; + private LongColumnVector longVec; + private DoubleColumnVector dblVec; + private BytesColumnVector strVec; + + private TypeInfo intType; + private TypeInfo longType; + private TypeInfo dblType; + private TypeInfo strType; + + /** + * Filter for this trial, built from the {@link #filter} @Param. {@code null} when + * {@code filter=none}, in which case the bench calls the 3-arg readBatch (pre-patch shape). + */ + private ParquetProbeFilter probeFilter; + + @Setup(Level.Trial) Review Comment: ## SonarCloud / Temporary files should not be created in publicly writable directories <!--SONAR_ISSUE_KEY:AaBxMK3N50elM9qDY5Wq-->Make sure publicly writable directories are used safely here. <p>See more on <a href="https://sonarcloud.io/project/issues?id=apache_hive&issues=AaBxMK3N50elM9qDY5Wq&open=AaBxMK3N50elM9qDY5Wq&pullRequest=6758";>SonarQube Cloud</a></p> [Show more details](https://github.com/apache/hive/security/code-scanning/457) -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
