This is an automated email from the ASF dual-hosted git repository. asf-gitbox-commits pushed a commit to branch parquet-io-cache in repository https://gitbox.apache.org/repos/asf/hive.git
commit a0924f4e975faad0a7aae5710c614bf56660870a Author: Laszlo Bodor <[email protected]> AuthorDate: Thu Sep 17 09:11:30 2026 +0200 HIVE_VECTORIZED_INPUT_FORMAT_SUPPORTS_DECIMAL_64 --- common/src/java/org/apache/hadoop/hive/conf/HiveConf.java | 9 ++++++++- .../apache/hadoop/hive/llap/io/api/impl/LlapRecordReader.java | 3 ++- .../hadoop/hive/llap/io/decode/ParquetEncodedDataConsumer.java | 3 ++- .../apache/hadoop/hive/llap/io/encoded/OrcEncodedDataReader.java | 3 ++- .../hadoop/hive/llap/io/encoded/SerDeEncodedDataReader.java | 3 ++- .../hadoop/hive/llap/io/encoded/VectorDeserializeOrcWriter.java | 3 ++- .../hive/llap/io/encoded/TestParquetEncodedDataReader.java | 3 ++- .../java/org/apache/hadoop/hive/ql/io/orc/RecordReaderImpl.java | 3 ++- .../hadoop/hive/ql/io/orc/VectorizedOrcAcidRowBatchReader.java | 6 ++++-- ql/src/java/org/apache/hadoop/hive/ql/io/orc/WriterImpl.java | 2 +- .../org/apache/hadoop/hive/ql/io/orc/TestInputOutputFormat.java | 6 ++++-- 11 files changed, 31 insertions(+), 13 deletions(-) diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index 2b5102adc97..80ae8b3b5c5 100644 --- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -261,6 +261,13 @@ private static URL checkConfigFile(File f) { public static final String HIVE_SERVER2_AUTHENTICATION_LDAP_USERMEMBERSHIPKEY_NAME = "hive.server2.authentication.ldap.userMembershipKey"; + /** + * Token that {@link ConfVars#HIVE_VECTORIZED_INPUT_FORMAT_SUPPORTS_ENABLED} must contain for + * decimal_64 vectorization to be enabled. Extracted here so call sites can reference the same + * constant instead of repeating the literal. + */ + public static final String HIVE_VECTORIZED_INPUT_FORMAT_SUPPORTS_DECIMAL_64 = "decimal_64"; + /** * dbVars are the parameters can be set per database. If these * parameters are set as a database property, when switching to that @@ -4623,7 +4630,7 @@ public static enum ConfVars { "evaluate call and turn them into NULLs. Assume, by default, this is not needed"), HIVE_VECTORIZED_INPUT_FORMAT_SUPPORTS_ENABLED( "hive.vectorized.input.format.supports.enabled", - "decimal_64", + HIVE_VECTORIZED_INPUT_FORMAT_SUPPORTS_DECIMAL_64, "Which vectorized input format support features are enabled for vectorization.\n" + "That is, if a VectorizedInputFormat input format does support \"decimal_64\" for example\n" + "this variable must enable that to be used in vectorization"), diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/io/api/impl/LlapRecordReader.java b/llap-server/src/java/org/apache/hadoop/hive/llap/io/api/impl/LlapRecordReader.java index 01fa9cd4705..af837e1c553 100644 --- a/llap-server/src/java/org/apache/hadoop/hive/llap/io/api/impl/LlapRecordReader.java +++ b/llap-server/src/java/org/apache/hadoop/hive/llap/io/api/impl/LlapRecordReader.java @@ -188,7 +188,8 @@ private LlapRecordReader(MapWork mapWork, JobConf job, FileSplit split, final boolean decimal64Support = - HiveConf.getVar(job, ConfVars.HIVE_VECTORIZED_INPUT_FORMAT_SUPPORTS_ENABLED).equalsIgnoreCase("decimal_64"); + HiveConf.getVar(job, ConfVars.HIVE_VECTORIZED_INPUT_FORMAT_SUPPORTS_ENABLED) + .equalsIgnoreCase(HiveConf.HIVE_VECTORIZED_INPUT_FORMAT_SUPPORTS_DECIMAL_64); int limit = determineQueueLimit(bestEffortSize, diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/ParquetEncodedDataConsumer.java b/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/ParquetEncodedDataConsumer.java index 1ab26aed50d..d25f665c8e7 100644 --- a/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/ParquetEncodedDataConsumer.java +++ b/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/ParquetEncodedDataConsumer.java @@ -94,7 +94,8 @@ public ParquetEncodedDataConsumer(Consumer<ColumnVectorBatch> consumer, Includes super(consumer, includes.getPhysicalColumnIds().size(), ioMetrics, counters); this.jobConf = jobConf; this.useDecimal64ColumnVectors = HiveConf.getVar(jobConf, - ConfVars.HIVE_VECTORIZED_INPUT_FORMAT_SUPPORTS_ENABLED).equalsIgnoreCase("decimal_64"); + ConfVars.HIVE_VECTORIZED_INPUT_FORMAT_SUPPORTS_ENABLED) + .equalsIgnoreCase(HiveConf.HIVE_VECTORIZED_INPUT_FORMAT_SUPPORTS_DECIMAL_64); ParquetReadOptions options = HadoopReadOptions.builder(jobConf).build(); this.codecFactory = options.getCodecFactory(); this.converter = new ParquetMetadataConverter(options); diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/OrcEncodedDataReader.java b/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/OrcEncodedDataReader.java index 24306d5db1f..8e8a917f20b 100644 --- a/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/OrcEncodedDataReader.java +++ b/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/OrcEncodedDataReader.java @@ -261,7 +261,8 @@ public OrcEncodedDataReader(LowLevelCache lowLevelCache, BufferUsageManager buff } consumer.setUseDecimal64ColumnVectors(HiveConf.getVar(jobConf, - ConfVars.HIVE_VECTORIZED_INPUT_FORMAT_SUPPORTS_ENABLED).equalsIgnoreCase("decimal_64")); + ConfVars.HIVE_VECTORIZED_INPUT_FORMAT_SUPPORTS_ENABLED) + .equalsIgnoreCase(HiveConf.HIVE_VECTORIZED_INPUT_FORMAT_SUPPORTS_DECIMAL_64)); consumer.setFileMetadata(fileMetadata); consumer.setSchemaEvolution(evolution); isReadCacheOnly = HiveConf.getBoolVar(jobConf, ConfVars.LLAP_IO_CACHE_ONLY); diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/SerDeEncodedDataReader.java b/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/SerDeEncodedDataReader.java index e3dc6806b42..0b454d00258 100644 --- a/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/SerDeEncodedDataReader.java +++ b/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/SerDeEncodedDataReader.java @@ -232,7 +232,8 @@ public MemoryBuffer create() { this.reporter = reporter; this.jobConf = jobConf; final boolean useDecimal64ColumnVectors = HiveConf.getVar(jobConf, ConfVars - .HIVE_VECTORIZED_INPUT_FORMAT_SUPPORTS_ENABLED).equalsIgnoreCase("decimal_64"); + .HIVE_VECTORIZED_INPUT_FORMAT_SUPPORTS_ENABLED) + .equalsIgnoreCase(HiveConf.HIVE_VECTORIZED_INPUT_FORMAT_SUPPORTS_DECIMAL_64); consumer.setUseDecimal64ColumnVectors(useDecimal64ColumnVectors); this.schema = schema; this.writerIncludes = OrcInputFormat.genIncludedColumns(schema, columnIds); diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/VectorDeserializeOrcWriter.java b/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/VectorDeserializeOrcWriter.java index 46e8a12bdb5..4181118daca 100644 --- a/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/VectorDeserializeOrcWriter.java +++ b/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/VectorDeserializeOrcWriter.java @@ -235,7 +235,8 @@ public void startAsync(AsyncCallback callback) { private static VectorizedRowBatchCtx createVrbCtx(StructObjectInspector oi, final Properties tblProps, final Configuration conf) throws IOException { final boolean useDecimal64ColumnVectors = HiveConf.getVar(conf, ConfVars - .HIVE_VECTORIZED_INPUT_FORMAT_SUPPORTS_ENABLED).equalsIgnoreCase("decimal_64"); + .HIVE_VECTORIZED_INPUT_FORMAT_SUPPORTS_ENABLED) + .equalsIgnoreCase(HiveConf.HIVE_VECTORIZED_INPUT_FORMAT_SUPPORTS_DECIMAL_64); final String serde = tblProps.getProperty(serdeConstants.SERIALIZATION_LIB); final String inputFormat = tblProps.getProperty(hive_metastoreConstants.FILE_INPUT_FORMAT); final boolean isTextFormat = TextInputFormat.class.getName().equals(inputFormat) diff --git a/llap-server/src/test/org/apache/hadoop/hive/llap/io/encoded/TestParquetEncodedDataReader.java b/llap-server/src/test/org/apache/hadoop/hive/llap/io/encoded/TestParquetEncodedDataReader.java index 3ba0afbdd70..8a8a4ac1bf6 100644 --- a/llap-server/src/test/org/apache/hadoop/hive/llap/io/encoded/TestParquetEncodedDataReader.java +++ b/llap-server/src/test/org/apache/hadoop/hive/llap/io/encoded/TestParquetEncodedDataReader.java @@ -689,7 +689,8 @@ private static JobConf jobConf(String columns, String types, int... readColumnId ids.add(id); } ColumnProjectionUtils.setReadColumns(job, ids); - HiveConf.setVar(job, ConfVars.HIVE_VECTORIZED_INPUT_FORMAT_SUPPORTS_ENABLED, "decimal_64"); + HiveConf.setVar(job, ConfVars.HIVE_VECTORIZED_INPUT_FORMAT_SUPPORTS_ENABLED, + HiveConf.HIVE_VECTORIZED_INPUT_FORMAT_SUPPORTS_DECIMAL_64); return job; } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderImpl.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderImpl.java index 190bec88218..25d00b315c1 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderImpl.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderImpl.java @@ -68,7 +68,8 @@ protected RecordReaderImpl(ReaderImpl fileReader, Reader.Options options, final Configuration conf) throws IOException { super(fileReader, options); final boolean useDecimal64ColumnVectors = conf != null && HiveConf.getVar(conf, - HiveConf.ConfVars.HIVE_VECTORIZED_INPUT_FORMAT_SUPPORTS_ENABLED).equalsIgnoreCase("decimal_64"); + HiveConf.ConfVars.HIVE_VECTORIZED_INPUT_FORMAT_SUPPORTS_ENABLED) + .equalsIgnoreCase(HiveConf.HIVE_VECTORIZED_INPUT_FORMAT_SUPPORTS_DECIMAL_64); if (useDecimal64ColumnVectors){ batch = this.schema.createRowBatchV2(); } else { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/VectorizedOrcAcidRowBatchReader.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/VectorizedOrcAcidRowBatchReader.java index c4a92f624e2..20d2c07e3df 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/VectorizedOrcAcidRowBatchReader.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/VectorizedOrcAcidRowBatchReader.java @@ -212,7 +212,8 @@ public float getProgress() throws IOException { } }; final boolean useDecimal64ColumnVectors = HiveConf - .getVar(conf, ConfVars.HIVE_VECTORIZED_INPUT_FORMAT_SUPPORTS_ENABLED).equalsIgnoreCase("decimal_64"); + .getVar(conf, ConfVars.HIVE_VECTORIZED_INPUT_FORMAT_SUPPORTS_ENABLED) + .equalsIgnoreCase(HiveConf.HIVE_VECTORIZED_INPUT_FORMAT_SUPPORTS_DECIMAL_64); if (useDecimal64ColumnVectors) { this.vectorizedRowBatchBase = ((RecordReaderImpl) innerReader).createRowBatch(true); } else { @@ -1514,7 +1515,8 @@ static class DeleteReaderValue { this.bucketForSplit = bucket; final boolean useDecimal64ColumnVector = HiveConf.getVar(conf, ConfVars - .HIVE_VECTORIZED_INPUT_FORMAT_SUPPORTS_ENABLED).equalsIgnoreCase("decimal_64"); + .HIVE_VECTORIZED_INPUT_FORMAT_SUPPORTS_ENABLED) + .equalsIgnoreCase(HiveConf.HIVE_VECTORIZED_INPUT_FORMAT_SUPPORTS_DECIMAL_64); if (useDecimal64ColumnVector) { this.batch = acidEmptyStructSchema.createRowBatchV2(); } else { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/WriterImpl.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/WriterImpl.java index 24083b3ab84..8b9865513d4 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/WriterImpl.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/WriterImpl.java @@ -97,7 +97,7 @@ public class WriterImpl extends org.apache.orc.impl.WriterImpl implements Writer this.inspector = opts.getInspector(); boolean useDecimal64ColumnVectors = opts.getConfiguration() != null && HiveConf.getVar(opts.getConfiguration(), HiveConf.ConfVars.HIVE_VECTORIZED_INPUT_FORMAT_SUPPORTS_ENABLED) - .equalsIgnoreCase("decimal_64"); + .equalsIgnoreCase(HiveConf.HIVE_VECTORIZED_INPUT_FORMAT_SUPPORTS_DECIMAL_64); if (useDecimal64ColumnVectors) { this.internalBatch = opts.getSchema().createRowBatch(TypeDescription.RowBatchVersion.USE_DECIMAL64, opts.getBatchSize()); diff --git a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestInputOutputFormat.java b/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestInputOutputFormat.java index a3b66396400..afc4e796d2b 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestInputOutputFormat.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestInputOutputFormat.java @@ -3874,7 +3874,8 @@ public void testRowNumberUniquenessInDifferentSplits() throws Exception { public void testSchemaEvolutionOldDecimal() throws Exception { TypeDescription fileSchema = TypeDescription.fromString("struct<a:int,b:struct<c:int>,d:string>"); - conf.set(ConfVars.HIVE_VECTORIZED_INPUT_FORMAT_SUPPORTS_ENABLED.varname, "decimal_64"); + conf.set(ConfVars.HIVE_VECTORIZED_INPUT_FORMAT_SUPPORTS_ENABLED.varname, + HiveConf.HIVE_VECTORIZED_INPUT_FORMAT_SUPPORTS_DECIMAL_64); Writer writer = OrcFile.createWriter(testFilePath, OrcFile.writerOptions(conf) .fileSystem(fs) @@ -3946,7 +3947,8 @@ public void testSchemaEvolutionOldDecimal() throws Exception { public void testSchemaEvolutionDecimal64() throws Exception { TypeDescription fileSchema = TypeDescription.fromString("struct<a:int,b:struct<c:int>,d:string>"); - conf.set(ConfVars.HIVE_VECTORIZED_INPUT_FORMAT_SUPPORTS_ENABLED.varname, "decimal_64"); + conf.set(ConfVars.HIVE_VECTORIZED_INPUT_FORMAT_SUPPORTS_ENABLED.varname, + HiveConf.HIVE_VECTORIZED_INPUT_FORMAT_SUPPORTS_DECIMAL_64); Writer writer = OrcFile.createWriter(testFilePath, OrcFile.writerOptions(conf) .fileSystem(fs)
