This is an automated email from the ASF dual-hosted git repository.
github-actions[bot] pushed a change to branch bot/trino-pin
in repository https://gitbox.apache.org/repos/asf/hudi.git
discard ad3a455d1666 chore(trino): advance trino master pin to a02b3257cebd
add 3b5bb09b1d36 fix(writer): reject directory-traversal partition paths
across all writers (#19342)
add 77f5e021b8e5 fix(build): remove duplicate dependency declarations
rejected by Maven 3.10 (#20212)
add ec2831ec932d perf(spark): check file existence in parallel in MOR
incremental fallback scan (#20211)
add 379f566530d0 fix(Spark): Fix nested struct predicate pushdown for
HoodieFileGroupReaderBasedFileFormat (#20029)
add 296f5d8283b9 fix(spark): fix incremental file index reporting
sizeInBytes=0 (#19912)
add 211ccba18c0f fix(spark): avoid full partition listing in incremental
reads (#19898)
add b6fe7c3fd8b8 fix(metrics): handle stale DistributedRegistry across
SparkContext restarts (#19790)
add 7e4c94434813 chore: fix javadoc and compiler warnings flagged in CI
(#20215)
add 7f35e7eedcc0 test(spark): guard against executor .hoodie access and
task payload growth on the read path (#20112)
new 9a6851e9439d chore(trino): advance trino master pin to 6b823e91c613
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (ad3a455d1666)
\
N -- N -- N refs/heads/bot/trino-pin (9a6851e9439d)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
hudi-aws/pom.xml | 6 -
.../apache/hudi/cli/commands/RepairsCommand.java | 4 +-
.../hudi/cli/utils/SparkTempViewProvider.java | 2 +-
.../apache/hudi/config/HoodieClusteringConfig.java | 2 +
.../java/org/apache/hudi/io/BaseCreateHandle.java | 2 +-
.../hudi/io/FileGroupReaderBasedMergeHandle.java | 2 +-
.../apache/hudi/io/HoodieAbstractMergeHandle.java | 2 +-
.../org/apache/hudi/io/HoodieAppendHandle.java | 2 +-
.../testutils/TaskDeserializationRecorder.java | 340 ++++++++++++
.../io/storage/row/HoodieRowDataCreateHandle.java | 2 +-
.../client/common/HoodieSparkEngineContext.java | 30 +-
.../hudi/client/utils/SparkValidatorUtils.java | 2 -
.../hudi/io/storage/row/HoodieRowCreateHandle.java | 2 +-
.../SparkHoodieBackedTableMetadataWriter.java | 4 +-
...ieBackedTableMetadataWriterTableVersionSix.java | 4 +-
.../apache/hudi/metrics/DistributedRegistry.java | 36 +-
.../hudi/metrics/TestDistributedRegistry.java | 102 ++++
.../table/action/commit/TestUpsertPartitioner.java | 4 +-
.../hudi/testutils/FunctionalTestHarness.java | 1 +
.../apache/hudi/testutils/SparkExecutorGuards.java | 515 ++++++++++++++++++
hudi-common/pom.xml | 6 -
.../org/apache/hudi/common/expression/Literal.java | 2 +-
.../hudi/common/expression/NameReference.java | 2 +-
.../hudi/common/model/HoodiePartitionMetadata.java | 12 +-
.../timeline/versioning/v1/ActiveTimelineV1.java | 12 +-
.../timeline/versioning/v1/ArchivedTimelineV1.java | 12 +-
.../timeline/versioning/v2/ActiveTimelineV2.java | 12 +-
.../table/view/FileSystemViewStorageConfig.java | 10 +-
.../hudi/common/util/PartitionPathEncodeUtils.java | 60 +++
.../hudi/common/model/TestHoodieWriteStat.java | 8 +-
.../common/testutils/HoodieTestDataGenerator.java | 2 +-
.../apache/hudi/common/util/TestBaseFileUtils.java | 8 +-
.../common/util/TestPartitionPathEncodeUtils.java | 59 +-
.../apache/hudi/util/ImplicitTypeConverter.java | 4 +-
.../org/apache/hudi/util/TestExpressionUtils.java | 14 +-
.../common/model/TestHoodiePartitionMetadata.java | 88 ++-
.../table/view/TestHoodieTableFileSystemView.java | 6 +-
.../hudi/common/testutils/HoodieTestTable.java | 2 +-
.../hudi/common/util/TestTablePathUtils.java | 4 +-
.../hudi/hadoop/testutils/InputFormatTestUtil.java | 2 +-
.../scala/org/apache/hudi/DataSourceOptions.scala | 14 +
.../scala/org/apache/hudi/HoodieBaseRelation.scala | 29 +
.../apache/hudi/HoodieIncrementalFileIndex.scala | 37 +-
.../org/apache/hudi/IncrementalRelationV1.scala | 15 +-
.../org/apache/hudi/IncrementalRelationV2.scala | 15 +-
.../hudi/MergeOnReadIncrementalRelationV1.scala | 25 +-
.../hudi/MergeOnReadIncrementalRelationV2.scala | 26 +-
.../apache/hudi/SparkHoodieTableFileIndex.scala | 173 +++---
.../HoodieFileGroupReaderBasedFileFormat.scala | 17 +-
.../parquet/ParquetSchemaEvolutionUtils.scala | 55 +-
.../AlterHoodieTableAddPartitionCommand.scala | 8 +-
.../parquet/TestParquetSchemaEvolutionUtils.scala | 77 ++-
.../RepairAddpartitionmetaProcedure.scala | 2 +-
.../RepairMigratePartitionMetaProcedure.scala | 2 +-
.../org/apache/hudi/ColumnStatsIndexHelper.java | 20 +-
.../TestColStatsRecordWithMetadataRecord.java | 48 +-
.../functional/TestSparkReadExecutorFootprint.java | 597 +++++++++++++++++++++
.../TestIncrementalFileIndexSizeInBytes.scala | 127 +++++
.../TestIncrementalReadWithPartitionFilter.scala | 131 +++++
.../TestNestedSchemaPruningOptimization.scala | 158 +++++-
.../apache/spark/sql/hudi/ddl/TestSpark3DDL.scala | 6 +
hudi-utilities/pom.xml | 7 -
pom.xml | 8 +-
63 files changed, 2731 insertions(+), 255 deletions(-)
create mode 100644
hudi-client/hudi-client-common/src/test/java/org/apache/hudi/testutils/TaskDeserializationRecorder.java
create mode 100644
hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/testutils/SparkExecutorGuards.java
create mode 100644
hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/functional/TestSparkReadExecutorFootprint.java
create mode 100644
hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestIncrementalFileIndexSizeInBytes.scala
create mode 100644
hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestIncrementalReadWithPartitionFilter.scala
