This is an automated email from the ASF dual-hosted git repository.
yihua pushed a change to branch
dependabot/maven/hudi-platform-service/hudi-metaserver/hudi-metaserver-server/mysql-mysql-connector-java-8.0.28
in repository https://gitbox.apache.org/repos/asf/hudi.git
omit c00d18e74a3 Bump mysql-connector-java
add d2a3d11977d [MINOR] Add database config for flink (#7682)
add 6cb4580defc [HUDI-4710] Fix flaky:
TestKeyRangeLookupTree#testFileGroupLookUpManyEntriesWithSameStartValue (#7571)
add c9bc03ed868 [HUDI-4148] Add client for Hudi table service manager
(TSM) (#6732)
add ec5022b4fdd [MINOR] Unify naming for record merger (#7660)
add 27a8866a1e1 [HUDI-5433] Fix the way we deduce the pending instants for
MDT writes (#7544)
add 124ab5fda66 [HUDI-5488] Make sure Disrupt queue start first, then
insert records (#7582)
add 43d23a49a02 [HUDI-5577] Validate option catalog.path in dfs mode
(#7698)
add 9ee36de98ab [Minor] add missing link and fix typo (#7696)
add f8028a400eb [HUDI-4911][HUDI-3301] Fixing
`HoodieMetadataLogRecordReader` to avoid flushing cache for every lookup (#6782)
add 86be8554820 [HUDI-5336] Fixing log file pattern match to ignore
extraneous files (#7612)
add 0e9bb024fb9 [HUDI-5559] Support CDC for flink bounded source (#7677)
add e62b9da66b6 [HUDI-5516] Reduce memory footprint on workload with
thousand active partitions (#7626)
add b1552eff7af [HUDI-5384] Adding optimization rule to appropriately push
down filters into the `HoodieFileIndex` (#7423)
add 6f4c60e1835 [HUDI-5589] Fix Hudi config inference (#7713)
add f0f8d618b30 [HUDI-5499] Fixing Spark SQL configs not being properly
propagated for CTAS and other commands (#7607)
add d03c8f9c155 [MINOR] Disable async clean in
testCleanerDeleteReplacedDataWithArchive (#7721)
add 6f6652a84aa [HUDI-5407][HUDI-5408] Fixing rollback in MDT to be eager
(#7490)
add febff4afd2a [HUDI-5417] support to read avro from non-legacy map/list
in parquet log (#7512)
add a70355f4457 [HUDI-5579] Fixing Kryo registration to be properly wired
into Spark sessions (#7702)
add 1d28d51d00c [HUDI-5596] Fix hudi-cli-bundle startup issue with gson
(#7728)
add 6593e8ba9d9 [minor] Fix flink 1.15 build profile (#7731)
add c18d6153e10 [HUDI-1575] Early Conflict Detection For Multi-writer
(#6133)
add 811dcc591cd [MINOR] Eliminating Kryo from `hudi-integ-test-bundle`
since it's being used w/ Spark (#7735)
add 8917971e9f7 [HUDI-5555] Set class-loader in parquet data block (#7670)
add 4f6b831ea11 [HUDI-5235] Clustering target size should larger than
small file limit (#7232)
add fc1831b22c3 Fixing FS `InputStream` leaks (#7741)
add 5e3ca834366 [MINOR] Fixing `TestStructuredStreaming` test (#7736)
add 146f39d49e5 [HUDI-5593] Fixing deadlocks due to async cleaner awaiting
for lock while main thread is acquired the lock and awaiting for async cleaner
to finish (#7739)
add d439fab2421 [HUDI-3673] Clean up hbase shading dependencies (#7371)
add 2fc20c186b7 [HUDI-5575] Adding/Fixing auto generation of record keys
w/ hudi (#7726)
add 25afb357df9 [HUDI-5401] Ensure user-provided hive metastore uri is set
in HiveConf if not already set (#7543)
add 20969c26059 [HUDI-5392] Fixing Bootstrapping flow handling of arrays
(#7461)
add 26b719a7fba [HUDI-2608] Support json schema in SchemaRegistryProvider
(#7727)
add 1769ff8fb90 [HUDI-5443] Fixing exception trying to read MOR table
after `NestedSchemaPruning` rule has been applied (#7528)
add a79f8093755 [HUDI-5582] Do not let users override internal metadata
configs (#7709)
add 65044d38fbe [HUDI-2118] Skip checking corrupt log blocks for
transactional write file systems (#6830)
add f49b5d34342 [HUDI-4991] Allow kafka-like configs to set truststore and
keystore for the SchemaProvider
add 2f22b07385f [HUDI-5276] Fix getting partition paths under relative
paths (#7744)
add c95abd3213f Revert "[HUDI-5575] Adding/Fixing auto generation of
record keys w/ hudi (#7726)" (#7747)
add 8c640b51331 [HUDI-5610] Fix hudi-cli-bundle startup conflict for spark
3.2.0 (#7746)
add 31b02d1798e [HUDI-5594] Add metaserver bundle validation (#7722)
add e00ee2d88e2 [HUDI-5620] Fix metaserver bundle validation (#7749)
add 7e35874c7ba [HUDI-5617] Rename configs for async conflict detector for
clarity (#7750)
add d4dcb3d1190 [HUDI-5618] Add `since version` to new configs for 0.13.0
release (#7751)
add 3a08bdc3f97 [HUDI-5363] Removing default value for shuffle parallelism
configs (#7723)
add 2e59c8e6b9d [MINOR] Correct RFC numbering (#7754)
add 98643c63056 [HUDI-5380] Fixing change table path but table location in
metastore … (#7445)
add 45da30dc3ec [HUDI-5485] Add File System View API for batch listing and
improve savepoint performance with metadata table (#7690)
add e969a4c7848 [HUDI-5592] Fixing some of the flaky tests in CI (#7720)
add 5ea2f8487c9 Moving to 0.14.0-SNAPSHOT on master branch. (#7755)
add e270924c5eb [HUDI-5534] Optimizing Bloom Index lookup when using Bloom
Filters from Metadata Table (#7642)
add a62f0f684fa [HUDI-5627] Improve the usability of Hudi CLI bundle
(#7762)
add 1ecc0401eef [HUDI-5623] Increase default time to wait between retries
by lock provider client (#7758)
add ff590c6d72c [HUDI-5023] Switching default Write Executor type to
`SIMPLE` (#7476)
add 67d661c2952 [HUDI-5630] Fixing flaky parquet projection tests (#7768)
add 6011de44d47 [HUDI-5629] Clean CDC log files for enable/disable
scenario (#7767)
add 7352661283e [MINOR] Add `hudi-platform-service` and
`hudi-metaserver-server-bundle` to root pom (#7774)
add 50b2cb5d8f7 [HUDI-5635] Fix release scripts (#7775)
add 1cc6442781e [MINOR] Add missing Apache License in source files (#7779)
add aebe3cc53e5 [MINOR] Fix validate_staged_release.sh (#7780)
add c21eca564c6 [HUDI-5626] Rename CDC logging mode options (#7760)
add 5d8c3a98189 [MINOR] Skip docs generation for table service manager
(#7773)
add d25058b36b2 Revert "[MINOR] Add `hudi-platform-service` and
`hudi-metaserver-server-bundle` to root pom (#7774)" (#7782)
add 2c56aa4ce99 [HUDI-5631] Improve defaults of early conflict detection
configs (#7770)
add 9bbd71eb67a [HUDI-5624] Fix HoodieAvroRecordMerger to use new
precombine API (#7759)
add 3979848a499 [HUDI-5628] Fixing log record reader scan V2 config name
(#7764)
add 4a9ae5c4091 [HUDI-5638] Fix metaserver test setup (#7785)
add 3b301aa7f03 [HUDI-5640] Add missing profiles in
`deploy_staging_jars.sh` (#7784)
add a1ba929b5e4 [HUDI-5639] Fixing stream identifier for single writer
with spark streaming ingest (#7783)
add 44b245dc820 [MINOR] Fix `HoodieCDCRDD` setting flag `usesVirtualKeys`
(#7777)
add b00dac54642 [HUDI-5503] Optimize flink table factory option check
(#7608)
add 88d8e5e96d5 [MINOR] Cleaning up recently introduced configs (#7772)
add 53b813a565b [HUDI-5629] Clean CDC log files for enable/disable
scenario (#7786)
add e6c0bd63b2f [HUDI-5637] Add Kryo for hive sync bundle (#7781)
add 32f45f0f7d6 [HUDI-5634] Rename CDC related classes (#7410)
add 22eab392fc7 [HUDI-5632] Fix failure launching Spark jobs from
hudi-cli-bundle (#7790)
add d968f393bd2 [MINOR] Make `data_before_after` the default cdc logging
mode (#7797)
add 1a72f504eef [HUDI-5563] Check table exist before drop table (#7679)
add d90f286a197 [HUDI-5568] Fix the BucketStreamWriteFunction to rebase
the local filesystem instance instead (#7685)
add 9906df48e7c [HUDI-5655] Closing write client for spark ds writer in
all cases (including exception) (#7799)
add 5acc6fe51ac [HUDI-5654] Fixing read of an empty rollback completed
meta files from data table timeline w/ metadata reads (#7798)
add 1377143656a [HUDI-5487] Reduce duplicate logs in ExternalSpillableMap
(#7579)
add 252c4033010 [MINOR] Standardise schema concepts on Flink Engine (#7761)
add d55a1cef330 [HUDI-5567] Make the bootstrapping exception message more
clear (#7684)
add d8576933a9b [HUDI-5553] Prevent partition(s) from being dropped if
there are pending… (#7669)
add 628dc8cd851 [HUDI-5633] Fixing performance regression in
`HoodieSparkRecord` (#7769)
add 9469882d80f [HUDI-5585][flink] Fix flink creates and writes the table,
the spark alter table reports an error (#7706)
add 255d40c5fb5 [HUDI-5540] Close write client after usage of
DeleteMarker/RollbackToInstantTime/RunClean/RunCompactionProcedure (#7655)
add 0a9a6d20471 [HUDI-5317] Fix insert overwrite table for partitioned
table (#7793)
add 5e616ab115c [HUDI-5646] Guard dropping columns by a config, do not
allow by default (#7787)
add 7064c380506 [MINOR] Restoring existing behavior for `DeltaStreamer`
Incremental Source (#7810)
add e93fbeee4ac [HUDI-5681] Fixing Kryo being instantiated w/ invalid
`SparkConf` (#7821)
add abe26d4169c [HUDI-5676] Fix BigQuerySyncTool standalone mode (#7816)
add 6fbf9d4f840 [HUDI-5647] Automate savepoint and restore tests (#7796)
add 1459eddb0e7 [HUDI-5684] Fix CTAS and Insert Into to avoid
combine-on-insert by default (#7813)
add e3b95e88a76 [HUDI-5678] Fix `deduceShuffleParallelism` in row-writing
Bulk Insert helper (#7818)
add 9c969380233 [HUDI-5691] Fixing `HoodiePruneFileSourcePartitions` to
properly handle non-partitioned tables (#7833)
add 3282caa2242 [HUDI-5671] BucketIndexPartitioner partition algorithm
skew (#7815)
add 0ea3c44110c [HUDI-5551] support seconds unit on event_time metrics
(#7664)
add fc031b9d3b0 [HUDI-5682] Bucket index does not work correctly for
multi-writer scenarios (#7838)
add 3e1171cf70d [HUDI-5653] Reset TestDataSource for
TestHoodieDeltaStreamerWithMultiWriter (#7831)
add f3ac50935bf [MINOR] Fixing CTAS configuration not propagated properly
(#7832)
add 5cdfacba1bd [HUDI-5496] Avoid unnecessary file system parsing to
initialize metadata table for a new data table (#7841)
add 6424d4ddb57 [HUDI-5689] Make CDC config working for DeltaStreamer path
(#7830)
add ef3a17e3d97 [HUDI-5692] SpillableMapBasePath should be lazily loaded
(#7837)
add ff832f4d860 [MINOR] Validate configs for OCC early conflict detection
(#7848)
add 13ed95a9fd2 Make most of the Spark SQL DML operations configs
overridable (#7850)
add 352445fb965 [HUDI-5700] Annotate config classes for docs generation
(#7853)
add 52e41052217 [HUDI-5701] Remove meta fields from cdc new record in
CDCLogger (#7852)
add 1d305c9f5da [HUDI-5702] Avoid to write useless change-data when
compaction (#7849)
add ea36857035f [HUDI-5703] Improve `ConfigProperty` APIs for docs
generation (#7854)
add 0c9465f2ab6 [MINOR] Improve configuration configs (#7855)
add 33dd8ae5153 [HUDI-5638] Activate platform service profile when deploy
(#7858)
add 21f58999a88 [HUDI-5704] De-coupling column drop flag and schema
validation flag (#7857)
add 2c38af5e627 [HUDI-5651] Sort the inputs by record keys for bulk insert
tasks (#7795)
add 48381df096a [HUDI-5717] Precision field is not need in
FixedLenBytesColumnReader (#7866)
add 6436ef3ee2c [HUDI-5329] Spark reads hudi table error when flink
creates the table without preCombine fields (#7378)
add 4609ed156b7 [MINOR] Added safety-net check to catch any potential
issue to deduce parallelism from the incoming `Dataset` appropriately (#7873)
add 5976a519303 [HUDI-5718] Unsupported Operation Exception for compaction
(#7874)
add 33212ccabb5 [HUDI-5720] Improve the validation script of the staged
source release (#7882)
add 08b47fcd9fa [MINOR] Fix bootstrap typos (#7676)
add dee020e18ce [HUDI-5657] Fix NPE if filters condition contains null
literal when using column stats data skipping for flink (#7801)
add 380d648d085 [MINOR] Fix indexer typos (#7863)
add 53fca761be1 [MINOR] fixed docs for WRITE_EXECUTOR_TYPE (#7880)
add a9f1be52ffc [HUDI-5721] Add Github actions on source validation (#7883)
add 8329414edfa [MINOR] Remove useless code (#7756)
add a1da9ec2e4f [HUDI-5270] Duplicate key error when insert_overwrite the
same partition in multi writer (#7281)
add 0dbc3450a95 [HUDI-4968] Update misleading
read.streaming.skip_compaction/skip_clustering config (#6856)
add b2e3805b3fb [MINOR] improve RunClusteringProcedure with partition
selected (#7876)
add 2b0615c52ba [HUDI-5725] Creating Hudi table with misspelled table type
in Flink leads to Flink cluster crash (#7888)
add 088d9a061e5 [MINOR] Description update for Flink Clustering and
Compaction configuration (#7887)
add 60dfe4d766d Cleaning up unnecessary relocation for com.google.common
packages (#7900)
add a7151ef53c2 [HUDI-5734] Fix flink batch read skip clustering data lost
(#7903)
add 653aa86145f [MINOR] Fix wrong assertion in TestHoodieTableFactory.java
(#7916)
add 67f4c78c5a0 [HUDI-5758] Restoring state of `HoodieKey` to make sure
it's binary compatible w/ its state in 0.12 (#7917)
add 3e31ca73828 [MINOR] Remove unnecessary TestCallExpressions which are
adapters for CallExpression (#7911)
add 1cb8ffe7264 [HUDI-5768] Fix Spark Datasource read of metadata table
(#7924)
add e25381c6966 [HUDI-5764] Rollback delta commits from `HoodieIndexer`
lazily in metadata table (#7921)
add a932e482408 [HUDI-5771] Improve deploy script of release artifacts
(#7927)
add d395f058183 [HUDI-3580] [RFC-48] Create RFC for LogCompaction support
to Hudi (#5041)
add 71a62627cdd [HUDI-5772] Align Flink clustering configuration with
HoodieClusteringConfig (#7928)
add ed6b7f6aedc [HUDI-4406] Support Flink compaction/clustering write
error resolvement to avoid data loss (#6121)
add 4f8f2d8dc5c [HUDI-5737] Fix Deletes issued without any prior commits
(#7633)
add 87b97369b68 [HUDI-5773] Support archive command for spark sql (#7931)
add 5d7edd5097e [MINOR] Fixing RFC 48 title (#7937)
add b59fa1bd7fd [HUDI-5792] Disabling flaky tests:
TestDisruptorExecutionInSpark (#7946)
add af61dea6f98 [MINOR] Enable Azure CI to publish test results (#7943)
add 25f6927b47d [HUDI-5787] HMSDDLExecutor should set table type to
EXTERNAL_TABLE when hoodie.datasource.hive_sync.create_managed_table of sync
config is false (#7940)
add da80b8c8d52 [HUDI-5673] Support multi writer for bucket index with
guarded lock (#7860)
add 91505853144 [MINOR] Fix spark sql run clean do not exit (#7918)
add c9395adeb21 [MINOR] Fix format name and remove redundant line in
examples (#7952)
add a584c6f97f4 [MINOR] Improving logging and tracking of failures for
metadata validator (#7792)
add e5ee6576cba [HUDI-5814] Disable flaky tests to unblock CI (#7986)
add 45017036ecf [HUDI-1593] Add support for "show restores" and "show
restore" commands in hudi-cli (#7868)
add 6356dbdca35 [HUDI-5809][RFC-56] Keep RFC-56 early conflict detection
update to date (#7972)
add aad6516da96 [HUDI-5664] Improve SqlQueryPreCommitValidator#queries
Parallelism. (#7809)
add c173edfaf0d [HUDI-5811] Specify the artifact of protoc to solve the
problem that cannot be compiled on m1 mac (#7977)
add 68135ad2296 [HUDI-5800] Fix test failure in TestHoodieMergeOnReadTable
(#7959)
add 73ca223e2e5 [HUDI-5799] Fix Spark partition validation in
TestBulkInsertInternalPartitionerForRows (#7958)
add 96032b9b768 [HUDI-5557]Avoid converting columns that are not indexed
in CSI (#7672)
add 25fc4f2ab52 [MINOR] De-duplicating Iterator implementations (#7752)
add 28d6bfef58c [HUDI-5795] Ensure we only have a single push gw client
and hence a single connection pool
add be92be657a3 [HUDI-5786] Add a new config to specific spark write rdd
storage level (#7941)
add d3840a0c02f [HUDI-5058] HoodieCatalog#getTable sets primary key with
hoodie.datasource.write.recordkey.field for table intialized via Spark (#7981)
add 688d947e44b [HUDI-5729] Fix RowDataKeyGen method getRecordKey (#7894)
add dfac12385bd [HUDI-5802] Allow configuration for deletes in
DefaultHoodieRecordPayload (#7961)
add 0c84482aa91 [HUDI-5774] Fix prometheus configs for metadata table and
support metric labels (#7933)
add 767da51baeb [HUDI-5821] Improve the resource release for
HoodieTableFileSystemView (#7995)
add 33987b1bc27 [HUDI-5792] fix CI test:TestDisruptorExecutionInSpark
timeout problem (#8000)
add 8ee354fcaf6 [HUDI-5778] support absolute path names for hierarchical
configs (#7920)
add 0f99315ab84 Handle empty payloads for AbstractDebeziumAvroPayload
(#7944)
add 9c8144045de [HUDI-5823] Claim RFC-65 for Partition TTL Management
(#8006)
add 6430e9d82a2 [MINOR] Close InflaterInputStream in finally when calling
decompressBytes in BitCaskDiskMap (#7840)
add 018ecbd1fa6 update hudi-flink exception string (#8003)
add ad2a37d6796 [HUDI-5817] Fix async indexer metadata writer to avoid
eager rollback and failed write cleaning (#8001)
add 24c9a86dae2 Fix usage of SchemaRegistryProvider with Row source (#7601)
add 812950bc9ea [HUDI-4690][HUDI-4503] Cleaning up Hudi custom Spark
`Rule`s (#7871)
add d705dcc4188 [HUDI-5173] Skip if there is only one file in
clusteringGroup (#7159)
add 265aac89584 [HUDI-5825] disable Spark UI in tests if SPARK_EVLOG_DIR
not set (#8005)
add 2770ff50714 [HUDI-915][HUDI-5656] Rebased `HoodieBootstrapRelation`
onto `HoodieBaseRelation` (#7804)
add b39d99c7e67 [MINOR] Update DOAP with 0.13.0 Release (#8012)
add 654600990b8 [HUDI-5455] Add commons-configuration2 in hudi cli bundle
(#7536)
add 828a85e8c2e [MINOR] Refactor tests - spark read only once to df, then
assert (#8045)
add 5af78c665bd [HUDI-5845] Remove usage of deprecated
getTableAvroSchemaWithoutMetadataFields. (#8032)
add e131505a4f0 [HUDI-5838] Mask sensitive info while printing hudi
properties in DeltaStreamer (#8027)
add f22d1c9f26a [HUDI-5794] Failing new commits on any pending restore
commits (#7948)
add ba3db4e1339 [MINOR] Reduce verbosity of the most noisy log messages
(#6269)
add 58e9ce710b4 [HUDI-5850] Fix timestamp(6) field long overflow (#8052)
add dd083c41942 [HUDI-5855] Release resource actively for Flink hive meta
sync (#8050)
add 3e12b46d717 [MINOR] fix the compilation failure for master branch.
(#8059)
add 31e94abf902 [HUDI-5835] After performing the update operation, the
hoodie table cannot be read normally by spark (#8026)
add 30175e992ef [MINOR] schema evolution relax decimal type conversion
conditions (#8063)
add b6fdcb7ada2 [HUDI-5853] Add infer functions to BQ sync configs (#8053)
add dc8b078af68 [HUDI-5080] Unpersist only relevant RDDs instead of all
(#7914)
add c7cebf445f4 [HUDI-5843] multiwriter deltastreamer checkpoints (#8043)
add b6490c160bf [HUDI-5813] Error table support for error events (#7982)
add bb240f7beaf [HUDI-5777] Support Metrics for Multiple Tables
Simultaneously (#7934)
add 04c2ddfd8f9 [HUDI-5852] Release the HoodieSyncTool actively in time
where feasible (#8064)
add a72ca04eeea [HUDI-5808] Add Support for kaffka ofsets in
jsonkafkasource and avrokafkasource (#8011)
add 47356a57930 [HUDI-5862] Strength MARKER based rollback for log files
with a single commit (#8077)
add f6518e87b55 [HUDI-5869] Fix Some Typos in Hudi-Common (#8083)
add eca57b51866 [HUDI-5867] Upgrade commons-io to v2.11.0 (#8081)
add 6b178ce978c [HUDI-4442] [HUDI-5001] Sanitize JsonConversion and
RowSource (#8010)
add 8e7524574ce [HUDI-5728] HoodieTimelineArchiver archives the latest
instant before inflight replacecommit (#7891)
add cfe490fcb23 [HUDI-5665] Adding support to re-use table configs (#7901)
add 81e6e854883 [HUDI-5847] Add support for multiple metric reporters and
metric labels (#8041)
add 2a52bc03d90 [HUDI-5812] Optimize the data size check in
HoodieBaseParquetWriter (#7978)
add 18d528f33d8 [HUDI-5736] Common de-coupling column drop flag and schema
validation flag (#7895)
add d40a6211f64 [HUDI-5796] Adding auto inferring partition from incoming
df (#7951)
add 2ddcf96cddb [HUDI-5863] Fix HoodieMetadataFileSystemView serving stale
view at the timeline server (#8079)
add 9bb6b55440c [HUDI-4372] Enable matadata table by default for flink
(#8070)
add bce038967e3 [HUDI-5866] Fix unnecessary log messages during bulk
insert in Spark (#8095)
add 1db22d84493 [HUDI-5876] Remove usage of deprecated TableConfig (#8094)
add 0fd3c2c6ca3 [HUDI-5878] Optimize the txn metadata for
BaseCommitActionExecutor (#8099)
add dcedf139d32 Fix some comments on schema change (#8084)
add 168005f020d [HUDI-5872] Abstraction for DeltaSyncService and
DeltaStreamerMetrics (#8086)
add 46a2ccf56ee [HUDI-5881] Handle pending clean instants while running
savepoint (#8105)
add 2ba1423ff5e [HUDI-5883] Avoid throwing error if data table does not
exist in Metadata Table Validator (#8108)
add c3bfd129f8e [HUDI-5882] Close table metadata and file system view
instances in Metadata Table Validator (#8106)
add c6441134ba6 [MINOR] Fix testHoodieFlinkClusteringScheduleAfterArchive
(#8110)
add a8312a9b8c3 [HUDI-5887] Distinguish the single writer enabling
metadata table and multi-writer use cases for lock guard (#8111)
add bab75b6c60c [HUDI-4911] Following the first patch, fix the inefficient
code (#8127)
add 2675118d95c [HUDI-5941] Fix for RowData class cast exception (#8145)
add 79428391bac [HUDI-5851] Improvement of data skipping, only converts
expressions to evaluators once (#8051)
add 8129a85eebb [HUDI-5333] Ignore file system type of basePath when using
RocksDbBasedFileSystemView (#7387)
add 51d03515728 [HUDI-5857] Insert overwrite into bucket table would
generate new file group id (#8072)
add 6589ebe8486 [HUDI-5904] support more than one update actions in merge
into table (#8133)
add 84fcc3851cc [HUDI-5913] Table can not read correctly when computed
column is in the midst (#8154)
add 5416f001dc7 [HUDI-5917] Fix HoodieRetryWrapperFileSystem
getDefaultReplication (#8150)
add 9c44339e4bb [HUDI-5909] Reuse hive client if possible (#8139)
add b162330a1f3 [HUDI-5919] Fix the validation of partition listing in
metadata table validator (#8156)
add 25c2f594938 [MINOR] Remove redundant variable in
MetadataBootstrapHandlerFactory (#8158)
add ca76bc5788b [HUDI-5713] Add advanced property for configs (#7869)
add 07c6d8b1845 [MINOR] Use ExecutorFactory in BootstrapHandler (#7808)
add 1b756d62e19 [HUDI-5915] Fixed load ckpMeatadata error when using minio
(#8149)
add c48db6d8ebe [HUDI-4675] add unittest for
RebalancedSparkHoodieHBaseIndex (#6458)
add c392fb7ff8c [HUDI-4630] Add transformer capability to individual feeds
in MultiTableDeltaStreamer (#6726)
add 38e4078d23b [MINOR] Ignoring warn msg for timeline server for metadata
table (#8168)
add bfb01bdfc1e [HUDI-5930] Fix build failure due to ClosableIterator
(#8177)
add bd4b13926ed [HUDI-5926] Improve cleaner parallelism (#8171)
add 17fdd10fdc3 [HUDI-5925] Improve bootstrap parallelism (#8170)
add 0d6f656e5bb [HUDI-5927] Improve parallelism of deleting invalid files
(#8172)
add d760ed99734 [HUDI-5933] Revert "[HUDI-4630] Add transformer capability
to individual feeds in MultiTableDeltaStreamer (#6726)" (#8183)
add ca552c6a8ae [MINOR] bot.yml ignore more filetype. (#7890)
add eb921591562 [HUDI-5938] No need to init the properties for each data
flushing (#8192)
add d889cc77347 [HUDI-5939] Revert "[HUDI-4675] add unittest for
RebalancedSparkHoodieHBaseIndex (#6458)" (#8197)
add b79ce80f709 [HUDI-5931] Improve the description of operation in
HoodieDeltaStreamer (#8175)
add b466e66bad2 [HUDI-5922] Reuse IMetaStoreClient between
HoodieHiveSyncClient and DDLExecutor (#8165)
add fbb51a6f154 [HUDI-5947] Update the README for flink jar building
(#8212)
add fc3e81f24e7 [HUDI-5407] Following the first patch, add changes for
Flink (#8204)
add e51b4575cb7 [HUDI-5920] Improve documentation of parallelism configs
(#8157)
add 344df73c6b5 [HUDI-5865] Fix table service client to instantiate with
timeline server (#8080)
add 102b535c970 [HUDI-5785] Enhance Spark Datasource tests (#7938)
add 42185a6118f [HUDI-5879] Extends evaluators to support evaluate based
on column values (#8218)
add ce21873f332 [MINOR] Fix typo in ValidateMetadataTableFilesProcedure
(#8202)
add f7c0d31fa3f [HUDI-5950] Fixing pending instant deduction to trigger
compaction in MDT (#8223)
add 607ca5a664d [HUDI-5965] Add fallback keys for Flink options (#8252)
add 749a93ba269 [HUDI-5781] Refactor SQL transformer configs to use
HoodieConfig and ConfigProperty (#8155)
add ff921568226 fix error info not show in CreateHoodieTableCommand (#8251)
add a75da8cd6aa [HUDI-5822] Fix bucket stream writer fileId not found
exception (#8263)
add e849ad828e5 [MINOR] Fix typos in hudi-client and hudi-spark-datasource
(#8230)
add 22ff13b9a14 [MINOR] Fix job name in prometheus.properties (#8239)
add 91fafcf3bc3 [HUDI-5960] Allow bootstrap procedure to throw an
exception when execution fails (#8241)
add 14719735c4e [HUDI-5961] SparkBulkInsertDeltaCommitActionExecutor lacks
bulkInsertPartitioner construction parameters (#8242)
add 23be4531e53 [HUDI-5962] Adding timeline server support to integ test
suite (#8248)
add bf2e64caa6b [HUDI-5957] Fix table props not being properly propagated
for HoodieCLIUtils (#8243)
add dbadacc9768 [HUDI-5958] Replace deprecated TableSchema with
ResolvedSchema (#8237)
add a36c8e0f973 [HUDI-5967] Add partition ordering for full table scans
(#8269)
add 6916803f7a4 [HUDI-5941] Support savepoint call procedure with base
path in Spark SQL (#8271)
add 41026ef1fea [HUDI-5289] Avoiding repeated trigger of clustering dag
(#8275)
add 7da1127c14f [HUDI-5978] Update timeline timezone when write in spark
(#8284)
add 1a526eea748 [HUDI-5979] Add dependencies to hudi-trino-bundle needed
for Trino connector (#8285)
add 099c05a0aee [HUDI-5985] Fix orc version for spark 3.3 (#8294)
add a1509e25f5c [HUDI-5891] Fix clustering on bootstrapped tables (#8206)
add cd4f7f551b8 [HUDI-5984] Enable FT for spark3.x versions in CI (#8293)
add 192ee432ee2 Remove duplicated WriteOperationType.INSERT.value (#7999)
add 04ec593413e [HUDI-5977] Fix Date to String column schema evolution
(#8280)
add e1741ddc7d9 [MINOR] Fix typo for method in AvroSchemaConverter (#8306)
add 2023302ebee [HUDI-5986] Empty preCombineKey should never be stored in
hoodie.properties (#8296)
add 7243393c688 [HUDI-5952] Fix NPE when use kafka callback (#8227)
add 21f83594a9c [HUDI-5976] Add fs in the constructor of
HoodieAvroHFileReader (#8277)
add 16848bc46cb [HUDI-5928] Fixing shutting down deltastreamer properly
when post write termination strategy is enabled (#8173)
add f4f329e1d19 Fix typo in use of ShellOption (#7963)
add f076c0a8a4a [HUDI-5993] Connection leak for lock provider (#8304)
add 8310e250ff7 [HUDI-5992] Fix (de)serialization for avro versions >
1.10.0 (#8307)
add 2e7a93cabb9 [HUDI-6003] Add schema evolution nullability test case
(#8320)
add cb1395a820f [HUDI-5893] Mark advanced configs (#8295)
add 09f5d4f583b [HUDI-5907] Allow skip saving checkpoint in deltastreamer
(#8137)
add 4b995a8c5d3 [HUDI-6005] Auto generate client id for Flink multi writer
(#8323)
add f7a3c26960e [HUDI-5999] Fix bootstrap relation for nonpartitioned
bootstrap table (#8317)
add 7c664fd1f82 [HUDI-6010] Always write parquets for insert overwrite
operation (#8339)
add c53d9fbe019 [HUDI-5900] Clean up unused metadata configs (#8125)
add f051c8a0ad2 [HUDI-5929] Automatically infer key generator type (#8176)
add e9c5b58838d [MINOR] Remove unnecessary KryoSerializable interface in
HoodieSparkReocrd class signature (#8333)
add b752df2a2f9 [HUDI-5921] Partition path should be considered in
BucketIndexConcurrentFileWritesConflictResolutionStrategy (#8163)
add 6fd885fb3dc [HUDI-5740] Refactor Deltastreamer and schema providers to
use HoodieConfig/ConfigProperty (#8152)
add c7397013f64 [HUDI-5954] Infer cleaning policy based on clean configs
(#8238)
add 9c97340e001 Bump protobuf-java from 3.21.5 to 3.21.7 (#6871)
add 1a2a3dec3dc Bump mysql-connector-java
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (c00d18e74a3)
\
N -- N -- N
refs/heads/dependabot/maven/hudi-platform-service/hudi-metaserver/hudi-metaserver-server/mysql-mysql-connector-java-8.0.28
(1a2a3dec3dc)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
No new revisions were added by this update.
Summary of changes:
.github/workflows/bot.yml | 48 +-
README.md | 36 +-
azure-pipelines.yml | 16 +-
doap_HUDI.rdf | 5 +
docker/demo/config/spark-defaults.conf | 2 +
.../demo/config/test-suite/compact-test.properties | 2 +-
.../config/test-suite/multi-writer-1.properties | 2 +-
.../config/test-suite/multi-writer-2.properties | 2 +-
.../test-suite/multi-writer-local-1.properties | 2 +-
.../test-suite/multi-writer-local-2.properties | 2 +-
.../test-suite/multi-writer-local-3.properties | 2 +-
.../test-suite/multi-writer-local-4.properties | 2 +-
.../templates/spark_command.txt.template | 1 +
.../test-suite/templates/test.properties.template | 2 +-
...essive-clean-archival-inline-compact.properties | 2 +-
.../test-aggressive-clean-archival.properties | 2 +-
...essive-clean-archival-inline-compact.properties | 2 +-
...clustering-aggressive-clean-archival.properties | 2 +-
.../test-clustering-inline-compact.properties | 2 +-
...essive-clean-archival-inline-compact.properties | 2 +-
...g-metadata-aggressive-clean-archival.properties | 2 +-
.../config/test-suite/test-clustering.properties | 2 +-
.../test-suite/test-inline-compact.properties | 2 +-
...essive-clean-archival-inline-compact.properties | 2 +-
...t-metadata-aggressive-clean-archival.properties | 2 +-
.../test-metadata-inline-compact.properties | 2 +-
.../config/test-suite/test-metadata.properties | 2 +-
.../test-nonpartitioned-inline-compact.properties | 3 +-
...npartitioned-metadata-inline-compact.properties | 3 +-
.../test-nonpartitioned-metadata.properties | 3 +-
.../test-suite/test-nonpartitioned.properties | 3 +-
docker/demo/config/test-suite/test.properties | 2 +-
docker/hoodie/hadoop/base/pom.xml | 2 +-
docker/hoodie/hadoop/base_java11/pom.xml | 2 +-
docker/hoodie/hadoop/datanode/pom.xml | 2 +-
docker/hoodie/hadoop/historyserver/pom.xml | 2 +-
docker/hoodie/hadoop/hive_base/pom.xml | 2 +-
docker/hoodie/hadoop/namenode/pom.xml | 2 +-
docker/hoodie/hadoop/pom.xml | 2 +-
docker/hoodie/hadoop/prestobase/pom.xml | 2 +-
docker/hoodie/hadoop/spark_base/pom.xml | 2 +-
docker/hoodie/hadoop/sparkadhoc/pom.xml | 2 +-
docker/hoodie/hadoop/sparkmaster/pom.xml | 2 +-
docker/hoodie/hadoop/sparkworker/pom.xml | 2 +-
docker/hoodie/hadoop/trinobase/pom.xml | 2 +-
docker/hoodie/hadoop/trinocoordinator/pom.xml | 2 +-
docker/hoodie/hadoop/trinoworker/pom.xml | 2 +-
hudi-aws/pom.xml | 4 +-
.../hudi/aws/sync/AwsGlueCatalogSyncTool.java | 11 +-
.../hudi/config/DynamoDbBasedLockConfig.java | 11 +-
hudi-cli/pom.xml | 2 +-
.../apache/hudi/cli/HoodieTableHeaderFields.java | 6 +
.../src/main/java/org/apache/hudi/cli/Main.java | 2 +-
.../hudi/cli/commands/ArchivedCommitsCommand.java | 2 +-
.../apache/hudi/cli/commands/CommitsCommand.java | 6 +-
.../apache/hudi/cli/commands/ExportCommand.java | 2 +-
.../hudi/cli/commands/HoodieLogFileCommand.java | 6 +-
.../apache/hudi/cli/commands/RestoresCommand.java | 172 ++++
.../hudi/cli/utils/SparkTempViewProvider.java | 5 +-
.../java/org/apache/hudi/cli/utils/SparkUtil.java | 15 +-
.../cli/commands/TestHoodieLogFileCommand.java | 7 +-
.../hudi/cli/commands/TestRestoresCommand.java | 207 +++++
.../cli/functional/CLIFunctionalTestHarness.java | 4 +-
.../hudi/cli/integ/ITTestRepairsCommand.java | 5 +-
hudi-client/hudi-client-common/pom.xml | 10 +-
.../org/apache/hudi/client/BaseHoodieClient.java | 71 +-
.../hudi/client/BaseHoodieTableServiceClient.java | 875 +++++++++++++++++++++
.../apache/hudi/client/BaseHoodieWriteClient.java | 689 ++++------------
.../apache/hudi/client/CompactionAdminClient.java | 5 -
.../client/HoodieTableServiceManagerClient.java | 169 ++++
.../apache/hudi/client/HoodieTimelineArchiver.java | 4 +-
.../org/apache/hudi/client/RunsTableService.java | 5 +
.../java/org/apache/hudi/client/WriteStatus.java | 13 +-
.../client/embedded/EmbeddedTimelineService.java | 11 +
.../hudi/client/heartbeat/HeartbeatUtils.java | 9 +-
.../client/heartbeat/HoodieHeartbeatClient.java | 25 +-
...urrentFileWritesConflictResolutionStrategy.java | 61 ++
.../client/transaction/ConcurrentOperation.java | 37 +-
.../DirectMarkerTransactionManager.java | 92 +++
...urrentFileWritesConflictResolutionStrategy.java | 9 +-
.../client/transaction/TransactionManager.java | 35 +-
.../hudi/client/transaction/lock/LockManager.java | 8 +-
.../lock/metrics/HoodieLockMetrics.java | 4 +-
.../hudi/client/utils/DeletePartitionUtils.java | 77 ++
.../apache/hudi/client/utils/MergingIterator.java | 12 +-
.../common/table/log/HoodieFileSliceReader.java | 2 +-
.../apache/hudi/config/HoodieArchivalConfig.java | 11 +-
.../apache/hudi/config/HoodieBootstrapConfig.java | 12 +-
.../org/apache/hudi/config/HoodieCleanConfig.java | 88 ++-
.../apache/hudi/config/HoodieClusteringConfig.java | 11 +
.../apache/hudi/config/HoodieCompactionConfig.java | 14 +-
.../apache/hudi/config/HoodieErrorTableConfig.java | 100 +++
.../apache/hudi/config/HoodieHBaseIndexConfig.java | 1 +
.../org/apache/hudi/config/HoodieIndexConfig.java | 37 +-
.../org/apache/hudi/config/HoodieLockConfig.java | 27 +-
.../org/apache/hudi/config/HoodieMemoryConfig.java | 16 +-
.../config/HoodieWriteCommitCallbackConfig.java | 5 +-
.../org/apache/hudi/config/HoodieWriteConfig.java | 372 +++++++--
.../hudi/config/metrics/HoodieMetricsConfig.java | 7 +
.../metrics/HoodieMetricsPrometheusConfig.java | 11 +
.../hudi/execution/HoodieLazyInsertIterable.java | 30 +-
.../org/apache/hudi/index/HoodieIndexUtils.java | 5 +-
.../index/bloom/BaseHoodieBloomIndexHelper.java | 5 +-
.../apache/hudi/index/bloom/HoodieBloomIndex.java | 77 +-
...ion.java => HoodieBloomIndexCheckFunction.java} | 59 +-
.../hudi/index/bloom/HoodieGlobalBloomIndex.java | 14 +-
.../bloom/ListBasedHoodieBloomIndexHelper.java | 26 +-
.../hudi/index/bloom/ListBasedIndexFileFilter.java | 2 +-
.../apache/hudi/index/bucket/BucketIdentifier.java | 6 +-
.../org/apache/hudi/io/HoodieAppendHandle.java | 32 +-
.../org/apache/hudi/io/HoodieBootstrapHandle.java | 24 +-
.../java/org/apache/hudi/io/HoodieCDCLogger.java | 9 +-
.../org/apache/hudi/io/HoodieCreateHandle.java | 27 +-
.../java/org/apache/hudi/io/HoodieIOHandle.java | 2 +-
.../java/org/apache/hudi/io/HoodieMergeHandle.java | 20 +-
.../apache/hudi/io/HoodieMergeHandleFactory.java | 18 +-
.../java/org/apache/hudi/io/HoodieReadHandle.java | 2 +-
.../apache/hudi/io/HoodieSortedMergeHandle.java | 3 +-
.../java/org/apache/hudi/io/HoodieWriteHandle.java | 28 +-
.../java/org/apache/hudi/keygen/KeyGenUtils.java | 31 +-
.../apache/hudi/keygen/KeylessKeyGenerator.java | 239 ------
.../keygen/NonpartitionedAvroKeyGenerator.java | 5 +-
.../metadata/HoodieBackedTableMetadataWriter.java | 162 ++--
.../org/apache/hudi/metrics/HoodieMetrics.java | 90 ++-
.../java/org/apache/hudi/metrics/MetricUtils.java | 81 ++
.../main/java/org/apache/hudi/metrics/Metrics.java | 136 ++--
.../hudi/metrics/MetricsReporterFactory.java | 14 +-
.../hudi/metrics/datadog/DatadogHttpClient.java | 20 +-
.../metrics/datadog/DatadogMetricsReporter.java | 2 +-
.../hudi/metrics/datadog/DatadogReporter.java | 27 +-
.../prometheus/PushGatewayMetricsReporter.java | 19 +
.../metrics/prometheus/PushGatewayReporter.java | 70 +-
.../apache/hudi/table/BulkInsertPartitioner.java | 2 +-
.../java/org/apache/hudi/table/HoodieTable.java | 118 ++-
.../table/action/bootstrap/BootstrapUtils.java | 15 +-
.../table/action/clean/CleanActionExecutor.java | 4 +-
.../action/clean/CleanPlanActionExecutor.java | 12 +-
.../hudi/table/action/clean/CleanPlanner.java | 81 +-
.../cluster/ClusteringPlanActionExecutor.java | 1 +
.../PartitionAwareClusteringPlanStrategy.java | 2 +
.../table/action/commit/BaseBulkInsertHelper.java | 7 +-
.../action/commit/BaseCommitActionExecutor.java | 38 +-
.../hudi/table/action/commit/BaseDeleteHelper.java | 7 +-
.../hudi/table/action/commit/BaseWriteHelper.java | 14 +-
.../table/action/commit/HoodieDeleteHelper.java | 35 +-
.../table/action/commit/HoodieMergeHelper.java | 69 +-
.../table/action/commit/HoodieWriteHelper.java | 3 +-
.../table/action/commit/ParallelismHelper.java | 41 +-
.../hudi/table/action/compact/CompactHelpers.java | 2 +
.../action/compact/CompactionExecutionHelper.java | 4 +-
.../hudi/table/action/compact/HoodieCompactor.java | 4 +-
.../compact/LogCompactionExecutionHelper.java | 2 +-
.../compact/RunCompactionActionExecutor.java | 4 +-
.../compact/ScheduleCompactionActionExecutor.java | 14 +-
.../BaseHoodieCompactionPlanGenerator.java | 3 +
.../HoodieLogCompactionPlanGenerator.java | 5 +-
.../table/action/index/RunIndexActionExecutor.java | 5 +-
.../action/restore/BaseRestoreActionExecutor.java | 2 +-
.../rollback/BaseRollbackActionExecutor.java | 4 +-
.../table/action/rollback/BaseRollbackHelper.java | 7 +-
.../rollback/BaseRollbackPlanActionExecutor.java | 2 +-
.../rollback/ListingBasedRollbackStrategy.java | 30 +-
.../rollback/MarkerBasedRollbackStrategy.java | 12 +-
.../SerializableHoodieRollbackRequest.java | 2 +-
.../action/savepoint/SavepointActionExecutor.java | 86 +-
.../hudi/table/marker/ConflictDetectionUtils.java | 39 +-
.../hudi/table/marker/DirectWriteMarkers.java | 24 +
.../SimpleDirectMarkerBasedDetectionStrategy.java | 85 ++
...nsactionDirectMarkerBasedDetectionStrategy.java | 65 ++
.../marker/TimelineServerBasedWriteMarkers.java | 77 +-
.../org/apache/hudi/table/marker/WriteMarkers.java | 55 +-
.../storage/HoodieConsistentBucketLayout.java | 1 +
.../table/storage/HoodieSimpleBucketLayout.java | 1 +
.../hudi/table/upgrade/DowngradeHandler.java | 2 +-
.../java/org/apache/hudi/util/ExecutorFactory.java | 52 +-
...urrentFileWritesConflictResolutionStrategy.java | 178 +++++
.../hudi/client/transaction/TestLockManager.java | 111 +++
...urrentFileWritesConflictResolutionStrategy.java | 20 +-
.../client/utils/TestDeletePartitionUtils.java | 110 +++
.../apache/hudi/config/TestHoodieWriteConfig.java | 106 ++-
.../org/apache/hudi/keygen/TestKeyGenUtils.java | 26 +
.../hudi/keygen/TestKeylessKeyGenerator.java | 119 ---
.../factory/TestHoodieAvroKeyGeneratorFactory.java | 3 +-
.../hudi/metrics/TestHoodieConsoleMetrics.java | 15 +-
.../hudi/metrics/TestHoodieGraphiteMetrics.java | 15 +-
.../apache/hudi/metrics/TestHoodieJmxMetrics.java | 40 +-
.../org/apache/hudi/metrics/TestHoodieMetrics.java | 129 +--
.../hudi/metrics/TestMetricsReporterFactory.java | 4 +-
.../datadog/TestDatadogMetricsReporter.java | 33 +-
.../metrics/prometheus/TestPrometheusReporter.java | 11 +-
.../prometheus/TestPushGateWayReporter.java | 121 ++-
.../src/test/resources/datadog.properties | 11 +-
.../src/test/resources/keyless_schema.avsc | 44 --
.../src/test/resources/prometheus.properties | 10 +-
hudi-client/hudi-flink-client/pom.xml | 4 +-
.../hudi/client/HoodieFlinkTableServiceClient.java | 225 ++++++
.../apache/hudi/client/HoodieFlinkWriteClient.java | 271 +++----
.../FlinkSizeBasedClusteringPlanStrategy.java | 32 +-
.../client/common/HoodieFlinkEngineContext.java | 16 +
.../hudi/execution/FlinkLazyInsertIterable.java | 2 +-
.../apache/hudi/index/FlinkHoodieIndexFactory.java | 4 +-
.../apache/hudi/io/FlinkWriteHandleFactory.java | 31 +-
.../io/storage/row/HoodieRowDataCreateHandle.java | 2 +-
.../FlinkHoodieBackedTableMetadataWriter.java | 27 +-
.../hudi/table/HoodieFlinkCopyOnWriteTable.java | 2 +-
.../org/apache/hudi/table/HoodieFlinkTable.java | 12 +-
.../table/action/commit/FlinkDeleteHelper.java | 1 +
.../FlinkDeletePartitionCommitActionExecutor.java | 3 +
.../hudi/table/action/commit/FlinkWriteHelper.java | 3 +-
.../HoodieFlinkMergeOnReadTableCompactor.java | 3 +-
.../apache/hudi/client/TestFlinkWriteClient.java | 64 ++
.../index/bloom/TestFlinkHoodieBloomIndex.java | 8 +-
.../TestFlinkSizeBasedClusteringPlanStrategy.java | 96 +++
hudi-client/hudi-java-client/pom.xml | 10 +-
.../hudi/client/HoodieJavaTableServiceClient.java | 68 ++
.../apache/hudi/client/HoodieJavaWriteClient.java | 9 +-
.../client/common/HoodieJavaEngineContext.java | 21 +-
.../hudi/execution/JavaLazyInsertIterable.java | 2 +-
.../hudi/table/HoodieJavaCopyOnWriteTable.java | 2 +-
.../table/action/commit/JavaBulkInsertHelper.java | 12 +-
.../hudi/table/action/commit/JavaDeleteHelper.java | 1 +
.../hudi/table/action/commit/JavaWriteHelper.java | 1 +
.../HoodieJavaMergeOnReadTableCompactor.java | 3 +-
.../client/TestHoodieJavaWriteClientInsert.java | 46 ++
hudi-client/hudi-spark-client/pom.xml | 11 +-
.../hudi/client/HoodieSparkClusteringClient.java | 2 +-
.../hudi/client/SparkRDDTableServiceClient.java | 298 +++++++
.../apache/hudi/client/SparkRDDWriteClient.java | 227 +-----
.../HoodieSparkBootstrapSchemaProvider.java | 4 +-
.../SparkSizeBasedClusteringPlanStrategy.java | 10 +-
.../MultipleSparkJobExecutionStrategy.java | 11 +-
.../strategy/SingleSparkJobExecutionStrategy.java | 6 +-
.../client/common/HoodieSparkEngineContext.java | 46 +-
.../client/utils/SparkInternalSchemaConverter.java | 4 +-
.../validator/SqlQueryPreCommitValidator.java | 6 +-
.../hudi/common/model/HoodieSparkRecord.java | 107 ++-
.../org/apache/hudi/data/HoodieJavaPairRDD.java | 5 +
.../java/org/apache/hudi/data/HoodieJavaRDD.java | 22 +
.../hudi/execution/SparkLazyInsertIterable.java | 16 +-
.../bloom/BucketizedBloomCheckPartitioner.java | 18 +-
.../bloom/HoodieBloomFilterProbingResult.java | 32 +-
.../index/bloom/HoodieBloomIndexCheckFunction.java | 120 ---
.../index/bloom/HoodieFileProbingFunction.java | 143 ++++
.../HoodieMetadataBloomFilterProbingFunction.java | 157 ++++
.../HoodieMetadataBloomIndexCheckFunction.java | 154 ----
.../index/bloom/SparkHoodieBloomIndexHelper.java | 229 +++++-
.../hudi/io/storage/HoodieSparkFileReader.java | 2 +-
.../hudi/io/storage/HoodieSparkFileWriter.java | 17 -
.../hudi/io/storage/HoodieSparkParquetReader.java | 9 +-
.../hudi/io/storage/HoodieSparkParquetWriter.java | 46 +-
.../apache/hudi/keygen/BuiltinKeyGenerator.java | 2 +-
.../factory/HoodieSparkKeyGeneratorFactory.java | 78 +-
.../SparkHoodieBackedTableMetadataWriter.java | 31 +-
.../hudi/table/HoodieSparkCopyOnWriteTable.java | 4 +-
.../org/apache/hudi/table/HoodieSparkTable.java | 12 +-
.../bootstrap/BaseBootstrapMetadataHandler.java | 9 +-
.../bootstrap/MetadataBootstrapHandlerFactory.java | 1 -
.../bootstrap/OrcBootstrapMetadataHandler.java | 28 +-
.../bootstrap/ParquetBootstrapMetadataHandler.java | 96 ++-
.../SparkBootstrapCommitActionExecutor.java | 5 +-
.../SparkBootstrapDeltaCommitActionExecutor.java | 1 +
.../commit/BaseSparkCommitActionExecutor.java | 9 +-
.../commit/BulkInsertDataInternalWriterHelper.java | 5 +-
.../action/commit/SparkBucketIndexPartitioner.java | 11 +
.../table/action/commit/SparkBulkInsertHelper.java | 24 +-
.../SparkDeletePartitionCommitActionExecutor.java | 3 +
.../SparkInsertOverwriteCommitActionExecutor.java | 14 +
.../commit/SparkInsertOverwritePartitioner.java | 15 +
.../table/action/commit/UpsertPartitioner.java | 19 +-
.../HoodieSparkMergeOnReadTableCompactor.java | 6 +-
.../apache/hudi/util/HoodieSparkRecordUtils.java | 69 --
.../org/apache/hudi/AvroConversionUtils.scala | 28 +-
.../hudi/HoodieDatasetBulkInsertHelper.scala | 29 +-
.../scala/org/apache/hudi/HoodieSparkUtils.scala | 83 +-
.../scala/org/apache/hudi/util/JFunction.scala | 9 +-
...ovider.scala => HoodieSparkKryoRegistrar.scala} | 57 +-
.../scala/org/apache/spark/sql/DataFrameUtil.scala | 2 +-
.../spark/sql/HoodieCatalystExpressionUtils.scala | 74 +-
.../spark/sql/HoodieCatalystPlansUtils.scala | 42 +-
.../apache/spark/sql/HoodieInternalRowUtils.scala | 595 ++++++++------
.../apache/spark/sql/HoodieUnsafeRowUtils.scala | 33 +-
.../org/apache/spark/sql/HoodieUnsafeUtils.scala | 25 +
.../sql/catalyst/expressions/AttributeEq.scala | 32 +-
.../org/apache/spark/sql/hudi/SparkAdapter.scala | 75 +-
.../spark/sql/hudi/execution/RangeSample.scala | 2 +-
.../sql/parser/HoodieExtendedParserInterface.scala | 24 +-
.../org/apache/hudi/client/TestClientRollback.java | 6 +-
.../hudi/client/TestHoodieClientMultiWriter.java | 193 ++++-
.../java/org/apache/hudi/client/TestSavepoint.java | 152 ++++
.../hudi/client/TestSparkRDDWriteClient.java | 168 ++++
.../hudi/client/TestTableSchemaEvolution.java | 125 +--
.../org/apache/hudi/client/TestWriteStatus.java | 92 +++
.../common/TestHoodieSparkEngineContext.java | 51 ++
.../functional/TestHoodieBackedMetadata.java | 375 ++++++---
.../functional/TestHoodieBackedTableMetadata.java | 23 +-
.../TestHoodieClientOnCopyOnWriteStorage.java | 103 ++-
.../TestHoodieClientOnMergeOnReadStorage.java | 79 +-
.../client/functional/TestHoodieMetadataBase.java | 31 +-
.../functional/TestHoodieMetadataBootstrap.java | 6 +-
.../TestRemoteFileSystemViewWithMetadataTable.java | 275 +++++++
.../TestSavepointRestoreCopyOnWrite.java | 173 ++++
.../TestSavepointRestoreMergeOnRead.java | 248 ++++++
.../TestBoundedInMemoryExecutorInSpark.java | 28 +-
.../hudi/execution/TestBoundedInMemoryQueue.java | 23 +-
.../execution/TestDisruptorExecutionInSpark.java | 52 +-
.../hudi/execution/TestDisruptorMessageQueue.java | 30 +-
.../hudi/execution/TestSimpleExecutionInSpark.java | 56 +-
.../TestBulkInsertInternalPartitionerForRows.java | 12 +-
.../bloom/TestBucketizedBloomCheckPartitioner.java | 43 +-
.../hudi/index/bloom/TestHoodieBloomIndex.java | 10 +-
.../index/bloom/TestHoodieGlobalBloomIndex.java | 10 +-
.../hudi/index/bloom/TestKeyRangeLookupTree.java | 2 +-
.../index/hbase/TestSparkHoodieHBaseIndex.java | 22 +-
.../apache/hudi/io/TestHoodieTimelineArchiver.java | 31 +-
.../hudi/table/TestHoodieMergeOnReadTable.java | 59 +-
...TestSparkBuildClusteringGroupsForPartition.java | 93 +++
.../table/action/compact/TestHoodieCompactor.java | 118 ++-
.../table/action/compact/TestInlineCompaction.java | 4 +-
.../strategy/TestHoodieCompactionStrategy.java | 2 +-
.../TestHoodieSparkMergeOnReadTableCompaction.java | 2 +-
.../TestHoodieSparkMergeOnReadTableRollback.java | 4 +-
.../TestMarkerBasedRollbackStrategy.java | 54 +-
.../hudi/testutils/FunctionalTestHarness.java | 4 +-
.../hudi/testutils/HoodieClientTestBase.java | 64 +-
.../hudi/testutils/HoodieClientTestHarness.java | 11 +-
.../hudi/testutils/HoodieClientTestUtils.java | 29 +-
.../SparkClientFunctionalTestHarness.java | 20 +-
.../hudi/testutils/providers/SparkProvider.java | 1 +
.../spark/sql/TestHoodieUnsafeRowUtils.scala | 36 +-
hudi-client/pom.xml | 2 +-
hudi-common/pom.xml | 13 +-
.../org/apache/hudi/BaseHoodieTableFileIndex.java | 4 +-
.../apache/hudi/avro/AvroSchemaCompatibility.java | 15 +-
.../java/org/apache/hudi/avro/AvroSchemaUtils.java | 93 ++-
.../apache/hudi/avro/GenericAvroSerializer.java | 146 ++++
.../java/org/apache/hudi/avro/HoodieAvroUtils.java | 86 +-
.../apache/hudi/avro/MercifulJsonConverter.java | 120 ++-
.../org/apache/hudi/common/HoodieJsonPayload.java | 4 +-
.../bootstrap/index/HFileBootstrapIndex.java | 10 +-
.../hudi/common/config/ConfigClassProperty.java | 6 +-
.../apache/hudi/common/config/ConfigGroups.java | 68 +-
.../apache/hudi/common/config/ConfigProperty.java | 69 +-
.../common/config/DFSPropertiesConfiguration.java | 27 +-
.../apache/hudi/common/config/HoodieConfig.java | 33 +-
.../hudi/common/config/HoodieMetadataConfig.java | 101 +--
.../hudi/common/config/HoodieMetaserverConfig.java | 20 +
.../config/HoodieTableServiceManagerConfig.java | 183 +++++
.../common/config/SerializableConfiguration.java | 1 +
.../DirectMarkerBasedDetectionStrategy.java | 111 +++
.../detection/EarlyConflictDetectionStrategy.java | 48 ++
.../TimelineServerBasedDetectionStrategy.java | 64 ++
.../org/apache/hudi/common/data/HoodieData.java | 73 +-
.../apache/hudi/common/data/HoodieListData.java | 21 +
.../hudi/common/data/HoodieListPairData.java | 23 +
.../apache/hudi/common/data/HoodiePairData.java | 11 +-
.../hudi/common/engine/HoodieEngineContext.java | 6 +
.../common/engine/HoodieLocalEngineContext.java | 16 +
.../apache/hudi/common/fs/ConsistencyGuard.java | 2 +-
.../hudi/common/fs/ConsistencyGuardConfig.java | 4 +-
.../java/org/apache/hudi/common/fs/FSUtils.java | 12 +-
.../hudi/common/fs/FailSafeConsistencyGuard.java | 2 +-
.../common/fs/HoodieRetryWrapperFileSystem.java | 11 +
.../hudi/common/fs/OptimisticConsistencyGuard.java | 2 +-
.../hudi/common/fs/SizeAwareDataOutputStream.java | 4 +-
.../org/apache/hudi/common/fs/StorageSchemes.java | 67 +-
.../ThrowingConsumer.java} | 18 +-
.../common/heartbeat/HoodieHeartbeatUtils.java | 75 ++
.../common/model/DefaultHoodieRecordPayload.java | 36 +-
.../org/apache/hudi/common/model/DeleteRecord.java | 18 +-
.../hudi/common/model/HoodieAvroIndexedRecord.java | 34 +-
.../apache/hudi/common/model/HoodieAvroRecord.java | 42 +-
.../hudi/common/model/HoodieAvroRecordMerger.java | 15 +-
.../hudi/common/model/HoodieEmptyRecord.java | 9 +-
.../org/apache/hudi/common/model/HoodieKey.java | 28 +-
.../org/apache/hudi/common/model/HoodieRecord.java | 20 +-
.../apache/hudi/common/model/MetadataValues.java | 76 +-
.../debezium/AbstractDebeziumAvroPayload.java | 16 +-
.../hudi/common/table/HoodieTableConfig.java | 53 +-
.../hudi/common/table/HoodieTableMetaClient.java | 65 +-
.../hudi/common/table/TableSchemaResolver.java | 36 -
.../hudi/common/table/cdc/HoodieCDCExtractor.java | 20 +-
.../hudi/common/table/cdc/HoodieCDCFileSplit.java | 22 +-
...CInferCase.java => HoodieCDCInferenceCase.java} | 8 +-
.../cdc/HoodieCDCSupplementalLoggingMode.java | 37 +-
.../hudi/common/table/cdc/HoodieCDCUtils.java | 14 +-
.../table/log/AbstractHoodieLogRecordReader.java | 252 +++---
.../table/log/HoodieCDCLogRecordIterator.java | 6 +-
.../hudi/common/table/log/HoodieLogFileReader.java | 22 +-
.../table/log/HoodieMergedLogRecordScanner.java | 209 +++--
.../table/log/HoodieUnMergedLogRecordScanner.java | 23 +-
.../table/log/block/HoodieAvroDataBlock.java | 9 +-
.../common/table/log/block/HoodieDataBlock.java | 2 +-
.../common/table/log/block/HoodieDeleteBlock.java | 2 +
.../table/log/block/HoodieHFileDataBlock.java | 16 +-
.../table/log/block/HoodieParquetDataBlock.java | 9 +-
.../hudi/common/table/marker/MarkerOperation.java | 1 +
.../table/timeline/HoodieActiveTimeline.java | 2 +-
.../table/timeline/HoodieArchivedTimeline.java | 2 +-
.../table/view/AbstractTableFileSystemView.java | 233 ++++--
.../table/view/FileSystemViewStorageConfig.java | 1 +
.../table/view/HoodieTableFileSystemView.java | 3 +-
.../IncrementalTimelineSyncFileSystemView.java | 20 +-
.../table/view/PriorityBasedFileSystemView.java | 7 +
.../view/RemoteHoodieTableFileSystemView.java | 45 +-
.../common/table/view/TableFileSystemView.java | 11 +
.../org/apache/hudi/common/util/BaseFileUtils.java | 1 +
.../apache/hudi/common/util/ClusteringUtils.java | 35 +-
.../apache/hudi/common/util/CollectionUtils.java | 27 +-
.../org/apache/hudi/common/util/CommitUtils.java | 23 +-
...rovider.java => HoodieCommonKryoRegistrar.java} | 13 +-
.../apache/hudi/common/util/HoodieRecordUtils.java | 4 +-
.../apache/hudi/common/util/MappingIterator.java | 47 --
.../org/apache/hudi/common/util/MarkerUtils.java | 116 ++-
.../apache/hudi/common/util/OrcReaderIterator.java | 1 +
.../java/org/apache/hudi/common/util/OrcUtils.java | 1 +
.../hudi/common/util/ParquetReaderIterator.java | 1 +
.../org/apache/hudi/common/util/ParquetUtils.java | 35 +-
.../apache/hudi/common/util/ReflectionUtils.java | 12 +
.../hudi/common/util/SerializationUtils.java | 7 +-
.../org/apache/hudi/common/util/StringUtils.java | 16 +
.../apache/hudi/common/util/TablePathUtils.java | 4 +
.../apache/hudi/common/util/ValidationUtils.java | 9 +
.../common/util/collection/BitCaskDiskMap.java | 4 +-
.../util/{ => collection}/ClosableIterator.java | 23 +-
.../util/collection/CloseableMappingIterator.java | 9 +-
.../util/collection/ExternalSpillableMap.java | 11 +-
.../common/util/collection/FlatteningIterator.java | 55 ++
.../common/util/collection/LazyFileIterable.java | 1 -
.../common/util/collection/MappingIterator.java | 4 +-
.../hudi/common/util/collection/RocksDBDAO.java | 3 +-
.../util/queue/BaseHoodieQueueBasedExecutor.java | 3 +
.../common/util/queue/BoundedInMemoryExecutor.java | 2 +-
.../hudi/common/util/queue/DisruptorExecutor.java | 28 +-
.../common/util/queue/DisruptorMessageQueue.java | 49 +-
.../hudi/common/util/queue/ExecutorType.java | 15 +-
.../util/queue/IteratorBasedQueueProducer.java | 2 +-
...mpleHoodieExecutor.java => SimpleExecutor.java} | 51 +-
.../common/util/queue/WaitStrategyFactory.java | 8 +-
.../HoodieEarlyConflictDetectionException.java} | 35 +-
.../hudi/internal/schema/InternalSchema.java | 27 +-
.../hudi/internal/schema/action/TableChange.java | 6 +
.../hudi/internal/schema/action/TableChanges.java | 23 +-
.../schema/utils/AvroSchemaEvolutionUtils.java | 66 +-
.../internal/schema/utils/InternalSchemaUtils.java | 4 +-
.../internal/schema/utils/SchemaChangeUtils.java | 9 +-
.../internal/schema/visitor/NameToIDVisitor.java | 2 +-
.../hudi/io/storage/HoodieAvroFileReaderBase.java | 6 +-
.../hudi/io/storage/HoodieAvroHFileReader.java | 8 +-
.../hudi/io/storage/HoodieAvroOrcReader.java | 2 +-
.../hudi/io/storage/HoodieAvroParquetReader.java | 10 +-
.../hudi/io/storage/HoodieBaseParquetWriter.java | 38 +-
.../apache/hudi/io/storage/HoodieFileReader.java | 2 +-
.../hudi/io/storage/HoodieSeekingFileReader.java | 2 +-
.../hudi/keygen/constant/KeyGeneratorOptions.java | 6 -
.../apache/hudi/metadata/BaseTableMetadata.java | 53 +-
.../metadata/FileSystemBackedTableMetadata.java | 15 +-
.../hudi/metadata/HoodieBackedTableMetadata.java | 154 ++--
.../metadata/HoodieMetadataFileSystemView.java | 50 +-
.../metadata/HoodieMetadataLogRecordReader.java | 238 ++++++
.../HoodieMetadataMergedLogRecordReader.java | 254 ------
.../apache/hudi/metadata/HoodieTableMetadata.java | 10 +-
.../hudi/metadata/HoodieTableMetadataUtil.java | 20 +
.../hudi/parquet/io/ByteBufferBackedInputFile.java | 4 -
.../main/java/org/apache/hudi/util/Transient.java | 108 +++
.../avro/HoodieAvroParquetReaderBuilder.java | 79 ++
.../apache/parquet/avro/HoodieAvroReadSupport.java | 133 ++++
.../org/apache/hudi/avro/TestAvroSchemaUtils.java | 57 ++
.../org/apache/hudi/avro/TestHoodieAvroUtils.java | 11 +
.../hudi/avro/TestMercifulJsonConverter.java | 102 +++
.../hudi/common/config/HoodieTestFakeConfig.java | 96 +++
.../hudi/common/config/TestConfigProperty.java | 54 +-
.../hudi/common/config/TestHoodieConfig.java | 70 ++
.../fs/TestFSUtilsWithRetryWrapperEnable.java | 22 +
.../common/functional/TestHoodieLogFormat.java | 457 +++++++++--
.../TestHoodieLogFormatAppendFailure.java | 4 +-
.../model/TestDefaultHoodieRecordPayload.java | 65 ++
.../apache/hudi/common/model/TestHoodieRecord.java | 3 +-
.../debezium/TestPostgresDebeziumAvroPayload.java | 19 +
.../table/view/TestHoodieTableFileSystemView.java | 4 +-
.../hudi/common/testutils/FileCreateUtils.java | 16 +-
.../common/testutils/HoodieTestDataGenerator.java | 29 +-
.../hudi/common/testutils/SchemaTestUtil.java | 25 +-
.../util/TestDFSPropertiesConfiguration.java | 4 +-
.../hudi/common/util/TestReflectionUtils.java | 45 ++
.../common/util/collection/TestBitCaskDiskMap.java | 26 +-
.../util/collection/TestExternalSpillableMap.java | 27 +-
.../hudi/common/util/collection/TestIterators.java | 48 ++
.../util/collection/TestRocksDbBasedMap.java | 3 +-
.../common/util/collection/TestRocksDbDiskMap.java | 14 +-
.../io/storage/TestHoodieBaseParquetWriter.java | 122 +++
.../HoodieBackedTestDelayedTableMetadata.java | 54 ++
hudi-examples/bin/hudi-delta-streamer | 2 +
hudi-examples/hudi-examples-common/pom.xml | 8 +-
hudi-examples/hudi-examples-flink/pom.xml | 2 +-
.../examples/quickstart/HoodieFlinkQuickstart.java | 9 +-
.../factory/CollectSinkTableFactory.java | 8 +-
.../quickstart/utils/QuickstartConfigurations.java | 16 +-
hudi-examples/hudi-examples-java/pom.xml | 8 +-
hudi-examples/hudi-examples-spark/pom.xml | 2 +-
.../examples/common/HoodieExampleSparkUtils.java | 3 +-
.../examples/quickstart/HoodieSparkQuickstart.java | 18 +-
.../examples/spark/HoodieDataSourceExample.scala | 54 +-
.../examples/spark/HoodieMorCompactionJob.scala | 4 +-
.../quickstart/TestHoodieSparkQuickstart.java | 10 +-
.../src/test/python/HoodiePySparkQuickstart.py | 1 +
hudi-examples/pom.xml | 2 +-
hudi-flink-datasource/hudi-flink/pom.xml | 28 +-
.../apache/hudi/configuration/FlinkOptions.java | 119 ++-
.../hudi/configuration/OptionsInference.java | 28 +
.../apache/hudi/configuration/OptionsResolver.java | 68 +-
.../org/apache/hudi/sink/StreamWriteFunction.java | 20 +-
.../hudi/sink/StreamWriteOperatorCoordinator.java | 35 +-
.../hudi/sink/bootstrap/BootstrapOperator.java | 11 +-
.../sink/bucket/BucketStreamWriteFunction.java | 22 +-
.../hudi/sink/bulk/BulkInsertWriteFunction.java | 1 -
.../org/apache/hudi/sink/bulk/RowDataKeyGen.java | 52 +-
.../hudi/sink/bulk/sort/SortOperatorGen.java | 2 +-
.../hudi/sink/clustering/ClusteringCommitSink.java | 11 +
.../hudi/sink/clustering/ClusteringOperator.java | 11 +-
.../sink/clustering/FlinkClusteringConfig.java | 20 +-
.../sink/clustering/HoodieFlinkClusteringJob.java | 5 +-
.../sink/common/AbstractStreamWriteFunction.java | 2 +-
.../apache/hudi/sink/compact/CompactOperator.java | 1 -
.../hudi/sink/compact/CompactionCommitSink.java | 11 +
.../hudi/sink/compact/FlinkCompactionConfig.java | 6 +-
.../hudi/sink/compact/HoodieFlinkCompactor.java | 3 +
.../org/apache/hudi/sink/meta/CkpMetadata.java | 32 +-
.../sink/partitioner/BucketIndexPartitioner.java | 5 +-
.../java/org/apache/hudi/sink/utils/Pipelines.java | 18 +-
.../java/org/apache/hudi/source/DataPruner.java | 140 ++++
.../apache/hudi/source/ExpressionEvaluators.java | 586 ++++++++++++++
.../java/org/apache/hudi/source/FileIndex.java | 46 +-
.../apache/hudi/source/IncrementalInputSplits.java | 179 +++--
.../org/apache/hudi/source/stats/ColumnStats.java | 72 ++
.../hudi/source/stats/ExpressionEvaluator.java | 556 -------------
.../apache/hudi/streamer/FlinkStreamerConfig.java | 31 +-
.../apache/hudi/streamer/HoodieFlinkStreamer.java | 1 +
.../org/apache/hudi/table/HoodieTableFactory.java | 72 +-
.../org/apache/hudi/table/HoodieTableSink.java | 2 +
.../org/apache/hudi/table/HoodieTableSource.java | 43 +-
.../apache/hudi/table/catalog/HiveSchemaUtils.java | 8 +-
.../apache/hudi/table/catalog/HoodieCatalog.java | 42 +-
.../hudi/table/catalog/HoodieHiveCatalog.java | 17 +-
.../hudi/table/catalog/TableOptionProperties.java | 25 +-
.../hudi/table/format/InternalSchemaManager.java | 57 +-
.../table/format/ParquetSplitRecordIterator.java | 2 +-
.../apache/hudi/table/format/RecordIterators.java | 10 +-
.../table/format/SchemaEvolvedRecordIterator.java | 2 +-
.../hudi/table/format/cdc/CdcInputFormat.java | 8 +-
.../table/format/cow/CopyOnWriteInputFormat.java | 2 +-
.../table/format/mor/MergeOnReadInputFormat.java | 4 +-
.../org/apache/hudi/util/AvroSchemaConverter.java | 4 +-
.../main/java/org/apache/hudi/util/ClientIds.java | 276 +++++++
.../java/org/apache/hudi/util/ClusteringUtil.java | 4 +-
.../java/org/apache/hudi/util/CompactionUtil.java | 19 +-
.../org/apache/hudi/util/FlinkWriteClients.java | 26 +-
.../apache/hudi/util/RowDataToAvroConverters.java | 4 +-
.../java/org/apache/hudi/util/StreamerUtil.java | 29 +-
.../apache/hudi/util/ViewStorageProperties.java | 13 +-
.../hudi/configuration/TestOptionsInference.java | 78 ++
.../apache/hudi/sink/ITTestDataStreamWrite.java | 54 +-
.../sink/TestStreamWriteOperatorCoordinator.java | 10 +-
.../org/apache/hudi/sink/TestWriteCopyOnWrite.java | 80 +-
.../org/apache/hudi/sink/TestWriteMergeOnRead.java | 10 +
.../hudi/sink/TestWriteMergeOnReadWithCompact.java | 11 +
.../hudi/sink/bucket/ITTestBucketStreamWrite.java | 184 +++++
.../apache/hudi/sink/bulk/TestRowDataKeyGen.java | 43 +
.../sink/cluster/ITTestHoodieFlinkClustering.java | 133 ++++
.../apache/hudi/sink/compact/TestCustomSerDe.java | 81 ++
.../org/apache/hudi/sink/meta/TestCkpMetadata.java | 21 +-
.../utils/BucketStreamWriteFunctionWrapper.java | 212 +++++
.../org/apache/hudi/sink/utils/TestWriteBase.java | 68 +-
.../hudi/source/TestExpressionEvaluators.java | 412 ++++++++++
.../hudi/source/TestIncrementalInputSplits.java | 80 ++
.../hudi/source/stats/TestExpressionEvaluator.java | 374 ---------
.../apache/hudi/table/ITTestHoodieDataSource.java | 140 +++-
.../apache/hudi/table/TestHoodieTableFactory.java | 122 ++-
.../hudi/table/catalog/TestHoodieCatalog.java | 43 +-
.../hudi/table/catalog/TestHoodieHiveCatalog.java | 33 +
.../apache/hudi/table/format/TestInputFormat.java | 100 ++-
.../org/apache/hudi/utils/TestClusteringUtil.java | 4 +
.../org/apache/hudi/utils/TestCompactionUtil.java | 13 +
.../test/java/org/apache/hudi/utils/TestData.java | 13 +
.../hudi/utils/TestViewStorageProperties.java | 10 +-
.../utils/factory/CollectSinkTableFactory.java | 11 +-
.../resources/test_read_schema_dropped_age.avsc | 51 +-
hudi-flink-datasource/hudi-flink1.13.x/pom.xml | 4 +-
.../table/format/cow/ParquetSplitReaderUtil.java | 2 +-
.../vector/reader/FixedLenBytesColumnReader.java | 2 +-
hudi-flink-datasource/hudi-flink1.14.x/pom.xml | 4 +-
.../table/format/cow/ParquetSplitReaderUtil.java | 2 +-
.../vector/reader/FixedLenBytesColumnReader.java | 2 +-
hudi-flink-datasource/hudi-flink1.15.x/pom.xml | 4 +-
.../table/format/cow/ParquetSplitReaderUtil.java | 2 +-
.../vector/reader/FixedLenBytesColumnReader.java | 2 +-
hudi-flink-datasource/hudi-flink1.16.x/pom.xml | 4 +-
.../table/format/cow/ParquetSplitReaderUtil.java | 2 +-
.../vector/reader/FixedLenBytesColumnReader.java | 2 +-
.../reader/ParquetColumnarRowSplitReader.java | 3 +-
hudi-flink-datasource/pom.xml | 4 +-
hudi-gcp/pom.xml | 2 +-
.../hudi/gcp/bigquery/BigQuerySyncConfig.java | 70 +-
.../hudi/gcp/bigquery/TestBigQuerySyncConfig.java | 86 +-
.../gcp/bigquery/TestBigQuerySyncToolArgs.java | 70 ++
hudi-hadoop-mr/pom.xml | 8 +-
.../HoodieMergeOnReadTableInputFormat.java | 4 +-
.../realtime/RealtimeCompactedRecordReader.java | 2 +-
.../apache/hudi/hadoop/TestInputPathHandler.java | 2 +-
.../realtime/TestHoodieRealtimeRecordReader.java | 1 +
hudi-integ-test/README.md | 6 +
hudi-integ-test/pom.xml | 12 +-
.../testsuite/HoodieContinousTestSuiteWriter.java | 10 +-
.../testsuite/HoodieDeltaStreamerWrapper.java | 8 +-
.../testsuite/HoodieInlineTestSuiteWriter.java | 14 +-
.../integ/testsuite/dag/nodes/HiveQueryNode.java | 6 +-
.../reader/DFSHoodieDatasetInputReader.java | 8 +-
.../testsuite/dag/nodes/SparkDeleteNode.scala | 2 +-
.../dag/nodes/SparkDeletePartitionNode.scala | 2 +-
.../testsuite/dag/nodes/SparkInsertNode.scala | 2 +-
hudi-kafka-connect/pom.xml | 11 +-
.../hudi-metaserver/hudi-metaserver-client/pom.xml | 21 +-
.../common/table/HoodieTableMetaserverClient.java | 8 +-
.../timeline/HoodieMetaserverBasedTimeline.java | 3 +-
.../metaserver/client/HoodieMetaserverClient.java | 3 +-
.../client/HoodieMetaserverClientImp.java | 3 +-
.../client/HoodieMetaserverClientProxy.java | 2 +-
.../hudi-metaserver/hudi-metaserver-server/pom.xml | 79 +-
.../apache/hudi/metaserver/HoodieMetaserver.java | 17 +-
.../metaserver/HoodieMetaserverPreparations.java | 25 +-
...erService.java => HoodieMetaserverGateway.java} | 28 +-
.../service/HoodieMetaserverProxyHandler.java | 8 +-
.../hudi/metaserver/service/TableService.java | 10 +-
.../hudi/metaserver/service/TimelineService.java | 22 +-
.../hudi/metaserver/store/MetaserverStorage.java | 3 +-
.../metaserver/store/RelationalDBBasedStorage.java | 46 +-
.../hudi/metaserver/store/bean/InstantBean.java | 7 +
.../hudi/metaserver/store/bean/TableBean.java | 29 +-
.../metaserver/store/jdbc/BatchDaoOperation.java | 4 -
.../src/main/resources/hikariPool.properties | 7 +-
.../src/main/resources/mybatis-config.xml | 15 +-
.../src/main/resources/mybatis/DDLMapper.xml | 2 +-
.../src/main/resources/mybatis/TableMapper.xml | 2 +-
.../store/TestRelationalDBBasedStore.java | 14 +-
hudi-platform-service/hudi-metaserver/pom.xml | 159 ++--
.../src/main/thrift/bin/thrift_binary.sh | 16 +-
.../src/main/thrift/bin/thrift_in_docker.sh | 8 +-
.../src/main/thrift/bin/thrift_in_mac_m1.sh | 5 +-
.../src/main/thrift/hudi-metaserver.thrift | 20 +-
hudi-platform-service/pom.xml | 2 +-
hudi-spark-datasource/hudi-spark-common/pom.xml | 10 +-
.../scala/org/apache/hudi/AvroProjection.scala | 36 +-
.../org/apache/hudi/BaseFileOnlyRelation.scala | 46 +-
.../org/apache/hudi/ColumnStatsIndexSupport.scala | 37 +-
.../scala/org/apache/hudi/DataSourceOptions.scala | 122 +--
.../main/scala/org/apache/hudi/DefaultSource.scala | 27 +-
.../scala/org/apache/hudi/HoodieBaseRelation.scala | 276 ++++---
.../scala/org/apache/hudi/HoodieBootstrapRDD.scala | 103 ++-
.../org/apache/hudi/HoodieBootstrapRelation.scala | 300 +++----
.../scala/org/apache/hudi/HoodieCLIUtils.scala | 10 +-
.../scala/org/apache/hudi/HoodieFileIndex.scala | 25 +-
.../org/apache/hudi/HoodieMergeOnReadRDD.scala | 65 +-
.../org/apache/hudi/HoodieSparkSqlWriter.scala | 344 ++++----
.../org/apache/hudi/HoodieStreamingSink.scala | 28 +-
.../scala/org/apache/hudi/HoodieWriterUtils.scala | 27 +-
.../org/apache/hudi/IncrementalRelation.scala | 4 +-
.../src/main/scala/org/apache/hudi/Iterators.scala | 109 +--
.../hudi/MergeOnReadIncrementalRelation.scala | 20 +-
.../apache/hudi/MergeOnReadSnapshotRelation.scala | 72 +-
.../scala/org/apache/hudi/SafeAvroProjection.scala | 73 --
.../apache/hudi/SparkHoodieTableFileIndex.scala | 6 +-
.../scala/org/apache/hudi/cdc/CDCRelation.scala | 21 +-
.../scala/org/apache/hudi/cdc/HoodieCDCRDD.scala | 131 ++-
.../datasources/HoodieInMemoryFileIndex.scala | 4 +-
.../sql/catalyst/catalog/HoodieCatalogTable.scala | 2 +-
.../plans/logical/HoodieUnaryLikeSham.scala | 5 +-
.../apache/spark/sql/hive/HiveClientUtils.scala | 17 +-
.../apache/spark/sql/hudi/HoodieOptionConfig.scala | 22 +-
.../spark/sql/hudi/HoodieSqlCommonUtils.scala | 76 +-
.../spark/sql/hudi/ProvidesHoodieConfig.scala | 331 ++++----
.../AlterHoodieTableChangeColumnCommand.scala | 2 +-
.../hudi/command/CreateHoodieTableCommand.scala | 2 +-
.../hudi/command/HoodieLeafRunnableCommand.scala | 16 +-
.../sql/hudi/streaming/HoodieStreamSource.scala | 2 +-
.../HoodieBulkInsertInternalWriterTestBase.java | 3 +-
.../spark/sql/hive/TestHiveClientUtils.scala} | 31 +-
hudi-spark-datasource/hudi-spark/pom.xml | 4 +-
.../org/apache/hudi/cli/ArchiveExecutorUtils.java | 69 ++
.../apache/hudi/cli/BootstrapExecutorUtils.java | 4 +-
.../apache/hudi/cli/HDFSParquetImporterUtils.java | 2 +-
.../sql/hudi/HoodieSparkSessionExtension.scala | 15 +-
.../org/apache/spark/sql/hudi/HoodieSqlUtils.scala | 51 --
.../spark/sql/hudi/analysis/HoodieAnalysis.scala | 765 ++++++++----------
.../analysis/HoodiePruneFileSourcePartitions.scala | 124 +++
.../command/CreateHoodieTableAsSelectCommand.scala | 7 +-
.../hudi/command/DeleteHoodieTableCommand.scala | 47 +-
.../spark/sql/hudi/command/IndexCommands.scala | 43 +-
.../command/InsertIntoHoodieTableCommand.scala | 18 +-
.../hudi/command/MergeIntoHoodieTableCommand.scala | 850 +++++++++++---------
.../hudi/command/UpdateHoodieTableCommand.scala | 78 +-
.../hudi/command/payload/ExpressionPayload.scala | 157 +++-
.../procedures/ArchiveCommitsProcedure.scala | 79 ++
.../procedures/CreateSavepointProcedure.scala | 8 +-
.../command/procedures/DeleteMarkerProcedure.scala | 8 +-
.../procedures/DeleteSavepointProcedure.scala | 8 +-
.../command/procedures/HiveSyncProcedure.scala | 3 +-
.../hudi/command/procedures/HoodieProcedures.scala | 1 +
.../RollbackToInstantTimeProcedure.scala | 50 +-
.../procedures/RollbackToSavepointProcedure.scala | 8 +-
.../command/procedures/RunBootstrapProcedure.scala | 8 +-
.../command/procedures/RunCleanProcedure.scala | 27 +-
.../procedures/RunClusteringProcedure.scala | 31 +-
.../procedures/RunCompactionProcedure.scala | 126 +--
.../ShowHoodieLogFileRecordsProcedure.scala | 2 +-
.../procedures/ShowSavepointsProcedure.scala | 6 +-
.../ValidateMetadataTableFilesProcedure.scala | 2 +-
.../spark/sql/parser/HoodieCommonSqlParser.scala | 25 +-
.../hudi-spark/src/test/java/HoodieJavaApp.java | 19 +-
.../src/test/java/HoodieJavaGenerateApp.java | 14 +-
.../src/test/java/HoodieJavaStreamingApp.java | 15 +-
.../org/apache/hudi/ColumnStatsIndexHelper.java | 7 +-
.../org/apache/hudi/functional/TestBootstrap.java | 17 +-
.../functional/TestHiveTableSchemaEvolution.java | 55 +-
.../TestHoodieDatasetBulkInsertHelper.java | 4 +-
.../apache/hudi/functional/TestOrcBootstrap.java | 24 +-
.../keygen/TestTimestampBasedKeyGenerator.java | 6 +-
.../TestHoodieSparkKeyGeneratorFactory.java | 56 +-
.../hudi/testutils/HoodieSparkClientTestBase.java | 20 +-
.../src/test/resources/sql-statements.sql | 8 +-
.../hudi/TestAvroSchemaResolutionSupport.scala | 33 +-
.../org/apache/hudi/TestDataSkippingUtils.scala | 6 +-
.../hudi/TestGenericRecordAndRowConsistency.scala | 4 +-
.../org/apache/hudi/TestHoodieFileIndex.scala | 151 +++-
.../apache/hudi/TestHoodieInternalRowUtils.scala | 89 ---
.../org/apache/hudi/TestHoodieSparkSqlWriter.scala | 129 +--
.../org/apache/hudi/TestHoodieSparkUtils.scala | 4 +
.../hudi/TestTableSchemaResolverWithSparkSQL.scala | 6 +-
.../hudi/functional/TestBasicSchemaEvolution.scala | 25 +-
.../apache/hudi/functional/TestCOWDataSource.scala | 345 +++++++-
.../hudi/functional/TestCOWDataSourceStorage.scala | 14 +-
.../hudi/functional/TestColumnStatsIndex.scala | 141 +++-
.../functional/TestDataSourceForBootstrap.scala | 301 ++++---
.../apache/hudi/functional/TestEmptyCommit.scala | 4 +-
.../hudi/functional/TestHoodieActiveTimeline.scala | 4 +-
.../TestIncrementalReadWithFullTableScan.scala | 4 +-
.../hudi/functional/TestLayoutOptimization.scala | 4 +-
.../apache/hudi/functional/TestMORDataSource.scala | 67 +-
.../hudi/functional/TestMORDataSourceStorage.scala | 10 +-
.../TestMORDataSourceWithBucketIndex.scala | 33 +-
.../TestMetadataTableWithSparkDataSource.scala | 17 +-
.../functional/TestParquetColumnProjection.scala | 25 +-
.../functional/TestPartialUpdateAvroPayload.scala | 125 +++
.../hudi/functional/TestStreamingSource.scala | 9 +-
.../hudi/functional/TestStructuredStreaming.scala | 109 ++-
.../hudi/functional/TestTimeTravelQuery.scala | 15 +-
.../hudi/functional/cdc/HoodieCDCTestBase.scala | 36 +-
.../functional/cdc/TestCDCDataFrameSuite.scala | 55 +-
.../functional/cdc/TestCDCStreamingSuite.scala | 16 +-
.../spark/sql/avro/TestSchemaConverters.scala | 2 +-
.../benchmark/BoundInMemoryExecutorBenchmark.scala | 1 +
.../benchmark/CowTableReadBenchmark.scala | 1 +
.../ReadAndWriteWithoutAvroBenchmark.scala | 7 +-
.../spark/sql/hudi/HoodieSparkSqlTestBase.scala | 51 +-
.../sql/hudi/TestAlterTableDropPartition.scala | 135 +++-
.../apache/spark/sql/hudi/TestCDCForSparkSQL.scala | 27 +-
.../apache/spark/sql/hudi/TestCreateTable.scala | 53 +-
.../apache/spark/sql/hudi/TestDeleteTable.scala | 2 -
.../org/apache/spark/sql/hudi/TestDropTable.scala | 11 +
.../sql/hudi/TestHoodieInternalRowUtils.scala} | 99 ++-
.../spark/sql/hudi/TestHoodieOptionConfig.scala | 4 +-
.../apache/spark/sql/hudi/TestInsertTable.scala | 349 +++++---
.../apache/spark/sql/hudi/TestMergeIntoTable.scala | 181 ++++-
.../spark/sql/hudi/TestMergeIntoTable2.scala | 55 +-
.../hudi/TestNestedSchemaPruningOptimization.scala | 130 ++-
.../sql/hudi/TestPartialUpdateForMergeInto.scala | 23 +-
.../org/apache/spark/sql/hudi/TestSpark3DDL.scala | 100 ++-
.../org/apache/spark/sql/hudi/TestSqlConf.scala | 2 +-
.../apache/spark/sql/hudi/TestTruncateTable.scala | 3 -
.../apache/spark/sql/hudi/TestUpdateTable.scala | 34 +
.../TestHoodiePruneFileSourcePartitions.scala | 240 ++++++
.../sql/hudi/command/index/TestIndexSyntax.scala | 8 +-
.../procedure/TestArchiveCommitsProcedure.scala | 71 ++
.../hudi/procedure/TestBootstrapProcedure.scala | 43 +-
.../hudi/procedure/TestClusteringProcedure.scala | 43 +-
.../sql/hudi/procedure/TestRepairsProcedure.scala | 5 +-
.../hudi/procedure/TestSavepointsProcedure.scala | 70 +-
hudi-spark-datasource/hudi-spark2-common/pom.xml | 2 +-
hudi-spark-datasource/hudi-spark2/pom.xml | 4 +-
.../internal/HoodieDataSourceInternalWriter.java | 4 +-
.../sql/HoodieSpark2CatalystExpressionUtils.scala | 60 +-
.../spark/sql/HoodieSpark2CatalystPlanUtils.scala | 35 +-
.../apache/spark/sql/adapter/Spark2Adapter.scala | 48 +-
.../catalyst/analysis/HoodieSpark2Analysis.scala | 208 +++++
.../sql/catalyst/plans/logical/MergeInto.scala | 7 +
.../parquet/Spark24HoodieParquetFileFormat.scala | 6 +-
.../parser/HoodieSpark2ExtendedSqlParser.scala | 3 +-
hudi-spark-datasource/hudi-spark3-common/pom.xml | 2 +-
.../sql/HoodieSpark3CatalystExpressionUtils.scala} | 21 +-
.../spark/sql/HoodieSpark3CatalystPlanUtils.scala | 57 +-
.../spark/sql/adapter/BaseSpark3Adapter.scala | 49 +-
hudi-spark-datasource/hudi-spark3.1.x/pom.xml | 4 +-
.../sql/HoodieSpark31CatalystExpressionUtils.scala | 10 +-
.../spark/sql/HoodieSpark31CatalystPlanUtils.scala | 14 +-
.../apache/spark/sql/adapter/Spark3_1Adapter.scala | 42 +-
.../parquet/Spark31HoodieParquetFileFormat.scala | 6 +-
...lder.scala => HoodieSpark31SqlAstBuilder.scala} | 2 +-
.../parser/HoodieSpark3_1ExtendedSqlParser.scala | 10 +-
hudi-spark-datasource/hudi-spark3.2.x/pom.xml | 4 +-
.../sql/HoodieSpark32CatalystExpressionUtils.scala | 11 +-
.../spark/sql/HoodieSpark32CatalystPlanUtils.scala | 16 +-
.../apache/spark/sql/adapter/Spark3_2Adapter.scala | 28 +-
.../HoodieSpark3_2ExtendedSqlAstBuilder.scala | 3 +-
.../parser/HoodieSpark3_2ExtendedSqlParser.scala | 2 +-
.../hudi-spark3.2plus-common/pom.xml | 2 +-
.../sql/catalyst/plans/logcal/HoodieQuery.scala | 46 +-
.../Spark32PlusHoodieParquetFileFormat.scala | 6 +-
.../hudi/analysis/HoodieSpark32PlusAnalysis.scala | 167 ++++
.../sql/hudi/analysis/HoodieSpark3Analysis.scala | 175 -----
.../spark/sql/hudi/catalog/HoodieCatalog.scala | 2 +-
.../sql/hudi/catalog/HoodieInternalV2Table.scala | 13 +-
.../sql/hudi/logical/TimeTravelRelation.scala | 16 +-
hudi-spark-datasource/hudi-spark3.3.x/pom.xml | 4 +-
.../sql/HoodieSpark33CatalystExpressionUtils.scala | 9 +-
.../spark/sql/HoodieSpark33CatalystPlanUtils.scala | 16 +-
.../apache/spark/sql/adapter/Spark3_3Adapter.scala | 39 +-
.../parser/HoodieSpark3_3ExtendedSqlParser.scala | 6 +-
hudi-spark-datasource/pom.xml | 2 +-
hudi-sync/hudi-adb-sync/pom.xml | 2 +-
hudi-sync/hudi-datahub-sync/pom.xml | 2 +-
.../sync/datahub/config/DataHubSyncConfig.java | 24 +-
hudi-sync/hudi-hive-sync/pom.xml | 2 +-
.../java/org/apache/hudi/hive/HiveSyncConfig.java | 10 +-
.../org/apache/hudi/hive/HiveSyncConfigHolder.java | 7 +-
.../java/org/apache/hudi/hive/HiveSyncTool.java | 29 +-
.../org/apache/hudi/hive/HoodieHiveSyncClient.java | 16 +-
.../org/apache/hudi/hive/ddl/HMSDDLExecutor.java | 15 +-
.../apache/hudi/hive/ddl/HiveQueryDDLExecutor.java | 15 +-
.../hive/replication/GlobalHiveSyncConfig.java | 8 +
.../lock/HiveMetastoreBasedLockProvider.java | 3 +-
.../hudi/hive/util/IMetaStoreClientUtil.java | 47 ++
.../org/apache/hudi/hive/TestHiveSyncTool.java | 164 +++-
.../testutils/HiveSyncFunctionalTestHarness.java | 5 +-
.../apache/hudi/hive/testutils/HiveTestUtil.java | 3 +-
hudi-sync/hudi-sync-common/pom.xml | 8 +-
.../apache/hudi/sync/common/HoodieSyncConfig.java | 9 +
.../apache/hudi/sync/common/util/ConfigUtils.java | 6 +
.../common/util/SparkDataSourceTableUtils.java | 3 +-
.../hudi/sync/common/util/SyncUtilHelpers.java | 4 +-
hudi-sync/pom.xml | 2 +-
hudi-tests-common/pom.xml | 2 +-
hudi-timeline-service/pom.xml | 8 +-
.../hudi/timeline/service/RequestHandler.java | 39 +-
.../hudi/timeline/service/TimelineService.java | 81 +-
.../timeline/service/handlers/BaseFileHandler.java | 11 +
.../timeline/service/handlers/MarkerHandler.java | 117 ++-
.../AsyncTimelineServerBasedDetectionStrategy.java | 86 ++
.../MarkerBasedEarlyConflictDetectionRunnable.java | 124 +++
.../service/handlers/marker/MarkerDirState.java | 69 +-
...tMarkerBasedEarlyConflictDetectionRunnable.java | 133 ++++
hudi-utilities/pom.xml | 2 +-
.../apache/hudi/utilities/HoodieClusteringJob.java | 9 +-
.../hudi/utilities/HoodieDropPartitionsTool.java | 5 +-
.../org/apache/hudi/utilities/HoodieIndexer.java | 4 +-
.../utilities/HoodieMetadataTableValidator.java | 112 ++-
.../hudi/utilities/HoodieSnapshotCopier.java | 5 +-
.../hudi/utilities/HoodieSnapshotExporter.java | 5 +-
.../org/apache/hudi/utilities/UtilHelpers.java | 64 +-
.../kafka/HoodieWriteCommitKafkaCallback.java | 4 +-
.../HoodieWriteCommitKafkaCallbackConfig.java | 1 +
.../HoodieWriteCommitPulsarCallbackConfig.java | 1 +
.../checkpointing/InitialCheckPointProvider.java | 18 +-
.../config/FilebasedSchemaProviderConfig.java | 49 ++
.../utilities/config/HiveSchemaProviderConfig.java | 59 ++
.../config/HoodieDeltaStreamerConfig.java | 89 +++
.../config/HoodieSchemaProviderConfig.java | 82 ++
.../config/JdbcbasedSchemaProviderConfig.java | 77 ++
.../ProtoClassBasedSchemaProviderConfig.java | 69 ++
.../config/SchemaProviderPostProcessorConfig.java | 76 ++
.../utilities/config/SqlTransformerConfig.java | 46 ++
.../deltastreamer/BaseErrorTableWriter.java | 70 ++
.../utilities/deltastreamer/BootstrapExecutor.java | 4 +-
.../hudi/utilities/deltastreamer/DeltaSync.java | 223 ++++--
.../DeltastreamerMultiWriterCkptUpdateFunc.java | 98 +++
.../hudi/utilities/deltastreamer/ErrorEvent.java | 58 ++
.../utilities/deltastreamer/ErrorTableUtils.java | 66 ++
.../deltastreamer/HoodieDeltaStreamer.java | 167 ++--
.../deltastreamer/HoodieDeltaStreamerMetrics.java | 123 ++-
.../HoodieMultiTableDeltaStreamer.java | 57 +-
.../deltastreamer/SourceFormatAdapter.java | 196 ++++-
.../ingestion/HoodieIngestionException.java | 28 +-
.../ingestion/HoodieIngestionMetrics.java | 57 ++
.../ingestion/HoodieIngestionService.java | 185 +++++
.../utilities/schema/FilebasedSchemaProvider.java | 39 +-
.../hudi/utilities/schema/HiveSchemaProvider.java | 23 +-
.../utilities/schema/JdbcbasedSchemaProvider.java | 36 +-
.../utilities/schema/KafkaOffsetPostProcessor.java | 71 ++
.../schema/ProtoClassBasedSchemaProvider.java | 52 +-
.../hudi/utilities/schema/SchemaPostProcessor.java | 9 +-
.../utilities/schema/SchemaRegistryProvider.java | 111 ++-
.../utilities/schema/SparkAvroPostProcessor.java | 5 +-
.../converter/JsonToAvroSchemaConverter.java | 254 ++++++
.../DropColumnSchemaPostProcessor.java | 12 +-
.../add/AddPrimitiveColumnSchemaPostProcessor.java | 15 +-
.../add/BaseSchemaPostProcessorConfig.java | 53 --
.../hudi/utilities/sources/AvroKafkaSource.java | 41 +-
.../sources/GcsEventsHoodieIncrSource.java | 2 +-
.../hudi/utilities/sources/GcsEventsSource.java | 7 +-
.../hudi/utilities/sources/HoodieIncrSource.java | 36 +-
.../hudi/utilities/sources/JsonKafkaSource.java | 47 +-
.../apache/hudi/utilities/sources/KafkaSource.java | 16 +-
.../hudi/utilities/sources/ProtoKafkaSource.java | 13 +-
.../utilities/sources/debezium/DebeziumSource.java | 28 +-
.../sources/debezium/MysqlDebeziumSource.java | 4 +-
.../sources/debezium/PostgresDebeziumSource.java | 4 +-
.../utilities/sources/helpers/AvroConvertor.java | 56 +-
.../utilities/sources/helpers/KafkaOffsetGen.java | 7 +-
.../sources/helpers/SanitizationUtils.java | 198 +++++
.../transform/SqlFileBasedTransformer.java | 16 +-
.../transform/SqlQueryBasedTransformer.java | 13 +-
.../hudi/utilities/TestHiveIncrementalPuller.java | 10 +-
.../apache/hudi/utilities/TestHoodieIndexer.java | 170 +++-
.../hudi/utilities/TestHoodieRepairTool.java | 4 +-
.../hudi/utilities/TestSchemaPostProcessor.java | 27 +-
.../callback/TestKafkaCallbackProvider.java | 90 +++
.../deltastreamer/HoodieDeltaStreamerTestBase.java | 33 +-
.../deltastreamer/TestHoodieDeltaStreamer.java | 245 +++++-
.../TestHoodieDeltaStreamerWithMultiWriter.java | 6 +
.../TestHoodieMultiTableDeltaStreamer.java | 8 +-
.../deltastreamer/TestSourceFormatAdapter.java | 189 +++++
.../schema/TestFilebasedSchemaProvider.java | 99 +++
.../schema/TestProtoClassBasedSchemaProvider.java | 17 +-
.../schema/TestSchemaRegistryProvider.java | 60 +-
.../converter/TestJsonToAvroSchemaConverter.java | 63 ++
.../utilities/sources/BaseTestKafkaSource.java | 4 +-
.../utilities/sources/TestAvroKafkaSource.java | 188 +++++
.../utilities/sources/TestGenericRddTransform.java | 57 ++
.../utilities/sources/TestJsonKafkaSource.java | 173 +++-
.../sources/TestJsonKafkaSourcePostProcessor.java | 14 +-
.../utilities/sources/TestProtoKafkaSource.java | 5 +-
.../debezium/TestAbstractDebeziumSource.java | 4 +-
.../sources/helpers/TestKafkaOffsetGen.java | 4 +-
.../sources/helpers/TestSanitizationUtils.java | 141 ++++
.../utilities/testutils/SanitizationTestUtils.java | 173 ++++
.../src/test/resources/data/avro_sanitization.json | 2 +
.../data/avro_sanitization_bad_naming_in.json | 2 +
...ro_sanitization_bad_naming_nested_array_in.json | 2 +
...o_sanitization_bad_naming_nested_array_out.json | 2 +
...avro_sanitization_bad_naming_nested_map_in.json | 2 +
...vro_sanitization_bad_naming_nested_map_out.json | 2 +
.../data/avro_sanitization_bad_naming_out.json | 2 +
.../file_schema_provider_invalid.avsc | 51 +-
.../file_schema_provider_valid.avsc | 59 +-
.../json/enum-properties/expected.json | 24 +
.../json/enum-properties/input.json | 19 +
.../json/example-address/expected.json | 39 +
.../json/example-address/input.json | 34 +
.../json/example-calendar/expected.json | 60 ++
.../json/example-calendar/input.json | 47 ++
.../json/example-card/expected.json | 162 ++++
.../schema-provider/json/example-card/input.json | 99 +++
.../example-geographical-location/expected.json | 15 +
.../json/example-geographical-location/input.json | 20 +
.../json/multiple-properties/expected.json | 15 +
.../json/multiple-properties/input.json | 18 +
.../json/nested-properties/expected.json | 70 ++
.../json/nested-properties/input.json | 63 ++
.../json/single-properties/expected.json | 11 +
.../json/single-properties/input.json | 14 +
.../schema-provider/proto/oneof_schema.avsc | 17 +
.../parent_schema_recursive_default_limit.avsc | 17 +
.../proto/parent_schema_recursive_depth_2.avsc | 17 +
.../proto/sample_schema_defaults.avsc | 17 +
...ple_schema_wrapped_and_timestamp_as_record.avsc | 17 +
packaging/bundle-validation/ci_run.sh | 1 +
.../bundle-validation/conf/spark-defaults.conf | 1 +
packaging/bundle-validation/flink/insert.sql | 1 +
.../bundle-validation/service/read.scala | 36 +-
.../{spark_hadoop_mr => service}/write.scala | 14 +-
.../bundle-validation/spark_hadoop_mr/write.scala | 1 -
packaging/bundle-validation/validate.sh | 64 +-
packaging/hudi-aws-bundle/pom.xml | 98 +--
packaging/hudi-cli-bundle/hudi-cli-with-bundle.sh | 27 +-
packaging/hudi-cli-bundle/pom.xml | 61 +-
packaging/hudi-datahub-sync-bundle/pom.xml | 98 +--
packaging/hudi-flink-bundle/pom.xml | 139 +---
packaging/hudi-gcp-bundle/pom.xml | 106 +--
packaging/hudi-hadoop-mr-bundle/pom.xml | 134 +---
packaging/hudi-hive-sync-bundle/pom.xml | 111 +--
packaging/hudi-integ-test-bundle/pom.xml | 112 +--
packaging/hudi-kafka-connect-bundle/pom.xml | 131 +--
packaging/hudi-metaserver-server-bundle/pom.xml | 69 +-
packaging/hudi-presto-bundle/pom.xml | 129 +--
packaging/hudi-spark-bundle/pom.xml | 111 +--
packaging/hudi-timeline-server-bundle/pom.xml | 98 +--
packaging/hudi-trino-bundle/pom.xml | 147 ++--
packaging/hudi-utilities-bundle/pom.xml | 98 +--
packaging/hudi-utilities-slim-bundle/README.md | 5 +
packaging/hudi-utilities-slim-bundle/pom.xml | 98 +--
pom.xml | 161 +++-
rfc/README.md | 5 +-
rfc/rfc-41/rfc-41.md | 2 +-
rfc/rfc-46/rfc-46.md | 2 +-
rfc/rfc-48/async_logcompaction_issues.jpeg | Bin 0 -> 67412 bytes
rfc/rfc-48/base_case.jpeg | Bin 0 -> 53991 bytes
...r_logcompaction_written_to_multiple_blocks.jpeg | Bin 0 -> 100262 bytes
rfc/rfc-48/rfc-48.md | 174 ++++
rfc/rfc-48/rollback_block_scenario.jpeg | Bin 0 -> 150939 bytes
rfc/rfc-51/rfc-51.md | 4 +-
rfc/rfc-56/rfc-56.md | 78 +-
scripts/release/create_source_directory.sh | 29 +
scripts/release/create_source_release.sh | 9 +-
scripts/release/cut_release_branch.sh | 20 +-
scripts/release/deploy_staging_jars.sh | 59 +-
scripts/release/validate_source_binary_files.sh | 34 +
scripts/release/validate_source_copyright.sh | 55 ++
scripts/release/validate_source_rat.sh | 24 +
scripts/release/validate_staged_bundles.sh | 4 +
scripts/release/validate_staged_release.sh | 76 +-
1017 files changed, 32964 insertions(+), 13462 deletions(-)
create mode 100644
hudi-cli/src/main/java/org/apache/hudi/cli/commands/RestoresCommand.java
create mode 100644
hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestRestoresCommand.java
create mode 100644
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/BaseHoodieTableServiceClient.java
create mode 100644
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/HoodieTableServiceManagerClient.java
create mode 100644
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/transaction/BucketIndexConcurrentFileWritesConflictResolutionStrategy.java
create mode 100644
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/transaction/DirectMarkerTransactionManager.java
create mode 100644
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/utils/DeletePartitionUtils.java
create mode 100644
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieErrorTableConfig.java
rename
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/index/bloom/{HoodieBaseBloomIndexCheckFunction.java
=> HoodieBloomIndexCheckFunction.java} (61%)
delete mode 100644
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/keygen/KeylessKeyGenerator.java
create mode 100644
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metrics/MetricUtils.java
rename
hudi-common/src/main/java/org/apache/hudi/common/util/ClosableIteratorWithSchema.java
=>
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/commit/ParallelismHelper.java
(51%)
copy packaging/bundle-validation/flink/insert.sql =>
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/marker/ConflictDetectionUtils.java
(51%)
create mode 100644
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/marker/SimpleDirectMarkerBasedDetectionStrategy.java
create mode 100644
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/marker/SimpleTransactionDirectMarkerBasedDetectionStrategy.java
create mode 100644
hudi-client/hudi-client-common/src/test/java/org/apache/hudi/client/transaction/TestBucketIndexConcurrentFileWritesConflictResolutionStrategy.java
create mode 100644
hudi-client/hudi-client-common/src/test/java/org/apache/hudi/client/transaction/TestLockManager.java
create mode 100644
hudi-client/hudi-client-common/src/test/java/org/apache/hudi/client/utils/TestDeletePartitionUtils.java
delete mode 100644
hudi-client/hudi-client-common/src/test/java/org/apache/hudi/keygen/TestKeylessKeyGenerator.java
copy
hudi-platform-service/hudi-metaserver/hudi-metaserver-server/src/main/resources/hikariPool.properties
=> hudi-client/hudi-client-common/src/test/resources/datadog.properties (69%)
delete mode 100644
hudi-client/hudi-client-common/src/test/resources/keyless_schema.avsc
copy
hudi-platform-service/hudi-metaserver/hudi-metaserver-server/src/main/resources/hikariPool.properties
=> hudi-client/hudi-client-common/src/test/resources/prometheus.properties
(71%)
create mode 100644
hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/client/HoodieFlinkTableServiceClient.java
create mode 100644
hudi-client/hudi-flink-client/src/test/java/org/apache/hudi/client/TestFlinkWriteClient.java
create mode 100644
hudi-client/hudi-flink-client/src/test/java/org/apache/hudi/table/action/cluster/strategy/TestFlinkSizeBasedClusteringPlanStrategy.java
create mode 100644
hudi-client/hudi-java-client/src/main/java/org/apache/hudi/client/HoodieJavaTableServiceClient.java
create mode 100644
hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/SparkRDDTableServiceClient.java
copy packaging/bundle-validation/flink/insert.sql =>
hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/index/bloom/HoodieBloomFilterProbingResult.java
(61%)
delete mode 100644
hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/index/bloom/HoodieBloomIndexCheckFunction.java
create mode 100644
hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/index/bloom/HoodieFileProbingFunction.java
create mode 100644
hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/index/bloom/HoodieMetadataBloomFilterProbingFunction.java
delete mode 100644
hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/index/bloom/HoodieMetadataBloomIndexCheckFunction.java
delete mode 100644
hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/util/HoodieSparkRecordUtils.java
rename
hudi-client/hudi-spark-client/src/main/scala/org/apache/spark/{HoodieSparkKryoProvider.scala
=> HoodieSparkKryoRegistrar.scala} (50%)
rename
hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/SerDeUtils.scala
=>
hudi-client/hudi-spark-client/src/main/scala/org/apache/spark/sql/catalyst/expressions/AttributeEq.scala
(53%)
copy hudi-cli/src/main/java/org/apache/hudi/cli/Main.java =>
hudi-client/hudi-spark-client/src/main/scala/org/apache/spark/sql/parser/HoodieExtendedParserInterface.scala
(55%)
create mode 100644
hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/TestSavepoint.java
create mode 100644
hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/TestSparkRDDWriteClient.java
create mode 100644
hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/common/TestHoodieSparkEngineContext.java
create mode 100644
hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestRemoteFileSystemViewWithMetadataTable.java
create mode 100644
hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestSavepointRestoreCopyOnWrite.java
create mode 100644
hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestSavepointRestoreMergeOnRead.java
create mode 100644
hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/action/cluster/strategy/TestSparkBuildClusteringGroupsForPartition.java
create mode 100644
hudi-common/src/main/java/org/apache/hudi/avro/GenericAvroSerializer.java
create mode 100644
hudi-common/src/main/java/org/apache/hudi/common/config/HoodieTableServiceManagerConfig.java
create mode 100644
hudi-common/src/main/java/org/apache/hudi/common/conflict/detection/DirectMarkerBasedDetectionStrategy.java
create mode 100644
hudi-common/src/main/java/org/apache/hudi/common/conflict/detection/EarlyConflictDetectionStrategy.java
create mode 100644
hudi-common/src/main/java/org/apache/hudi/common/conflict/detection/TimelineServerBasedDetectionStrategy.java
copy
hudi-common/src/main/java/org/apache/hudi/common/{util/ClosableIterator.java =>
function/ThrowingConsumer.java} (73%)
create mode 100644
hudi-common/src/main/java/org/apache/hudi/common/heartbeat/HoodieHeartbeatUtils.java
rename
hudi-common/src/main/java/org/apache/hudi/common/table/cdc/{HoodieCDCInferCase.java
=> HoodieCDCInferenceCase.java} (91%)
rename
hudi-common/src/main/java/org/apache/hudi/common/util/{HoodieCommonKryoProvider.java
=> HoodieCommonKryoRegistrar.java} (93%)
delete mode 100644
hudi-common/src/main/java/org/apache/hudi/common/util/MappingIterator.java
rename hudi-common/src/main/java/org/apache/hudi/common/util/{ =>
collection}/ClosableIterator.java (70%)
create mode 100644
hudi-common/src/main/java/org/apache/hudi/common/util/collection/FlatteningIterator.java
rename
hudi-common/src/main/java/org/apache/hudi/common/util/queue/{SimpleHoodieExecutor.java
=> SimpleExecutor.java} (53%)
copy
hudi-common/src/main/java/org/apache/hudi/{common/util/collection/MappingIterator.java
=> exception/HoodieEarlyConflictDetectionException.java} (58%)
create mode 100644
hudi-common/src/main/java/org/apache/hudi/metadata/HoodieMetadataLogRecordReader.java
delete mode 100644
hudi-common/src/main/java/org/apache/hudi/metadata/HoodieMetadataMergedLogRecordReader.java
create mode 100644
hudi-common/src/main/java/org/apache/hudi/util/Transient.java
create mode 100644
hudi-common/src/main/java/org/apache/parquet/avro/HoodieAvroParquetReaderBuilder.java
create mode 100644
hudi-common/src/main/java/org/apache/parquet/avro/HoodieAvroReadSupport.java
create mode 100644
hudi-common/src/test/java/org/apache/hudi/avro/TestMercifulJsonConverter.java
create mode 100644
hudi-common/src/test/java/org/apache/hudi/common/config/HoodieTestFakeConfig.java
create mode 100644
hudi-common/src/test/java/org/apache/hudi/common/config/TestHoodieConfig.java
create mode 100644
hudi-common/src/test/java/org/apache/hudi/common/util/TestReflectionUtils.java
create mode 100644
hudi-common/src/test/java/org/apache/hudi/common/util/collection/TestIterators.java
create mode 100644
hudi-common/src/test/java/org/apache/hudi/io/storage/TestHoodieBaseParquetWriter.java
create mode 100644
hudi-common/src/test/java/org/apache/hudi/metadata/HoodieBackedTestDelayedTableMetadata.java
create mode 100644
hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/source/DataPruner.java
create mode 100644
hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/source/ExpressionEvaluators.java
create mode 100644
hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/source/stats/ColumnStats.java
delete mode 100644
hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/source/stats/ExpressionEvaluator.java
create mode 100644
hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/util/ClientIds.java
create mode 100644
hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/configuration/TestOptionsInference.java
create mode 100644
hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/sink/bucket/ITTestBucketStreamWrite.java
create mode 100644
hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/sink/compact/TestCustomSerDe.java
create mode 100644
hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/sink/utils/BucketStreamWriteFunctionWrapper.java
create mode 100644
hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/source/TestExpressionEvaluators.java
delete mode 100644
hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/source/stats/TestExpressionEvaluator.java
copy
hudi-common/src/main/java/org/apache/hudi/common/util/IdentityIterator.java =>
hudi-flink-datasource/hudi-flink/src/test/resources/test_read_schema_dropped_age.avsc
(59%)
create mode 100644
hudi-gcp/src/test/java/org/apache/hudi/gcp/bigquery/TestBigQuerySyncToolArgs.java
copy hudi-cli/src/main/java/org/apache/hudi/cli/Main.java =>
hudi-platform-service/hudi-metaserver/hudi-metaserver-server/src/main/java/org/apache/hudi/metaserver/HoodieMetaserverPreparations.java
(54%)
rename
hudi-platform-service/hudi-metaserver/hudi-metaserver-server/src/main/java/org/apache/hudi/metaserver/service/{HoodieMetaserverService.java
=> HoodieMetaserverGateway.java} (68%)
mode change 100644 => 100755
hudi-platform-service/hudi-metaserver/src/main/thrift/bin/thrift_in_mac_m1.sh
copy
hudi-client/hudi-spark-client/src/main/scala/org/apache/spark/sql/DataFrameUtil.scala
=>
hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/AvroProjection.scala
(50%)
delete mode 100644
hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/SafeAvroProjection.scala
rename hudi-spark-datasource/{hudi-spark3-common =>
hudi-spark-common}/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/HoodieUnaryLikeSham.scala
(94%)
rename
hudi-spark-datasource/{hudi-spark/src/test/java/org/apache/hudi/functional/SparkRDDWriteClientOverride.java
=>
hudi-spark-common/src/test/scala/org/apache/spark/sql/hive/TestHiveClientUtils.scala}
(50%)
create mode 100644
hudi-spark-datasource/hudi-spark/src/main/java/org/apache/hudi/cli/ArchiveExecutorUtils.java
delete mode 100644
hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/HoodieSqlUtils.scala
create mode 100644
hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/analysis/HoodiePruneFileSourcePartitions.scala
create mode 100644
hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ArchiveCommitsProcedure.scala
copy
hudi-common/src/main/java/org/apache/hudi/common/util/collection/CloseableMappingIterator.java
=>
hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/testutils/HoodieSparkClientTestBase.java
(63%)
delete mode 100644
hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestHoodieInternalRowUtils.scala
create mode 100644
hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestPartialUpdateAvroPayload.scala
rename
hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/{hudi/TestStructTypeSchemaEvolutionUtils.scala
=> spark/sql/hudi/TestHoodieInternalRowUtils.scala} (77%)
create mode 100644
hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/analysis/TestHoodiePruneFileSourcePartitions.scala
create mode 100644
hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/procedure/TestArchiveCommitsProcedure.scala
create mode 100644
hudi-spark-datasource/hudi-spark2/src/main/scala/org/apache/spark/sql/catalyst/analysis/HoodieSpark2Analysis.scala
copy
hudi-spark-datasource/{hudi-spark3.2plus-common/src/main/scala/org/apache/spark/sql/hudi/logical/TimeTravelRelation.scala
=>
hudi-spark3-common/src/main/scala/org/apache/spark/sql/HoodieSpark3CatalystExpressionUtils.scala}
(52%)
rename
hudi-spark-datasource/hudi-spark3.1.x/src/main/scala/org/apache/spark/sql/parser/{HoodieSpark312SqlAstBuilder.scala
=> HoodieSpark31SqlAstBuilder.scala} (98%)
create mode 100644
hudi-spark-datasource/hudi-spark3.2plus-common/src/main/scala/org/apache/spark/sql/hudi/analysis/HoodieSpark32PlusAnalysis.scala
delete mode 100644
hudi-spark-datasource/hudi-spark3.2plus-common/src/main/scala/org/apache/spark/sql/hudi/analysis/HoodieSpark3Analysis.scala
create mode 100644
hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/util/IMetaStoreClientUtil.java
create mode 100644
hudi-timeline-service/src/main/java/org/apache/hudi/timeline/service/handlers/marker/AsyncTimelineServerBasedDetectionStrategy.java
create mode 100644
hudi-timeline-service/src/main/java/org/apache/hudi/timeline/service/handlers/marker/MarkerBasedEarlyConflictDetectionRunnable.java
create mode 100644
hudi-timeline-service/src/test/java/org/apache/hudi/timeline/service/handlers/marker/TestMarkerBasedEarlyConflictDetectionRunnable.java
create mode 100644
hudi-utilities/src/main/java/org/apache/hudi/utilities/config/FilebasedSchemaProviderConfig.java
create mode 100644
hudi-utilities/src/main/java/org/apache/hudi/utilities/config/HiveSchemaProviderConfig.java
create mode 100644
hudi-utilities/src/main/java/org/apache/hudi/utilities/config/HoodieDeltaStreamerConfig.java
create mode 100644
hudi-utilities/src/main/java/org/apache/hudi/utilities/config/HoodieSchemaProviderConfig.java
create mode 100644
hudi-utilities/src/main/java/org/apache/hudi/utilities/config/JdbcbasedSchemaProviderConfig.java
create mode 100644
hudi-utilities/src/main/java/org/apache/hudi/utilities/config/ProtoClassBasedSchemaProviderConfig.java
create mode 100644
hudi-utilities/src/main/java/org/apache/hudi/utilities/config/SchemaProviderPostProcessorConfig.java
create mode 100644
hudi-utilities/src/main/java/org/apache/hudi/utilities/config/SqlTransformerConfig.java
create mode 100644
hudi-utilities/src/main/java/org/apache/hudi/utilities/deltastreamer/BaseErrorTableWriter.java
create mode 100644
hudi-utilities/src/main/java/org/apache/hudi/utilities/deltastreamer/DeltastreamerMultiWriterCkptUpdateFunc.java
create mode 100644
hudi-utilities/src/main/java/org/apache/hudi/utilities/deltastreamer/ErrorEvent.java
create mode 100644
hudi-utilities/src/main/java/org/apache/hudi/utilities/deltastreamer/ErrorTableUtils.java
copy hudi-cli/src/main/java/org/apache/hudi/cli/Main.java =>
hudi-utilities/src/main/java/org/apache/hudi/utilities/ingestion/HoodieIngestionException.java
(61%)
create mode 100644
hudi-utilities/src/main/java/org/apache/hudi/utilities/ingestion/HoodieIngestionMetrics.java
create mode 100644
hudi-utilities/src/main/java/org/apache/hudi/utilities/ingestion/HoodieIngestionService.java
create mode 100644
hudi-utilities/src/main/java/org/apache/hudi/utilities/schema/KafkaOffsetPostProcessor.java
create mode 100644
hudi-utilities/src/main/java/org/apache/hudi/utilities/schema/converter/JsonToAvroSchemaConverter.java
delete mode 100644
hudi-utilities/src/main/java/org/apache/hudi/utilities/schema/postprocessor/add/BaseSchemaPostProcessorConfig.java
create mode 100644
hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/helpers/SanitizationUtils.java
create mode 100644
hudi-utilities/src/test/java/org/apache/hudi/utilities/callback/TestKafkaCallbackProvider.java
create mode 100644
hudi-utilities/src/test/java/org/apache/hudi/utilities/deltastreamer/TestSourceFormatAdapter.java
create mode 100644
hudi-utilities/src/test/java/org/apache/hudi/utilities/schema/TestFilebasedSchemaProvider.java
create mode 100644
hudi-utilities/src/test/java/org/apache/hudi/utilities/schema/converter/TestJsonToAvroSchemaConverter.java
create mode 100644
hudi-utilities/src/test/java/org/apache/hudi/utilities/sources/TestAvroKafkaSource.java
create mode 100644
hudi-utilities/src/test/java/org/apache/hudi/utilities/sources/TestGenericRddTransform.java
create mode 100644
hudi-utilities/src/test/java/org/apache/hudi/utilities/sources/helpers/TestSanitizationUtils.java
create mode 100644
hudi-utilities/src/test/java/org/apache/hudi/utilities/testutils/SanitizationTestUtils.java
create mode 100644
hudi-utilities/src/test/resources/data/avro_sanitization.json
create mode 100644
hudi-utilities/src/test/resources/data/avro_sanitization_bad_naming_in.json
create mode 100644
hudi-utilities/src/test/resources/data/avro_sanitization_bad_naming_nested_array_in.json
create mode 100644
hudi-utilities/src/test/resources/data/avro_sanitization_bad_naming_nested_array_out.json
create mode 100644
hudi-utilities/src/test/resources/data/avro_sanitization_bad_naming_nested_map_in.json
create mode 100644
hudi-utilities/src/test/resources/data/avro_sanitization_bad_naming_nested_map_out.json
create mode 100644
hudi-utilities/src/test/resources/data/avro_sanitization_bad_naming_out.json
copy
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/RunsTableService.java
=>
hudi-utilities/src/test/resources/delta-streamer-config/file_schema_provider_invalid.avsc
(55%)
copy
hudi-common/src/main/java/org/apache/hudi/common/util/IdentityIterator.java =>
hudi-utilities/src/test/resources/delta-streamer-config/file_schema_provider_valid.avsc
(57%)
create mode 100644
hudi-utilities/src/test/resources/schema-provider/json/enum-properties/expected.json
create mode 100644
hudi-utilities/src/test/resources/schema-provider/json/enum-properties/input.json
create mode 100644
hudi-utilities/src/test/resources/schema-provider/json/example-address/expected.json
create mode 100644
hudi-utilities/src/test/resources/schema-provider/json/example-address/input.json
create mode 100644
hudi-utilities/src/test/resources/schema-provider/json/example-calendar/expected.json
create mode 100644
hudi-utilities/src/test/resources/schema-provider/json/example-calendar/input.json
create mode 100644
hudi-utilities/src/test/resources/schema-provider/json/example-card/expected.json
create mode 100644
hudi-utilities/src/test/resources/schema-provider/json/example-card/input.json
create mode 100644
hudi-utilities/src/test/resources/schema-provider/json/example-geographical-location/expected.json
create mode 100644
hudi-utilities/src/test/resources/schema-provider/json/example-geographical-location/input.json
create mode 100644
hudi-utilities/src/test/resources/schema-provider/json/multiple-properties/expected.json
create mode 100644
hudi-utilities/src/test/resources/schema-provider/json/multiple-properties/input.json
create mode 100644
hudi-utilities/src/test/resources/schema-provider/json/nested-properties/expected.json
create mode 100644
hudi-utilities/src/test/resources/schema-provider/json/nested-properties/input.json
create mode 100644
hudi-utilities/src/test/resources/schema-provider/json/single-properties/expected.json
create mode 100644
hudi-utilities/src/test/resources/schema-provider/json/single-properties/input.json
rename
hudi-common/src/main/java/org/apache/hudi/common/util/IdentityIterator.java =>
packaging/bundle-validation/service/read.scala (63%)
copy packaging/bundle-validation/{spark_hadoop_mr => service}/write.scala (78%)
create mode 100644 rfc/rfc-48/async_logcompaction_issues.jpeg
create mode 100644 rfc/rfc-48/base_case.jpeg
create mode 100644
rfc/rfc-48/huge_data_after_logcompaction_written_to_multiple_blocks.jpeg
create mode 100644 rfc/rfc-48/rfc-48.md
create mode 100644 rfc/rfc-48/rollback_block_scenario.jpeg
create mode 100755 scripts/release/create_source_directory.sh
create mode 100755 scripts/release/validate_source_binary_files.sh
create mode 100755 scripts/release/validate_source_copyright.sh
create mode 100755 scripts/release/validate_source_rat.sh
