nsivabalan commented on code in PR #20604: URL: https://github.com/apache/hudi/pull/20604#discussion_r4608307893
########## hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestDeferredRecordIndexInit.scala: ########## @@ -0,0 +1,331 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hudi.functional + +import org.apache.hudi.{DataSourceWriteOptions, SparkDatasetMixin} +import org.apache.hudi.DataSourceWriteOptions._ +import org.apache.hudi.common.config.HoodieMetadataConfig +import org.apache.hudi.common.model.{HoodieRecord, HoodieTableType} +import org.apache.hudi.common.table.{HoodieTableConfig, HoodieTableMetaClient, HoodieTableVersion} +import org.apache.hudi.common.testutils.HoodieTestDataGenerator +import org.apache.hudi.common.util.{Option => HOption} +import org.apache.hudi.config.{HoodieCompactionConfig, HoodieIndexConfig, HoodieWriteConfig} +import org.apache.hudi.functional.TestDeferredRecordIndexInit.{expectedFileGroupCount, DeferredInitTestCase} +import org.apache.hudi.metadata.{HoodieTableMetadata, HoodieTableMetadataUtil, MetadataPartitionType} + +import org.apache.spark.sql.SaveMode +import org.junit.jupiter.api.Tag +import org.junit.jupiter.api.Assertions.{assertEquals, assertFalse, assertTrue} +import org.junit.jupiter.params.ParameterizedTest +import org.junit.jupiter.params.provider.{Arguments, MethodSource} + +import java.util + +import scala.collection.JavaConverters._ + +/** + * End to end coverage of hoodie.metadata.record.level.index.defer.init on its own, with no record threshold: + * the record index is not built by the first commit, is built by the second one and sized from the first commit's + * data, and the switch from the fallback index produces no duplicates. + */ +@Tag("functional") +class TestDeferredRecordIndexInit extends RecordLevelIndexTestBase with SparkDatasetMixin { + + private def deferredInitOptions(testCase: DeferredInitTestCase): Map[String, String] = Map( + HoodieWriteConfig.TBL_NAME.key -> "hoodie_test", + DataSourceWriteOptions.TABLE_TYPE.key -> testCase.tableType.name(), + RECORDKEY_FIELD.key -> "_row_key", + PARTITIONPATH_FIELD.key -> "partition_path", + HoodieTableConfig.ORDERING_FIELDS.key -> "timestamp", + HoodieWriteConfig.WRITE_TABLE_VERSION.key -> testCase.tableVersion.versionCode().toString, + HoodieWriteConfig.AUTO_UPGRADE_VERSION.key -> "false", + HoodieMetadataConfig.ENABLE.key -> "true", + HoodieMetadataConfig.GLOBAL_RECORD_LEVEL_INDEX_ENABLE_PROP.key -> (!testCase.partitioned).toString, + HoodieMetadataConfig.RECORD_LEVEL_INDEX_ENABLE_PROP.key -> testCase.partitioned.toString, + HoodieMetadataConfig.STREAMING_WRITE_ENABLED.key -> testCase.streamingWrites.toString, + // Pinned low so the sizing result is not simply the floor. + HoodieMetadataConfig.GLOBAL_RECORD_LEVEL_INDEX_MIN_FILE_GROUP_COUNT_PROP.key -> "1", + HoodieMetadataConfig.GLOBAL_RECORD_LEVEL_INDEX_MAX_FILE_GROUP_COUNT_PROP.key -> "1000", + HoodieMetadataConfig.RECORD_LEVEL_INDEX_MIN_FILE_GROUP_COUNT_PROP.key -> "1", + HoodieMetadataConfig.RECORD_LEVEL_INDEX_MAX_FILE_GROUP_COUNT_PROP.key -> "1000", + // 9600 bytes / 48 bytes per entry = 200 entries per file group. + HoodieMetadataConfig.RECORD_INDEX_MAX_FILE_GROUP_SIZE_BYTES_PROP.key -> "9600", + HoodieMetadataConfig.DEFER_RLI_INIT_FOR_FRESH_TABLE.key -> "true", + HoodieCompactionConfig.INLINE_COMPACT.key -> "false", + HoodieIndexConfig.INDEX_TYPE.key -> (if (testCase.partitioned) "RECORD_LEVEL_INDEX" else "GLOBAL_RECORD_LEVEL_INDEX")) + + @ParameterizedTest + @MethodSource(Array("testArgsForDeferredInit")) + def testRecordIndexDeferredUntilSecondCommit(testCase: DeferredInitTestCase): Unit = { + val options = deferredInitOptions(testCase) + val dataGen = new HoodieTestDataGenerator() + val allRecords = new util.ArrayList[HoodieRecord[_]]() + val write = (records: util.List[HoodieRecord[_]], operation: String, mode: SaveMode) => + toDataset(spark, records).write.format("hudi").options(options) + .option(DataSourceWriteOptions.OPERATION.key, operation).mode(mode).save(basePath) + + // Commit 1: the record index must not be built, since the table had no completed commit when the write started. + val firstBatch = dataGen.generateInserts("001", 2000) + allRecords.addAll(firstBatch) + write(firstBatch.asInstanceOf[util.List[HoodieRecord[_]]], testCase.firstOperation, SaveMode.Overwrite) + assertTableVersion(testCase.tableVersion) + assertFalse(isRecordIndexAvailable, "Record index should not be built by the first commit") Review Comment: This only reads `tableConfig.getMetadataPartitions`. Please also assert, after commit 1, that (a) the MDT `record_index` partition has no file slices / nothing on storage, (b) the MDT timeline has no RLI writes, and (c) the index metadata has no `record_index` definition. (a) fails today in the partitioned case: `maybeInitializeNewFileGroupsForPartitionedRLI` only checks `isRecordLevelIndexEnabled()`, so min-count file groups get created during the deferral and are cleared at build time. After commit 2, assert the definition exists with the right `isPartitioned` option. ########## hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestDeferredRecordIndexInit.scala: ########## @@ -0,0 +1,331 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hudi.functional + +import org.apache.hudi.{DataSourceWriteOptions, SparkDatasetMixin} +import org.apache.hudi.DataSourceWriteOptions._ +import org.apache.hudi.common.config.HoodieMetadataConfig +import org.apache.hudi.common.model.{HoodieRecord, HoodieTableType} +import org.apache.hudi.common.table.{HoodieTableConfig, HoodieTableMetaClient, HoodieTableVersion} +import org.apache.hudi.common.testutils.HoodieTestDataGenerator +import org.apache.hudi.common.util.{Option => HOption} +import org.apache.hudi.config.{HoodieCompactionConfig, HoodieIndexConfig, HoodieWriteConfig} +import org.apache.hudi.functional.TestDeferredRecordIndexInit.{expectedFileGroupCount, DeferredInitTestCase} +import org.apache.hudi.metadata.{HoodieTableMetadata, HoodieTableMetadataUtil, MetadataPartitionType} + +import org.apache.spark.sql.SaveMode +import org.junit.jupiter.api.Tag +import org.junit.jupiter.api.Assertions.{assertEquals, assertFalse, assertTrue} +import org.junit.jupiter.params.ParameterizedTest +import org.junit.jupiter.params.provider.{Arguments, MethodSource} + +import java.util + +import scala.collection.JavaConverters._ + +/** + * End to end coverage of hoodie.metadata.record.level.index.defer.init on its own, with no record threshold: + * the record index is not built by the first commit, is built by the second one and sized from the first commit's + * data, and the switch from the fallback index produces no duplicates. + */ +@Tag("functional") +class TestDeferredRecordIndexInit extends RecordLevelIndexTestBase with SparkDatasetMixin { + + private def deferredInitOptions(testCase: DeferredInitTestCase): Map[String, String] = Map( + HoodieWriteConfig.TBL_NAME.key -> "hoodie_test", + DataSourceWriteOptions.TABLE_TYPE.key -> testCase.tableType.name(), + RECORDKEY_FIELD.key -> "_row_key", + PARTITIONPATH_FIELD.key -> "partition_path", + HoodieTableConfig.ORDERING_FIELDS.key -> "timestamp", + HoodieWriteConfig.WRITE_TABLE_VERSION.key -> testCase.tableVersion.versionCode().toString, + HoodieWriteConfig.AUTO_UPGRADE_VERSION.key -> "false", + HoodieMetadataConfig.ENABLE.key -> "true", + HoodieMetadataConfig.GLOBAL_RECORD_LEVEL_INDEX_ENABLE_PROP.key -> (!testCase.partitioned).toString, + HoodieMetadataConfig.RECORD_LEVEL_INDEX_ENABLE_PROP.key -> testCase.partitioned.toString, + HoodieMetadataConfig.STREAMING_WRITE_ENABLED.key -> testCase.streamingWrites.toString, + // Pinned low so the sizing result is not simply the floor. + HoodieMetadataConfig.GLOBAL_RECORD_LEVEL_INDEX_MIN_FILE_GROUP_COUNT_PROP.key -> "1", + HoodieMetadataConfig.GLOBAL_RECORD_LEVEL_INDEX_MAX_FILE_GROUP_COUNT_PROP.key -> "1000", + HoodieMetadataConfig.RECORD_LEVEL_INDEX_MIN_FILE_GROUP_COUNT_PROP.key -> "1", + HoodieMetadataConfig.RECORD_LEVEL_INDEX_MAX_FILE_GROUP_COUNT_PROP.key -> "1000", + // 9600 bytes / 48 bytes per entry = 200 entries per file group. + HoodieMetadataConfig.RECORD_INDEX_MAX_FILE_GROUP_SIZE_BYTES_PROP.key -> "9600", + HoodieMetadataConfig.DEFER_RLI_INIT_FOR_FRESH_TABLE.key -> "true", + HoodieCompactionConfig.INLINE_COMPACT.key -> "false", + HoodieIndexConfig.INDEX_TYPE.key -> (if (testCase.partitioned) "RECORD_LEVEL_INDEX" else "GLOBAL_RECORD_LEVEL_INDEX")) + + @ParameterizedTest + @MethodSource(Array("testArgsForDeferredInit")) + def testRecordIndexDeferredUntilSecondCommit(testCase: DeferredInitTestCase): Unit = { + val options = deferredInitOptions(testCase) + val dataGen = new HoodieTestDataGenerator() + val allRecords = new util.ArrayList[HoodieRecord[_]]() + val write = (records: util.List[HoodieRecord[_]], operation: String, mode: SaveMode) => + toDataset(spark, records).write.format("hudi").options(options) + .option(DataSourceWriteOptions.OPERATION.key, operation).mode(mode).save(basePath) + + // Commit 1: the record index must not be built, since the table had no completed commit when the write started. + val firstBatch = dataGen.generateInserts("001", 2000) + allRecords.addAll(firstBatch) + write(firstBatch.asInstanceOf[util.List[HoodieRecord[_]]], testCase.firstOperation, SaveMode.Overwrite) + assertTableVersion(testCase.tableVersion) + assertFalse(isRecordIndexAvailable, "Record index should not be built by the first commit") + + // What the index should be sized from once it is built. + val rowsPerPartition = spark.read.format("hudi").load(basePath).groupBy("_hoodie_partition_path").count().collect() + .map(row => row.getString(0) -> row.getLong(1)).toMap + val rowsBeforeBuild = rowsPerPartition.values.sum + + // Commit 2: updates of records tagged through the fallback index, plus new records. The index is built here. + val updates = dataGen.generateUpdates("002", allRecords.subList(0, 300)) + val newRecords = dataGen.generateInserts("002", 10) + val secondBatch = new util.ArrayList[HoodieRecord[_]](updates) + secondBatch.addAll(newRecords) + allRecords.addAll(newRecords) + write(secondBatch, UPSERT_OPERATION_OPT_VAL, SaveMode.Append) + assertTrue(isRecordIndexAvailable, "Record index should be built by the second commit") + + // Commit 3: upserts tagged through the record index itself. + val laterUpdates = dataGen.generateUpdates("003", allRecords.subList(0, 300)) Review Comment: This updates only first-batch keys again. To exercise the switch, commit 3 should update the keys that commit 2 *inserted*, and re-insert a key that commit 2 deleted. Also assert that the RLI entry count equals the distinct key count after commits 2 and 3. Same ask for the SQL, structured streaming and Streamer cases, which currently have no write after the build. ########## hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestDeferredRecordIndexInit.scala: ########## @@ -0,0 +1,331 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hudi.functional + +import org.apache.hudi.{DataSourceWriteOptions, SparkDatasetMixin} +import org.apache.hudi.DataSourceWriteOptions._ +import org.apache.hudi.common.config.HoodieMetadataConfig +import org.apache.hudi.common.model.{HoodieRecord, HoodieTableType} +import org.apache.hudi.common.table.{HoodieTableConfig, HoodieTableMetaClient, HoodieTableVersion} +import org.apache.hudi.common.testutils.HoodieTestDataGenerator +import org.apache.hudi.common.util.{Option => HOption} +import org.apache.hudi.config.{HoodieCompactionConfig, HoodieIndexConfig, HoodieWriteConfig} +import org.apache.hudi.functional.TestDeferredRecordIndexInit.{expectedFileGroupCount, DeferredInitTestCase} +import org.apache.hudi.metadata.{HoodieTableMetadata, HoodieTableMetadataUtil, MetadataPartitionType} + +import org.apache.spark.sql.SaveMode +import org.junit.jupiter.api.Tag +import org.junit.jupiter.api.Assertions.{assertEquals, assertFalse, assertTrue} +import org.junit.jupiter.params.ParameterizedTest +import org.junit.jupiter.params.provider.{Arguments, MethodSource} + +import java.util + +import scala.collection.JavaConverters._ + +/** + * End to end coverage of hoodie.metadata.record.level.index.defer.init on its own, with no record threshold: + * the record index is not built by the first commit, is built by the second one and sized from the first commit's + * data, and the switch from the fallback index produces no duplicates. + */ +@Tag("functional") +class TestDeferredRecordIndexInit extends RecordLevelIndexTestBase with SparkDatasetMixin { + + private def deferredInitOptions(testCase: DeferredInitTestCase): Map[String, String] = Map( + HoodieWriteConfig.TBL_NAME.key -> "hoodie_test", + DataSourceWriteOptions.TABLE_TYPE.key -> testCase.tableType.name(), + RECORDKEY_FIELD.key -> "_row_key", + PARTITIONPATH_FIELD.key -> "partition_path", + HoodieTableConfig.ORDERING_FIELDS.key -> "timestamp", + HoodieWriteConfig.WRITE_TABLE_VERSION.key -> testCase.tableVersion.versionCode().toString, + HoodieWriteConfig.AUTO_UPGRADE_VERSION.key -> "false", + HoodieMetadataConfig.ENABLE.key -> "true", + HoodieMetadataConfig.GLOBAL_RECORD_LEVEL_INDEX_ENABLE_PROP.key -> (!testCase.partitioned).toString, + HoodieMetadataConfig.RECORD_LEVEL_INDEX_ENABLE_PROP.key -> testCase.partitioned.toString, + HoodieMetadataConfig.STREAMING_WRITE_ENABLED.key -> testCase.streamingWrites.toString, + // Pinned low so the sizing result is not simply the floor. + HoodieMetadataConfig.GLOBAL_RECORD_LEVEL_INDEX_MIN_FILE_GROUP_COUNT_PROP.key -> "1", + HoodieMetadataConfig.GLOBAL_RECORD_LEVEL_INDEX_MAX_FILE_GROUP_COUNT_PROP.key -> "1000", + HoodieMetadataConfig.RECORD_LEVEL_INDEX_MIN_FILE_GROUP_COUNT_PROP.key -> "1", + HoodieMetadataConfig.RECORD_LEVEL_INDEX_MAX_FILE_GROUP_COUNT_PROP.key -> "1000", + // 9600 bytes / 48 bytes per entry = 200 entries per file group. + HoodieMetadataConfig.RECORD_INDEX_MAX_FILE_GROUP_SIZE_BYTES_PROP.key -> "9600", + HoodieMetadataConfig.DEFER_RLI_INIT_FOR_FRESH_TABLE.key -> "true", + HoodieCompactionConfig.INLINE_COMPACT.key -> "false", + HoodieIndexConfig.INDEX_TYPE.key -> (if (testCase.partitioned) "RECORD_LEVEL_INDEX" else "GLOBAL_RECORD_LEVEL_INDEX")) + + @ParameterizedTest + @MethodSource(Array("testArgsForDeferredInit")) + def testRecordIndexDeferredUntilSecondCommit(testCase: DeferredInitTestCase): Unit = { + val options = deferredInitOptions(testCase) + val dataGen = new HoodieTestDataGenerator() + val allRecords = new util.ArrayList[HoodieRecord[_]]() + val write = (records: util.List[HoodieRecord[_]], operation: String, mode: SaveMode) => + toDataset(spark, records).write.format("hudi").options(options) + .option(DataSourceWriteOptions.OPERATION.key, operation).mode(mode).save(basePath) + + // Commit 1: the record index must not be built, since the table had no completed commit when the write started. + val firstBatch = dataGen.generateInserts("001", 2000) + allRecords.addAll(firstBatch) + write(firstBatch.asInstanceOf[util.List[HoodieRecord[_]]], testCase.firstOperation, SaveMode.Overwrite) + assertTableVersion(testCase.tableVersion) + assertFalse(isRecordIndexAvailable, "Record index should not be built by the first commit") + + // What the index should be sized from once it is built. + val rowsPerPartition = spark.read.format("hudi").load(basePath).groupBy("_hoodie_partition_path").count().collect() + .map(row => row.getString(0) -> row.getLong(1)).toMap + val rowsBeforeBuild = rowsPerPartition.values.sum + + // Commit 2: updates of records tagged through the fallback index, plus new records. The index is built here. + val updates = dataGen.generateUpdates("002", allRecords.subList(0, 300)) + val newRecords = dataGen.generateInserts("002", 10) + val secondBatch = new util.ArrayList[HoodieRecord[_]](updates) + secondBatch.addAll(newRecords) + allRecords.addAll(newRecords) + write(secondBatch, UPSERT_OPERATION_OPT_VAL, SaveMode.Append) + assertTrue(isRecordIndexAvailable, "Record index should be built by the second commit") + + // Commit 3: upserts tagged through the record index itself. + val laterUpdates = dataGen.generateUpdates("003", allRecords.subList(0, 300)) + write(laterUpdates.asInstanceOf[util.List[HoodieRecord[_]]], UPSERT_OPERATION_OPT_VAL, SaveMode.Append) + + // No duplicates across the fallback to record index switch. + val keys = spark.read.format("hudi").load(basePath).select("_row_key").collect().map(_.getString(0)) + assertEquals(allRecords.asScala.map(_.getRecordKey).sorted.toSeq, keys.sorted.toSeq) + + // The index is sized from the data the first commit left behind, not from an empty table. + val fileGroupIds = recordIndexFileGroupIds() + if (testCase.partitioned) { + val perPartition = fileGroupIds.groupBy(HoodieTableMetadataUtil.getDataTablePartitionNameFromFileGroupName) + .mapValues(_.size).toMap + assertEquals(rowsPerPartition.mapValues(rows => expectedFileGroupCount(rows)).toMap, perPartition) + } else { + assertEquals(expectedFileGroupCount(rowsBeforeBuild), fileGroupIds.size) Review Comment: 2000 and 2010 rows both give 20 file groups (and roughly 667 vs 670 per partition both give 6), so this can't tell "sized from commit 1's data" from "sized after commit 2". It does catch the built-from-empty regression. Make commit 2 add enough rows to move the count, e.g. +2000. ########## hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestStructuredStreaming.scala: ########## @@ -228,6 +229,94 @@ class TestStructuredStreaming extends HoodieSparkClientTestBase { tableType, addCompactionConfigs = false, isAsyncCompaction = false) } + /** + * Deferred record index initialization through the streaming sink: the first micro-batch leaves the index unbuilt + * and the second one builds it, with no duplicates across the switch from the fallback index. + */ + @Tag("functional") Review Comment: nit: this is the only tagged method in an untagged class. Please confirm the CI profiles run it where you expect. ########## hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestDeferredRecordIndexInit.scala: ########## @@ -0,0 +1,331 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hudi.functional + +import org.apache.hudi.{DataSourceWriteOptions, SparkDatasetMixin} +import org.apache.hudi.DataSourceWriteOptions._ +import org.apache.hudi.common.config.HoodieMetadataConfig +import org.apache.hudi.common.model.{HoodieRecord, HoodieTableType} +import org.apache.hudi.common.table.{HoodieTableConfig, HoodieTableMetaClient, HoodieTableVersion} +import org.apache.hudi.common.testutils.HoodieTestDataGenerator +import org.apache.hudi.common.util.{Option => HOption} +import org.apache.hudi.config.{HoodieCompactionConfig, HoodieIndexConfig, HoodieWriteConfig} +import org.apache.hudi.functional.TestDeferredRecordIndexInit.{expectedFileGroupCount, DeferredInitTestCase} +import org.apache.hudi.metadata.{HoodieTableMetadata, HoodieTableMetadataUtil, MetadataPartitionType} + +import org.apache.spark.sql.SaveMode +import org.junit.jupiter.api.Tag +import org.junit.jupiter.api.Assertions.{assertEquals, assertFalse, assertTrue} +import org.junit.jupiter.params.ParameterizedTest +import org.junit.jupiter.params.provider.{Arguments, MethodSource} + +import java.util + +import scala.collection.JavaConverters._ + +/** + * End to end coverage of hoodie.metadata.record.level.index.defer.init on its own, with no record threshold: + * the record index is not built by the first commit, is built by the second one and sized from the first commit's + * data, and the switch from the fallback index produces no duplicates. + */ +@Tag("functional") +class TestDeferredRecordIndexInit extends RecordLevelIndexTestBase with SparkDatasetMixin { + + private def deferredInitOptions(testCase: DeferredInitTestCase): Map[String, String] = Map( + HoodieWriteConfig.TBL_NAME.key -> "hoodie_test", + DataSourceWriteOptions.TABLE_TYPE.key -> testCase.tableType.name(), + RECORDKEY_FIELD.key -> "_row_key", + PARTITIONPATH_FIELD.key -> "partition_path", + HoodieTableConfig.ORDERING_FIELDS.key -> "timestamp", + HoodieWriteConfig.WRITE_TABLE_VERSION.key -> testCase.tableVersion.versionCode().toString, + HoodieWriteConfig.AUTO_UPGRADE_VERSION.key -> "false", + HoodieMetadataConfig.ENABLE.key -> "true", + HoodieMetadataConfig.GLOBAL_RECORD_LEVEL_INDEX_ENABLE_PROP.key -> (!testCase.partitioned).toString, + HoodieMetadataConfig.RECORD_LEVEL_INDEX_ENABLE_PROP.key -> testCase.partitioned.toString, + HoodieMetadataConfig.STREAMING_WRITE_ENABLED.key -> testCase.streamingWrites.toString, + // Pinned low so the sizing result is not simply the floor. + HoodieMetadataConfig.GLOBAL_RECORD_LEVEL_INDEX_MIN_FILE_GROUP_COUNT_PROP.key -> "1", + HoodieMetadataConfig.GLOBAL_RECORD_LEVEL_INDEX_MAX_FILE_GROUP_COUNT_PROP.key -> "1000", + HoodieMetadataConfig.RECORD_LEVEL_INDEX_MIN_FILE_GROUP_COUNT_PROP.key -> "1", + HoodieMetadataConfig.RECORD_LEVEL_INDEX_MAX_FILE_GROUP_COUNT_PROP.key -> "1000", + // 9600 bytes / 48 bytes per entry = 200 entries per file group. + HoodieMetadataConfig.RECORD_INDEX_MAX_FILE_GROUP_SIZE_BYTES_PROP.key -> "9600", + HoodieMetadataConfig.DEFER_RLI_INIT_FOR_FRESH_TABLE.key -> "true", + HoodieCompactionConfig.INLINE_COMPACT.key -> "false", + HoodieIndexConfig.INDEX_TYPE.key -> (if (testCase.partitioned) "RECORD_LEVEL_INDEX" else "GLOBAL_RECORD_LEVEL_INDEX")) + + @ParameterizedTest + @MethodSource(Array("testArgsForDeferredInit")) + def testRecordIndexDeferredUntilSecondCommit(testCase: DeferredInitTestCase): Unit = { + val options = deferredInitOptions(testCase) + val dataGen = new HoodieTestDataGenerator() + val allRecords = new util.ArrayList[HoodieRecord[_]]() + val write = (records: util.List[HoodieRecord[_]], operation: String, mode: SaveMode) => + toDataset(spark, records).write.format("hudi").options(options) + .option(DataSourceWriteOptions.OPERATION.key, operation).mode(mode).save(basePath) + + // Commit 1: the record index must not be built, since the table had no completed commit when the write started. + val firstBatch = dataGen.generateInserts("001", 2000) + allRecords.addAll(firstBatch) + write(firstBatch.asInstanceOf[util.List[HoodieRecord[_]]], testCase.firstOperation, SaveMode.Overwrite) + assertTableVersion(testCase.tableVersion) + assertFalse(isRecordIndexAvailable, "Record index should not be built by the first commit") + + // What the index should be sized from once it is built. + val rowsPerPartition = spark.read.format("hudi").load(basePath).groupBy("_hoodie_partition_path").count().collect() + .map(row => row.getString(0) -> row.getLong(1)).toMap + val rowsBeforeBuild = rowsPerPartition.values.sum + + // Commit 2: updates of records tagged through the fallback index, plus new records. The index is built here. + val updates = dataGen.generateUpdates("002", allRecords.subList(0, 300)) + val newRecords = dataGen.generateInserts("002", 10) + val secondBatch = new util.ArrayList[HoodieRecord[_]](updates) + secondBatch.addAll(newRecords) + allRecords.addAll(newRecords) + write(secondBatch, UPSERT_OPERATION_OPT_VAL, SaveMode.Append) + assertTrue(isRecordIndexAvailable, "Record index should be built by the second commit") + + // Commit 3: upserts tagged through the record index itself. + val laterUpdates = dataGen.generateUpdates("003", allRecords.subList(0, 300)) + write(laterUpdates.asInstanceOf[util.List[HoodieRecord[_]]], UPSERT_OPERATION_OPT_VAL, SaveMode.Append) + + // No duplicates across the fallback to record index switch. + val keys = spark.read.format("hudi").load(basePath).select("_row_key").collect().map(_.getString(0)) + assertEquals(allRecords.asScala.map(_.getRecordKey).sorted.toSeq, keys.sorted.toSeq) + + // The index is sized from the data the first commit left behind, not from an empty table. + val fileGroupIds = recordIndexFileGroupIds() + if (testCase.partitioned) { + val perPartition = fileGroupIds.groupBy(HoodieTableMetadataUtil.getDataTablePartitionNameFromFileGroupName) + .mapValues(_.size).toMap + assertEquals(rowsPerPartition.mapValues(rows => expectedFileGroupCount(rows)).toMap, perPartition) + } else { + assertEquals(expectedFileGroupCount(rowsBeforeBuild), fileGroupIds.size) + } + } + + /** + * The commit that builds the record index replaces the data the index was just sized from, so the index has to + * carry the replacing records and none of the replaced ones. + */ + @ParameterizedTest + @MethodSource(Array("testArgsForInsertOverwrite")) + def testRecordIndexBuiltByInsertOverwrite(testCase: DeferredInitTestCase): Unit = { + val options = deferredInitOptions(testCase) + val dataGen = new HoodieTestDataGenerator() + val write = (records: util.List[HoodieRecord[_]], operation: String, mode: SaveMode) => + toDataset(spark, records).write.format("hudi").options(options) + .option(DataSourceWriteOptions.OPERATION.key, operation).mode(mode).save(basePath) + + val firstBatch = dataGen.generateInserts("001", 2000) + write(firstBatch.asInstanceOf[util.List[HoodieRecord[_]]], testCase.firstOperation, SaveMode.Overwrite) + assertTableVersion(testCase.tableVersion) + assertFalse(isRecordIndexAvailable, "Record index should not be built by the first commit") + + // Commit 2 builds the record index and then replaces every partition the first commit wrote. + val replacingBatch = dataGen.generateInserts("002", 500) + write(replacingBatch.asInstanceOf[util.List[HoodieRecord[_]]], INSERT_OVERWRITE_OPERATION_OPT_VAL, SaveMode.Append) + assertTrue(isRecordIndexAvailable, "Record index should be built by the second commit") + + val replacingKeys = replacingBatch.asScala.map(_.getRecordKey).sorted.toSeq + assertEquals(replacingKeys, spark.read.format("hudi").load(basePath).select("_row_key") + .collect().map(_.getString(0)).sorted.toSeq) + + // Commit 3 is tagged through the record index. Replaced records must not come back and survivors must not double. + val laterUpdates = dataGen.generateUpdates("003", replacingBatch.subList(0, 100)) + write(laterUpdates.asInstanceOf[util.List[HoodieRecord[_]]], UPSERT_OPERATION_OPT_VAL, SaveMode.Append) + assertEquals(replacingKeys, spark.read.format("hudi").load(basePath).select("_row_key") + .collect().map(_.getString(0)).sorted.toSeq) + } + + /** + * The same deferral through Spark SQL: INSERT INTO leaves the record index unbuilt and the following MERGE INTO + * builds it, sized from the rows the first statement wrote. + */ + @ParameterizedTest + @MethodSource(Array("testArgsForSql")) + def testRecordIndexDeferredUntilSecondStatement(testCase: DeferredInitTestCase): Unit = { + val tableName = s"rli_deferred_sql_${if (testCase.partitioned) "partitioned" else "global"}_v${testCase.tableVersion.versionCode()}" + val tablePath = tempDir.resolve(tableName).toAbsolutePath.toString + val indexEnableProp = if (testCase.partitioned) { + HoodieMetadataConfig.RECORD_LEVEL_INDEX_ENABLE_PROP.key + } else { + HoodieMetadataConfig.GLOBAL_RECORD_LEVEL_INDEX_ENABLE_PROP.key + } + val minFileGroupProp = if (testCase.partitioned) { + HoodieMetadataConfig.RECORD_LEVEL_INDEX_MIN_FILE_GROUP_COUNT_PROP.key + } else { + HoodieMetadataConfig.GLOBAL_RECORD_LEVEL_INDEX_MIN_FILE_GROUP_COUNT_PROP.key + } + val maxFileGroupProp = if (testCase.partitioned) { + HoodieMetadataConfig.RECORD_LEVEL_INDEX_MAX_FILE_GROUP_COUNT_PROP.key + } else { + HoodieMetadataConfig.GLOBAL_RECORD_LEVEL_INDEX_MAX_FILE_GROUP_COUNT_PROP.key + } + val indexType = if (testCase.partitioned) "RECORD_LEVEL_INDEX" else "GLOBAL_RECORD_LEVEL_INDEX" + spark.sql( + s""" + |create table $tableName ( + | id string, + | ts long, + | name string, + | part string + |) using hudi + | options ( + | primaryKey = 'id', + | orderingFields = 'ts', + | hoodie.metadata.enable = 'true', + | ${HoodieWriteConfig.WRITE_TABLE_VERSION.key} = '${testCase.tableVersion.versionCode()}', + | ${HoodieWriteConfig.AUTO_UPGRADE_VERSION.key} = 'false', + | ${HoodieMetadataConfig.STREAMING_WRITE_ENABLED.key} = '${testCase.streamingWrites}', + | $indexEnableProp = 'true', + | hoodie.index.type = '$indexType', + | ${HoodieMetadataConfig.DEFER_RLI_INIT_FOR_FRESH_TABLE.key} = 'true', + | ${HoodieMetadataConfig.RECORD_INDEX_MAX_FILE_GROUP_SIZE_BYTES_PROP.key} = '9600', + | $minFileGroupProp = '1', + | $maxFileGroupProp = '1000' + | ) + | partitioned by (part) + | location '$tablePath' + """.stripMargin) + val sqlMetaClient = HoodieTableMetaClient.builder().setBasePath(tablePath).setConf(storageConf).build() + val recordIndexBuilt = () => HoodieTableMetaClient.reload(sqlMetaClient).getTableConfig.getMetadataPartitions + .contains(MetadataPartitionType.RECORD_INDEX.getPartitionPath) + val assertSqlTableVersion = () => assertEquals(testCase.tableVersion, + HoodieTableMetaClient.reload(sqlMetaClient).getTableConfig.getTableVersion, + "Table was written at an unexpected version") + val rowsOf = (from: Int, to: Int, ts: Long, name: String) => + s"select cast(id as string) as id, ${ts}L as ts, '$name' as name, concat('p', cast(id % 3 as string)) as part from range($from, $to)" + + try { + spark.conf.set("hoodie.spark.sql.insert.into.operation", testCase.firstOperation) + spark.sql(s"insert into $tableName ${rowsOf(0, 2000, 1L, "v1")}") + } finally { + spark.conf.unset("hoodie.spark.sql.insert.into.operation") + } + assertSqlTableVersion() + assertFalse(recordIndexBuilt(), "Record index should not be built by the first statement") + + // The index is sized from the rows the first statement wrote, so capture them before the merge adds more. + // Keyed by the partition path the table writes, which is Hive styled for a SQL created table. + val rowsPerPartition = spark + .sql(s"select _hoodie_partition_path, count(*) as rows from $tableName group by _hoodie_partition_path") + .collect().map(row => row.getString(0) -> row.getLong(1)).toMap + val rowsBeforeBuild = rowsPerPartition.values.sum + + spark.sql( + s""" + |merge into $tableName t + |using (${rowsOf(0, 300, 4L, "v4")} union all ${rowsOf(2000, 2010, 4L, "v4")}) s + |on t.id = s.id + |when matched then update set * + |when not matched then insert * + """.stripMargin) + assertTrue(recordIndexBuilt(), "Record index should be built by the second statement") + + // Every key appears once, and the merge updated exactly the rows it matched. + assertEquals(2010L, spark.sql(s"select id from $tableName").count()) + assertEquals(2010L, spark.sql(s"select distinct id from $tableName").count()) + assertEquals(310L, spark.sql(s"select id from $tableName where name = 'v4'").count()) + + // The index is sized from the rows the first statement left behind, not from the table as the merge left it. + val fileGroupIds = recordIndexFileGroupIds(tablePath) + if (testCase.partitioned) { + val perPartition = fileGroupIds.groupBy(HoodieTableMetadataUtil.getDataTablePartitionNameFromFileGroupName) + .mapValues(_.size).toMap + assertEquals(rowsPerPartition.mapValues(rows => expectedFileGroupCount(rows)).toMap, perPartition) + } else { + assertEquals(expectedFileGroupCount(rowsBeforeBuild), fileGroupIds.size) + } + } + + private def assertTableVersion(expected: HoodieTableVersion): Unit = { + metaClient = HoodieTableMetaClient.reload(metaClient) + assertEquals(expected, metaClient.getTableConfig.getTableVersion, "Table was written at an unexpected version") + } + + private def isRecordIndexAvailable: Boolean = { + metaClient = HoodieTableMetaClient.reload(metaClient) + metaClient.getTableConfig.getMetadataPartitions.contains(MetadataPartitionType.RECORD_INDEX.getPartitionPath) + } + + private def recordIndexFileGroupIds(tablePath: String = basePath): Seq[String] = { + val metadataMetaClient = HoodieTableMetaClient.builder() + .setBasePath(HoodieTableMetadata.getMetadataTableBasePath(tablePath)) + .setConf(storageConf) + .build() + HoodieTableMetadataUtil.getPartitionLatestFileSlices(metadataMetaClient, HOption.empty(), + MetadataPartitionType.RECORD_INDEX.getPartitionPath).asScala.map(_.getFileId).toSeq + } +} + +object TestDeferredRecordIndexInit { + + case class DeferredInitTestCase(tableType: HoodieTableType, + partitioned: Boolean, + tableVersion: HoodieTableVersion, + streamingWrites: Boolean, + firstOperation: String) { + override def toString: String = + s"${tableType.name()}/${if (partitioned) "partitioned" else "global"}/v${tableVersion.versionCode()}" + + s"/streaming=$streamingWrites/$firstOperation" + } + + // The partitioned record index and streaming metadata writes both need a table version above six. + def testArgsForDeferredInit: java.util.stream.Stream[Arguments] = { + java.util.stream.Stream.of( + Arguments.arguments(DeferredInitTestCase(HoodieTableType.COPY_ON_WRITE, partitioned = false, HoodieTableVersion.SIX, streamingWrites = false, BULK_INSERT_OPERATION_OPT_VAL)), + Arguments.arguments(DeferredInitTestCase(HoodieTableType.COPY_ON_WRITE, partitioned = false, HoodieTableVersion.SIX, streamingWrites = false, INSERT_OPERATION_OPT_VAL)), + Arguments.arguments(DeferredInitTestCase(HoodieTableType.MERGE_ON_READ, partitioned = false, HoodieTableVersion.SIX, streamingWrites = false, BULK_INSERT_OPERATION_OPT_VAL)), + Arguments.arguments(DeferredInitTestCase(HoodieTableType.COPY_ON_WRITE, partitioned = false, HoodieTableVersion.TEN, streamingWrites = true, BULK_INSERT_OPERATION_OPT_VAL)), + Arguments.arguments(DeferredInitTestCase(HoodieTableType.COPY_ON_WRITE, partitioned = false, HoodieTableVersion.TEN, streamingWrites = false, BULK_INSERT_OPERATION_OPT_VAL)), + Arguments.arguments(DeferredInitTestCase(HoodieTableType.COPY_ON_WRITE, partitioned = false, HoodieTableVersion.TEN, streamingWrites = true, INSERT_OPERATION_OPT_VAL)), + Arguments.arguments(DeferredInitTestCase(HoodieTableType.COPY_ON_WRITE, partitioned = true, HoodieTableVersion.TEN, streamingWrites = true, BULK_INSERT_OPERATION_OPT_VAL)), + Arguments.arguments(DeferredInitTestCase(HoodieTableType.COPY_ON_WRITE, partitioned = true, HoodieTableVersion.TEN, streamingWrites = false, BULK_INSERT_OPERATION_OPT_VAL)), + Arguments.arguments(DeferredInitTestCase(HoodieTableType.MERGE_ON_READ, partitioned = false, HoodieTableVersion.TEN, streamingWrites = true, BULK_INSERT_OPERATION_OPT_VAL)), + Arguments.arguments(DeferredInitTestCase(HoodieTableType.MERGE_ON_READ, partitioned = true, HoodieTableVersion.TEN, streamingWrites = true, BULK_INSERT_OPERATION_OPT_VAL)) + ) + } + + def testArgsForSql: java.util.stream.Stream[Arguments] = { + java.util.stream.Stream.of( + Arguments.arguments(DeferredInitTestCase(HoodieTableType.COPY_ON_WRITE, partitioned = false, HoodieTableVersion.SIX, streamingWrites = false, "bulk_insert")), + Arguments.arguments(DeferredInitTestCase(HoodieTableType.COPY_ON_WRITE, partitioned = false, HoodieTableVersion.TEN, streamingWrites = true, "bulk_insert")), + Arguments.arguments(DeferredInitTestCase(HoodieTableType.COPY_ON_WRITE, partitioned = false, HoodieTableVersion.TEN, streamingWrites = true, "insert")), + Arguments.arguments(DeferredInitTestCase(HoodieTableType.COPY_ON_WRITE, partitioned = false, HoodieTableVersion.TEN, streamingWrites = true, "upsert")), + Arguments.arguments(DeferredInitTestCase(HoodieTableType.COPY_ON_WRITE, partitioned = true, HoodieTableVersion.TEN, streamingWrites = true, "bulk_insert")) + ) + } + + def testArgsForInsertOverwrite: java.util.stream.Stream[Arguments] = { + java.util.stream.Stream.of( + Arguments.arguments(DeferredInitTestCase(HoodieTableType.COPY_ON_WRITE, partitioned = false, HoodieTableVersion.SIX, streamingWrites = false, BULK_INSERT_OPERATION_OPT_VAL)), + Arguments.arguments(DeferredInitTestCase(HoodieTableType.COPY_ON_WRITE, partitioned = false, HoodieTableVersion.TEN, streamingWrites = true, BULK_INSERT_OPERATION_OPT_VAL)), + Arguments.arguments(DeferredInitTestCase(HoodieTableType.COPY_ON_WRITE, partitioned = true, HoodieTableVersion.TEN, streamingWrites = true, BULK_INSERT_OPERATION_OPT_VAL)), + Arguments.arguments(DeferredInitTestCase(HoodieTableType.MERGE_ON_READ, partitioned = false, HoodieTableVersion.TEN, streamingWrites = true, BULK_INSERT_OPERATION_OPT_VAL)) + ) + } + + /** Mirrors HoodieTableMetadataUtil.estimateFileGroupCount with growth factor 2 and 200 entries per file group. */ + def expectedFileGroupCount(records: Long): Int = { Review Comment: This formula is duplicated in all three files with growth factor 2 and 48 bytes/entry hard-coded. Suggest calling `HoodieTableMetadataUtil.estimateFileGroupCount(RECORD_INDEX, ...)` with the configured values. ########## hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestDeferredRecordIndexInit.scala: ########## @@ -0,0 +1,331 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hudi.functional + +import org.apache.hudi.{DataSourceWriteOptions, SparkDatasetMixin} +import org.apache.hudi.DataSourceWriteOptions._ +import org.apache.hudi.common.config.HoodieMetadataConfig +import org.apache.hudi.common.model.{HoodieRecord, HoodieTableType} +import org.apache.hudi.common.table.{HoodieTableConfig, HoodieTableMetaClient, HoodieTableVersion} +import org.apache.hudi.common.testutils.HoodieTestDataGenerator +import org.apache.hudi.common.util.{Option => HOption} +import org.apache.hudi.config.{HoodieCompactionConfig, HoodieIndexConfig, HoodieWriteConfig} +import org.apache.hudi.functional.TestDeferredRecordIndexInit.{expectedFileGroupCount, DeferredInitTestCase} +import org.apache.hudi.metadata.{HoodieTableMetadata, HoodieTableMetadataUtil, MetadataPartitionType} + +import org.apache.spark.sql.SaveMode +import org.junit.jupiter.api.Tag +import org.junit.jupiter.api.Assertions.{assertEquals, assertFalse, assertTrue} +import org.junit.jupiter.params.ParameterizedTest +import org.junit.jupiter.params.provider.{Arguments, MethodSource} + +import java.util + +import scala.collection.JavaConverters._ + +/** + * End to end coverage of hoodie.metadata.record.level.index.defer.init on its own, with no record threshold: + * the record index is not built by the first commit, is built by the second one and sized from the first commit's + * data, and the switch from the fallback index produces no duplicates. + */ +@Tag("functional") +class TestDeferredRecordIndexInit extends RecordLevelIndexTestBase with SparkDatasetMixin { + + private def deferredInitOptions(testCase: DeferredInitTestCase): Map[String, String] = Map( + HoodieWriteConfig.TBL_NAME.key -> "hoodie_test", + DataSourceWriteOptions.TABLE_TYPE.key -> testCase.tableType.name(), + RECORDKEY_FIELD.key -> "_row_key", + PARTITIONPATH_FIELD.key -> "partition_path", + HoodieTableConfig.ORDERING_FIELDS.key -> "timestamp", + HoodieWriteConfig.WRITE_TABLE_VERSION.key -> testCase.tableVersion.versionCode().toString, + HoodieWriteConfig.AUTO_UPGRADE_VERSION.key -> "false", + HoodieMetadataConfig.ENABLE.key -> "true", + HoodieMetadataConfig.GLOBAL_RECORD_LEVEL_INDEX_ENABLE_PROP.key -> (!testCase.partitioned).toString, + HoodieMetadataConfig.RECORD_LEVEL_INDEX_ENABLE_PROP.key -> testCase.partitioned.toString, + HoodieMetadataConfig.STREAMING_WRITE_ENABLED.key -> testCase.streamingWrites.toString, + // Pinned low so the sizing result is not simply the floor. + HoodieMetadataConfig.GLOBAL_RECORD_LEVEL_INDEX_MIN_FILE_GROUP_COUNT_PROP.key -> "1", + HoodieMetadataConfig.GLOBAL_RECORD_LEVEL_INDEX_MAX_FILE_GROUP_COUNT_PROP.key -> "1000", + HoodieMetadataConfig.RECORD_LEVEL_INDEX_MIN_FILE_GROUP_COUNT_PROP.key -> "1", + HoodieMetadataConfig.RECORD_LEVEL_INDEX_MAX_FILE_GROUP_COUNT_PROP.key -> "1000", + // 9600 bytes / 48 bytes per entry = 200 entries per file group. + HoodieMetadataConfig.RECORD_INDEX_MAX_FILE_GROUP_SIZE_BYTES_PROP.key -> "9600", + HoodieMetadataConfig.DEFER_RLI_INIT_FOR_FRESH_TABLE.key -> "true", + HoodieCompactionConfig.INLINE_COMPACT.key -> "false", + HoodieIndexConfig.INDEX_TYPE.key -> (if (testCase.partitioned) "RECORD_LEVEL_INDEX" else "GLOBAL_RECORD_LEVEL_INDEX")) + + @ParameterizedTest + @MethodSource(Array("testArgsForDeferredInit")) + def testRecordIndexDeferredUntilSecondCommit(testCase: DeferredInitTestCase): Unit = { + val options = deferredInitOptions(testCase) + val dataGen = new HoodieTestDataGenerator() + val allRecords = new util.ArrayList[HoodieRecord[_]]() + val write = (records: util.List[HoodieRecord[_]], operation: String, mode: SaveMode) => + toDataset(spark, records).write.format("hudi").options(options) + .option(DataSourceWriteOptions.OPERATION.key, operation).mode(mode).save(basePath) + + // Commit 1: the record index must not be built, since the table had no completed commit when the write started. + val firstBatch = dataGen.generateInserts("001", 2000) + allRecords.addAll(firstBatch) + write(firstBatch.asInstanceOf[util.List[HoodieRecord[_]]], testCase.firstOperation, SaveMode.Overwrite) + assertTableVersion(testCase.tableVersion) + assertFalse(isRecordIndexAvailable, "Record index should not be built by the first commit") + + // What the index should be sized from once it is built. + val rowsPerPartition = spark.read.format("hudi").load(basePath).groupBy("_hoodie_partition_path").count().collect() + .map(row => row.getString(0) -> row.getLong(1)).toMap + val rowsBeforeBuild = rowsPerPartition.values.sum + + // Commit 2: updates of records tagged through the fallback index, plus new records. The index is built here. + val updates = dataGen.generateUpdates("002", allRecords.subList(0, 300)) + val newRecords = dataGen.generateInserts("002", 10) + val secondBatch = new util.ArrayList[HoodieRecord[_]](updates) + secondBatch.addAll(newRecords) + allRecords.addAll(newRecords) + write(secondBatch, UPSERT_OPERATION_OPT_VAL, SaveMode.Append) + assertTrue(isRecordIndexAvailable, "Record index should be built by the second commit") + + // Commit 3: upserts tagged through the record index itself. + val laterUpdates = dataGen.generateUpdates("003", allRecords.subList(0, 300)) + write(laterUpdates.asInstanceOf[util.List[HoodieRecord[_]]], UPSERT_OPERATION_OPT_VAL, SaveMode.Append) + + // No duplicates across the fallback to record index switch. + val keys = spark.read.format("hudi").load(basePath).select("_row_key").collect().map(_.getString(0)) + assertEquals(allRecords.asScala.map(_.getRecordKey).sorted.toSeq, keys.sorted.toSeq) + + // The index is sized from the data the first commit left behind, not from an empty table. + val fileGroupIds = recordIndexFileGroupIds() + if (testCase.partitioned) { + val perPartition = fileGroupIds.groupBy(HoodieTableMetadataUtil.getDataTablePartitionNameFromFileGroupName) + .mapValues(_.size).toMap + assertEquals(rowsPerPartition.mapValues(rows => expectedFileGroupCount(rows)).toMap, perPartition) + } else { + assertEquals(expectedFileGroupCount(rowsBeforeBuild), fileGroupIds.size) + } + } + + /** + * The commit that builds the record index replaces the data the index was just sized from, so the index has to + * carry the replacing records and none of the replaced ones. + */ + @ParameterizedTest + @MethodSource(Array("testArgsForInsertOverwrite")) + def testRecordIndexBuiltByInsertOverwrite(testCase: DeferredInitTestCase): Unit = { + val options = deferredInitOptions(testCase) + val dataGen = new HoodieTestDataGenerator() + val write = (records: util.List[HoodieRecord[_]], operation: String, mode: SaveMode) => + toDataset(spark, records).write.format("hudi").options(options) + .option(DataSourceWriteOptions.OPERATION.key, operation).mode(mode).save(basePath) + + val firstBatch = dataGen.generateInserts("001", 2000) + write(firstBatch.asInstanceOf[util.List[HoodieRecord[_]]], testCase.firstOperation, SaveMode.Overwrite) + assertTableVersion(testCase.tableVersion) + assertFalse(isRecordIndexAvailable, "Record index should not be built by the first commit") + + // Commit 2 builds the record index and then replaces every partition the first commit wrote. + val replacingBatch = dataGen.generateInserts("002", 500) + write(replacingBatch.asInstanceOf[util.List[HoodieRecord[_]]], INSERT_OVERWRITE_OPERATION_OPT_VAL, SaveMode.Append) + assertTrue(isRecordIndexAvailable, "Record index should be built by the second commit") + + val replacingKeys = replacingBatch.asScala.map(_.getRecordKey).sorted.toSeq + assertEquals(replacingKeys, spark.read.format("hudi").load(basePath).select("_row_key") + .collect().map(_.getString(0)).sorted.toSeq) + + // Commit 3 is tagged through the record index. Replaced records must not come back and survivors must not double. + val laterUpdates = dataGen.generateUpdates("003", replacingBatch.subList(0, 100)) + write(laterUpdates.asInstanceOf[util.List[HoodieRecord[_]]], UPSERT_OPERATION_OPT_VAL, SaveMode.Append) + assertEquals(replacingKeys, spark.read.format("hudi").load(basePath).select("_row_key") + .collect().map(_.getString(0)).sorted.toSeq) + } + + /** + * The same deferral through Spark SQL: INSERT INTO leaves the record index unbuilt and the following MERGE INTO + * builds it, sized from the rows the first statement wrote. + */ + @ParameterizedTest + @MethodSource(Array("testArgsForSql")) + def testRecordIndexDeferredUntilSecondStatement(testCase: DeferredInitTestCase): Unit = { + val tableName = s"rli_deferred_sql_${if (testCase.partitioned) "partitioned" else "global"}_v${testCase.tableVersion.versionCode()}" + val tablePath = tempDir.resolve(tableName).toAbsolutePath.toString + val indexEnableProp = if (testCase.partitioned) { + HoodieMetadataConfig.RECORD_LEVEL_INDEX_ENABLE_PROP.key + } else { + HoodieMetadataConfig.GLOBAL_RECORD_LEVEL_INDEX_ENABLE_PROP.key + } + val minFileGroupProp = if (testCase.partitioned) { + HoodieMetadataConfig.RECORD_LEVEL_INDEX_MIN_FILE_GROUP_COUNT_PROP.key + } else { + HoodieMetadataConfig.GLOBAL_RECORD_LEVEL_INDEX_MIN_FILE_GROUP_COUNT_PROP.key + } + val maxFileGroupProp = if (testCase.partitioned) { + HoodieMetadataConfig.RECORD_LEVEL_INDEX_MAX_FILE_GROUP_COUNT_PROP.key + } else { + HoodieMetadataConfig.GLOBAL_RECORD_LEVEL_INDEX_MAX_FILE_GROUP_COUNT_PROP.key + } + val indexType = if (testCase.partitioned) "RECORD_LEVEL_INDEX" else "GLOBAL_RECORD_LEVEL_INDEX" + spark.sql( + s""" + |create table $tableName ( + | id string, + | ts long, + | name string, + | part string + |) using hudi + | options ( + | primaryKey = 'id', + | orderingFields = 'ts', + | hoodie.metadata.enable = 'true', + | ${HoodieWriteConfig.WRITE_TABLE_VERSION.key} = '${testCase.tableVersion.versionCode()}', + | ${HoodieWriteConfig.AUTO_UPGRADE_VERSION.key} = 'false', + | ${HoodieMetadataConfig.STREAMING_WRITE_ENABLED.key} = '${testCase.streamingWrites}', + | $indexEnableProp = 'true', + | hoodie.index.type = '$indexType', + | ${HoodieMetadataConfig.DEFER_RLI_INIT_FOR_FRESH_TABLE.key} = 'true', + | ${HoodieMetadataConfig.RECORD_INDEX_MAX_FILE_GROUP_SIZE_BYTES_PROP.key} = '9600', + | $minFileGroupProp = '1', + | $maxFileGroupProp = '1000' + | ) + | partitioned by (part) + | location '$tablePath' + """.stripMargin) + val sqlMetaClient = HoodieTableMetaClient.builder().setBasePath(tablePath).setConf(storageConf).build() + val recordIndexBuilt = () => HoodieTableMetaClient.reload(sqlMetaClient).getTableConfig.getMetadataPartitions + .contains(MetadataPartitionType.RECORD_INDEX.getPartitionPath) + val assertSqlTableVersion = () => assertEquals(testCase.tableVersion, + HoodieTableMetaClient.reload(sqlMetaClient).getTableConfig.getTableVersion, + "Table was written at an unexpected version") + val rowsOf = (from: Int, to: Int, ts: Long, name: String) => + s"select cast(id as string) as id, ${ts}L as ts, '$name' as name, concat('p', cast(id % 3 as string)) as part from range($from, $to)" + + try { + spark.conf.set("hoodie.spark.sql.insert.into.operation", testCase.firstOperation) + spark.sql(s"insert into $tableName ${rowsOf(0, 2000, 1L, "v1")}") + } finally { + spark.conf.unset("hoodie.spark.sql.insert.into.operation") + } + assertSqlTableVersion() + assertFalse(recordIndexBuilt(), "Record index should not be built by the first statement") + + // The index is sized from the rows the first statement wrote, so capture them before the merge adds more. + // Keyed by the partition path the table writes, which is Hive styled for a SQL created table. + val rowsPerPartition = spark + .sql(s"select _hoodie_partition_path, count(*) as rows from $tableName group by _hoodie_partition_path") + .collect().map(row => row.getString(0) -> row.getLong(1)).toMap + val rowsBeforeBuild = rowsPerPartition.values.sum + + spark.sql( + s""" + |merge into $tableName t + |using (${rowsOf(0, 300, 4L, "v4")} union all ${rowsOf(2000, 2010, 4L, "v4")}) s + |on t.id = s.id + |when matched then update set * + |when not matched then insert * + """.stripMargin) + assertTrue(recordIndexBuilt(), "Record index should be built by the second statement") + + // Every key appears once, and the merge updated exactly the rows it matched. + assertEquals(2010L, spark.sql(s"select id from $tableName").count()) + assertEquals(2010L, spark.sql(s"select distinct id from $tableName").count()) + assertEquals(310L, spark.sql(s"select id from $tableName where name = 'v4'").count()) + + // The index is sized from the rows the first statement left behind, not from the table as the merge left it. Review Comment: Same as above: the numbers here give the same count either way, so this comment claims more than the assertion can show. ########## hudi-utilities/src/test/java/org/apache/hudi/utilities/deltastreamer/TestDeferredRecordIndexOnStreamerPath.java: ########## @@ -0,0 +1,157 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.utilities.deltastreamer; + +import org.apache.hudi.common.config.HoodieMetadataConfig; +import org.apache.hudi.common.model.FileSlice; +import org.apache.hudi.common.model.WriteOperationType; +import org.apache.hudi.common.table.HoodieTableMetaClient; +import org.apache.hudi.common.table.HoodieTableVersion; +import org.apache.hudi.common.testutils.HoodieTestUtils; +import org.apache.hudi.common.util.Option; +import org.apache.hudi.config.HoodieIndexConfig; +import org.apache.hudi.config.HoodieWriteConfig; +import org.apache.hudi.metadata.HoodieTableMetadata; +import org.apache.hudi.metadata.HoodieTableMetadataUtil; +import org.apache.hudi.metadata.MetadataPartitionType; + +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Deferred record index initialization on the HoodieStreamer path: the first sync leaves the index unbuilt and the + * second one builds it, with no duplicates across the switch from the fallback index. + */ +@Tag("functional") +public class TestDeferredRecordIndexOnStreamerPath extends HoodieDeltaStreamerTestBase { + + private static void enableDeferredRecordIndex(HoodieDeltaStreamer.Config cfg, boolean partitioned, int tableVersion) { + cfg.configs.add(HoodieMetadataConfig.ENABLE.key() + "=true"); + cfg.configs.add(HoodieWriteConfig.WRITE_TABLE_VERSION.key() + "=" + tableVersion); + cfg.configs.add(HoodieWriteConfig.AUTO_UPGRADE_VERSION.key() + "=false"); + cfg.configs.add(HoodieMetadataConfig.GLOBAL_RECORD_LEVEL_INDEX_ENABLE_PROP.key() + "=" + !partitioned); + cfg.configs.add(HoodieMetadataConfig.RECORD_LEVEL_INDEX_ENABLE_PROP.key() + "=" + partitioned); + cfg.configs.add(HoodieIndexConfig.INDEX_TYPE.key() + "=" + + (partitioned ? "RECORD_LEVEL_INDEX" : "GLOBAL_RECORD_LEVEL_INDEX")); + cfg.configs.add(HoodieMetadataConfig.DEFER_RLI_INIT_FOR_FRESH_TABLE.key() + "=true"); + // Pin the sizing inputs so the expected file group count follows from the record count alone: 9600 bytes over + // 48 bytes per entry is 200 entries per file group, and the floor is kept out of the way. + cfg.configs.add(HoodieMetadataConfig.RECORD_INDEX_MAX_FILE_GROUP_SIZE_BYTES_PROP.key() + "=9600"); + cfg.configs.add(HoodieMetadataConfig.GLOBAL_RECORD_LEVEL_INDEX_MIN_FILE_GROUP_COUNT_PROP.key() + "=1"); + cfg.configs.add(HoodieMetadataConfig.RECORD_LEVEL_INDEX_MIN_FILE_GROUP_COUNT_PROP.key() + "=1"); + cfg.configs.add(HoodieMetadataConfig.GLOBAL_RECORD_LEVEL_INDEX_MAX_FILE_GROUP_COUNT_PROP.key() + "=1000"); + cfg.configs.add(HoodieMetadataConfig.RECORD_LEVEL_INDEX_MAX_FILE_GROUP_COUNT_PROP.key() + "=1000"); + } + + /** + * The file group count the record index is sized to for the given number of records, applying the growth factor and + * the entries that fit in one file group. + */ + private static int expectedFileGroupCount(long records) { + long estimated = (long) Math.ceil(records * 2.0) / 200; + return (int) Math.max(1, Math.min(1000, estimated)); + } + + private static List<String> recordIndexFileGroupIds(String tableBasePath) { + HoodieTableMetaClient metadataMetaClient = HoodieTableMetaClient.builder() + .setBasePath(HoodieTableMetadata.getMetadataTableBasePath(tableBasePath)) + .setConf(HoodieTestUtils.getDefaultStorageConf()) + .build(); + return HoodieTableMetadataUtil.getPartitionLatestFileSlices(metadataMetaClient, Option.empty(), + MetadataPartitionType.RECORD_INDEX.getPartitionPath()) + .stream().map(FileSlice::getFileId).collect(Collectors.toList()); + } + + private static boolean isRecordIndexBuilt(String tableBasePath) { + return HoodieTableMetaClient.builder() + .setBasePath(tableBasePath) + .setConf(HoodieTestUtils.getDefaultStorageConf()) + .build() + .getTableConfig().getMetadataPartitions() + .contains(MetadataPartitionType.RECORD_INDEX.getPartitionPath()); + } + + static Stream<Arguments> testArgs() { + return Stream.of( + Arguments.arguments(false, HoodieTableVersion.SIX.versionCode()), + Arguments.arguments(false, HoodieTableVersion.TEN.versionCode()), + Arguments.arguments(true, HoodieTableVersion.TEN.versionCode())); + } + + @ParameterizedTest + @MethodSource("testArgs") + public void testRecordIndexDeferredUntilSecondSync(boolean partitioned, int tableVersion) throws Exception { + String tableBasePath = basePath + "/test_rli_deferred_streamer_" + + (partitioned ? "partitioned" : "global") + "_v" + tableVersion; + + HoodieDeltaStreamer.Config insertCfg = + TestHoodieDeltaStreamer.TestHelpers.makeConfig(tableBasePath, WriteOperationType.INSERT); + enableDeferredRecordIndex(insertCfg, partitioned, tableVersion); + new HoodieDeltaStreamer(insertCfg, jsc).sync(); + assertEquals(tableVersion, HoodieTableMetaClient.builder().setBasePath(tableBasePath) + .setConf(HoodieTestUtils.getDefaultStorageConf()).build().getTableConfig().getTableVersion().versionCode()); + assertFalse(isRecordIndexBuilt(tableBasePath), "Record index should not be built by the first sync"); + + // The index is sized from the data the first sync left behind, so capture it before the second sync adds to it. + assertRecordCount(1000, tableBasePath, sqlContext); + sqlContext.clearCache(); + Map<String, Long> rowsPerPartition = sqlContext.read().format("hudi").load(tableBasePath) + .groupBy("_hoodie_partition_path").count() + .collectAsList().stream().collect(Collectors.toMap(row -> row.getString(0), row -> row.getLong(1))); + long rowsBeforeBuild = rowsPerPartition.values().stream().mapToLong(Long::longValue).sum(); + + HoodieDeltaStreamer.Config upsertCfg = + TestHoodieDeltaStreamer.TestHelpers.makeConfig(tableBasePath, WriteOperationType.UPSERT); + enableDeferredRecordIndex(upsertCfg, partitioned, tableVersion); + new HoodieDeltaStreamer(upsertCfg, jsc).sync(); + assertTrue(isRecordIndexBuilt(tableBasePath), "Record index should be built by the second sync"); + + // The second sync carries 1000 records from the test source: 500 inserts, 450 updates and 50 deletes, so the + // 1000 the first sync wrote become 1450, one row per key, and the deletes make this cover a key removed through + // the fallback index as well. The same 1450 comes out with the deferral turned off, so this pins what the two Review Comment: This case does discriminate on sizing (1000 vs 1450), which is good. It still needs a third sync that updates the second sync's inserts and re-inserts its deletes, plus the RLI entry count check. ########## hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestDeferredRecordIndexInit.scala: ########## @@ -0,0 +1,331 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hudi.functional + +import org.apache.hudi.{DataSourceWriteOptions, SparkDatasetMixin} +import org.apache.hudi.DataSourceWriteOptions._ +import org.apache.hudi.common.config.HoodieMetadataConfig +import org.apache.hudi.common.model.{HoodieRecord, HoodieTableType} +import org.apache.hudi.common.table.{HoodieTableConfig, HoodieTableMetaClient, HoodieTableVersion} +import org.apache.hudi.common.testutils.HoodieTestDataGenerator +import org.apache.hudi.common.util.{Option => HOption} +import org.apache.hudi.config.{HoodieCompactionConfig, HoodieIndexConfig, HoodieWriteConfig} +import org.apache.hudi.functional.TestDeferredRecordIndexInit.{expectedFileGroupCount, DeferredInitTestCase} +import org.apache.hudi.metadata.{HoodieTableMetadata, HoodieTableMetadataUtil, MetadataPartitionType} + +import org.apache.spark.sql.SaveMode +import org.junit.jupiter.api.Tag +import org.junit.jupiter.api.Assertions.{assertEquals, assertFalse, assertTrue} +import org.junit.jupiter.params.ParameterizedTest +import org.junit.jupiter.params.provider.{Arguments, MethodSource} + +import java.util + +import scala.collection.JavaConverters._ + +/** + * End to end coverage of hoodie.metadata.record.level.index.defer.init on its own, with no record threshold: + * the record index is not built by the first commit, is built by the second one and sized from the first commit's + * data, and the switch from the fallback index produces no duplicates. + */ +@Tag("functional") +class TestDeferredRecordIndexInit extends RecordLevelIndexTestBase with SparkDatasetMixin { + + private def deferredInitOptions(testCase: DeferredInitTestCase): Map[String, String] = Map( + HoodieWriteConfig.TBL_NAME.key -> "hoodie_test", + DataSourceWriteOptions.TABLE_TYPE.key -> testCase.tableType.name(), + RECORDKEY_FIELD.key -> "_row_key", + PARTITIONPATH_FIELD.key -> "partition_path", + HoodieTableConfig.ORDERING_FIELDS.key -> "timestamp", + HoodieWriteConfig.WRITE_TABLE_VERSION.key -> testCase.tableVersion.versionCode().toString, + HoodieWriteConfig.AUTO_UPGRADE_VERSION.key -> "false", + HoodieMetadataConfig.ENABLE.key -> "true", + HoodieMetadataConfig.GLOBAL_RECORD_LEVEL_INDEX_ENABLE_PROP.key -> (!testCase.partitioned).toString, + HoodieMetadataConfig.RECORD_LEVEL_INDEX_ENABLE_PROP.key -> testCase.partitioned.toString, + HoodieMetadataConfig.STREAMING_WRITE_ENABLED.key -> testCase.streamingWrites.toString, + // Pinned low so the sizing result is not simply the floor. + HoodieMetadataConfig.GLOBAL_RECORD_LEVEL_INDEX_MIN_FILE_GROUP_COUNT_PROP.key -> "1", + HoodieMetadataConfig.GLOBAL_RECORD_LEVEL_INDEX_MAX_FILE_GROUP_COUNT_PROP.key -> "1000", + HoodieMetadataConfig.RECORD_LEVEL_INDEX_MIN_FILE_GROUP_COUNT_PROP.key -> "1", + HoodieMetadataConfig.RECORD_LEVEL_INDEX_MAX_FILE_GROUP_COUNT_PROP.key -> "1000", + // 9600 bytes / 48 bytes per entry = 200 entries per file group. + HoodieMetadataConfig.RECORD_INDEX_MAX_FILE_GROUP_SIZE_BYTES_PROP.key -> "9600", + HoodieMetadataConfig.DEFER_RLI_INIT_FOR_FRESH_TABLE.key -> "true", + HoodieCompactionConfig.INLINE_COMPACT.key -> "false", + HoodieIndexConfig.INDEX_TYPE.key -> (if (testCase.partitioned) "RECORD_LEVEL_INDEX" else "GLOBAL_RECORD_LEVEL_INDEX")) + + @ParameterizedTest + @MethodSource(Array("testArgsForDeferredInit")) + def testRecordIndexDeferredUntilSecondCommit(testCase: DeferredInitTestCase): Unit = { + val options = deferredInitOptions(testCase) + val dataGen = new HoodieTestDataGenerator() + val allRecords = new util.ArrayList[HoodieRecord[_]]() + val write = (records: util.List[HoodieRecord[_]], operation: String, mode: SaveMode) => + toDataset(spark, records).write.format("hudi").options(options) + .option(DataSourceWriteOptions.OPERATION.key, operation).mode(mode).save(basePath) + + // Commit 1: the record index must not be built, since the table had no completed commit when the write started. + val firstBatch = dataGen.generateInserts("001", 2000) + allRecords.addAll(firstBatch) + write(firstBatch.asInstanceOf[util.List[HoodieRecord[_]]], testCase.firstOperation, SaveMode.Overwrite) + assertTableVersion(testCase.tableVersion) + assertFalse(isRecordIndexAvailable, "Record index should not be built by the first commit") + + // What the index should be sized from once it is built. + val rowsPerPartition = spark.read.format("hudi").load(basePath).groupBy("_hoodie_partition_path").count().collect() + .map(row => row.getString(0) -> row.getLong(1)).toMap + val rowsBeforeBuild = rowsPerPartition.values.sum + + // Commit 2: updates of records tagged through the fallback index, plus new records. The index is built here. Review Comment: This comment isn't accurate. `doInitTable` runs `initMetadataTable` before the write, so the index is built at the start of commit 2 and these updates are tagged through the new record index, not the fallback. The fallback only ever tags commit 1's inserts. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
