prathik-onehouse commented on code in PR #20604: URL: https://github.com/apache/hudi/pull/20604#discussion_r4622658847
########## hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestDeferredRecordIndexInit.scala: ########## @@ -0,0 +1,331 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hudi.functional + +import org.apache.hudi.{DataSourceWriteOptions, SparkDatasetMixin} +import org.apache.hudi.DataSourceWriteOptions._ +import org.apache.hudi.common.config.HoodieMetadataConfig +import org.apache.hudi.common.model.{HoodieRecord, HoodieTableType} +import org.apache.hudi.common.table.{HoodieTableConfig, HoodieTableMetaClient, HoodieTableVersion} +import org.apache.hudi.common.testutils.HoodieTestDataGenerator +import org.apache.hudi.common.util.{Option => HOption} +import org.apache.hudi.config.{HoodieCompactionConfig, HoodieIndexConfig, HoodieWriteConfig} +import org.apache.hudi.functional.TestDeferredRecordIndexInit.{expectedFileGroupCount, DeferredInitTestCase} +import org.apache.hudi.metadata.{HoodieTableMetadata, HoodieTableMetadataUtil, MetadataPartitionType} + +import org.apache.spark.sql.SaveMode +import org.junit.jupiter.api.Tag +import org.junit.jupiter.api.Assertions.{assertEquals, assertFalse, assertTrue} +import org.junit.jupiter.params.ParameterizedTest +import org.junit.jupiter.params.provider.{Arguments, MethodSource} + +import java.util + +import scala.collection.JavaConverters._ + +/** + * End to end coverage of hoodie.metadata.record.level.index.defer.init on its own, with no record threshold: + * the record index is not built by the first commit, is built by the second one and sized from the first commit's + * data, and the switch from the fallback index produces no duplicates. + */ +@Tag("functional") +class TestDeferredRecordIndexInit extends RecordLevelIndexTestBase with SparkDatasetMixin { + + private def deferredInitOptions(testCase: DeferredInitTestCase): Map[String, String] = Map( + HoodieWriteConfig.TBL_NAME.key -> "hoodie_test", + DataSourceWriteOptions.TABLE_TYPE.key -> testCase.tableType.name(), + RECORDKEY_FIELD.key -> "_row_key", + PARTITIONPATH_FIELD.key -> "partition_path", + HoodieTableConfig.ORDERING_FIELDS.key -> "timestamp", + HoodieWriteConfig.WRITE_TABLE_VERSION.key -> testCase.tableVersion.versionCode().toString, + HoodieWriteConfig.AUTO_UPGRADE_VERSION.key -> "false", + HoodieMetadataConfig.ENABLE.key -> "true", + HoodieMetadataConfig.GLOBAL_RECORD_LEVEL_INDEX_ENABLE_PROP.key -> (!testCase.partitioned).toString, + HoodieMetadataConfig.RECORD_LEVEL_INDEX_ENABLE_PROP.key -> testCase.partitioned.toString, + HoodieMetadataConfig.STREAMING_WRITE_ENABLED.key -> testCase.streamingWrites.toString, + // Pinned low so the sizing result is not simply the floor. + HoodieMetadataConfig.GLOBAL_RECORD_LEVEL_INDEX_MIN_FILE_GROUP_COUNT_PROP.key -> "1", + HoodieMetadataConfig.GLOBAL_RECORD_LEVEL_INDEX_MAX_FILE_GROUP_COUNT_PROP.key -> "1000", + HoodieMetadataConfig.RECORD_LEVEL_INDEX_MIN_FILE_GROUP_COUNT_PROP.key -> "1", + HoodieMetadataConfig.RECORD_LEVEL_INDEX_MAX_FILE_GROUP_COUNT_PROP.key -> "1000", + // 9600 bytes / 48 bytes per entry = 200 entries per file group. + HoodieMetadataConfig.RECORD_INDEX_MAX_FILE_GROUP_SIZE_BYTES_PROP.key -> "9600", + HoodieMetadataConfig.DEFER_RLI_INIT_FOR_FRESH_TABLE.key -> "true", + HoodieCompactionConfig.INLINE_COMPACT.key -> "false", + HoodieIndexConfig.INDEX_TYPE.key -> (if (testCase.partitioned) "RECORD_LEVEL_INDEX" else "GLOBAL_RECORD_LEVEL_INDEX")) + + @ParameterizedTest + @MethodSource(Array("testArgsForDeferredInit")) + def testRecordIndexDeferredUntilSecondCommit(testCase: DeferredInitTestCase): Unit = { + val options = deferredInitOptions(testCase) + val dataGen = new HoodieTestDataGenerator() + val allRecords = new util.ArrayList[HoodieRecord[_]]() + val write = (records: util.List[HoodieRecord[_]], operation: String, mode: SaveMode) => + toDataset(spark, records).write.format("hudi").options(options) + .option(DataSourceWriteOptions.OPERATION.key, operation).mode(mode).save(basePath) + + // Commit 1: the record index must not be built, since the table had no completed commit when the write started. + val firstBatch = dataGen.generateInserts("001", 2000) + allRecords.addAll(firstBatch) + write(firstBatch.asInstanceOf[util.List[HoodieRecord[_]]], testCase.firstOperation, SaveMode.Overwrite) + assertTableVersion(testCase.tableVersion) + assertFalse(isRecordIndexAvailable, "Record index should not be built by the first commit") Review Comment: Done. After commit 1 every case now checks that the MDT has no `record_index` files, no metadata commit writes to it, and there is no index definition; after commit 2 it checks the definition and its `isPartitioned` option. As you expected, the partitioned case with batch metadata writes failed on master, so `maybeInitializeNewFileGroupsForPartitionedRLI` now also requires the record index to be initialized (or being built by the async indexer). ########## hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestDeferredRecordIndexInit.scala: ########## @@ -0,0 +1,331 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hudi.functional + +import org.apache.hudi.{DataSourceWriteOptions, SparkDatasetMixin} +import org.apache.hudi.DataSourceWriteOptions._ +import org.apache.hudi.common.config.HoodieMetadataConfig +import org.apache.hudi.common.model.{HoodieRecord, HoodieTableType} +import org.apache.hudi.common.table.{HoodieTableConfig, HoodieTableMetaClient, HoodieTableVersion} +import org.apache.hudi.common.testutils.HoodieTestDataGenerator +import org.apache.hudi.common.util.{Option => HOption} +import org.apache.hudi.config.{HoodieCompactionConfig, HoodieIndexConfig, HoodieWriteConfig} +import org.apache.hudi.functional.TestDeferredRecordIndexInit.{expectedFileGroupCount, DeferredInitTestCase} +import org.apache.hudi.metadata.{HoodieTableMetadata, HoodieTableMetadataUtil, MetadataPartitionType} + +import org.apache.spark.sql.SaveMode +import org.junit.jupiter.api.Tag +import org.junit.jupiter.api.Assertions.{assertEquals, assertFalse, assertTrue} +import org.junit.jupiter.params.ParameterizedTest +import org.junit.jupiter.params.provider.{Arguments, MethodSource} + +import java.util + +import scala.collection.JavaConverters._ + +/** + * End to end coverage of hoodie.metadata.record.level.index.defer.init on its own, with no record threshold: + * the record index is not built by the first commit, is built by the second one and sized from the first commit's + * data, and the switch from the fallback index produces no duplicates. + */ +@Tag("functional") +class TestDeferredRecordIndexInit extends RecordLevelIndexTestBase with SparkDatasetMixin { + + private def deferredInitOptions(testCase: DeferredInitTestCase): Map[String, String] = Map( + HoodieWriteConfig.TBL_NAME.key -> "hoodie_test", + DataSourceWriteOptions.TABLE_TYPE.key -> testCase.tableType.name(), + RECORDKEY_FIELD.key -> "_row_key", + PARTITIONPATH_FIELD.key -> "partition_path", + HoodieTableConfig.ORDERING_FIELDS.key -> "timestamp", + HoodieWriteConfig.WRITE_TABLE_VERSION.key -> testCase.tableVersion.versionCode().toString, + HoodieWriteConfig.AUTO_UPGRADE_VERSION.key -> "false", + HoodieMetadataConfig.ENABLE.key -> "true", + HoodieMetadataConfig.GLOBAL_RECORD_LEVEL_INDEX_ENABLE_PROP.key -> (!testCase.partitioned).toString, + HoodieMetadataConfig.RECORD_LEVEL_INDEX_ENABLE_PROP.key -> testCase.partitioned.toString, + HoodieMetadataConfig.STREAMING_WRITE_ENABLED.key -> testCase.streamingWrites.toString, + // Pinned low so the sizing result is not simply the floor. + HoodieMetadataConfig.GLOBAL_RECORD_LEVEL_INDEX_MIN_FILE_GROUP_COUNT_PROP.key -> "1", + HoodieMetadataConfig.GLOBAL_RECORD_LEVEL_INDEX_MAX_FILE_GROUP_COUNT_PROP.key -> "1000", + HoodieMetadataConfig.RECORD_LEVEL_INDEX_MIN_FILE_GROUP_COUNT_PROP.key -> "1", + HoodieMetadataConfig.RECORD_LEVEL_INDEX_MAX_FILE_GROUP_COUNT_PROP.key -> "1000", + // 9600 bytes / 48 bytes per entry = 200 entries per file group. + HoodieMetadataConfig.RECORD_INDEX_MAX_FILE_GROUP_SIZE_BYTES_PROP.key -> "9600", + HoodieMetadataConfig.DEFER_RLI_INIT_FOR_FRESH_TABLE.key -> "true", + HoodieCompactionConfig.INLINE_COMPACT.key -> "false", + HoodieIndexConfig.INDEX_TYPE.key -> (if (testCase.partitioned) "RECORD_LEVEL_INDEX" else "GLOBAL_RECORD_LEVEL_INDEX")) + + @ParameterizedTest + @MethodSource(Array("testArgsForDeferredInit")) + def testRecordIndexDeferredUntilSecondCommit(testCase: DeferredInitTestCase): Unit = { + val options = deferredInitOptions(testCase) + val dataGen = new HoodieTestDataGenerator() + val allRecords = new util.ArrayList[HoodieRecord[_]]() + val write = (records: util.List[HoodieRecord[_]], operation: String, mode: SaveMode) => + toDataset(spark, records).write.format("hudi").options(options) + .option(DataSourceWriteOptions.OPERATION.key, operation).mode(mode).save(basePath) + + // Commit 1: the record index must not be built, since the table had no completed commit when the write started. + val firstBatch = dataGen.generateInserts("001", 2000) + allRecords.addAll(firstBatch) + write(firstBatch.asInstanceOf[util.List[HoodieRecord[_]]], testCase.firstOperation, SaveMode.Overwrite) + assertTableVersion(testCase.tableVersion) + assertFalse(isRecordIndexAvailable, "Record index should not be built by the first commit") + + // What the index should be sized from once it is built. + val rowsPerPartition = spark.read.format("hudi").load(basePath).groupBy("_hoodie_partition_path").count().collect() + .map(row => row.getString(0) -> row.getLong(1)).toMap + val rowsBeforeBuild = rowsPerPartition.values.sum + + // Commit 2: updates of records tagged through the fallback index, plus new records. The index is built here. Review Comment: Fixed, thanks for catching it. The comment is gone, and commit 2 is now described as building the index from commit 1's data and then writing through it. ########## hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestDeferredRecordIndexInit.scala: ########## @@ -0,0 +1,331 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hudi.functional + +import org.apache.hudi.{DataSourceWriteOptions, SparkDatasetMixin} +import org.apache.hudi.DataSourceWriteOptions._ +import org.apache.hudi.common.config.HoodieMetadataConfig +import org.apache.hudi.common.model.{HoodieRecord, HoodieTableType} +import org.apache.hudi.common.table.{HoodieTableConfig, HoodieTableMetaClient, HoodieTableVersion} +import org.apache.hudi.common.testutils.HoodieTestDataGenerator +import org.apache.hudi.common.util.{Option => HOption} +import org.apache.hudi.config.{HoodieCompactionConfig, HoodieIndexConfig, HoodieWriteConfig} +import org.apache.hudi.functional.TestDeferredRecordIndexInit.{expectedFileGroupCount, DeferredInitTestCase} +import org.apache.hudi.metadata.{HoodieTableMetadata, HoodieTableMetadataUtil, MetadataPartitionType} + +import org.apache.spark.sql.SaveMode +import org.junit.jupiter.api.Tag +import org.junit.jupiter.api.Assertions.{assertEquals, assertFalse, assertTrue} +import org.junit.jupiter.params.ParameterizedTest +import org.junit.jupiter.params.provider.{Arguments, MethodSource} + +import java.util + +import scala.collection.JavaConverters._ + +/** + * End to end coverage of hoodie.metadata.record.level.index.defer.init on its own, with no record threshold: + * the record index is not built by the first commit, is built by the second one and sized from the first commit's + * data, and the switch from the fallback index produces no duplicates. + */ +@Tag("functional") +class TestDeferredRecordIndexInit extends RecordLevelIndexTestBase with SparkDatasetMixin { + + private def deferredInitOptions(testCase: DeferredInitTestCase): Map[String, String] = Map( + HoodieWriteConfig.TBL_NAME.key -> "hoodie_test", + DataSourceWriteOptions.TABLE_TYPE.key -> testCase.tableType.name(), + RECORDKEY_FIELD.key -> "_row_key", + PARTITIONPATH_FIELD.key -> "partition_path", + HoodieTableConfig.ORDERING_FIELDS.key -> "timestamp", + HoodieWriteConfig.WRITE_TABLE_VERSION.key -> testCase.tableVersion.versionCode().toString, + HoodieWriteConfig.AUTO_UPGRADE_VERSION.key -> "false", + HoodieMetadataConfig.ENABLE.key -> "true", + HoodieMetadataConfig.GLOBAL_RECORD_LEVEL_INDEX_ENABLE_PROP.key -> (!testCase.partitioned).toString, + HoodieMetadataConfig.RECORD_LEVEL_INDEX_ENABLE_PROP.key -> testCase.partitioned.toString, + HoodieMetadataConfig.STREAMING_WRITE_ENABLED.key -> testCase.streamingWrites.toString, + // Pinned low so the sizing result is not simply the floor. + HoodieMetadataConfig.GLOBAL_RECORD_LEVEL_INDEX_MIN_FILE_GROUP_COUNT_PROP.key -> "1", + HoodieMetadataConfig.GLOBAL_RECORD_LEVEL_INDEX_MAX_FILE_GROUP_COUNT_PROP.key -> "1000", + HoodieMetadataConfig.RECORD_LEVEL_INDEX_MIN_FILE_GROUP_COUNT_PROP.key -> "1", + HoodieMetadataConfig.RECORD_LEVEL_INDEX_MAX_FILE_GROUP_COUNT_PROP.key -> "1000", + // 9600 bytes / 48 bytes per entry = 200 entries per file group. + HoodieMetadataConfig.RECORD_INDEX_MAX_FILE_GROUP_SIZE_BYTES_PROP.key -> "9600", + HoodieMetadataConfig.DEFER_RLI_INIT_FOR_FRESH_TABLE.key -> "true", + HoodieCompactionConfig.INLINE_COMPACT.key -> "false", + HoodieIndexConfig.INDEX_TYPE.key -> (if (testCase.partitioned) "RECORD_LEVEL_INDEX" else "GLOBAL_RECORD_LEVEL_INDEX")) + + @ParameterizedTest + @MethodSource(Array("testArgsForDeferredInit")) + def testRecordIndexDeferredUntilSecondCommit(testCase: DeferredInitTestCase): Unit = { + val options = deferredInitOptions(testCase) + val dataGen = new HoodieTestDataGenerator() + val allRecords = new util.ArrayList[HoodieRecord[_]]() + val write = (records: util.List[HoodieRecord[_]], operation: String, mode: SaveMode) => + toDataset(spark, records).write.format("hudi").options(options) + .option(DataSourceWriteOptions.OPERATION.key, operation).mode(mode).save(basePath) + + // Commit 1: the record index must not be built, since the table had no completed commit when the write started. + val firstBatch = dataGen.generateInserts("001", 2000) + allRecords.addAll(firstBatch) + write(firstBatch.asInstanceOf[util.List[HoodieRecord[_]]], testCase.firstOperation, SaveMode.Overwrite) + assertTableVersion(testCase.tableVersion) + assertFalse(isRecordIndexAvailable, "Record index should not be built by the first commit") + + // What the index should be sized from once it is built. + val rowsPerPartition = spark.read.format("hudi").load(basePath).groupBy("_hoodie_partition_path").count().collect() + .map(row => row.getString(0) -> row.getLong(1)).toMap + val rowsBeforeBuild = rowsPerPartition.values.sum + + // Commit 2: updates of records tagged through the fallback index, plus new records. The index is built here. + val updates = dataGen.generateUpdates("002", allRecords.subList(0, 300)) + val newRecords = dataGen.generateInserts("002", 10) + val secondBatch = new util.ArrayList[HoodieRecord[_]](updates) + secondBatch.addAll(newRecords) + allRecords.addAll(newRecords) + write(secondBatch, UPSERT_OPERATION_OPT_VAL, SaveMode.Append) + assertTrue(isRecordIndexAvailable, "Record index should be built by the second commit") + + // Commit 3: upserts tagged through the record index itself. + val laterUpdates = dataGen.generateUpdates("003", allRecords.subList(0, 300)) Review Comment: Done for every writer. Commit 2 now also deletes 50 rows, and commit 3 updates commit 2's inserts and brings back those keys. After commits 2 and 3 the test compares a full RLI scan to the table, so beyond the entry count it checks one entry per row, in the row's partition and file group. SQL and structured streaming now have a write after the build as well. ########## hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestDeferredRecordIndexInit.scala: ########## @@ -0,0 +1,331 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hudi.functional + +import org.apache.hudi.{DataSourceWriteOptions, SparkDatasetMixin} +import org.apache.hudi.DataSourceWriteOptions._ +import org.apache.hudi.common.config.HoodieMetadataConfig +import org.apache.hudi.common.model.{HoodieRecord, HoodieTableType} +import org.apache.hudi.common.table.{HoodieTableConfig, HoodieTableMetaClient, HoodieTableVersion} +import org.apache.hudi.common.testutils.HoodieTestDataGenerator +import org.apache.hudi.common.util.{Option => HOption} +import org.apache.hudi.config.{HoodieCompactionConfig, HoodieIndexConfig, HoodieWriteConfig} +import org.apache.hudi.functional.TestDeferredRecordIndexInit.{expectedFileGroupCount, DeferredInitTestCase} +import org.apache.hudi.metadata.{HoodieTableMetadata, HoodieTableMetadataUtil, MetadataPartitionType} + +import org.apache.spark.sql.SaveMode +import org.junit.jupiter.api.Tag +import org.junit.jupiter.api.Assertions.{assertEquals, assertFalse, assertTrue} +import org.junit.jupiter.params.ParameterizedTest +import org.junit.jupiter.params.provider.{Arguments, MethodSource} + +import java.util + +import scala.collection.JavaConverters._ + +/** + * End to end coverage of hoodie.metadata.record.level.index.defer.init on its own, with no record threshold: + * the record index is not built by the first commit, is built by the second one and sized from the first commit's + * data, and the switch from the fallback index produces no duplicates. + */ +@Tag("functional") +class TestDeferredRecordIndexInit extends RecordLevelIndexTestBase with SparkDatasetMixin { + + private def deferredInitOptions(testCase: DeferredInitTestCase): Map[String, String] = Map( + HoodieWriteConfig.TBL_NAME.key -> "hoodie_test", + DataSourceWriteOptions.TABLE_TYPE.key -> testCase.tableType.name(), + RECORDKEY_FIELD.key -> "_row_key", + PARTITIONPATH_FIELD.key -> "partition_path", + HoodieTableConfig.ORDERING_FIELDS.key -> "timestamp", + HoodieWriteConfig.WRITE_TABLE_VERSION.key -> testCase.tableVersion.versionCode().toString, + HoodieWriteConfig.AUTO_UPGRADE_VERSION.key -> "false", + HoodieMetadataConfig.ENABLE.key -> "true", + HoodieMetadataConfig.GLOBAL_RECORD_LEVEL_INDEX_ENABLE_PROP.key -> (!testCase.partitioned).toString, + HoodieMetadataConfig.RECORD_LEVEL_INDEX_ENABLE_PROP.key -> testCase.partitioned.toString, + HoodieMetadataConfig.STREAMING_WRITE_ENABLED.key -> testCase.streamingWrites.toString, + // Pinned low so the sizing result is not simply the floor. + HoodieMetadataConfig.GLOBAL_RECORD_LEVEL_INDEX_MIN_FILE_GROUP_COUNT_PROP.key -> "1", + HoodieMetadataConfig.GLOBAL_RECORD_LEVEL_INDEX_MAX_FILE_GROUP_COUNT_PROP.key -> "1000", + HoodieMetadataConfig.RECORD_LEVEL_INDEX_MIN_FILE_GROUP_COUNT_PROP.key -> "1", + HoodieMetadataConfig.RECORD_LEVEL_INDEX_MAX_FILE_GROUP_COUNT_PROP.key -> "1000", + // 9600 bytes / 48 bytes per entry = 200 entries per file group. + HoodieMetadataConfig.RECORD_INDEX_MAX_FILE_GROUP_SIZE_BYTES_PROP.key -> "9600", + HoodieMetadataConfig.DEFER_RLI_INIT_FOR_FRESH_TABLE.key -> "true", + HoodieCompactionConfig.INLINE_COMPACT.key -> "false", + HoodieIndexConfig.INDEX_TYPE.key -> (if (testCase.partitioned) "RECORD_LEVEL_INDEX" else "GLOBAL_RECORD_LEVEL_INDEX")) + + @ParameterizedTest + @MethodSource(Array("testArgsForDeferredInit")) + def testRecordIndexDeferredUntilSecondCommit(testCase: DeferredInitTestCase): Unit = { + val options = deferredInitOptions(testCase) + val dataGen = new HoodieTestDataGenerator() + val allRecords = new util.ArrayList[HoodieRecord[_]]() + val write = (records: util.List[HoodieRecord[_]], operation: String, mode: SaveMode) => + toDataset(spark, records).write.format("hudi").options(options) + .option(DataSourceWriteOptions.OPERATION.key, operation).mode(mode).save(basePath) + + // Commit 1: the record index must not be built, since the table had no completed commit when the write started. + val firstBatch = dataGen.generateInserts("001", 2000) + allRecords.addAll(firstBatch) + write(firstBatch.asInstanceOf[util.List[HoodieRecord[_]]], testCase.firstOperation, SaveMode.Overwrite) + assertTableVersion(testCase.tableVersion) + assertFalse(isRecordIndexAvailable, "Record index should not be built by the first commit") + + // What the index should be sized from once it is built. + val rowsPerPartition = spark.read.format("hudi").load(basePath).groupBy("_hoodie_partition_path").count().collect() + .map(row => row.getString(0) -> row.getLong(1)).toMap + val rowsBeforeBuild = rowsPerPartition.values.sum + + // Commit 2: updates of records tagged through the fallback index, plus new records. The index is built here. + val updates = dataGen.generateUpdates("002", allRecords.subList(0, 300)) + val newRecords = dataGen.generateInserts("002", 10) + val secondBatch = new util.ArrayList[HoodieRecord[_]](updates) + secondBatch.addAll(newRecords) + allRecords.addAll(newRecords) + write(secondBatch, UPSERT_OPERATION_OPT_VAL, SaveMode.Append) + assertTrue(isRecordIndexAvailable, "Record index should be built by the second commit") + + // Commit 3: upserts tagged through the record index itself. + val laterUpdates = dataGen.generateUpdates("003", allRecords.subList(0, 300)) + write(laterUpdates.asInstanceOf[util.List[HoodieRecord[_]]], UPSERT_OPERATION_OPT_VAL, SaveMode.Append) + + // No duplicates across the fallback to record index switch. + val keys = spark.read.format("hudi").load(basePath).select("_row_key").collect().map(_.getString(0)) + assertEquals(allRecords.asScala.map(_.getRecordKey).sorted.toSeq, keys.sorted.toSeq) + + // The index is sized from the data the first commit left behind, not from an empty table. + val fileGroupIds = recordIndexFileGroupIds() + if (testCase.partitioned) { + val perPartition = fileGroupIds.groupBy(HoodieTableMetadataUtil.getDataTablePartitionNameFromFileGroupName) + .mapValues(_.size).toMap + assertEquals(rowsPerPartition.mapValues(rows => expectedFileGroupCount(rows)).toMap, perPartition) + } else { + assertEquals(expectedFileGroupCount(rowsBeforeBuild), fileGroupIds.size) Review Comment: Done. Commit 2 now adds 2000 rows, so the expected count is 20 sized from commit 1 vs 40 after commit 2 (about 6 vs 13 per partition). ########## hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestDeferredRecordIndexInit.scala: ########## @@ -0,0 +1,331 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hudi.functional + +import org.apache.hudi.{DataSourceWriteOptions, SparkDatasetMixin} +import org.apache.hudi.DataSourceWriteOptions._ +import org.apache.hudi.common.config.HoodieMetadataConfig +import org.apache.hudi.common.model.{HoodieRecord, HoodieTableType} +import org.apache.hudi.common.table.{HoodieTableConfig, HoodieTableMetaClient, HoodieTableVersion} +import org.apache.hudi.common.testutils.HoodieTestDataGenerator +import org.apache.hudi.common.util.{Option => HOption} +import org.apache.hudi.config.{HoodieCompactionConfig, HoodieIndexConfig, HoodieWriteConfig} +import org.apache.hudi.functional.TestDeferredRecordIndexInit.{expectedFileGroupCount, DeferredInitTestCase} +import org.apache.hudi.metadata.{HoodieTableMetadata, HoodieTableMetadataUtil, MetadataPartitionType} + +import org.apache.spark.sql.SaveMode +import org.junit.jupiter.api.Tag +import org.junit.jupiter.api.Assertions.{assertEquals, assertFalse, assertTrue} +import org.junit.jupiter.params.ParameterizedTest +import org.junit.jupiter.params.provider.{Arguments, MethodSource} + +import java.util + +import scala.collection.JavaConverters._ + +/** + * End to end coverage of hoodie.metadata.record.level.index.defer.init on its own, with no record threshold: + * the record index is not built by the first commit, is built by the second one and sized from the first commit's + * data, and the switch from the fallback index produces no duplicates. + */ +@Tag("functional") +class TestDeferredRecordIndexInit extends RecordLevelIndexTestBase with SparkDatasetMixin { + + private def deferredInitOptions(testCase: DeferredInitTestCase): Map[String, String] = Map( + HoodieWriteConfig.TBL_NAME.key -> "hoodie_test", + DataSourceWriteOptions.TABLE_TYPE.key -> testCase.tableType.name(), + RECORDKEY_FIELD.key -> "_row_key", + PARTITIONPATH_FIELD.key -> "partition_path", + HoodieTableConfig.ORDERING_FIELDS.key -> "timestamp", + HoodieWriteConfig.WRITE_TABLE_VERSION.key -> testCase.tableVersion.versionCode().toString, + HoodieWriteConfig.AUTO_UPGRADE_VERSION.key -> "false", + HoodieMetadataConfig.ENABLE.key -> "true", + HoodieMetadataConfig.GLOBAL_RECORD_LEVEL_INDEX_ENABLE_PROP.key -> (!testCase.partitioned).toString, + HoodieMetadataConfig.RECORD_LEVEL_INDEX_ENABLE_PROP.key -> testCase.partitioned.toString, + HoodieMetadataConfig.STREAMING_WRITE_ENABLED.key -> testCase.streamingWrites.toString, + // Pinned low so the sizing result is not simply the floor. + HoodieMetadataConfig.GLOBAL_RECORD_LEVEL_INDEX_MIN_FILE_GROUP_COUNT_PROP.key -> "1", + HoodieMetadataConfig.GLOBAL_RECORD_LEVEL_INDEX_MAX_FILE_GROUP_COUNT_PROP.key -> "1000", + HoodieMetadataConfig.RECORD_LEVEL_INDEX_MIN_FILE_GROUP_COUNT_PROP.key -> "1", + HoodieMetadataConfig.RECORD_LEVEL_INDEX_MAX_FILE_GROUP_COUNT_PROP.key -> "1000", + // 9600 bytes / 48 bytes per entry = 200 entries per file group. + HoodieMetadataConfig.RECORD_INDEX_MAX_FILE_GROUP_SIZE_BYTES_PROP.key -> "9600", + HoodieMetadataConfig.DEFER_RLI_INIT_FOR_FRESH_TABLE.key -> "true", + HoodieCompactionConfig.INLINE_COMPACT.key -> "false", + HoodieIndexConfig.INDEX_TYPE.key -> (if (testCase.partitioned) "RECORD_LEVEL_INDEX" else "GLOBAL_RECORD_LEVEL_INDEX")) + + @ParameterizedTest + @MethodSource(Array("testArgsForDeferredInit")) + def testRecordIndexDeferredUntilSecondCommit(testCase: DeferredInitTestCase): Unit = { + val options = deferredInitOptions(testCase) + val dataGen = new HoodieTestDataGenerator() + val allRecords = new util.ArrayList[HoodieRecord[_]]() + val write = (records: util.List[HoodieRecord[_]], operation: String, mode: SaveMode) => + toDataset(spark, records).write.format("hudi").options(options) + .option(DataSourceWriteOptions.OPERATION.key, operation).mode(mode).save(basePath) + + // Commit 1: the record index must not be built, since the table had no completed commit when the write started. + val firstBatch = dataGen.generateInserts("001", 2000) + allRecords.addAll(firstBatch) + write(firstBatch.asInstanceOf[util.List[HoodieRecord[_]]], testCase.firstOperation, SaveMode.Overwrite) + assertTableVersion(testCase.tableVersion) + assertFalse(isRecordIndexAvailable, "Record index should not be built by the first commit") + + // What the index should be sized from once it is built. + val rowsPerPartition = spark.read.format("hudi").load(basePath).groupBy("_hoodie_partition_path").count().collect() + .map(row => row.getString(0) -> row.getLong(1)).toMap + val rowsBeforeBuild = rowsPerPartition.values.sum + + // Commit 2: updates of records tagged through the fallback index, plus new records. The index is built here. + val updates = dataGen.generateUpdates("002", allRecords.subList(0, 300)) + val newRecords = dataGen.generateInserts("002", 10) + val secondBatch = new util.ArrayList[HoodieRecord[_]](updates) + secondBatch.addAll(newRecords) + allRecords.addAll(newRecords) + write(secondBatch, UPSERT_OPERATION_OPT_VAL, SaveMode.Append) + assertTrue(isRecordIndexAvailable, "Record index should be built by the second commit") + + // Commit 3: upserts tagged through the record index itself. + val laterUpdates = dataGen.generateUpdates("003", allRecords.subList(0, 300)) + write(laterUpdates.asInstanceOf[util.List[HoodieRecord[_]]], UPSERT_OPERATION_OPT_VAL, SaveMode.Append) + + // No duplicates across the fallback to record index switch. + val keys = spark.read.format("hudi").load(basePath).select("_row_key").collect().map(_.getString(0)) + assertEquals(allRecords.asScala.map(_.getRecordKey).sorted.toSeq, keys.sorted.toSeq) + + // The index is sized from the data the first commit left behind, not from an empty table. + val fileGroupIds = recordIndexFileGroupIds() + if (testCase.partitioned) { + val perPartition = fileGroupIds.groupBy(HoodieTableMetadataUtil.getDataTablePartitionNameFromFileGroupName) + .mapValues(_.size).toMap + assertEquals(rowsPerPartition.mapValues(rows => expectedFileGroupCount(rows)).toMap, perPartition) + } else { + assertEquals(expectedFileGroupCount(rowsBeforeBuild), fileGroupIds.size) + } + } + + /** + * The commit that builds the record index replaces the data the index was just sized from, so the index has to + * carry the replacing records and none of the replaced ones. + */ + @ParameterizedTest + @MethodSource(Array("testArgsForInsertOverwrite")) + def testRecordIndexBuiltByInsertOverwrite(testCase: DeferredInitTestCase): Unit = { + val options = deferredInitOptions(testCase) + val dataGen = new HoodieTestDataGenerator() + val write = (records: util.List[HoodieRecord[_]], operation: String, mode: SaveMode) => + toDataset(spark, records).write.format("hudi").options(options) + .option(DataSourceWriteOptions.OPERATION.key, operation).mode(mode).save(basePath) + + val firstBatch = dataGen.generateInserts("001", 2000) + write(firstBatch.asInstanceOf[util.List[HoodieRecord[_]]], testCase.firstOperation, SaveMode.Overwrite) + assertTableVersion(testCase.tableVersion) + assertFalse(isRecordIndexAvailable, "Record index should not be built by the first commit") + + // Commit 2 builds the record index and then replaces every partition the first commit wrote. + val replacingBatch = dataGen.generateInserts("002", 500) + write(replacingBatch.asInstanceOf[util.List[HoodieRecord[_]]], INSERT_OVERWRITE_OPERATION_OPT_VAL, SaveMode.Append) + assertTrue(isRecordIndexAvailable, "Record index should be built by the second commit") + + val replacingKeys = replacingBatch.asScala.map(_.getRecordKey).sorted.toSeq + assertEquals(replacingKeys, spark.read.format("hudi").load(basePath).select("_row_key") + .collect().map(_.getString(0)).sorted.toSeq) + + // Commit 3 is tagged through the record index. Replaced records must not come back and survivors must not double. + val laterUpdates = dataGen.generateUpdates("003", replacingBatch.subList(0, 100)) + write(laterUpdates.asInstanceOf[util.List[HoodieRecord[_]]], UPSERT_OPERATION_OPT_VAL, SaveMode.Append) + assertEquals(replacingKeys, spark.read.format("hudi").load(basePath).select("_row_key") + .collect().map(_.getString(0)).sorted.toSeq) + } + + /** + * The same deferral through Spark SQL: INSERT INTO leaves the record index unbuilt and the following MERGE INTO + * builds it, sized from the rows the first statement wrote. + */ + @ParameterizedTest + @MethodSource(Array("testArgsForSql")) + def testRecordIndexDeferredUntilSecondStatement(testCase: DeferredInitTestCase): Unit = { + val tableName = s"rli_deferred_sql_${if (testCase.partitioned) "partitioned" else "global"}_v${testCase.tableVersion.versionCode()}" + val tablePath = tempDir.resolve(tableName).toAbsolutePath.toString + val indexEnableProp = if (testCase.partitioned) { + HoodieMetadataConfig.RECORD_LEVEL_INDEX_ENABLE_PROP.key + } else { + HoodieMetadataConfig.GLOBAL_RECORD_LEVEL_INDEX_ENABLE_PROP.key + } + val minFileGroupProp = if (testCase.partitioned) { + HoodieMetadataConfig.RECORD_LEVEL_INDEX_MIN_FILE_GROUP_COUNT_PROP.key + } else { + HoodieMetadataConfig.GLOBAL_RECORD_LEVEL_INDEX_MIN_FILE_GROUP_COUNT_PROP.key + } + val maxFileGroupProp = if (testCase.partitioned) { + HoodieMetadataConfig.RECORD_LEVEL_INDEX_MAX_FILE_GROUP_COUNT_PROP.key + } else { + HoodieMetadataConfig.GLOBAL_RECORD_LEVEL_INDEX_MAX_FILE_GROUP_COUNT_PROP.key + } + val indexType = if (testCase.partitioned) "RECORD_LEVEL_INDEX" else "GLOBAL_RECORD_LEVEL_INDEX" + spark.sql( + s""" + |create table $tableName ( + | id string, + | ts long, + | name string, + | part string + |) using hudi + | options ( + | primaryKey = 'id', + | orderingFields = 'ts', + | hoodie.metadata.enable = 'true', + | ${HoodieWriteConfig.WRITE_TABLE_VERSION.key} = '${testCase.tableVersion.versionCode()}', + | ${HoodieWriteConfig.AUTO_UPGRADE_VERSION.key} = 'false', + | ${HoodieMetadataConfig.STREAMING_WRITE_ENABLED.key} = '${testCase.streamingWrites}', + | $indexEnableProp = 'true', + | hoodie.index.type = '$indexType', + | ${HoodieMetadataConfig.DEFER_RLI_INIT_FOR_FRESH_TABLE.key} = 'true', + | ${HoodieMetadataConfig.RECORD_INDEX_MAX_FILE_GROUP_SIZE_BYTES_PROP.key} = '9600', + | $minFileGroupProp = '1', + | $maxFileGroupProp = '1000' + | ) + | partitioned by (part) + | location '$tablePath' + """.stripMargin) + val sqlMetaClient = HoodieTableMetaClient.builder().setBasePath(tablePath).setConf(storageConf).build() + val recordIndexBuilt = () => HoodieTableMetaClient.reload(sqlMetaClient).getTableConfig.getMetadataPartitions + .contains(MetadataPartitionType.RECORD_INDEX.getPartitionPath) + val assertSqlTableVersion = () => assertEquals(testCase.tableVersion, + HoodieTableMetaClient.reload(sqlMetaClient).getTableConfig.getTableVersion, + "Table was written at an unexpected version") + val rowsOf = (from: Int, to: Int, ts: Long, name: String) => + s"select cast(id as string) as id, ${ts}L as ts, '$name' as name, concat('p', cast(id % 3 as string)) as part from range($from, $to)" + + try { + spark.conf.set("hoodie.spark.sql.insert.into.operation", testCase.firstOperation) + spark.sql(s"insert into $tableName ${rowsOf(0, 2000, 1L, "v1")}") + } finally { + spark.conf.unset("hoodie.spark.sql.insert.into.operation") + } + assertSqlTableVersion() + assertFalse(recordIndexBuilt(), "Record index should not be built by the first statement") + + // The index is sized from the rows the first statement wrote, so capture them before the merge adds more. + // Keyed by the partition path the table writes, which is Hive styled for a SQL created table. + val rowsPerPartition = spark + .sql(s"select _hoodie_partition_path, count(*) as rows from $tableName group by _hoodie_partition_path") + .collect().map(row => row.getString(0) -> row.getLong(1)).toMap + val rowsBeforeBuild = rowsPerPartition.values.sum + + spark.sql( + s""" + |merge into $tableName t + |using (${rowsOf(0, 300, 4L, "v4")} union all ${rowsOf(2000, 2010, 4L, "v4")}) s + |on t.id = s.id + |when matched then update set * + |when not matched then insert * + """.stripMargin) + assertTrue(recordIndexBuilt(), "Record index should be built by the second statement") + + // Every key appears once, and the merge updated exactly the rows it matched. + assertEquals(2010L, spark.sql(s"select id from $tableName").count()) + assertEquals(2010L, spark.sql(s"select distinct id from $tableName").count()) + assertEquals(310L, spark.sql(s"select id from $tableName where name = 'v4'").count()) + + // The index is sized from the rows the first statement left behind, not from the table as the merge left it. Review Comment: Done. The merge now adds 2000 rows, so the assertion can tell the two apart. ########## hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestDeferredRecordIndexInit.scala: ########## @@ -0,0 +1,331 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hudi.functional + +import org.apache.hudi.{DataSourceWriteOptions, SparkDatasetMixin} +import org.apache.hudi.DataSourceWriteOptions._ +import org.apache.hudi.common.config.HoodieMetadataConfig +import org.apache.hudi.common.model.{HoodieRecord, HoodieTableType} +import org.apache.hudi.common.table.{HoodieTableConfig, HoodieTableMetaClient, HoodieTableVersion} +import org.apache.hudi.common.testutils.HoodieTestDataGenerator +import org.apache.hudi.common.util.{Option => HOption} +import org.apache.hudi.config.{HoodieCompactionConfig, HoodieIndexConfig, HoodieWriteConfig} +import org.apache.hudi.functional.TestDeferredRecordIndexInit.{expectedFileGroupCount, DeferredInitTestCase} +import org.apache.hudi.metadata.{HoodieTableMetadata, HoodieTableMetadataUtil, MetadataPartitionType} + +import org.apache.spark.sql.SaveMode +import org.junit.jupiter.api.Tag +import org.junit.jupiter.api.Assertions.{assertEquals, assertFalse, assertTrue} +import org.junit.jupiter.params.ParameterizedTest +import org.junit.jupiter.params.provider.{Arguments, MethodSource} + +import java.util + +import scala.collection.JavaConverters._ + +/** + * End to end coverage of hoodie.metadata.record.level.index.defer.init on its own, with no record threshold: + * the record index is not built by the first commit, is built by the second one and sized from the first commit's + * data, and the switch from the fallback index produces no duplicates. + */ +@Tag("functional") +class TestDeferredRecordIndexInit extends RecordLevelIndexTestBase with SparkDatasetMixin { + + private def deferredInitOptions(testCase: DeferredInitTestCase): Map[String, String] = Map( + HoodieWriteConfig.TBL_NAME.key -> "hoodie_test", + DataSourceWriteOptions.TABLE_TYPE.key -> testCase.tableType.name(), + RECORDKEY_FIELD.key -> "_row_key", + PARTITIONPATH_FIELD.key -> "partition_path", + HoodieTableConfig.ORDERING_FIELDS.key -> "timestamp", + HoodieWriteConfig.WRITE_TABLE_VERSION.key -> testCase.tableVersion.versionCode().toString, + HoodieWriteConfig.AUTO_UPGRADE_VERSION.key -> "false", + HoodieMetadataConfig.ENABLE.key -> "true", + HoodieMetadataConfig.GLOBAL_RECORD_LEVEL_INDEX_ENABLE_PROP.key -> (!testCase.partitioned).toString, + HoodieMetadataConfig.RECORD_LEVEL_INDEX_ENABLE_PROP.key -> testCase.partitioned.toString, + HoodieMetadataConfig.STREAMING_WRITE_ENABLED.key -> testCase.streamingWrites.toString, + // Pinned low so the sizing result is not simply the floor. + HoodieMetadataConfig.GLOBAL_RECORD_LEVEL_INDEX_MIN_FILE_GROUP_COUNT_PROP.key -> "1", + HoodieMetadataConfig.GLOBAL_RECORD_LEVEL_INDEX_MAX_FILE_GROUP_COUNT_PROP.key -> "1000", + HoodieMetadataConfig.RECORD_LEVEL_INDEX_MIN_FILE_GROUP_COUNT_PROP.key -> "1", + HoodieMetadataConfig.RECORD_LEVEL_INDEX_MAX_FILE_GROUP_COUNT_PROP.key -> "1000", + // 9600 bytes / 48 bytes per entry = 200 entries per file group. + HoodieMetadataConfig.RECORD_INDEX_MAX_FILE_GROUP_SIZE_BYTES_PROP.key -> "9600", + HoodieMetadataConfig.DEFER_RLI_INIT_FOR_FRESH_TABLE.key -> "true", + HoodieCompactionConfig.INLINE_COMPACT.key -> "false", + HoodieIndexConfig.INDEX_TYPE.key -> (if (testCase.partitioned) "RECORD_LEVEL_INDEX" else "GLOBAL_RECORD_LEVEL_INDEX")) + + @ParameterizedTest + @MethodSource(Array("testArgsForDeferredInit")) + def testRecordIndexDeferredUntilSecondCommit(testCase: DeferredInitTestCase): Unit = { + val options = deferredInitOptions(testCase) + val dataGen = new HoodieTestDataGenerator() + val allRecords = new util.ArrayList[HoodieRecord[_]]() + val write = (records: util.List[HoodieRecord[_]], operation: String, mode: SaveMode) => + toDataset(spark, records).write.format("hudi").options(options) + .option(DataSourceWriteOptions.OPERATION.key, operation).mode(mode).save(basePath) + + // Commit 1: the record index must not be built, since the table had no completed commit when the write started. + val firstBatch = dataGen.generateInserts("001", 2000) + allRecords.addAll(firstBatch) + write(firstBatch.asInstanceOf[util.List[HoodieRecord[_]]], testCase.firstOperation, SaveMode.Overwrite) + assertTableVersion(testCase.tableVersion) + assertFalse(isRecordIndexAvailable, "Record index should not be built by the first commit") + + // What the index should be sized from once it is built. + val rowsPerPartition = spark.read.format("hudi").load(basePath).groupBy("_hoodie_partition_path").count().collect() + .map(row => row.getString(0) -> row.getLong(1)).toMap + val rowsBeforeBuild = rowsPerPartition.values.sum + + // Commit 2: updates of records tagged through the fallback index, plus new records. The index is built here. + val updates = dataGen.generateUpdates("002", allRecords.subList(0, 300)) + val newRecords = dataGen.generateInserts("002", 10) + val secondBatch = new util.ArrayList[HoodieRecord[_]](updates) + secondBatch.addAll(newRecords) + allRecords.addAll(newRecords) + write(secondBatch, UPSERT_OPERATION_OPT_VAL, SaveMode.Append) + assertTrue(isRecordIndexAvailable, "Record index should be built by the second commit") + + // Commit 3: upserts tagged through the record index itself. + val laterUpdates = dataGen.generateUpdates("003", allRecords.subList(0, 300)) + write(laterUpdates.asInstanceOf[util.List[HoodieRecord[_]]], UPSERT_OPERATION_OPT_VAL, SaveMode.Append) + + // No duplicates across the fallback to record index switch. + val keys = spark.read.format("hudi").load(basePath).select("_row_key").collect().map(_.getString(0)) + assertEquals(allRecords.asScala.map(_.getRecordKey).sorted.toSeq, keys.sorted.toSeq) + + // The index is sized from the data the first commit left behind, not from an empty table. + val fileGroupIds = recordIndexFileGroupIds() + if (testCase.partitioned) { + val perPartition = fileGroupIds.groupBy(HoodieTableMetadataUtil.getDataTablePartitionNameFromFileGroupName) + .mapValues(_.size).toMap + assertEquals(rowsPerPartition.mapValues(rows => expectedFileGroupCount(rows)).toMap, perPartition) + } else { + assertEquals(expectedFileGroupCount(rowsBeforeBuild), fileGroupIds.size) + } + } + + /** + * The commit that builds the record index replaces the data the index was just sized from, so the index has to + * carry the replacing records and none of the replaced ones. + */ + @ParameterizedTest + @MethodSource(Array("testArgsForInsertOverwrite")) + def testRecordIndexBuiltByInsertOverwrite(testCase: DeferredInitTestCase): Unit = { + val options = deferredInitOptions(testCase) + val dataGen = new HoodieTestDataGenerator() + val write = (records: util.List[HoodieRecord[_]], operation: String, mode: SaveMode) => + toDataset(spark, records).write.format("hudi").options(options) + .option(DataSourceWriteOptions.OPERATION.key, operation).mode(mode).save(basePath) + + val firstBatch = dataGen.generateInserts("001", 2000) + write(firstBatch.asInstanceOf[util.List[HoodieRecord[_]]], testCase.firstOperation, SaveMode.Overwrite) + assertTableVersion(testCase.tableVersion) + assertFalse(isRecordIndexAvailable, "Record index should not be built by the first commit") + + // Commit 2 builds the record index and then replaces every partition the first commit wrote. + val replacingBatch = dataGen.generateInserts("002", 500) + write(replacingBatch.asInstanceOf[util.List[HoodieRecord[_]]], INSERT_OVERWRITE_OPERATION_OPT_VAL, SaveMode.Append) + assertTrue(isRecordIndexAvailable, "Record index should be built by the second commit") + + val replacingKeys = replacingBatch.asScala.map(_.getRecordKey).sorted.toSeq + assertEquals(replacingKeys, spark.read.format("hudi").load(basePath).select("_row_key") + .collect().map(_.getString(0)).sorted.toSeq) + + // Commit 3 is tagged through the record index. Replaced records must not come back and survivors must not double. + val laterUpdates = dataGen.generateUpdates("003", replacingBatch.subList(0, 100)) + write(laterUpdates.asInstanceOf[util.List[HoodieRecord[_]]], UPSERT_OPERATION_OPT_VAL, SaveMode.Append) + assertEquals(replacingKeys, spark.read.format("hudi").load(basePath).select("_row_key") + .collect().map(_.getString(0)).sorted.toSeq) + } + + /** + * The same deferral through Spark SQL: INSERT INTO leaves the record index unbuilt and the following MERGE INTO + * builds it, sized from the rows the first statement wrote. + */ + @ParameterizedTest + @MethodSource(Array("testArgsForSql")) + def testRecordIndexDeferredUntilSecondStatement(testCase: DeferredInitTestCase): Unit = { + val tableName = s"rli_deferred_sql_${if (testCase.partitioned) "partitioned" else "global"}_v${testCase.tableVersion.versionCode()}" + val tablePath = tempDir.resolve(tableName).toAbsolutePath.toString + val indexEnableProp = if (testCase.partitioned) { + HoodieMetadataConfig.RECORD_LEVEL_INDEX_ENABLE_PROP.key + } else { + HoodieMetadataConfig.GLOBAL_RECORD_LEVEL_INDEX_ENABLE_PROP.key + } + val minFileGroupProp = if (testCase.partitioned) { + HoodieMetadataConfig.RECORD_LEVEL_INDEX_MIN_FILE_GROUP_COUNT_PROP.key + } else { + HoodieMetadataConfig.GLOBAL_RECORD_LEVEL_INDEX_MIN_FILE_GROUP_COUNT_PROP.key + } + val maxFileGroupProp = if (testCase.partitioned) { + HoodieMetadataConfig.RECORD_LEVEL_INDEX_MAX_FILE_GROUP_COUNT_PROP.key + } else { + HoodieMetadataConfig.GLOBAL_RECORD_LEVEL_INDEX_MAX_FILE_GROUP_COUNT_PROP.key + } + val indexType = if (testCase.partitioned) "RECORD_LEVEL_INDEX" else "GLOBAL_RECORD_LEVEL_INDEX" + spark.sql( + s""" + |create table $tableName ( + | id string, + | ts long, + | name string, + | part string + |) using hudi + | options ( + | primaryKey = 'id', + | orderingFields = 'ts', + | hoodie.metadata.enable = 'true', + | ${HoodieWriteConfig.WRITE_TABLE_VERSION.key} = '${testCase.tableVersion.versionCode()}', + | ${HoodieWriteConfig.AUTO_UPGRADE_VERSION.key} = 'false', + | ${HoodieMetadataConfig.STREAMING_WRITE_ENABLED.key} = '${testCase.streamingWrites}', + | $indexEnableProp = 'true', + | hoodie.index.type = '$indexType', + | ${HoodieMetadataConfig.DEFER_RLI_INIT_FOR_FRESH_TABLE.key} = 'true', + | ${HoodieMetadataConfig.RECORD_INDEX_MAX_FILE_GROUP_SIZE_BYTES_PROP.key} = '9600', + | $minFileGroupProp = '1', + | $maxFileGroupProp = '1000' + | ) + | partitioned by (part) + | location '$tablePath' + """.stripMargin) + val sqlMetaClient = HoodieTableMetaClient.builder().setBasePath(tablePath).setConf(storageConf).build() + val recordIndexBuilt = () => HoodieTableMetaClient.reload(sqlMetaClient).getTableConfig.getMetadataPartitions + .contains(MetadataPartitionType.RECORD_INDEX.getPartitionPath) + val assertSqlTableVersion = () => assertEquals(testCase.tableVersion, + HoodieTableMetaClient.reload(sqlMetaClient).getTableConfig.getTableVersion, + "Table was written at an unexpected version") + val rowsOf = (from: Int, to: Int, ts: Long, name: String) => + s"select cast(id as string) as id, ${ts}L as ts, '$name' as name, concat('p', cast(id % 3 as string)) as part from range($from, $to)" + + try { + spark.conf.set("hoodie.spark.sql.insert.into.operation", testCase.firstOperation) + spark.sql(s"insert into $tableName ${rowsOf(0, 2000, 1L, "v1")}") + } finally { + spark.conf.unset("hoodie.spark.sql.insert.into.operation") + } + assertSqlTableVersion() + assertFalse(recordIndexBuilt(), "Record index should not be built by the first statement") + + // The index is sized from the rows the first statement wrote, so capture them before the merge adds more. + // Keyed by the partition path the table writes, which is Hive styled for a SQL created table. + val rowsPerPartition = spark + .sql(s"select _hoodie_partition_path, count(*) as rows from $tableName group by _hoodie_partition_path") + .collect().map(row => row.getString(0) -> row.getLong(1)).toMap + val rowsBeforeBuild = rowsPerPartition.values.sum + + spark.sql( + s""" + |merge into $tableName t + |using (${rowsOf(0, 300, 4L, "v4")} union all ${rowsOf(2000, 2010, 4L, "v4")}) s + |on t.id = s.id + |when matched then update set * + |when not matched then insert * + """.stripMargin) + assertTrue(recordIndexBuilt(), "Record index should be built by the second statement") + + // Every key appears once, and the merge updated exactly the rows it matched. + assertEquals(2010L, spark.sql(s"select id from $tableName").count()) + assertEquals(2010L, spark.sql(s"select distinct id from $tableName").count()) + assertEquals(310L, spark.sql(s"select id from $tableName where name = 'v4'").count()) + + // The index is sized from the rows the first statement left behind, not from the table as the merge left it. + val fileGroupIds = recordIndexFileGroupIds(tablePath) + if (testCase.partitioned) { + val perPartition = fileGroupIds.groupBy(HoodieTableMetadataUtil.getDataTablePartitionNameFromFileGroupName) + .mapValues(_.size).toMap + assertEquals(rowsPerPartition.mapValues(rows => expectedFileGroupCount(rows)).toMap, perPartition) + } else { + assertEquals(expectedFileGroupCount(rowsBeforeBuild), fileGroupIds.size) + } + } + + private def assertTableVersion(expected: HoodieTableVersion): Unit = { + metaClient = HoodieTableMetaClient.reload(metaClient) + assertEquals(expected, metaClient.getTableConfig.getTableVersion, "Table was written at an unexpected version") + } + + private def isRecordIndexAvailable: Boolean = { + metaClient = HoodieTableMetaClient.reload(metaClient) + metaClient.getTableConfig.getMetadataPartitions.contains(MetadataPartitionType.RECORD_INDEX.getPartitionPath) + } + + private def recordIndexFileGroupIds(tablePath: String = basePath): Seq[String] = { + val metadataMetaClient = HoodieTableMetaClient.builder() + .setBasePath(HoodieTableMetadata.getMetadataTableBasePath(tablePath)) + .setConf(storageConf) + .build() + HoodieTableMetadataUtil.getPartitionLatestFileSlices(metadataMetaClient, HOption.empty(), + MetadataPartitionType.RECORD_INDEX.getPartitionPath).asScala.map(_.getFileId).toSeq + } +} + +object TestDeferredRecordIndexInit { + + case class DeferredInitTestCase(tableType: HoodieTableType, + partitioned: Boolean, + tableVersion: HoodieTableVersion, + streamingWrites: Boolean, + firstOperation: String) { + override def toString: String = + s"${tableType.name()}/${if (partitioned) "partitioned" else "global"}/v${tableVersion.versionCode()}" + + s"/streaming=$streamingWrites/$firstOperation" + } + + // The partitioned record index and streaming metadata writes both need a table version above six. + def testArgsForDeferredInit: java.util.stream.Stream[Arguments] = { + java.util.stream.Stream.of( + Arguments.arguments(DeferredInitTestCase(HoodieTableType.COPY_ON_WRITE, partitioned = false, HoodieTableVersion.SIX, streamingWrites = false, BULK_INSERT_OPERATION_OPT_VAL)), + Arguments.arguments(DeferredInitTestCase(HoodieTableType.COPY_ON_WRITE, partitioned = false, HoodieTableVersion.SIX, streamingWrites = false, INSERT_OPERATION_OPT_VAL)), + Arguments.arguments(DeferredInitTestCase(HoodieTableType.MERGE_ON_READ, partitioned = false, HoodieTableVersion.SIX, streamingWrites = false, BULK_INSERT_OPERATION_OPT_VAL)), + Arguments.arguments(DeferredInitTestCase(HoodieTableType.COPY_ON_WRITE, partitioned = false, HoodieTableVersion.TEN, streamingWrites = true, BULK_INSERT_OPERATION_OPT_VAL)), + Arguments.arguments(DeferredInitTestCase(HoodieTableType.COPY_ON_WRITE, partitioned = false, HoodieTableVersion.TEN, streamingWrites = false, BULK_INSERT_OPERATION_OPT_VAL)), + Arguments.arguments(DeferredInitTestCase(HoodieTableType.COPY_ON_WRITE, partitioned = false, HoodieTableVersion.TEN, streamingWrites = true, INSERT_OPERATION_OPT_VAL)), + Arguments.arguments(DeferredInitTestCase(HoodieTableType.COPY_ON_WRITE, partitioned = true, HoodieTableVersion.TEN, streamingWrites = true, BULK_INSERT_OPERATION_OPT_VAL)), + Arguments.arguments(DeferredInitTestCase(HoodieTableType.COPY_ON_WRITE, partitioned = true, HoodieTableVersion.TEN, streamingWrites = false, BULK_INSERT_OPERATION_OPT_VAL)), + Arguments.arguments(DeferredInitTestCase(HoodieTableType.MERGE_ON_READ, partitioned = false, HoodieTableVersion.TEN, streamingWrites = true, BULK_INSERT_OPERATION_OPT_VAL)), + Arguments.arguments(DeferredInitTestCase(HoodieTableType.MERGE_ON_READ, partitioned = true, HoodieTableVersion.TEN, streamingWrites = true, BULK_INSERT_OPERATION_OPT_VAL)) + ) + } + + def testArgsForSql: java.util.stream.Stream[Arguments] = { + java.util.stream.Stream.of( + Arguments.arguments(DeferredInitTestCase(HoodieTableType.COPY_ON_WRITE, partitioned = false, HoodieTableVersion.SIX, streamingWrites = false, "bulk_insert")), + Arguments.arguments(DeferredInitTestCase(HoodieTableType.COPY_ON_WRITE, partitioned = false, HoodieTableVersion.TEN, streamingWrites = true, "bulk_insert")), + Arguments.arguments(DeferredInitTestCase(HoodieTableType.COPY_ON_WRITE, partitioned = false, HoodieTableVersion.TEN, streamingWrites = true, "insert")), + Arguments.arguments(DeferredInitTestCase(HoodieTableType.COPY_ON_WRITE, partitioned = false, HoodieTableVersion.TEN, streamingWrites = true, "upsert")), + Arguments.arguments(DeferredInitTestCase(HoodieTableType.COPY_ON_WRITE, partitioned = true, HoodieTableVersion.TEN, streamingWrites = true, "bulk_insert")) + ) + } + + def testArgsForInsertOverwrite: java.util.stream.Stream[Arguments] = { + java.util.stream.Stream.of( + Arguments.arguments(DeferredInitTestCase(HoodieTableType.COPY_ON_WRITE, partitioned = false, HoodieTableVersion.SIX, streamingWrites = false, BULK_INSERT_OPERATION_OPT_VAL)), + Arguments.arguments(DeferredInitTestCase(HoodieTableType.COPY_ON_WRITE, partitioned = false, HoodieTableVersion.TEN, streamingWrites = true, BULK_INSERT_OPERATION_OPT_VAL)), + Arguments.arguments(DeferredInitTestCase(HoodieTableType.COPY_ON_WRITE, partitioned = true, HoodieTableVersion.TEN, streamingWrites = true, BULK_INSERT_OPERATION_OPT_VAL)), + Arguments.arguments(DeferredInitTestCase(HoodieTableType.MERGE_ON_READ, partitioned = false, HoodieTableVersion.TEN, streamingWrites = true, BULK_INSERT_OPERATION_OPT_VAL)) + ) + } + + /** Mirrors HoodieTableMetadataUtil.estimateFileGroupCount with growth factor 2 and 200 entries per file group. */ + def expectedFileGroupCount(records: Long): Int = { Review Comment: Done. The three copies are gone; a shared helper calls `HoodieTableMetadataUtil.estimateFileGroupCount` with the sizing configs the writer used. ########## hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestStructuredStreaming.scala: ########## @@ -228,6 +229,94 @@ class TestStructuredStreaming extends HoodieSparkClientTestBase { tableType, addCompactionConfigs = false, isAsyncCompaction = false) } + /** + * Deferred record index initialization through the streaming sink: the first micro-batch leaves the index unbuilt + * and the second one builds it, with no duplicates across the switch from the fallback index. + */ + @Tag("functional") Review Comment: Done. `unit-tests` excludes the `functional` group and `functional-tests` includes it, so it did run, just in a different profile from the rest of the class. I dropped the tag so it runs with its class. ########## hudi-utilities/src/test/java/org/apache/hudi/utilities/deltastreamer/TestDeferredRecordIndexOnStreamerPath.java: ########## @@ -0,0 +1,157 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.utilities.deltastreamer; + +import org.apache.hudi.common.config.HoodieMetadataConfig; +import org.apache.hudi.common.model.FileSlice; +import org.apache.hudi.common.model.WriteOperationType; +import org.apache.hudi.common.table.HoodieTableMetaClient; +import org.apache.hudi.common.table.HoodieTableVersion; +import org.apache.hudi.common.testutils.HoodieTestUtils; +import org.apache.hudi.common.util.Option; +import org.apache.hudi.config.HoodieIndexConfig; +import org.apache.hudi.config.HoodieWriteConfig; +import org.apache.hudi.metadata.HoodieTableMetadata; +import org.apache.hudi.metadata.HoodieTableMetadataUtil; +import org.apache.hudi.metadata.MetadataPartitionType; + +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Deferred record index initialization on the HoodieStreamer path: the first sync leaves the index unbuilt and the + * second one builds it, with no duplicates across the switch from the fallback index. + */ +@Tag("functional") +public class TestDeferredRecordIndexOnStreamerPath extends HoodieDeltaStreamerTestBase { + + private static void enableDeferredRecordIndex(HoodieDeltaStreamer.Config cfg, boolean partitioned, int tableVersion) { + cfg.configs.add(HoodieMetadataConfig.ENABLE.key() + "=true"); + cfg.configs.add(HoodieWriteConfig.WRITE_TABLE_VERSION.key() + "=" + tableVersion); + cfg.configs.add(HoodieWriteConfig.AUTO_UPGRADE_VERSION.key() + "=false"); + cfg.configs.add(HoodieMetadataConfig.GLOBAL_RECORD_LEVEL_INDEX_ENABLE_PROP.key() + "=" + !partitioned); + cfg.configs.add(HoodieMetadataConfig.RECORD_LEVEL_INDEX_ENABLE_PROP.key() + "=" + partitioned); + cfg.configs.add(HoodieIndexConfig.INDEX_TYPE.key() + "=" + + (partitioned ? "RECORD_LEVEL_INDEX" : "GLOBAL_RECORD_LEVEL_INDEX")); + cfg.configs.add(HoodieMetadataConfig.DEFER_RLI_INIT_FOR_FRESH_TABLE.key() + "=true"); + // Pin the sizing inputs so the expected file group count follows from the record count alone: 9600 bytes over + // 48 bytes per entry is 200 entries per file group, and the floor is kept out of the way. + cfg.configs.add(HoodieMetadataConfig.RECORD_INDEX_MAX_FILE_GROUP_SIZE_BYTES_PROP.key() + "=9600"); + cfg.configs.add(HoodieMetadataConfig.GLOBAL_RECORD_LEVEL_INDEX_MIN_FILE_GROUP_COUNT_PROP.key() + "=1"); + cfg.configs.add(HoodieMetadataConfig.RECORD_LEVEL_INDEX_MIN_FILE_GROUP_COUNT_PROP.key() + "=1"); + cfg.configs.add(HoodieMetadataConfig.GLOBAL_RECORD_LEVEL_INDEX_MAX_FILE_GROUP_COUNT_PROP.key() + "=1000"); + cfg.configs.add(HoodieMetadataConfig.RECORD_LEVEL_INDEX_MAX_FILE_GROUP_COUNT_PROP.key() + "=1000"); + } + + /** + * The file group count the record index is sized to for the given number of records, applying the growth factor and + * the entries that fit in one file group. + */ + private static int expectedFileGroupCount(long records) { + long estimated = (long) Math.ceil(records * 2.0) / 200; + return (int) Math.max(1, Math.min(1000, estimated)); + } + + private static List<String> recordIndexFileGroupIds(String tableBasePath) { + HoodieTableMetaClient metadataMetaClient = HoodieTableMetaClient.builder() + .setBasePath(HoodieTableMetadata.getMetadataTableBasePath(tableBasePath)) + .setConf(HoodieTestUtils.getDefaultStorageConf()) + .build(); + return HoodieTableMetadataUtil.getPartitionLatestFileSlices(metadataMetaClient, Option.empty(), + MetadataPartitionType.RECORD_INDEX.getPartitionPath()) + .stream().map(FileSlice::getFileId).collect(Collectors.toList()); + } + + private static boolean isRecordIndexBuilt(String tableBasePath) { + return HoodieTableMetaClient.builder() + .setBasePath(tableBasePath) + .setConf(HoodieTestUtils.getDefaultStorageConf()) + .build() + .getTableConfig().getMetadataPartitions() + .contains(MetadataPartitionType.RECORD_INDEX.getPartitionPath()); + } + + static Stream<Arguments> testArgs() { + return Stream.of( + Arguments.arguments(false, HoodieTableVersion.SIX.versionCode()), + Arguments.arguments(false, HoodieTableVersion.TEN.versionCode()), + Arguments.arguments(true, HoodieTableVersion.TEN.versionCode())); + } + + @ParameterizedTest + @MethodSource("testArgs") + public void testRecordIndexDeferredUntilSecondSync(boolean partitioned, int tableVersion) throws Exception { + String tableBasePath = basePath + "/test_rli_deferred_streamer_" + + (partitioned ? "partitioned" : "global") + "_v" + tableVersion; + + HoodieDeltaStreamer.Config insertCfg = + TestHoodieDeltaStreamer.TestHelpers.makeConfig(tableBasePath, WriteOperationType.INSERT); + enableDeferredRecordIndex(insertCfg, partitioned, tableVersion); + new HoodieDeltaStreamer(insertCfg, jsc).sync(); + assertEquals(tableVersion, HoodieTableMetaClient.builder().setBasePath(tableBasePath) + .setConf(HoodieTestUtils.getDefaultStorageConf()).build().getTableConfig().getTableVersion().versionCode()); + assertFalse(isRecordIndexBuilt(tableBasePath), "Record index should not be built by the first sync"); + + // The index is sized from the data the first sync left behind, so capture it before the second sync adds to it. + assertRecordCount(1000, tableBasePath, sqlContext); + sqlContext.clearCache(); + Map<String, Long> rowsPerPartition = sqlContext.read().format("hudi").load(tableBasePath) + .groupBy("_hoodie_partition_path").count() + .collectAsList().stream().collect(Collectors.toMap(row -> row.getString(0), row -> row.getLong(1))); + long rowsBeforeBuild = rowsPerPartition.values().stream().mapToLong(Long::longValue).sum(); + + HoodieDeltaStreamer.Config upsertCfg = + TestHoodieDeltaStreamer.TestHelpers.makeConfig(tableBasePath, WriteOperationType.UPSERT); + enableDeferredRecordIndex(upsertCfg, partitioned, tableVersion); + new HoodieDeltaStreamer(upsertCfg, jsc).sync(); + assertTrue(isRecordIndexBuilt(tableBasePath), "Record index should be built by the second sync"); + + // The second sync carries 1000 records from the test source: 500 inserts, 450 updates and 50 deletes, so the + // 1000 the first sync wrote become 1450, one row per key, and the deletes make this cover a key removed through + // the fallback index as well. The same 1450 comes out with the deferral turned off, so this pins what the two Review Comment: Done, with a caveat. There is now a third sync, with the RLI-vs-table check after the second and third. The test source picks its updates and deletes at random, so I can't aim the third sync at the second sync's inserts or make it re-insert deleted keys; the datasource, SQL and structured streaming cases cover that targeted path. I also added a continuous-mode case, where one write client spans the deferral, the build and the writes after it. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
