prathik-onehouse commented on code in PR #20604:
URL: https://github.com/apache/hudi/pull/20604#discussion_r4622658847


##########
hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestDeferredRecordIndexInit.scala:
##########
@@ -0,0 +1,331 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hudi.functional
+
+import org.apache.hudi.{DataSourceWriteOptions, SparkDatasetMixin}
+import org.apache.hudi.DataSourceWriteOptions._
+import org.apache.hudi.common.config.HoodieMetadataConfig
+import org.apache.hudi.common.model.{HoodieRecord, HoodieTableType}
+import org.apache.hudi.common.table.{HoodieTableConfig, HoodieTableMetaClient, 
HoodieTableVersion}
+import org.apache.hudi.common.testutils.HoodieTestDataGenerator
+import org.apache.hudi.common.util.{Option => HOption}
+import org.apache.hudi.config.{HoodieCompactionConfig, HoodieIndexConfig, 
HoodieWriteConfig}
+import 
org.apache.hudi.functional.TestDeferredRecordIndexInit.{expectedFileGroupCount, 
DeferredInitTestCase}
+import org.apache.hudi.metadata.{HoodieTableMetadata, HoodieTableMetadataUtil, 
MetadataPartitionType}
+
+import org.apache.spark.sql.SaveMode
+import org.junit.jupiter.api.Tag
+import org.junit.jupiter.api.Assertions.{assertEquals, assertFalse, assertTrue}
+import org.junit.jupiter.params.ParameterizedTest
+import org.junit.jupiter.params.provider.{Arguments, MethodSource}
+
+import java.util
+
+import scala.collection.JavaConverters._
+
+/**
+ * End to end coverage of hoodie.metadata.record.level.index.defer.init on its 
own, with no record threshold:
+ * the record index is not built by the first commit, is built by the second 
one and sized from the first commit's
+ * data, and the switch from the fallback index produces no duplicates.
+ */
+@Tag("functional")
+class TestDeferredRecordIndexInit extends RecordLevelIndexTestBase with 
SparkDatasetMixin {
+
+  private def deferredInitOptions(testCase: DeferredInitTestCase): Map[String, 
String] = Map(
+    HoodieWriteConfig.TBL_NAME.key -> "hoodie_test",
+    DataSourceWriteOptions.TABLE_TYPE.key -> testCase.tableType.name(),
+    RECORDKEY_FIELD.key -> "_row_key",
+    PARTITIONPATH_FIELD.key -> "partition_path",
+    HoodieTableConfig.ORDERING_FIELDS.key -> "timestamp",
+    HoodieWriteConfig.WRITE_TABLE_VERSION.key -> 
testCase.tableVersion.versionCode().toString,
+    HoodieWriteConfig.AUTO_UPGRADE_VERSION.key -> "false",
+    HoodieMetadataConfig.ENABLE.key -> "true",
+    HoodieMetadataConfig.GLOBAL_RECORD_LEVEL_INDEX_ENABLE_PROP.key -> 
(!testCase.partitioned).toString,
+    HoodieMetadataConfig.RECORD_LEVEL_INDEX_ENABLE_PROP.key -> 
testCase.partitioned.toString,
+    HoodieMetadataConfig.STREAMING_WRITE_ENABLED.key -> 
testCase.streamingWrites.toString,
+    // Pinned low so the sizing result is not simply the floor.
+    
HoodieMetadataConfig.GLOBAL_RECORD_LEVEL_INDEX_MIN_FILE_GROUP_COUNT_PROP.key -> 
"1",
+    
HoodieMetadataConfig.GLOBAL_RECORD_LEVEL_INDEX_MAX_FILE_GROUP_COUNT_PROP.key -> 
"1000",
+    HoodieMetadataConfig.RECORD_LEVEL_INDEX_MIN_FILE_GROUP_COUNT_PROP.key -> 
"1",
+    HoodieMetadataConfig.RECORD_LEVEL_INDEX_MAX_FILE_GROUP_COUNT_PROP.key -> 
"1000",
+    // 9600 bytes / 48 bytes per entry = 200 entries per file group.
+    HoodieMetadataConfig.RECORD_INDEX_MAX_FILE_GROUP_SIZE_BYTES_PROP.key -> 
"9600",
+    HoodieMetadataConfig.DEFER_RLI_INIT_FOR_FRESH_TABLE.key -> "true",
+    HoodieCompactionConfig.INLINE_COMPACT.key -> "false",
+    HoodieIndexConfig.INDEX_TYPE.key -> (if (testCase.partitioned) 
"RECORD_LEVEL_INDEX" else "GLOBAL_RECORD_LEVEL_INDEX"))
+
+  @ParameterizedTest
+  @MethodSource(Array("testArgsForDeferredInit"))
+  def testRecordIndexDeferredUntilSecondCommit(testCase: 
DeferredInitTestCase): Unit = {
+    val options = deferredInitOptions(testCase)
+    val dataGen = new HoodieTestDataGenerator()
+    val allRecords = new util.ArrayList[HoodieRecord[_]]()
+    val write = (records: util.List[HoodieRecord[_]], operation: String, mode: 
SaveMode) =>
+      toDataset(spark, records).write.format("hudi").options(options)
+        .option(DataSourceWriteOptions.OPERATION.key, 
operation).mode(mode).save(basePath)
+
+    // Commit 1: the record index must not be built, since the table had no 
completed commit when the write started.
+    val firstBatch = dataGen.generateInserts("001", 2000)
+    allRecords.addAll(firstBatch)
+    write(firstBatch.asInstanceOf[util.List[HoodieRecord[_]]], 
testCase.firstOperation, SaveMode.Overwrite)
+    assertTableVersion(testCase.tableVersion)
+    assertFalse(isRecordIndexAvailable, "Record index should not be built by 
the first commit")

Review Comment:
   Done. After commit 1 every case now checks that the MDT has no 
`record_index` files, no metadata commit writes to it, and there is no index 
definition; after commit 2 it checks the definition and its `isPartitioned` 
option. As you expected, the partitioned case with batch metadata writes failed 
on master, so `maybeInitializeNewFileGroupsForPartitionedRLI` now also requires 
the record index to be initialized (or being built by the async indexer).
   



##########
hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestDeferredRecordIndexInit.scala:
##########
@@ -0,0 +1,331 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hudi.functional
+
+import org.apache.hudi.{DataSourceWriteOptions, SparkDatasetMixin}
+import org.apache.hudi.DataSourceWriteOptions._
+import org.apache.hudi.common.config.HoodieMetadataConfig
+import org.apache.hudi.common.model.{HoodieRecord, HoodieTableType}
+import org.apache.hudi.common.table.{HoodieTableConfig, HoodieTableMetaClient, 
HoodieTableVersion}
+import org.apache.hudi.common.testutils.HoodieTestDataGenerator
+import org.apache.hudi.common.util.{Option => HOption}
+import org.apache.hudi.config.{HoodieCompactionConfig, HoodieIndexConfig, 
HoodieWriteConfig}
+import 
org.apache.hudi.functional.TestDeferredRecordIndexInit.{expectedFileGroupCount, 
DeferredInitTestCase}
+import org.apache.hudi.metadata.{HoodieTableMetadata, HoodieTableMetadataUtil, 
MetadataPartitionType}
+
+import org.apache.spark.sql.SaveMode
+import org.junit.jupiter.api.Tag
+import org.junit.jupiter.api.Assertions.{assertEquals, assertFalse, assertTrue}
+import org.junit.jupiter.params.ParameterizedTest
+import org.junit.jupiter.params.provider.{Arguments, MethodSource}
+
+import java.util
+
+import scala.collection.JavaConverters._
+
+/**
+ * End to end coverage of hoodie.metadata.record.level.index.defer.init on its 
own, with no record threshold:
+ * the record index is not built by the first commit, is built by the second 
one and sized from the first commit's
+ * data, and the switch from the fallback index produces no duplicates.
+ */
+@Tag("functional")
+class TestDeferredRecordIndexInit extends RecordLevelIndexTestBase with 
SparkDatasetMixin {
+
+  private def deferredInitOptions(testCase: DeferredInitTestCase): Map[String, 
String] = Map(
+    HoodieWriteConfig.TBL_NAME.key -> "hoodie_test",
+    DataSourceWriteOptions.TABLE_TYPE.key -> testCase.tableType.name(),
+    RECORDKEY_FIELD.key -> "_row_key",
+    PARTITIONPATH_FIELD.key -> "partition_path",
+    HoodieTableConfig.ORDERING_FIELDS.key -> "timestamp",
+    HoodieWriteConfig.WRITE_TABLE_VERSION.key -> 
testCase.tableVersion.versionCode().toString,
+    HoodieWriteConfig.AUTO_UPGRADE_VERSION.key -> "false",
+    HoodieMetadataConfig.ENABLE.key -> "true",
+    HoodieMetadataConfig.GLOBAL_RECORD_LEVEL_INDEX_ENABLE_PROP.key -> 
(!testCase.partitioned).toString,
+    HoodieMetadataConfig.RECORD_LEVEL_INDEX_ENABLE_PROP.key -> 
testCase.partitioned.toString,
+    HoodieMetadataConfig.STREAMING_WRITE_ENABLED.key -> 
testCase.streamingWrites.toString,
+    // Pinned low so the sizing result is not simply the floor.
+    
HoodieMetadataConfig.GLOBAL_RECORD_LEVEL_INDEX_MIN_FILE_GROUP_COUNT_PROP.key -> 
"1",
+    
HoodieMetadataConfig.GLOBAL_RECORD_LEVEL_INDEX_MAX_FILE_GROUP_COUNT_PROP.key -> 
"1000",
+    HoodieMetadataConfig.RECORD_LEVEL_INDEX_MIN_FILE_GROUP_COUNT_PROP.key -> 
"1",
+    HoodieMetadataConfig.RECORD_LEVEL_INDEX_MAX_FILE_GROUP_COUNT_PROP.key -> 
"1000",
+    // 9600 bytes / 48 bytes per entry = 200 entries per file group.
+    HoodieMetadataConfig.RECORD_INDEX_MAX_FILE_GROUP_SIZE_BYTES_PROP.key -> 
"9600",
+    HoodieMetadataConfig.DEFER_RLI_INIT_FOR_FRESH_TABLE.key -> "true",
+    HoodieCompactionConfig.INLINE_COMPACT.key -> "false",
+    HoodieIndexConfig.INDEX_TYPE.key -> (if (testCase.partitioned) 
"RECORD_LEVEL_INDEX" else "GLOBAL_RECORD_LEVEL_INDEX"))
+
+  @ParameterizedTest
+  @MethodSource(Array("testArgsForDeferredInit"))
+  def testRecordIndexDeferredUntilSecondCommit(testCase: 
DeferredInitTestCase): Unit = {
+    val options = deferredInitOptions(testCase)
+    val dataGen = new HoodieTestDataGenerator()
+    val allRecords = new util.ArrayList[HoodieRecord[_]]()
+    val write = (records: util.List[HoodieRecord[_]], operation: String, mode: 
SaveMode) =>
+      toDataset(spark, records).write.format("hudi").options(options)
+        .option(DataSourceWriteOptions.OPERATION.key, 
operation).mode(mode).save(basePath)
+
+    // Commit 1: the record index must not be built, since the table had no 
completed commit when the write started.
+    val firstBatch = dataGen.generateInserts("001", 2000)
+    allRecords.addAll(firstBatch)
+    write(firstBatch.asInstanceOf[util.List[HoodieRecord[_]]], 
testCase.firstOperation, SaveMode.Overwrite)
+    assertTableVersion(testCase.tableVersion)
+    assertFalse(isRecordIndexAvailable, "Record index should not be built by 
the first commit")
+
+    // What the index should be sized from once it is built.
+    val rowsPerPartition = 
spark.read.format("hudi").load(basePath).groupBy("_hoodie_partition_path").count().collect()
+      .map(row => row.getString(0) -> row.getLong(1)).toMap
+    val rowsBeforeBuild = rowsPerPartition.values.sum
+
+    // Commit 2: updates of records tagged through the fallback index, plus 
new records. The index is built here.

Review Comment:
   Fixed, thanks for catching it. The comment is gone, and commit 2 is now 
described as building the index from commit 1's data and then writing through 
it.
   



##########
hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestDeferredRecordIndexInit.scala:
##########
@@ -0,0 +1,331 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hudi.functional
+
+import org.apache.hudi.{DataSourceWriteOptions, SparkDatasetMixin}
+import org.apache.hudi.DataSourceWriteOptions._
+import org.apache.hudi.common.config.HoodieMetadataConfig
+import org.apache.hudi.common.model.{HoodieRecord, HoodieTableType}
+import org.apache.hudi.common.table.{HoodieTableConfig, HoodieTableMetaClient, 
HoodieTableVersion}
+import org.apache.hudi.common.testutils.HoodieTestDataGenerator
+import org.apache.hudi.common.util.{Option => HOption}
+import org.apache.hudi.config.{HoodieCompactionConfig, HoodieIndexConfig, 
HoodieWriteConfig}
+import 
org.apache.hudi.functional.TestDeferredRecordIndexInit.{expectedFileGroupCount, 
DeferredInitTestCase}
+import org.apache.hudi.metadata.{HoodieTableMetadata, HoodieTableMetadataUtil, 
MetadataPartitionType}
+
+import org.apache.spark.sql.SaveMode
+import org.junit.jupiter.api.Tag
+import org.junit.jupiter.api.Assertions.{assertEquals, assertFalse, assertTrue}
+import org.junit.jupiter.params.ParameterizedTest
+import org.junit.jupiter.params.provider.{Arguments, MethodSource}
+
+import java.util
+
+import scala.collection.JavaConverters._
+
+/**
+ * End to end coverage of hoodie.metadata.record.level.index.defer.init on its 
own, with no record threshold:
+ * the record index is not built by the first commit, is built by the second 
one and sized from the first commit's
+ * data, and the switch from the fallback index produces no duplicates.
+ */
+@Tag("functional")
+class TestDeferredRecordIndexInit extends RecordLevelIndexTestBase with 
SparkDatasetMixin {
+
+  private def deferredInitOptions(testCase: DeferredInitTestCase): Map[String, 
String] = Map(
+    HoodieWriteConfig.TBL_NAME.key -> "hoodie_test",
+    DataSourceWriteOptions.TABLE_TYPE.key -> testCase.tableType.name(),
+    RECORDKEY_FIELD.key -> "_row_key",
+    PARTITIONPATH_FIELD.key -> "partition_path",
+    HoodieTableConfig.ORDERING_FIELDS.key -> "timestamp",
+    HoodieWriteConfig.WRITE_TABLE_VERSION.key -> 
testCase.tableVersion.versionCode().toString,
+    HoodieWriteConfig.AUTO_UPGRADE_VERSION.key -> "false",
+    HoodieMetadataConfig.ENABLE.key -> "true",
+    HoodieMetadataConfig.GLOBAL_RECORD_LEVEL_INDEX_ENABLE_PROP.key -> 
(!testCase.partitioned).toString,
+    HoodieMetadataConfig.RECORD_LEVEL_INDEX_ENABLE_PROP.key -> 
testCase.partitioned.toString,
+    HoodieMetadataConfig.STREAMING_WRITE_ENABLED.key -> 
testCase.streamingWrites.toString,
+    // Pinned low so the sizing result is not simply the floor.
+    
HoodieMetadataConfig.GLOBAL_RECORD_LEVEL_INDEX_MIN_FILE_GROUP_COUNT_PROP.key -> 
"1",
+    
HoodieMetadataConfig.GLOBAL_RECORD_LEVEL_INDEX_MAX_FILE_GROUP_COUNT_PROP.key -> 
"1000",
+    HoodieMetadataConfig.RECORD_LEVEL_INDEX_MIN_FILE_GROUP_COUNT_PROP.key -> 
"1",
+    HoodieMetadataConfig.RECORD_LEVEL_INDEX_MAX_FILE_GROUP_COUNT_PROP.key -> 
"1000",
+    // 9600 bytes / 48 bytes per entry = 200 entries per file group.
+    HoodieMetadataConfig.RECORD_INDEX_MAX_FILE_GROUP_SIZE_BYTES_PROP.key -> 
"9600",
+    HoodieMetadataConfig.DEFER_RLI_INIT_FOR_FRESH_TABLE.key -> "true",
+    HoodieCompactionConfig.INLINE_COMPACT.key -> "false",
+    HoodieIndexConfig.INDEX_TYPE.key -> (if (testCase.partitioned) 
"RECORD_LEVEL_INDEX" else "GLOBAL_RECORD_LEVEL_INDEX"))
+
+  @ParameterizedTest
+  @MethodSource(Array("testArgsForDeferredInit"))
+  def testRecordIndexDeferredUntilSecondCommit(testCase: 
DeferredInitTestCase): Unit = {
+    val options = deferredInitOptions(testCase)
+    val dataGen = new HoodieTestDataGenerator()
+    val allRecords = new util.ArrayList[HoodieRecord[_]]()
+    val write = (records: util.List[HoodieRecord[_]], operation: String, mode: 
SaveMode) =>
+      toDataset(spark, records).write.format("hudi").options(options)
+        .option(DataSourceWriteOptions.OPERATION.key, 
operation).mode(mode).save(basePath)
+
+    // Commit 1: the record index must not be built, since the table had no 
completed commit when the write started.
+    val firstBatch = dataGen.generateInserts("001", 2000)
+    allRecords.addAll(firstBatch)
+    write(firstBatch.asInstanceOf[util.List[HoodieRecord[_]]], 
testCase.firstOperation, SaveMode.Overwrite)
+    assertTableVersion(testCase.tableVersion)
+    assertFalse(isRecordIndexAvailable, "Record index should not be built by 
the first commit")
+
+    // What the index should be sized from once it is built.
+    val rowsPerPartition = 
spark.read.format("hudi").load(basePath).groupBy("_hoodie_partition_path").count().collect()
+      .map(row => row.getString(0) -> row.getLong(1)).toMap
+    val rowsBeforeBuild = rowsPerPartition.values.sum
+
+    // Commit 2: updates of records tagged through the fallback index, plus 
new records. The index is built here.
+    val updates = dataGen.generateUpdates("002", allRecords.subList(0, 300))
+    val newRecords = dataGen.generateInserts("002", 10)
+    val secondBatch = new util.ArrayList[HoodieRecord[_]](updates)
+    secondBatch.addAll(newRecords)
+    allRecords.addAll(newRecords)
+    write(secondBatch, UPSERT_OPERATION_OPT_VAL, SaveMode.Append)
+    assertTrue(isRecordIndexAvailable, "Record index should be built by the 
second commit")
+
+    // Commit 3: upserts tagged through the record index itself.
+    val laterUpdates = dataGen.generateUpdates("003", allRecords.subList(0, 
300))

Review Comment:
   Done for every writer. Commit 2 now also deletes 50 rows, and commit 3 
updates commit 2's inserts and brings back those keys. After commits 2 and 3 
the test compares a full RLI scan to the table, so beyond the entry count it 
checks one entry per row, in the row's partition and file group. SQL and 
structured streaming now have a write after the build as well.
   



##########
hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestDeferredRecordIndexInit.scala:
##########
@@ -0,0 +1,331 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hudi.functional
+
+import org.apache.hudi.{DataSourceWriteOptions, SparkDatasetMixin}
+import org.apache.hudi.DataSourceWriteOptions._
+import org.apache.hudi.common.config.HoodieMetadataConfig
+import org.apache.hudi.common.model.{HoodieRecord, HoodieTableType}
+import org.apache.hudi.common.table.{HoodieTableConfig, HoodieTableMetaClient, 
HoodieTableVersion}
+import org.apache.hudi.common.testutils.HoodieTestDataGenerator
+import org.apache.hudi.common.util.{Option => HOption}
+import org.apache.hudi.config.{HoodieCompactionConfig, HoodieIndexConfig, 
HoodieWriteConfig}
+import 
org.apache.hudi.functional.TestDeferredRecordIndexInit.{expectedFileGroupCount, 
DeferredInitTestCase}
+import org.apache.hudi.metadata.{HoodieTableMetadata, HoodieTableMetadataUtil, 
MetadataPartitionType}
+
+import org.apache.spark.sql.SaveMode
+import org.junit.jupiter.api.Tag
+import org.junit.jupiter.api.Assertions.{assertEquals, assertFalse, assertTrue}
+import org.junit.jupiter.params.ParameterizedTest
+import org.junit.jupiter.params.provider.{Arguments, MethodSource}
+
+import java.util
+
+import scala.collection.JavaConverters._
+
+/**
+ * End to end coverage of hoodie.metadata.record.level.index.defer.init on its 
own, with no record threshold:
+ * the record index is not built by the first commit, is built by the second 
one and sized from the first commit's
+ * data, and the switch from the fallback index produces no duplicates.
+ */
+@Tag("functional")
+class TestDeferredRecordIndexInit extends RecordLevelIndexTestBase with 
SparkDatasetMixin {
+
+  private def deferredInitOptions(testCase: DeferredInitTestCase): Map[String, 
String] = Map(
+    HoodieWriteConfig.TBL_NAME.key -> "hoodie_test",
+    DataSourceWriteOptions.TABLE_TYPE.key -> testCase.tableType.name(),
+    RECORDKEY_FIELD.key -> "_row_key",
+    PARTITIONPATH_FIELD.key -> "partition_path",
+    HoodieTableConfig.ORDERING_FIELDS.key -> "timestamp",
+    HoodieWriteConfig.WRITE_TABLE_VERSION.key -> 
testCase.tableVersion.versionCode().toString,
+    HoodieWriteConfig.AUTO_UPGRADE_VERSION.key -> "false",
+    HoodieMetadataConfig.ENABLE.key -> "true",
+    HoodieMetadataConfig.GLOBAL_RECORD_LEVEL_INDEX_ENABLE_PROP.key -> 
(!testCase.partitioned).toString,
+    HoodieMetadataConfig.RECORD_LEVEL_INDEX_ENABLE_PROP.key -> 
testCase.partitioned.toString,
+    HoodieMetadataConfig.STREAMING_WRITE_ENABLED.key -> 
testCase.streamingWrites.toString,
+    // Pinned low so the sizing result is not simply the floor.
+    
HoodieMetadataConfig.GLOBAL_RECORD_LEVEL_INDEX_MIN_FILE_GROUP_COUNT_PROP.key -> 
"1",
+    
HoodieMetadataConfig.GLOBAL_RECORD_LEVEL_INDEX_MAX_FILE_GROUP_COUNT_PROP.key -> 
"1000",
+    HoodieMetadataConfig.RECORD_LEVEL_INDEX_MIN_FILE_GROUP_COUNT_PROP.key -> 
"1",
+    HoodieMetadataConfig.RECORD_LEVEL_INDEX_MAX_FILE_GROUP_COUNT_PROP.key -> 
"1000",
+    // 9600 bytes / 48 bytes per entry = 200 entries per file group.
+    HoodieMetadataConfig.RECORD_INDEX_MAX_FILE_GROUP_SIZE_BYTES_PROP.key -> 
"9600",
+    HoodieMetadataConfig.DEFER_RLI_INIT_FOR_FRESH_TABLE.key -> "true",
+    HoodieCompactionConfig.INLINE_COMPACT.key -> "false",
+    HoodieIndexConfig.INDEX_TYPE.key -> (if (testCase.partitioned) 
"RECORD_LEVEL_INDEX" else "GLOBAL_RECORD_LEVEL_INDEX"))
+
+  @ParameterizedTest
+  @MethodSource(Array("testArgsForDeferredInit"))
+  def testRecordIndexDeferredUntilSecondCommit(testCase: 
DeferredInitTestCase): Unit = {
+    val options = deferredInitOptions(testCase)
+    val dataGen = new HoodieTestDataGenerator()
+    val allRecords = new util.ArrayList[HoodieRecord[_]]()
+    val write = (records: util.List[HoodieRecord[_]], operation: String, mode: 
SaveMode) =>
+      toDataset(spark, records).write.format("hudi").options(options)
+        .option(DataSourceWriteOptions.OPERATION.key, 
operation).mode(mode).save(basePath)
+
+    // Commit 1: the record index must not be built, since the table had no 
completed commit when the write started.
+    val firstBatch = dataGen.generateInserts("001", 2000)
+    allRecords.addAll(firstBatch)
+    write(firstBatch.asInstanceOf[util.List[HoodieRecord[_]]], 
testCase.firstOperation, SaveMode.Overwrite)
+    assertTableVersion(testCase.tableVersion)
+    assertFalse(isRecordIndexAvailable, "Record index should not be built by 
the first commit")
+
+    // What the index should be sized from once it is built.
+    val rowsPerPartition = 
spark.read.format("hudi").load(basePath).groupBy("_hoodie_partition_path").count().collect()
+      .map(row => row.getString(0) -> row.getLong(1)).toMap
+    val rowsBeforeBuild = rowsPerPartition.values.sum
+
+    // Commit 2: updates of records tagged through the fallback index, plus 
new records. The index is built here.
+    val updates = dataGen.generateUpdates("002", allRecords.subList(0, 300))
+    val newRecords = dataGen.generateInserts("002", 10)
+    val secondBatch = new util.ArrayList[HoodieRecord[_]](updates)
+    secondBatch.addAll(newRecords)
+    allRecords.addAll(newRecords)
+    write(secondBatch, UPSERT_OPERATION_OPT_VAL, SaveMode.Append)
+    assertTrue(isRecordIndexAvailable, "Record index should be built by the 
second commit")
+
+    // Commit 3: upserts tagged through the record index itself.
+    val laterUpdates = dataGen.generateUpdates("003", allRecords.subList(0, 
300))
+    write(laterUpdates.asInstanceOf[util.List[HoodieRecord[_]]], 
UPSERT_OPERATION_OPT_VAL, SaveMode.Append)
+
+    // No duplicates across the fallback to record index switch.
+    val keys = 
spark.read.format("hudi").load(basePath).select("_row_key").collect().map(_.getString(0))
+    assertEquals(allRecords.asScala.map(_.getRecordKey).sorted.toSeq, 
keys.sorted.toSeq)
+
+    // The index is sized from the data the first commit left behind, not from 
an empty table.
+    val fileGroupIds = recordIndexFileGroupIds()
+    if (testCase.partitioned) {
+      val perPartition = 
fileGroupIds.groupBy(HoodieTableMetadataUtil.getDataTablePartitionNameFromFileGroupName)
+        .mapValues(_.size).toMap
+      assertEquals(rowsPerPartition.mapValues(rows => 
expectedFileGroupCount(rows)).toMap, perPartition)
+    } else {
+      assertEquals(expectedFileGroupCount(rowsBeforeBuild), fileGroupIds.size)

Review Comment:
   Done. Commit 2 now adds 2000 rows, so the expected count is 20 sized from 
commit 1 vs 40 after commit 2 (about 6 vs 13 per partition).
   



##########
hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestDeferredRecordIndexInit.scala:
##########
@@ -0,0 +1,331 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hudi.functional
+
+import org.apache.hudi.{DataSourceWriteOptions, SparkDatasetMixin}
+import org.apache.hudi.DataSourceWriteOptions._
+import org.apache.hudi.common.config.HoodieMetadataConfig
+import org.apache.hudi.common.model.{HoodieRecord, HoodieTableType}
+import org.apache.hudi.common.table.{HoodieTableConfig, HoodieTableMetaClient, 
HoodieTableVersion}
+import org.apache.hudi.common.testutils.HoodieTestDataGenerator
+import org.apache.hudi.common.util.{Option => HOption}
+import org.apache.hudi.config.{HoodieCompactionConfig, HoodieIndexConfig, 
HoodieWriteConfig}
+import 
org.apache.hudi.functional.TestDeferredRecordIndexInit.{expectedFileGroupCount, 
DeferredInitTestCase}
+import org.apache.hudi.metadata.{HoodieTableMetadata, HoodieTableMetadataUtil, 
MetadataPartitionType}
+
+import org.apache.spark.sql.SaveMode
+import org.junit.jupiter.api.Tag
+import org.junit.jupiter.api.Assertions.{assertEquals, assertFalse, assertTrue}
+import org.junit.jupiter.params.ParameterizedTest
+import org.junit.jupiter.params.provider.{Arguments, MethodSource}
+
+import java.util
+
+import scala.collection.JavaConverters._
+
+/**
+ * End to end coverage of hoodie.metadata.record.level.index.defer.init on its 
own, with no record threshold:
+ * the record index is not built by the first commit, is built by the second 
one and sized from the first commit's
+ * data, and the switch from the fallback index produces no duplicates.
+ */
+@Tag("functional")
+class TestDeferredRecordIndexInit extends RecordLevelIndexTestBase with 
SparkDatasetMixin {
+
+  private def deferredInitOptions(testCase: DeferredInitTestCase): Map[String, 
String] = Map(
+    HoodieWriteConfig.TBL_NAME.key -> "hoodie_test",
+    DataSourceWriteOptions.TABLE_TYPE.key -> testCase.tableType.name(),
+    RECORDKEY_FIELD.key -> "_row_key",
+    PARTITIONPATH_FIELD.key -> "partition_path",
+    HoodieTableConfig.ORDERING_FIELDS.key -> "timestamp",
+    HoodieWriteConfig.WRITE_TABLE_VERSION.key -> 
testCase.tableVersion.versionCode().toString,
+    HoodieWriteConfig.AUTO_UPGRADE_VERSION.key -> "false",
+    HoodieMetadataConfig.ENABLE.key -> "true",
+    HoodieMetadataConfig.GLOBAL_RECORD_LEVEL_INDEX_ENABLE_PROP.key -> 
(!testCase.partitioned).toString,
+    HoodieMetadataConfig.RECORD_LEVEL_INDEX_ENABLE_PROP.key -> 
testCase.partitioned.toString,
+    HoodieMetadataConfig.STREAMING_WRITE_ENABLED.key -> 
testCase.streamingWrites.toString,
+    // Pinned low so the sizing result is not simply the floor.
+    
HoodieMetadataConfig.GLOBAL_RECORD_LEVEL_INDEX_MIN_FILE_GROUP_COUNT_PROP.key -> 
"1",
+    
HoodieMetadataConfig.GLOBAL_RECORD_LEVEL_INDEX_MAX_FILE_GROUP_COUNT_PROP.key -> 
"1000",
+    HoodieMetadataConfig.RECORD_LEVEL_INDEX_MIN_FILE_GROUP_COUNT_PROP.key -> 
"1",
+    HoodieMetadataConfig.RECORD_LEVEL_INDEX_MAX_FILE_GROUP_COUNT_PROP.key -> 
"1000",
+    // 9600 bytes / 48 bytes per entry = 200 entries per file group.
+    HoodieMetadataConfig.RECORD_INDEX_MAX_FILE_GROUP_SIZE_BYTES_PROP.key -> 
"9600",
+    HoodieMetadataConfig.DEFER_RLI_INIT_FOR_FRESH_TABLE.key -> "true",
+    HoodieCompactionConfig.INLINE_COMPACT.key -> "false",
+    HoodieIndexConfig.INDEX_TYPE.key -> (if (testCase.partitioned) 
"RECORD_LEVEL_INDEX" else "GLOBAL_RECORD_LEVEL_INDEX"))
+
+  @ParameterizedTest
+  @MethodSource(Array("testArgsForDeferredInit"))
+  def testRecordIndexDeferredUntilSecondCommit(testCase: 
DeferredInitTestCase): Unit = {
+    val options = deferredInitOptions(testCase)
+    val dataGen = new HoodieTestDataGenerator()
+    val allRecords = new util.ArrayList[HoodieRecord[_]]()
+    val write = (records: util.List[HoodieRecord[_]], operation: String, mode: 
SaveMode) =>
+      toDataset(spark, records).write.format("hudi").options(options)
+        .option(DataSourceWriteOptions.OPERATION.key, 
operation).mode(mode).save(basePath)
+
+    // Commit 1: the record index must not be built, since the table had no 
completed commit when the write started.
+    val firstBatch = dataGen.generateInserts("001", 2000)
+    allRecords.addAll(firstBatch)
+    write(firstBatch.asInstanceOf[util.List[HoodieRecord[_]]], 
testCase.firstOperation, SaveMode.Overwrite)
+    assertTableVersion(testCase.tableVersion)
+    assertFalse(isRecordIndexAvailable, "Record index should not be built by 
the first commit")
+
+    // What the index should be sized from once it is built.
+    val rowsPerPartition = 
spark.read.format("hudi").load(basePath).groupBy("_hoodie_partition_path").count().collect()
+      .map(row => row.getString(0) -> row.getLong(1)).toMap
+    val rowsBeforeBuild = rowsPerPartition.values.sum
+
+    // Commit 2: updates of records tagged through the fallback index, plus 
new records. The index is built here.
+    val updates = dataGen.generateUpdates("002", allRecords.subList(0, 300))
+    val newRecords = dataGen.generateInserts("002", 10)
+    val secondBatch = new util.ArrayList[HoodieRecord[_]](updates)
+    secondBatch.addAll(newRecords)
+    allRecords.addAll(newRecords)
+    write(secondBatch, UPSERT_OPERATION_OPT_VAL, SaveMode.Append)
+    assertTrue(isRecordIndexAvailable, "Record index should be built by the 
second commit")
+
+    // Commit 3: upserts tagged through the record index itself.
+    val laterUpdates = dataGen.generateUpdates("003", allRecords.subList(0, 
300))
+    write(laterUpdates.asInstanceOf[util.List[HoodieRecord[_]]], 
UPSERT_OPERATION_OPT_VAL, SaveMode.Append)
+
+    // No duplicates across the fallback to record index switch.
+    val keys = 
spark.read.format("hudi").load(basePath).select("_row_key").collect().map(_.getString(0))
+    assertEquals(allRecords.asScala.map(_.getRecordKey).sorted.toSeq, 
keys.sorted.toSeq)
+
+    // The index is sized from the data the first commit left behind, not from 
an empty table.
+    val fileGroupIds = recordIndexFileGroupIds()
+    if (testCase.partitioned) {
+      val perPartition = 
fileGroupIds.groupBy(HoodieTableMetadataUtil.getDataTablePartitionNameFromFileGroupName)
+        .mapValues(_.size).toMap
+      assertEquals(rowsPerPartition.mapValues(rows => 
expectedFileGroupCount(rows)).toMap, perPartition)
+    } else {
+      assertEquals(expectedFileGroupCount(rowsBeforeBuild), fileGroupIds.size)
+    }
+  }
+
+  /**
+   * The commit that builds the record index replaces the data the index was 
just sized from, so the index has to
+   * carry the replacing records and none of the replaced ones.
+   */
+  @ParameterizedTest
+  @MethodSource(Array("testArgsForInsertOverwrite"))
+  def testRecordIndexBuiltByInsertOverwrite(testCase: DeferredInitTestCase): 
Unit = {
+    val options = deferredInitOptions(testCase)
+    val dataGen = new HoodieTestDataGenerator()
+    val write = (records: util.List[HoodieRecord[_]], operation: String, mode: 
SaveMode) =>
+      toDataset(spark, records).write.format("hudi").options(options)
+        .option(DataSourceWriteOptions.OPERATION.key, 
operation).mode(mode).save(basePath)
+
+    val firstBatch = dataGen.generateInserts("001", 2000)
+    write(firstBatch.asInstanceOf[util.List[HoodieRecord[_]]], 
testCase.firstOperation, SaveMode.Overwrite)
+    assertTableVersion(testCase.tableVersion)
+    assertFalse(isRecordIndexAvailable, "Record index should not be built by 
the first commit")
+
+    // Commit 2 builds the record index and then replaces every partition the 
first commit wrote.
+    val replacingBatch = dataGen.generateInserts("002", 500)
+    write(replacingBatch.asInstanceOf[util.List[HoodieRecord[_]]], 
INSERT_OVERWRITE_OPERATION_OPT_VAL, SaveMode.Append)
+    assertTrue(isRecordIndexAvailable, "Record index should be built by the 
second commit")
+
+    val replacingKeys = replacingBatch.asScala.map(_.getRecordKey).sorted.toSeq
+    assertEquals(replacingKeys, 
spark.read.format("hudi").load(basePath).select("_row_key")
+      .collect().map(_.getString(0)).sorted.toSeq)
+
+    // Commit 3 is tagged through the record index. Replaced records must not 
come back and survivors must not double.
+    val laterUpdates = dataGen.generateUpdates("003", 
replacingBatch.subList(0, 100))
+    write(laterUpdates.asInstanceOf[util.List[HoodieRecord[_]]], 
UPSERT_OPERATION_OPT_VAL, SaveMode.Append)
+    assertEquals(replacingKeys, 
spark.read.format("hudi").load(basePath).select("_row_key")
+      .collect().map(_.getString(0)).sorted.toSeq)
+  }
+
+  /**
+   * The same deferral through Spark SQL: INSERT INTO leaves the record index 
unbuilt and the following MERGE INTO
+   * builds it, sized from the rows the first statement wrote.
+   */
+  @ParameterizedTest
+  @MethodSource(Array("testArgsForSql"))
+  def testRecordIndexDeferredUntilSecondStatement(testCase: 
DeferredInitTestCase): Unit = {
+    val tableName = s"rli_deferred_sql_${if (testCase.partitioned) 
"partitioned" else "global"}_v${testCase.tableVersion.versionCode()}"
+    val tablePath = tempDir.resolve(tableName).toAbsolutePath.toString
+    val indexEnableProp = if (testCase.partitioned) {
+      HoodieMetadataConfig.RECORD_LEVEL_INDEX_ENABLE_PROP.key
+    } else {
+      HoodieMetadataConfig.GLOBAL_RECORD_LEVEL_INDEX_ENABLE_PROP.key
+    }
+    val minFileGroupProp = if (testCase.partitioned) {
+      HoodieMetadataConfig.RECORD_LEVEL_INDEX_MIN_FILE_GROUP_COUNT_PROP.key
+    } else {
+      
HoodieMetadataConfig.GLOBAL_RECORD_LEVEL_INDEX_MIN_FILE_GROUP_COUNT_PROP.key
+    }
+    val maxFileGroupProp = if (testCase.partitioned) {
+      HoodieMetadataConfig.RECORD_LEVEL_INDEX_MAX_FILE_GROUP_COUNT_PROP.key
+    } else {
+      
HoodieMetadataConfig.GLOBAL_RECORD_LEVEL_INDEX_MAX_FILE_GROUP_COUNT_PROP.key
+    }
+    val indexType = if (testCase.partitioned) "RECORD_LEVEL_INDEX" else 
"GLOBAL_RECORD_LEVEL_INDEX"
+    spark.sql(
+      s"""
+         |create table $tableName (
+         |  id string,
+         |  ts long,
+         |  name string,
+         |  part string
+         |) using hudi
+         | options (
+         |  primaryKey = 'id',
+         |  orderingFields = 'ts',
+         |  hoodie.metadata.enable = 'true',
+         |  ${HoodieWriteConfig.WRITE_TABLE_VERSION.key} = 
'${testCase.tableVersion.versionCode()}',
+         |  ${HoodieWriteConfig.AUTO_UPGRADE_VERSION.key} = 'false',
+         |  ${HoodieMetadataConfig.STREAMING_WRITE_ENABLED.key} = 
'${testCase.streamingWrites}',
+         |  $indexEnableProp = 'true',
+         |  hoodie.index.type = '$indexType',
+         |  ${HoodieMetadataConfig.DEFER_RLI_INIT_FOR_FRESH_TABLE.key} = 
'true',
+         |  
${HoodieMetadataConfig.RECORD_INDEX_MAX_FILE_GROUP_SIZE_BYTES_PROP.key} = 
'9600',
+         |  $minFileGroupProp = '1',
+         |  $maxFileGroupProp = '1000'
+         | )
+         | partitioned by (part)
+         | location '$tablePath'
+       """.stripMargin)
+    val sqlMetaClient = 
HoodieTableMetaClient.builder().setBasePath(tablePath).setConf(storageConf).build()
+    val recordIndexBuilt = () => 
HoodieTableMetaClient.reload(sqlMetaClient).getTableConfig.getMetadataPartitions
+      .contains(MetadataPartitionType.RECORD_INDEX.getPartitionPath)
+    val assertSqlTableVersion = () => assertEquals(testCase.tableVersion,
+      
HoodieTableMetaClient.reload(sqlMetaClient).getTableConfig.getTableVersion,
+      "Table was written at an unexpected version")
+    val rowsOf = (from: Int, to: Int, ts: Long, name: String) =>
+      s"select cast(id as string) as id, ${ts}L as ts, '$name' as name, 
concat('p', cast(id % 3 as string)) as part from range($from, $to)"
+
+    try {
+      spark.conf.set("hoodie.spark.sql.insert.into.operation", 
testCase.firstOperation)
+      spark.sql(s"insert into $tableName ${rowsOf(0, 2000, 1L, "v1")}")
+    } finally {
+      spark.conf.unset("hoodie.spark.sql.insert.into.operation")
+    }
+    assertSqlTableVersion()
+    assertFalse(recordIndexBuilt(), "Record index should not be built by the 
first statement")
+
+    // The index is sized from the rows the first statement wrote, so capture 
them before the merge adds more.
+    // Keyed by the partition path the table writes, which is Hive styled for 
a SQL created table.
+    val rowsPerPartition = spark
+      .sql(s"select _hoodie_partition_path, count(*) as rows from $tableName 
group by _hoodie_partition_path")
+      .collect().map(row => row.getString(0) -> row.getLong(1)).toMap
+    val rowsBeforeBuild = rowsPerPartition.values.sum
+
+    spark.sql(
+      s"""
+         |merge into $tableName t
+         |using (${rowsOf(0, 300, 4L, "v4")} union all ${rowsOf(2000, 2010, 
4L, "v4")}) s
+         |on t.id = s.id
+         |when matched then update set *
+         |when not matched then insert *
+       """.stripMargin)
+    assertTrue(recordIndexBuilt(), "Record index should be built by the second 
statement")
+
+    // Every key appears once, and the merge updated exactly the rows it 
matched.
+    assertEquals(2010L, spark.sql(s"select id from $tableName").count())
+    assertEquals(2010L, spark.sql(s"select distinct id from 
$tableName").count())
+    assertEquals(310L, spark.sql(s"select id from $tableName where name = 
'v4'").count())
+
+    // The index is sized from the rows the first statement left behind, not 
from the table as the merge left it.

Review Comment:
   Done. The merge now adds 2000 rows, so the assertion can tell the two apart.
   



##########
hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestDeferredRecordIndexInit.scala:
##########
@@ -0,0 +1,331 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hudi.functional
+
+import org.apache.hudi.{DataSourceWriteOptions, SparkDatasetMixin}
+import org.apache.hudi.DataSourceWriteOptions._
+import org.apache.hudi.common.config.HoodieMetadataConfig
+import org.apache.hudi.common.model.{HoodieRecord, HoodieTableType}
+import org.apache.hudi.common.table.{HoodieTableConfig, HoodieTableMetaClient, 
HoodieTableVersion}
+import org.apache.hudi.common.testutils.HoodieTestDataGenerator
+import org.apache.hudi.common.util.{Option => HOption}
+import org.apache.hudi.config.{HoodieCompactionConfig, HoodieIndexConfig, 
HoodieWriteConfig}
+import 
org.apache.hudi.functional.TestDeferredRecordIndexInit.{expectedFileGroupCount, 
DeferredInitTestCase}
+import org.apache.hudi.metadata.{HoodieTableMetadata, HoodieTableMetadataUtil, 
MetadataPartitionType}
+
+import org.apache.spark.sql.SaveMode
+import org.junit.jupiter.api.Tag
+import org.junit.jupiter.api.Assertions.{assertEquals, assertFalse, assertTrue}
+import org.junit.jupiter.params.ParameterizedTest
+import org.junit.jupiter.params.provider.{Arguments, MethodSource}
+
+import java.util
+
+import scala.collection.JavaConverters._
+
+/**
+ * End to end coverage of hoodie.metadata.record.level.index.defer.init on its 
own, with no record threshold:
+ * the record index is not built by the first commit, is built by the second 
one and sized from the first commit's
+ * data, and the switch from the fallback index produces no duplicates.
+ */
+@Tag("functional")
+class TestDeferredRecordIndexInit extends RecordLevelIndexTestBase with 
SparkDatasetMixin {
+
+  private def deferredInitOptions(testCase: DeferredInitTestCase): Map[String, 
String] = Map(
+    HoodieWriteConfig.TBL_NAME.key -> "hoodie_test",
+    DataSourceWriteOptions.TABLE_TYPE.key -> testCase.tableType.name(),
+    RECORDKEY_FIELD.key -> "_row_key",
+    PARTITIONPATH_FIELD.key -> "partition_path",
+    HoodieTableConfig.ORDERING_FIELDS.key -> "timestamp",
+    HoodieWriteConfig.WRITE_TABLE_VERSION.key -> 
testCase.tableVersion.versionCode().toString,
+    HoodieWriteConfig.AUTO_UPGRADE_VERSION.key -> "false",
+    HoodieMetadataConfig.ENABLE.key -> "true",
+    HoodieMetadataConfig.GLOBAL_RECORD_LEVEL_INDEX_ENABLE_PROP.key -> 
(!testCase.partitioned).toString,
+    HoodieMetadataConfig.RECORD_LEVEL_INDEX_ENABLE_PROP.key -> 
testCase.partitioned.toString,
+    HoodieMetadataConfig.STREAMING_WRITE_ENABLED.key -> 
testCase.streamingWrites.toString,
+    // Pinned low so the sizing result is not simply the floor.
+    
HoodieMetadataConfig.GLOBAL_RECORD_LEVEL_INDEX_MIN_FILE_GROUP_COUNT_PROP.key -> 
"1",
+    
HoodieMetadataConfig.GLOBAL_RECORD_LEVEL_INDEX_MAX_FILE_GROUP_COUNT_PROP.key -> 
"1000",
+    HoodieMetadataConfig.RECORD_LEVEL_INDEX_MIN_FILE_GROUP_COUNT_PROP.key -> 
"1",
+    HoodieMetadataConfig.RECORD_LEVEL_INDEX_MAX_FILE_GROUP_COUNT_PROP.key -> 
"1000",
+    // 9600 bytes / 48 bytes per entry = 200 entries per file group.
+    HoodieMetadataConfig.RECORD_INDEX_MAX_FILE_GROUP_SIZE_BYTES_PROP.key -> 
"9600",
+    HoodieMetadataConfig.DEFER_RLI_INIT_FOR_FRESH_TABLE.key -> "true",
+    HoodieCompactionConfig.INLINE_COMPACT.key -> "false",
+    HoodieIndexConfig.INDEX_TYPE.key -> (if (testCase.partitioned) 
"RECORD_LEVEL_INDEX" else "GLOBAL_RECORD_LEVEL_INDEX"))
+
+  @ParameterizedTest
+  @MethodSource(Array("testArgsForDeferredInit"))
+  def testRecordIndexDeferredUntilSecondCommit(testCase: 
DeferredInitTestCase): Unit = {
+    val options = deferredInitOptions(testCase)
+    val dataGen = new HoodieTestDataGenerator()
+    val allRecords = new util.ArrayList[HoodieRecord[_]]()
+    val write = (records: util.List[HoodieRecord[_]], operation: String, mode: 
SaveMode) =>
+      toDataset(spark, records).write.format("hudi").options(options)
+        .option(DataSourceWriteOptions.OPERATION.key, 
operation).mode(mode).save(basePath)
+
+    // Commit 1: the record index must not be built, since the table had no 
completed commit when the write started.
+    val firstBatch = dataGen.generateInserts("001", 2000)
+    allRecords.addAll(firstBatch)
+    write(firstBatch.asInstanceOf[util.List[HoodieRecord[_]]], 
testCase.firstOperation, SaveMode.Overwrite)
+    assertTableVersion(testCase.tableVersion)
+    assertFalse(isRecordIndexAvailable, "Record index should not be built by 
the first commit")
+
+    // What the index should be sized from once it is built.
+    val rowsPerPartition = 
spark.read.format("hudi").load(basePath).groupBy("_hoodie_partition_path").count().collect()
+      .map(row => row.getString(0) -> row.getLong(1)).toMap
+    val rowsBeforeBuild = rowsPerPartition.values.sum
+
+    // Commit 2: updates of records tagged through the fallback index, plus 
new records. The index is built here.
+    val updates = dataGen.generateUpdates("002", allRecords.subList(0, 300))
+    val newRecords = dataGen.generateInserts("002", 10)
+    val secondBatch = new util.ArrayList[HoodieRecord[_]](updates)
+    secondBatch.addAll(newRecords)
+    allRecords.addAll(newRecords)
+    write(secondBatch, UPSERT_OPERATION_OPT_VAL, SaveMode.Append)
+    assertTrue(isRecordIndexAvailable, "Record index should be built by the 
second commit")
+
+    // Commit 3: upserts tagged through the record index itself.
+    val laterUpdates = dataGen.generateUpdates("003", allRecords.subList(0, 
300))
+    write(laterUpdates.asInstanceOf[util.List[HoodieRecord[_]]], 
UPSERT_OPERATION_OPT_VAL, SaveMode.Append)
+
+    // No duplicates across the fallback to record index switch.
+    val keys = 
spark.read.format("hudi").load(basePath).select("_row_key").collect().map(_.getString(0))
+    assertEquals(allRecords.asScala.map(_.getRecordKey).sorted.toSeq, 
keys.sorted.toSeq)
+
+    // The index is sized from the data the first commit left behind, not from 
an empty table.
+    val fileGroupIds = recordIndexFileGroupIds()
+    if (testCase.partitioned) {
+      val perPartition = 
fileGroupIds.groupBy(HoodieTableMetadataUtil.getDataTablePartitionNameFromFileGroupName)
+        .mapValues(_.size).toMap
+      assertEquals(rowsPerPartition.mapValues(rows => 
expectedFileGroupCount(rows)).toMap, perPartition)
+    } else {
+      assertEquals(expectedFileGroupCount(rowsBeforeBuild), fileGroupIds.size)
+    }
+  }
+
+  /**
+   * The commit that builds the record index replaces the data the index was 
just sized from, so the index has to
+   * carry the replacing records and none of the replaced ones.
+   */
+  @ParameterizedTest
+  @MethodSource(Array("testArgsForInsertOverwrite"))
+  def testRecordIndexBuiltByInsertOverwrite(testCase: DeferredInitTestCase): 
Unit = {
+    val options = deferredInitOptions(testCase)
+    val dataGen = new HoodieTestDataGenerator()
+    val write = (records: util.List[HoodieRecord[_]], operation: String, mode: 
SaveMode) =>
+      toDataset(spark, records).write.format("hudi").options(options)
+        .option(DataSourceWriteOptions.OPERATION.key, 
operation).mode(mode).save(basePath)
+
+    val firstBatch = dataGen.generateInserts("001", 2000)
+    write(firstBatch.asInstanceOf[util.List[HoodieRecord[_]]], 
testCase.firstOperation, SaveMode.Overwrite)
+    assertTableVersion(testCase.tableVersion)
+    assertFalse(isRecordIndexAvailable, "Record index should not be built by 
the first commit")
+
+    // Commit 2 builds the record index and then replaces every partition the 
first commit wrote.
+    val replacingBatch = dataGen.generateInserts("002", 500)
+    write(replacingBatch.asInstanceOf[util.List[HoodieRecord[_]]], 
INSERT_OVERWRITE_OPERATION_OPT_VAL, SaveMode.Append)
+    assertTrue(isRecordIndexAvailable, "Record index should be built by the 
second commit")
+
+    val replacingKeys = replacingBatch.asScala.map(_.getRecordKey).sorted.toSeq
+    assertEquals(replacingKeys, 
spark.read.format("hudi").load(basePath).select("_row_key")
+      .collect().map(_.getString(0)).sorted.toSeq)
+
+    // Commit 3 is tagged through the record index. Replaced records must not 
come back and survivors must not double.
+    val laterUpdates = dataGen.generateUpdates("003", 
replacingBatch.subList(0, 100))
+    write(laterUpdates.asInstanceOf[util.List[HoodieRecord[_]]], 
UPSERT_OPERATION_OPT_VAL, SaveMode.Append)
+    assertEquals(replacingKeys, 
spark.read.format("hudi").load(basePath).select("_row_key")
+      .collect().map(_.getString(0)).sorted.toSeq)
+  }
+
+  /**
+   * The same deferral through Spark SQL: INSERT INTO leaves the record index 
unbuilt and the following MERGE INTO
+   * builds it, sized from the rows the first statement wrote.
+   */
+  @ParameterizedTest
+  @MethodSource(Array("testArgsForSql"))
+  def testRecordIndexDeferredUntilSecondStatement(testCase: 
DeferredInitTestCase): Unit = {
+    val tableName = s"rli_deferred_sql_${if (testCase.partitioned) 
"partitioned" else "global"}_v${testCase.tableVersion.versionCode()}"
+    val tablePath = tempDir.resolve(tableName).toAbsolutePath.toString
+    val indexEnableProp = if (testCase.partitioned) {
+      HoodieMetadataConfig.RECORD_LEVEL_INDEX_ENABLE_PROP.key
+    } else {
+      HoodieMetadataConfig.GLOBAL_RECORD_LEVEL_INDEX_ENABLE_PROP.key
+    }
+    val minFileGroupProp = if (testCase.partitioned) {
+      HoodieMetadataConfig.RECORD_LEVEL_INDEX_MIN_FILE_GROUP_COUNT_PROP.key
+    } else {
+      
HoodieMetadataConfig.GLOBAL_RECORD_LEVEL_INDEX_MIN_FILE_GROUP_COUNT_PROP.key
+    }
+    val maxFileGroupProp = if (testCase.partitioned) {
+      HoodieMetadataConfig.RECORD_LEVEL_INDEX_MAX_FILE_GROUP_COUNT_PROP.key
+    } else {
+      
HoodieMetadataConfig.GLOBAL_RECORD_LEVEL_INDEX_MAX_FILE_GROUP_COUNT_PROP.key
+    }
+    val indexType = if (testCase.partitioned) "RECORD_LEVEL_INDEX" else 
"GLOBAL_RECORD_LEVEL_INDEX"
+    spark.sql(
+      s"""
+         |create table $tableName (
+         |  id string,
+         |  ts long,
+         |  name string,
+         |  part string
+         |) using hudi
+         | options (
+         |  primaryKey = 'id',
+         |  orderingFields = 'ts',
+         |  hoodie.metadata.enable = 'true',
+         |  ${HoodieWriteConfig.WRITE_TABLE_VERSION.key} = 
'${testCase.tableVersion.versionCode()}',
+         |  ${HoodieWriteConfig.AUTO_UPGRADE_VERSION.key} = 'false',
+         |  ${HoodieMetadataConfig.STREAMING_WRITE_ENABLED.key} = 
'${testCase.streamingWrites}',
+         |  $indexEnableProp = 'true',
+         |  hoodie.index.type = '$indexType',
+         |  ${HoodieMetadataConfig.DEFER_RLI_INIT_FOR_FRESH_TABLE.key} = 
'true',
+         |  
${HoodieMetadataConfig.RECORD_INDEX_MAX_FILE_GROUP_SIZE_BYTES_PROP.key} = 
'9600',
+         |  $minFileGroupProp = '1',
+         |  $maxFileGroupProp = '1000'
+         | )
+         | partitioned by (part)
+         | location '$tablePath'
+       """.stripMargin)
+    val sqlMetaClient = 
HoodieTableMetaClient.builder().setBasePath(tablePath).setConf(storageConf).build()
+    val recordIndexBuilt = () => 
HoodieTableMetaClient.reload(sqlMetaClient).getTableConfig.getMetadataPartitions
+      .contains(MetadataPartitionType.RECORD_INDEX.getPartitionPath)
+    val assertSqlTableVersion = () => assertEquals(testCase.tableVersion,
+      
HoodieTableMetaClient.reload(sqlMetaClient).getTableConfig.getTableVersion,
+      "Table was written at an unexpected version")
+    val rowsOf = (from: Int, to: Int, ts: Long, name: String) =>
+      s"select cast(id as string) as id, ${ts}L as ts, '$name' as name, 
concat('p', cast(id % 3 as string)) as part from range($from, $to)"
+
+    try {
+      spark.conf.set("hoodie.spark.sql.insert.into.operation", 
testCase.firstOperation)
+      spark.sql(s"insert into $tableName ${rowsOf(0, 2000, 1L, "v1")}")
+    } finally {
+      spark.conf.unset("hoodie.spark.sql.insert.into.operation")
+    }
+    assertSqlTableVersion()
+    assertFalse(recordIndexBuilt(), "Record index should not be built by the 
first statement")
+
+    // The index is sized from the rows the first statement wrote, so capture 
them before the merge adds more.
+    // Keyed by the partition path the table writes, which is Hive styled for 
a SQL created table.
+    val rowsPerPartition = spark
+      .sql(s"select _hoodie_partition_path, count(*) as rows from $tableName 
group by _hoodie_partition_path")
+      .collect().map(row => row.getString(0) -> row.getLong(1)).toMap
+    val rowsBeforeBuild = rowsPerPartition.values.sum
+
+    spark.sql(
+      s"""
+         |merge into $tableName t
+         |using (${rowsOf(0, 300, 4L, "v4")} union all ${rowsOf(2000, 2010, 
4L, "v4")}) s
+         |on t.id = s.id
+         |when matched then update set *
+         |when not matched then insert *
+       """.stripMargin)
+    assertTrue(recordIndexBuilt(), "Record index should be built by the second 
statement")
+
+    // Every key appears once, and the merge updated exactly the rows it 
matched.
+    assertEquals(2010L, spark.sql(s"select id from $tableName").count())
+    assertEquals(2010L, spark.sql(s"select distinct id from 
$tableName").count())
+    assertEquals(310L, spark.sql(s"select id from $tableName where name = 
'v4'").count())
+
+    // The index is sized from the rows the first statement left behind, not 
from the table as the merge left it.
+    val fileGroupIds = recordIndexFileGroupIds(tablePath)
+    if (testCase.partitioned) {
+      val perPartition = 
fileGroupIds.groupBy(HoodieTableMetadataUtil.getDataTablePartitionNameFromFileGroupName)
+        .mapValues(_.size).toMap
+      assertEquals(rowsPerPartition.mapValues(rows => 
expectedFileGroupCount(rows)).toMap, perPartition)
+    } else {
+      assertEquals(expectedFileGroupCount(rowsBeforeBuild), fileGroupIds.size)
+    }
+  }
+
+  private def assertTableVersion(expected: HoodieTableVersion): Unit = {
+    metaClient = HoodieTableMetaClient.reload(metaClient)
+    assertEquals(expected, metaClient.getTableConfig.getTableVersion, "Table 
was written at an unexpected version")
+  }
+
+  private def isRecordIndexAvailable: Boolean = {
+    metaClient = HoodieTableMetaClient.reload(metaClient)
+    
metaClient.getTableConfig.getMetadataPartitions.contains(MetadataPartitionType.RECORD_INDEX.getPartitionPath)
+  }
+
+  private def recordIndexFileGroupIds(tablePath: String = basePath): 
Seq[String] = {
+    val metadataMetaClient = HoodieTableMetaClient.builder()
+      .setBasePath(HoodieTableMetadata.getMetadataTableBasePath(tablePath))
+      .setConf(storageConf)
+      .build()
+    HoodieTableMetadataUtil.getPartitionLatestFileSlices(metadataMetaClient, 
HOption.empty(),
+      
MetadataPartitionType.RECORD_INDEX.getPartitionPath).asScala.map(_.getFileId).toSeq
+  }
+}
+
+object TestDeferredRecordIndexInit {
+
+  case class DeferredInitTestCase(tableType: HoodieTableType,
+                                  partitioned: Boolean,
+                                  tableVersion: HoodieTableVersion,
+                                  streamingWrites: Boolean,
+                                  firstOperation: String) {
+    override def toString: String =
+      s"${tableType.name()}/${if (partitioned) "partitioned" else 
"global"}/v${tableVersion.versionCode()}" +
+        s"/streaming=$streamingWrites/$firstOperation"
+  }
+
+  // The partitioned record index and streaming metadata writes both need a 
table version above six.
+  def testArgsForDeferredInit: java.util.stream.Stream[Arguments] = {
+    java.util.stream.Stream.of(
+      Arguments.arguments(DeferredInitTestCase(HoodieTableType.COPY_ON_WRITE, 
partitioned = false, HoodieTableVersion.SIX, streamingWrites = false, 
BULK_INSERT_OPERATION_OPT_VAL)),
+      Arguments.arguments(DeferredInitTestCase(HoodieTableType.COPY_ON_WRITE, 
partitioned = false, HoodieTableVersion.SIX, streamingWrites = false, 
INSERT_OPERATION_OPT_VAL)),
+      Arguments.arguments(DeferredInitTestCase(HoodieTableType.MERGE_ON_READ, 
partitioned = false, HoodieTableVersion.SIX, streamingWrites = false, 
BULK_INSERT_OPERATION_OPT_VAL)),
+      Arguments.arguments(DeferredInitTestCase(HoodieTableType.COPY_ON_WRITE, 
partitioned = false, HoodieTableVersion.TEN, streamingWrites = true, 
BULK_INSERT_OPERATION_OPT_VAL)),
+      Arguments.arguments(DeferredInitTestCase(HoodieTableType.COPY_ON_WRITE, 
partitioned = false, HoodieTableVersion.TEN, streamingWrites = false, 
BULK_INSERT_OPERATION_OPT_VAL)),
+      Arguments.arguments(DeferredInitTestCase(HoodieTableType.COPY_ON_WRITE, 
partitioned = false, HoodieTableVersion.TEN, streamingWrites = true, 
INSERT_OPERATION_OPT_VAL)),
+      Arguments.arguments(DeferredInitTestCase(HoodieTableType.COPY_ON_WRITE, 
partitioned = true, HoodieTableVersion.TEN, streamingWrites = true, 
BULK_INSERT_OPERATION_OPT_VAL)),
+      Arguments.arguments(DeferredInitTestCase(HoodieTableType.COPY_ON_WRITE, 
partitioned = true, HoodieTableVersion.TEN, streamingWrites = false, 
BULK_INSERT_OPERATION_OPT_VAL)),
+      Arguments.arguments(DeferredInitTestCase(HoodieTableType.MERGE_ON_READ, 
partitioned = false, HoodieTableVersion.TEN, streamingWrites = true, 
BULK_INSERT_OPERATION_OPT_VAL)),
+      Arguments.arguments(DeferredInitTestCase(HoodieTableType.MERGE_ON_READ, 
partitioned = true, HoodieTableVersion.TEN, streamingWrites = true, 
BULK_INSERT_OPERATION_OPT_VAL))
+    )
+  }
+
+  def testArgsForSql: java.util.stream.Stream[Arguments] = {
+    java.util.stream.Stream.of(
+      Arguments.arguments(DeferredInitTestCase(HoodieTableType.COPY_ON_WRITE, 
partitioned = false, HoodieTableVersion.SIX, streamingWrites = false, 
"bulk_insert")),
+      Arguments.arguments(DeferredInitTestCase(HoodieTableType.COPY_ON_WRITE, 
partitioned = false, HoodieTableVersion.TEN, streamingWrites = true, 
"bulk_insert")),
+      Arguments.arguments(DeferredInitTestCase(HoodieTableType.COPY_ON_WRITE, 
partitioned = false, HoodieTableVersion.TEN, streamingWrites = true, "insert")),
+      Arguments.arguments(DeferredInitTestCase(HoodieTableType.COPY_ON_WRITE, 
partitioned = false, HoodieTableVersion.TEN, streamingWrites = true, "upsert")),
+      Arguments.arguments(DeferredInitTestCase(HoodieTableType.COPY_ON_WRITE, 
partitioned = true, HoodieTableVersion.TEN, streamingWrites = true, 
"bulk_insert"))
+    )
+  }
+
+  def testArgsForInsertOverwrite: java.util.stream.Stream[Arguments] = {
+    java.util.stream.Stream.of(
+      Arguments.arguments(DeferredInitTestCase(HoodieTableType.COPY_ON_WRITE, 
partitioned = false, HoodieTableVersion.SIX, streamingWrites = false, 
BULK_INSERT_OPERATION_OPT_VAL)),
+      Arguments.arguments(DeferredInitTestCase(HoodieTableType.COPY_ON_WRITE, 
partitioned = false, HoodieTableVersion.TEN, streamingWrites = true, 
BULK_INSERT_OPERATION_OPT_VAL)),
+      Arguments.arguments(DeferredInitTestCase(HoodieTableType.COPY_ON_WRITE, 
partitioned = true, HoodieTableVersion.TEN, streamingWrites = true, 
BULK_INSERT_OPERATION_OPT_VAL)),
+      Arguments.arguments(DeferredInitTestCase(HoodieTableType.MERGE_ON_READ, 
partitioned = false, HoodieTableVersion.TEN, streamingWrites = true, 
BULK_INSERT_OPERATION_OPT_VAL))
+    )
+  }
+
+  /** Mirrors HoodieTableMetadataUtil.estimateFileGroupCount with growth 
factor 2 and 200 entries per file group. */
+  def expectedFileGroupCount(records: Long): Int = {

Review Comment:
   Done. The three copies are gone; a shared helper calls 
`HoodieTableMetadataUtil.estimateFileGroupCount` with the sizing configs the 
writer used.
   



##########
hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestStructuredStreaming.scala:
##########
@@ -228,6 +229,94 @@ class TestStructuredStreaming extends 
HoodieSparkClientTestBase {
       tableType, addCompactionConfigs = false, isAsyncCompaction = false)
   }
 
+  /**
+   * Deferred record index initialization through the streaming sink: the 
first micro-batch leaves the index unbuilt
+   * and the second one builds it, with no duplicates across the switch from 
the fallback index.
+   */
+  @Tag("functional")

Review Comment:
   Done. `unit-tests` excludes the `functional` group and `functional-tests` 
includes it, so it did run, just in a different profile from the rest of the 
class. I dropped the tag so it runs with its class.
   



##########
hudi-utilities/src/test/java/org/apache/hudi/utilities/deltastreamer/TestDeferredRecordIndexOnStreamerPath.java:
##########
@@ -0,0 +1,157 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.utilities.deltastreamer;
+
+import org.apache.hudi.common.config.HoodieMetadataConfig;
+import org.apache.hudi.common.model.FileSlice;
+import org.apache.hudi.common.model.WriteOperationType;
+import org.apache.hudi.common.table.HoodieTableMetaClient;
+import org.apache.hudi.common.table.HoodieTableVersion;
+import org.apache.hudi.common.testutils.HoodieTestUtils;
+import org.apache.hudi.common.util.Option;
+import org.apache.hudi.config.HoodieIndexConfig;
+import org.apache.hudi.config.HoodieWriteConfig;
+import org.apache.hudi.metadata.HoodieTableMetadata;
+import org.apache.hudi.metadata.HoodieTableMetadataUtil;
+import org.apache.hudi.metadata.MetadataPartitionType;
+
+import org.junit.jupiter.api.Tag;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.Arguments;
+import org.junit.jupiter.params.provider.MethodSource;
+
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+/**
+ * Deferred record index initialization on the HoodieStreamer path: the first 
sync leaves the index unbuilt and the
+ * second one builds it, with no duplicates across the switch from the 
fallback index.
+ */
+@Tag("functional")
+public class TestDeferredRecordIndexOnStreamerPath extends 
HoodieDeltaStreamerTestBase {
+
+  private static void enableDeferredRecordIndex(HoodieDeltaStreamer.Config 
cfg, boolean partitioned, int tableVersion) {
+    cfg.configs.add(HoodieMetadataConfig.ENABLE.key() + "=true");
+    cfg.configs.add(HoodieWriteConfig.WRITE_TABLE_VERSION.key() + "=" + 
tableVersion);
+    cfg.configs.add(HoodieWriteConfig.AUTO_UPGRADE_VERSION.key() + "=false");
+    
cfg.configs.add(HoodieMetadataConfig.GLOBAL_RECORD_LEVEL_INDEX_ENABLE_PROP.key()
 + "=" + !partitioned);
+    cfg.configs.add(HoodieMetadataConfig.RECORD_LEVEL_INDEX_ENABLE_PROP.key() 
+ "=" + partitioned);
+    cfg.configs.add(HoodieIndexConfig.INDEX_TYPE.key() + "="
+        + (partitioned ? "RECORD_LEVEL_INDEX" : "GLOBAL_RECORD_LEVEL_INDEX"));
+    cfg.configs.add(HoodieMetadataConfig.DEFER_RLI_INIT_FOR_FRESH_TABLE.key() 
+ "=true");
+    // Pin the sizing inputs so the expected file group count follows from the 
record count alone: 9600 bytes over
+    // 48 bytes per entry is 200 entries per file group, and the floor is kept 
out of the way.
+    
cfg.configs.add(HoodieMetadataConfig.RECORD_INDEX_MAX_FILE_GROUP_SIZE_BYTES_PROP.key()
 + "=9600");
+    
cfg.configs.add(HoodieMetadataConfig.GLOBAL_RECORD_LEVEL_INDEX_MIN_FILE_GROUP_COUNT_PROP.key()
 + "=1");
+    
cfg.configs.add(HoodieMetadataConfig.RECORD_LEVEL_INDEX_MIN_FILE_GROUP_COUNT_PROP.key()
 + "=1");
+    
cfg.configs.add(HoodieMetadataConfig.GLOBAL_RECORD_LEVEL_INDEX_MAX_FILE_GROUP_COUNT_PROP.key()
 + "=1000");
+    
cfg.configs.add(HoodieMetadataConfig.RECORD_LEVEL_INDEX_MAX_FILE_GROUP_COUNT_PROP.key()
 + "=1000");
+  }
+
+  /**
+   * The file group count the record index is sized to for the given number of 
records, applying the growth factor and
+   * the entries that fit in one file group.
+   */
+  private static int expectedFileGroupCount(long records) {
+    long estimated = (long) Math.ceil(records * 2.0) / 200;
+    return (int) Math.max(1, Math.min(1000, estimated));
+  }
+
+  private static List<String> recordIndexFileGroupIds(String tableBasePath) {
+    HoodieTableMetaClient metadataMetaClient = HoodieTableMetaClient.builder()
+        
.setBasePath(HoodieTableMetadata.getMetadataTableBasePath(tableBasePath))
+        .setConf(HoodieTestUtils.getDefaultStorageConf())
+        .build();
+    return 
HoodieTableMetadataUtil.getPartitionLatestFileSlices(metadataMetaClient, 
Option.empty(),
+            MetadataPartitionType.RECORD_INDEX.getPartitionPath())
+        .stream().map(FileSlice::getFileId).collect(Collectors.toList());
+  }
+
+  private static boolean isRecordIndexBuilt(String tableBasePath) {
+    return HoodieTableMetaClient.builder()
+        .setBasePath(tableBasePath)
+        .setConf(HoodieTestUtils.getDefaultStorageConf())
+        .build()
+        .getTableConfig().getMetadataPartitions()
+        .contains(MetadataPartitionType.RECORD_INDEX.getPartitionPath());
+  }
+
+  static Stream<Arguments> testArgs() {
+    return Stream.of(
+        Arguments.arguments(false, HoodieTableVersion.SIX.versionCode()),
+        Arguments.arguments(false, HoodieTableVersion.TEN.versionCode()),
+        Arguments.arguments(true, HoodieTableVersion.TEN.versionCode()));
+  }
+
+  @ParameterizedTest
+  @MethodSource("testArgs")
+  public void testRecordIndexDeferredUntilSecondSync(boolean partitioned, int 
tableVersion) throws Exception {
+    String tableBasePath = basePath + "/test_rli_deferred_streamer_"
+        + (partitioned ? "partitioned" : "global") + "_v" + tableVersion;
+
+    HoodieDeltaStreamer.Config insertCfg =
+        TestHoodieDeltaStreamer.TestHelpers.makeConfig(tableBasePath, 
WriteOperationType.INSERT);
+    enableDeferredRecordIndex(insertCfg, partitioned, tableVersion);
+    new HoodieDeltaStreamer(insertCfg, jsc).sync();
+    assertEquals(tableVersion, 
HoodieTableMetaClient.builder().setBasePath(tableBasePath)
+        
.setConf(HoodieTestUtils.getDefaultStorageConf()).build().getTableConfig().getTableVersion().versionCode());
+    assertFalse(isRecordIndexBuilt(tableBasePath), "Record index should not be 
built by the first sync");
+
+    // The index is sized from the data the first sync left behind, so capture 
it before the second sync adds to it.
+    assertRecordCount(1000, tableBasePath, sqlContext);
+    sqlContext.clearCache();
+    Map<String, Long> rowsPerPartition = 
sqlContext.read().format("hudi").load(tableBasePath)
+        .groupBy("_hoodie_partition_path").count()
+        .collectAsList().stream().collect(Collectors.toMap(row -> 
row.getString(0), row -> row.getLong(1)));
+    long rowsBeforeBuild = 
rowsPerPartition.values().stream().mapToLong(Long::longValue).sum();
+
+    HoodieDeltaStreamer.Config upsertCfg =
+        TestHoodieDeltaStreamer.TestHelpers.makeConfig(tableBasePath, 
WriteOperationType.UPSERT);
+    enableDeferredRecordIndex(upsertCfg, partitioned, tableVersion);
+    new HoodieDeltaStreamer(upsertCfg, jsc).sync();
+    assertTrue(isRecordIndexBuilt(tableBasePath), "Record index should be 
built by the second sync");
+
+    // The second sync carries 1000 records from the test source: 500 inserts, 
450 updates and 50 deletes, so the
+    // 1000 the first sync wrote become 1450, one row per key, and the deletes 
make this cover a key removed through
+    // the fallback index as well. The same 1450 comes out with the deferral 
turned off, so this pins what the two

Review Comment:
   Done, with a caveat. There is now a third sync, with the RLI-vs-table check 
after the second and third. The test source picks its updates and deletes at 
random, so I can't aim the third sync at the second sync's inserts or make it 
re-insert deleted keys; the datasource, SQL and structured streaming cases 
cover that targeted path. I also added a continuous-mode case, where one write 
client spans the deferral, the build and the writes after it.
   



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]

Reply via email to