yihua commented on code in PR #11678: URL: https://github.com/apache/hudi/pull/11678#discussion_r1691800246
########## hudi-spark-datasource/hudi-spark-common/src/test/scala/org/apache/hudi/TestRecordLevelIndexSupport.scala: ########## @@ -43,46 +48,62 @@ class TestRecordLevelIndexSupport { assertTrue(result.isEmpty) // Case 3: EqualTo filters on simple AttributeReference and non-Literal should return empty result - testFilter = EqualTo(AttributeReference("_row_key", StringType, nullable = true)(), fromUnixTime) + testFilter = EqualTo(AttributeReference(filterColumnName, StringType, nullable = true)(), fromUnixTime) result = RecordLevelIndexSupport.filterQueryWithRecordKey(testFilter, Option.empty) assertTrue(result.isEmpty) // Case 4: EqualTo filters on simple AttributeReference and Literal which should return non-empty result - testFilter = EqualTo(AttributeReference("_row_key", StringType, nullable = true)(), Literal("row1")) - result = RecordLevelIndexSupport.filterQueryWithRecordKey(testFilter, Option.apply(HoodieMetadataField.RECORD_KEY_METADATA_FIELD.getFieldName)) + testFilter = EqualTo(AttributeReference(filterColumnName, StringType, nullable = true)(), Literal("row1")) + result = RecordLevelIndexSupport.filterQueryWithRecordKey(testFilter, Option.apply(recordKeyField)) assertTrue(result.isDefined) assertEquals(result, Option.apply(testFilter, List.apply("row1"))) - // case 5: EqualTo on fields other than record key should return empty result + // case 5: EqualTo on fields other than record key should return empty result unless it's _hoodie_record_key result = RecordLevelIndexSupport.filterQueryWithRecordKey(testFilter, Option.apply("blah")) - assertTrue(result.isEmpty) + if (filterColumnName.equals(HoodieMetadataField.RECORD_KEY_METADATA_FIELD.getFieldName)) { + assertTrue(result.isDefined) + assertEquals(result, Option.apply(testFilter, List.apply("row1"))) + } else { + assertTrue(result.isEmpty) + } - // Case 6: In filter on fields other than record key should return empty result - testFilter = In(AttributeReference("_row_key", StringType, nullable = true)(), List.apply(Literal("xyz"), Literal("abc"))) + // Case 6: In filter on fields other than record key should return empty result unless it's _hoodie_record_key + testFilter = In(AttributeReference(filterColumnName, StringType, nullable = true)(), List.apply(Literal("xyz"), Literal("abc"))) result = RecordLevelIndexSupport.filterQueryWithRecordKey(testFilter, Option.apply("blah")) - assertTrue(result.isEmpty) + if (filterColumnName.equals(HoodieMetadataField.RECORD_KEY_METADATA_FIELD.getFieldName)) { + assertTrue(result.isDefined) + assertEquals(result, Option.apply(testFilter, List.apply("xyz", "abc"))) + } else { + assertTrue(result.isEmpty) + } // Case 7: In filter on record key should return non-empty result - testFilter = In(AttributeReference("_row_key", StringType, nullable = true)(), List.apply(Literal("xyz"), Literal("abc"))) - result = RecordLevelIndexSupport.filterQueryWithRecordKey(testFilter, Option.apply(HoodieMetadataField.RECORD_KEY_METADATA_FIELD.getFieldName)) + testFilter = In(AttributeReference(filterColumnName, StringType, nullable = true)(), List.apply(Literal("xyz"), Literal("abc"))) + result = RecordLevelIndexSupport.filterQueryWithRecordKey(testFilter, Option.apply(recordKeyField)) assertTrue(result.isDefined) + assertEquals(result, Option.apply(testFilter, List.apply("xyz", "abc"))) // Case 8: In filter on simple AttributeReference(on record-key) and non-Literal should return empty result - testFilter = In(AttributeReference("_row_key", StringType, nullable = true)(), List.apply(fromUnixTime)) + testFilter = In(AttributeReference(filterColumnName, StringType, nullable = true)(), List.apply(fromUnixTime)) result = RecordLevelIndexSupport.filterQueryWithRecordKey(testFilter, Option.apply(HoodieMetadataField.RECORD_KEY_METADATA_FIELD.getFieldName)) assertTrue(result.isEmpty) // Case 9: Anything other than EqualTo and In predicate is not supported. Hence it returns empty result - testFilter = Not(In(AttributeReference("_row_key", StringType, nullable = true)(), List.apply(Literal("xyz"), Literal("abc")))) + testFilter = Not(In(AttributeReference(filterColumnName, StringType, nullable = true)(), List.apply(Literal("xyz"), Literal("abc")))) result = RecordLevelIndexSupport.filterQueryWithRecordKey(testFilter, Option.apply(HoodieMetadataField.RECORD_KEY_METADATA_FIELD.getFieldName)) assertTrue(result.isEmpty) - testFilter = Not(In(AttributeReference("_row_key", StringType, nullable = true)(), List.apply(fromUnixTime))) + testFilter = Not(In(AttributeReference(filterColumnName, StringType, nullable = true)(), List.apply(fromUnixTime))) result = RecordLevelIndexSupport.filterQueryWithRecordKey(testFilter, Option.apply(HoodieMetadataField.RECORD_KEY_METADATA_FIELD.getFieldName)) assertTrue(result.isEmpty) - testFilter = GreaterThan(AttributeReference("_row_key", StringType, nullable = true)(), Literal("row1")) + testFilter = GreaterThan(AttributeReference(filterColumnName, StringType, nullable = true)(), Literal("row1")) result = RecordLevelIndexSupport.filterQueryWithRecordKey(testFilter, Option.apply(HoodieMetadataField.RECORD_KEY_METADATA_FIELD.getFieldName)) assertTrue(result.isEmpty) + + testFilter = EqualTo(AttributeReference(HoodieMetadataField.RECORD_KEY_METADATA_FIELD.getFieldName, StringType, nullable = true)(), Literal("row1")) + result = RecordLevelIndexSupport.filterQueryWithRecordKey(testFilter, Option.apply(recordKeyField)) + assertTrue(result.isDefined) + assertEquals(result, Option.apply(testFilter, List.apply("row1"))) Review Comment: This is already covered so we can remove this case. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@hudi.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org