JeonDaehong commented on code in PR #18027:
URL: https://github.com/apache/iceberg/pull/18027#discussion_r4005851410
##########
spark/v4.2/spark/src/test/java/org/apache/iceberg/spark/data/vectorized/TestColumnarBatchUtil.java:
##########
@@ -288,6 +267,116 @@ void testRemoveExtraColumnsNotNeeded() {
assertThat(result.length).isEqualTo(3);
}
+ @Test
+ void testBuildRowIdMappingNonZeroBatchStart() {
+ // batches after the first one start at a non-zero position in the file
+ long rowStartPosInBatch = 1_000_000L;
+ when(deleteFilter.hasPosDeletes()).thenReturn(true);
+ when(deleteFilter.deletedRowPositions())
+ .thenReturn(positionIndex(1_000_000L, 1_000_003L, 1_000_009L));
+
+ var rowIdMapping =
+ ColumnarBatchUtil.buildRowIdMapping(columnVectors, deleteFilter,
rowStartPosInBatch, 10);
+
+ assertThat(rowIdMapping).isNotNull();
+ int[] rowIds = (int[]) rowIdMapping.first();
+ int liveRows = (Integer) rowIdMapping.second();
+
+ assertThat(liveRows).isEqualTo(7);
+ assertThat(Arrays.copyOf(rowIds, liveRows)).containsExactly(1, 2, 4, 5, 6,
7, 8);
+ verify(deleteFilter, times(3)).incrementDeleteCount();
+ }
+
+ @Test
+ void testBuildRowIdMappingDeletesOutsideBatch() {
+ // the index covers the whole file, so most batches see no deletes at all
+ when(deleteFilter.hasPosDeletes()).thenReturn(true);
+ when(deleteFilter.deletedRowPositions()).thenReturn(positionIndex(5L, 6L,
500L, 501L));
+
+ var rowIdMapping = ColumnarBatchUtil.buildRowIdMapping(columnVectors,
deleteFilter, 100L, 100);
+
+ assertThat(rowIdMapping).isNull();
+ verify(deleteFilter, times(0)).incrementDeleteCount();
+ }
+
+ @Test
+ void testBuildRowIdMappingAllRowsDeletedByPosition() {
+ when(deleteFilter.hasPosDeletes()).thenReturn(true);
+ when(deleteFilter.deletedRowPositions()).thenReturn(positionIndex(0L, 1L,
2L, 3L, 4L));
+
+ var rowIdMapping = ColumnarBatchUtil.buildRowIdMapping(columnVectors,
deleteFilter, 0, 5);
+
+ assertThat(rowIdMapping).isNotNull();
+ assertThat((Integer) rowIdMapping.second()).isEqualTo(0);
+ verify(deleteFilter, times(5)).incrementDeleteCount();
+ }
+
+ @Test
+ void testBuildIsDeletedNonZeroBatchStart() {
+
when(deleteFilter.deletedRowPositions()).thenReturn(positionIndex(1_000_002L,
1_000_004L));
+
+ var isDeleted = ColumnarBatchUtil.buildIsDeleted(columnVectors,
deleteFilter, 1_000_000L, 5);
+
+ assertThat(isDeleted).containsExactly(false, false, true, false, true);
+ verify(deleteFilter, times(2)).incrementDeleteCount();
+ }
+
+ @Test
+ void testPositionOnlyPathMatchesPerRowPath() {
+ // the range traversal must agree with a straightforward per-row probe on
every batch
+ java.util.Random random = new java.util.Random(20260818L);
Review Comment:
Done !
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
