jackjlli commented on a change in pull request #8071:
URL: https://github.com/apache/pinot/pull/8071#discussion_r792310488
##########
File path:
pinot-controller/src/test/java/org/apache/pinot/controller/helix/core/PinotHelixResourceManagerTest.java
##########
@@ -816,6 +819,48 @@ public void testSegmentReplacementForRefresh()
Assert.assertEquals(new
HashSet<>(ControllerTestUtils.getHelixResourceManager()
.getSegmentsFor(OFFLINE_SEGMENTS_REPLACE_TEST_REFRESH_TABLE_NAME,
true)),
new HashSet<>(Arrays.asList("s9", "s10", "s11")));
+
+ // Call revert segment replacements
+ ControllerTestUtils.getHelixResourceManager()
+
.revertReplaceSegments(OFFLINE_SEGMENTS_REPLACE_TEST_REFRESH_TABLE_NAME,
lineageEntryId3, false);
+ waitForSegmentsToDelete(OFFLINE_SEGMENTS_REPLACE_TEST_REFRESH_TABLE_NAME,
3, MAX_TIMEOUT_IN_MILLISECOND);
+ Assert.assertEquals(new
HashSet<>(ControllerTestUtils.getHelixResourceManager()
+ .getSegmentsFor(OFFLINE_SEGMENTS_REPLACE_TEST_REFRESH_TABLE_NAME,
true)),
+ new HashSet<>(Arrays.asList("s3", "s4", "s5")));
+
+ // Re-upload (s9, s10, s11) to test the segment clean up from
startReplaceSegments
+ for (int i = 9; i < 12; i++) {
+
ControllerTestUtils.getHelixResourceManager().addNewSegment(OFFLINE_SEGMENTS_REPLACE_TEST_REFRESH_TABLE_NAME,
+
SegmentMetadataMockUtils.mockSegmentMetadata(OFFLINE_SEGMENTS_REPLACE_TEST_REFRESH_TABLE_NAME,
"s" + i),
+ "downloadUrl");
+ }
+ Assert.assertEquals(ControllerTestUtils.getHelixResourceManager()
+ .getSegmentsFor(OFFLINE_SEGMENTS_REPLACE_TEST_REFRESH_TABLE_NAME,
false).size(), 6);
+
+ // Call startReplaceSegments with (s3, s4, s5) -> (s12, s13, s14)
+ segmentsTo = Arrays.asList("s12", "s13", "s14");
+ String lineageEntryId4 = ControllerTestUtils.getHelixResourceManager()
+
.startReplaceSegments(OFFLINE_SEGMENTS_REPLACE_TEST_REFRESH_TABLE_NAME,
segmentsFrom, segmentsTo, true);
+ waitForSegmentsToDelete(OFFLINE_SEGMENTS_REPLACE_TEST_REFRESH_TABLE_NAME,
3, MAX_TIMEOUT_IN_MILLISECOND);
+ Assert.assertEquals(new
HashSet<>(ControllerTestUtils.getHelixResourceManager()
+ .getSegmentsFor(OFFLINE_SEGMENTS_REPLACE_TEST_REFRESH_TABLE_NAME,
false)),
+ new HashSet<>(Arrays.asList("s3", "s4", "s5")));
+
+ // Upload to test the segment clean up from startReplaceSegments
Review comment:
So the segment clean-up is done before the newer batch gets uploaded
right? The following lines just generally uploads the newer batch.
##########
File path:
pinot-controller/src/test/java/org/apache/pinot/controller/helix/core/PinotHelixResourceManagerTest.java
##########
@@ -816,6 +819,48 @@ public void testSegmentReplacementForRefresh()
Assert.assertEquals(new
HashSet<>(ControllerTestUtils.getHelixResourceManager()
.getSegmentsFor(OFFLINE_SEGMENTS_REPLACE_TEST_REFRESH_TABLE_NAME,
true)),
new HashSet<>(Arrays.asList("s9", "s10", "s11")));
+
+ // Call revert segment replacements
+ ControllerTestUtils.getHelixResourceManager()
+
.revertReplaceSegments(OFFLINE_SEGMENTS_REPLACE_TEST_REFRESH_TABLE_NAME,
lineageEntryId3, false);
+ waitForSegmentsToDelete(OFFLINE_SEGMENTS_REPLACE_TEST_REFRESH_TABLE_NAME,
3, MAX_TIMEOUT_IN_MILLISECOND);
+ Assert.assertEquals(new
HashSet<>(ControllerTestUtils.getHelixResourceManager()
+ .getSegmentsFor(OFFLINE_SEGMENTS_REPLACE_TEST_REFRESH_TABLE_NAME,
true)),
+ new HashSet<>(Arrays.asList("s3", "s4", "s5")));
+
+ // Re-upload (s9, s10, s11) to test the segment clean up from
startReplaceSegments
Review comment:
What's the purpose of this step? In which circumstance will we upload
the original segments again?
##########
File path:
pinot-controller/src/test/java/org/apache/pinot/controller/helix/core/PinotHelixResourceManagerTest.java
##########
@@ -767,9 +768,11 @@ public void testSegmentReplacementForRefresh()
.getSegmentsFor(OFFLINE_SEGMENTS_REPLACE_TEST_REFRESH_TABLE_NAME,
true)),
new HashSet<>(Arrays.asList("s3", "s4", "s5")));
- // Start the new protocol with "forceCleanup = true" to check if 2
different proactive clean-up mechanism works:
+ // Start the new protocol (s3, s4, s5) -> (s9, s10, s11) with
"forceCleanup = true" to check if 2 different
+ // proactive clean-up mechanism works:
+ //
// 1. the previous lineage entry (s3, s4, s5) -> (s6, s7, s8) should be
"REVERTED"
- // 2. the older snapshot (s0, s1, s2) needs to be cleaned up because we
are about to upload the 3rd data snapshot
+ // 2. the older segments (s0, s1, s2) needs to be cleaned up because we
are about to upload the 3rd data snapshot
Review comment:
nit: `sed 's/needs/need/'`
##########
File path:
pinot-controller/src/test/java/org/apache/pinot/controller/helix/core/PinotHelixResourceManagerTest.java
##########
@@ -816,6 +819,48 @@ public void testSegmentReplacementForRefresh()
Assert.assertEquals(new
HashSet<>(ControllerTestUtils.getHelixResourceManager()
.getSegmentsFor(OFFLINE_SEGMENTS_REPLACE_TEST_REFRESH_TABLE_NAME,
true)),
new HashSet<>(Arrays.asList("s9", "s10", "s11")));
+
+ // Call revert segment replacements
+ ControllerTestUtils.getHelixResourceManager()
+
.revertReplaceSegments(OFFLINE_SEGMENTS_REPLACE_TEST_REFRESH_TABLE_NAME,
lineageEntryId3, false);
+ waitForSegmentsToDelete(OFFLINE_SEGMENTS_REPLACE_TEST_REFRESH_TABLE_NAME,
3, MAX_TIMEOUT_IN_MILLISECOND);
+ Assert.assertEquals(new
HashSet<>(ControllerTestUtils.getHelixResourceManager()
+ .getSegmentsFor(OFFLINE_SEGMENTS_REPLACE_TEST_REFRESH_TABLE_NAME,
true)),
+ new HashSet<>(Arrays.asList("s3", "s4", "s5")));
+
+ // Re-upload (s9, s10, s11) to test the segment clean up from
startReplaceSegments
+ for (int i = 9; i < 12; i++) {
+
ControllerTestUtils.getHelixResourceManager().addNewSegment(OFFLINE_SEGMENTS_REPLACE_TEST_REFRESH_TABLE_NAME,
+
SegmentMetadataMockUtils.mockSegmentMetadata(OFFLINE_SEGMENTS_REPLACE_TEST_REFRESH_TABLE_NAME,
"s" + i),
+ "downloadUrl");
+ }
+ Assert.assertEquals(ControllerTestUtils.getHelixResourceManager()
+ .getSegmentsFor(OFFLINE_SEGMENTS_REPLACE_TEST_REFRESH_TABLE_NAME,
false).size(), 6);
+
+ // Call startReplaceSegments with (s3, s4, s5) -> (s12, s13, s14)
+ segmentsTo = Arrays.asList("s12", "s13", "s14");
+ String lineageEntryId4 = ControllerTestUtils.getHelixResourceManager()
+
.startReplaceSegments(OFFLINE_SEGMENTS_REPLACE_TEST_REFRESH_TABLE_NAME,
segmentsFrom, segmentsTo, true);
+ waitForSegmentsToDelete(OFFLINE_SEGMENTS_REPLACE_TEST_REFRESH_TABLE_NAME,
3, MAX_TIMEOUT_IN_MILLISECOND);
+ Assert.assertEquals(new
HashSet<>(ControllerTestUtils.getHelixResourceManager()
+ .getSegmentsFor(OFFLINE_SEGMENTS_REPLACE_TEST_REFRESH_TABLE_NAME,
false)),
+ new HashSet<>(Arrays.asList("s3", "s4", "s5")));
+
+ // Upload to test the segment clean up from startReplaceSegments
+ for (int i = 12; i < 15; i++) {
+
ControllerTestUtils.getHelixResourceManager().addNewSegment(OFFLINE_SEGMENTS_REPLACE_TEST_REFRESH_TABLE_NAME,
+
SegmentMetadataMockUtils.mockSegmentMetadata(OFFLINE_SEGMENTS_REPLACE_TEST_REFRESH_TABLE_NAME,
"s" + i),
+ "downloadUrl");
+ }
+
+ // Call endReplaceSegments
+ ControllerTestUtils.getHelixResourceManager()
+ .endReplaceSegments(OFFLINE_SEGMENTS_REPLACE_TEST_REFRESH_TABLE_NAME,
lineageEntryId4);
+ Assert.assertEquals(ControllerTestUtils.getHelixResourceManager()
+ .getSegmentsFor(OFFLINE_SEGMENTS_REPLACE_TEST_REFRESH_TABLE_NAME,
false).size(), 6);
Review comment:
It'd be good to add one more assertion on validating the actual existing
segments in the table with `shouldExcludeReplacedSegments` being false.
##########
File path:
pinot-controller/src/test/java/org/apache/pinot/controller/helix/core/PinotHelixResourceManagerTest.java
##########
@@ -816,6 +819,48 @@ public void testSegmentReplacementForRefresh()
Assert.assertEquals(new
HashSet<>(ControllerTestUtils.getHelixResourceManager()
.getSegmentsFor(OFFLINE_SEGMENTS_REPLACE_TEST_REFRESH_TABLE_NAME,
true)),
new HashSet<>(Arrays.asList("s9", "s10", "s11")));
+
+ // Call revert segment replacements
Review comment:
It'd be good to add a comment about what the current segment lineage
looks like for now.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
