jfrazee commented on a change in pull request #4883:
URL: https://github.com/apache/nifi/pull/4883#discussion_r591980559
##########
File path:
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/status/history/EmbeddedQuestDbRolloverHandlerTest.java
##########
@@ -75,68 +95,146 @@ public void tearDown() throws Exception {
}
@Test
- public void testRollOverWhenWithEmptyDatabase() throws Exception {
+ public void testNoOffsetTimeZoneWhenPartitionNeedsToBeRolled() throws
Exception {
+ // given
+ givenTableIsCreated(dbContext);
+ givenTableIsPopulated(UTC_MAR_5_1200, UTC_MAR_6_1200, UTC_MAR_7_1200,
UTC_MAR_8_1200);
+
+ // when
+ whenRollOverIsExecuted(UTC_MAR_8_1700);
+
+ // then
+ thenTheRemainingPartitionsAre("2021-03-06", "2021-03-07",
"2021-03-08");
+ }
+
+ @Test
+ // This scenario might occur when the NiFi was stopped and the persistent
storage remains in place
+ public void testNoOffsetTimeZoneAndNonConsecutive() throws Exception {
+ // given
+ givenTableIsCreated(dbContext);
+ givenTableIsPopulated(UTC_MAR_1_1200, UTC_MAR_2_1200, UTC_MAR_7_1200,
UTC_MAR_8_1200);
+
+ // when
+ whenRollOverIsExecuted(UTC_MAR_8_1700);
+
+ // then
+ thenTheRemainingPartitionsAre("2021-03-07", "2021-03-08");
+ }
+
+ @Test
+ public void testNoOffsetTimeWhenNoPartitionsNeedToBeDropped() throws
Exception {
+ // given
+ givenTableIsCreated(dbContext);
+ givenTableIsPopulated(UTC_MAR_6_1200, UTC_MAR_7_1200, UTC_MAR_8_1200);
+
+ // when
+ whenRollOverIsExecuted(UTC_MAR_8_1700);
+
+ // then
+ thenTheRemainingPartitionsAre("2021-03-06", "2021-03-07",
"2021-03-08");
+ }
+
+ @Test
+ public void testNoOffsetTimeZoneAndLessPartitionThanNeeded() throws
Exception {
+ // given
+ givenTableIsCreated(dbContext);
+ givenTableIsPopulated(UTC_MAR_7_1200, UTC_MAR_8_1200);
+
+ // when
+ whenRollOverIsExecuted(UTC_MAR_8_1700);
+
+ // then
+ thenTheRemainingPartitionsAre("2021-03-07", "2021-03-08");
+ }
+
+ @Test
+ public void testNoOffsetTimeZoneAndOldPartitionsOnly() throws Exception {
// given
givenTableIsCreated(dbContext);
+ givenTableIsPopulated(UTC_MAR_1_1200, UTC_MAR_2_1200);
// when
- whenRollOverIsExecuted();
+ whenRollOverIsExecuted(UTC_MAR_8_1700);
+
+ // then - QuestDB will not remove the active partition if presents
+ thenTheRemainingPartitionsAre("2021-03-02");
+ }
+
+ @Test
+ public void testNoOffsetTimeZoneAndEmptyDatabase() throws Exception {
+ // given
+ givenTableIsCreated(dbContext);
+
+ // when
+ whenRollOverIsExecuted(UTC_MAR_8_1700);
// then
- thenRemainingPartitionsAre(Arrays.asList());
+ thenNoPartitionsExpected();
}
@Test
- public void testRollOverWhenLessPartitionThanNeeded() throws Exception {
+ public void testNegativeOffsetTimeZoneWhenOverlaps() throws Exception {
// given
givenTableIsCreated(dbContext);
- givenTableIsPopulated(givenMeasurementTimes(Arrays.asList(0, 1)));
+ givenTableIsPopulated(EST_MAR_5_1200, EST_MAR_6_1200, EST_MAR_7_1200,
EST_MAR_8_1200, EST_MAR_8_1600);
// when
- whenRollOverIsExecuted();
+ whenRollOverIsExecuted(EST_MAR_8_1700);
// then
- thenRemainingPartitionsAre(Arrays.asList(0, 1));
+ thenTheRemainingPartitionsAre("2021-03-06", "2021-03-07",
"2021-03-08");
+ }
+
+ @Test
+ public void testNegativeOffsetTimeZoneWhenOverlapsAndRolledLater() throws
Exception {
+ // given
+ givenTableIsCreated(dbContext);
+ givenTableIsPopulated(EST_MAR_5_1200, EST_MAR_6_1200, EST_MAR_7_1200,
EST_MAR_8_1200, EST_MAR_8_1600);
+
+ // when
+ whenRollOverIsExecuted(EST_MAR_8_2300);
+
+ // then (there is no data inserted into the time range after the
partition 2021-03-08, so 2021-03-09 is not created)
+ thenTheRemainingPartitionsAre("2021-03-07", "2021-03-08");
}
@Test
- public void testRollOverWhenNoPartitionToDrop() throws Exception {
+ public void testNegativeOffsetTimeZoneWhenHangsOver() throws Exception {
// given
givenTableIsCreated(dbContext);
- givenTableIsPopulated(givenMeasurementTimes(Arrays.asList(0, 1, 2)));
+ givenTableIsPopulated(EST_MAR_6_1200, EST_MAR_7_1200, EST_MAR_8_1200,
EST_MAR_8_2200);
// when
- whenRollOverIsExecuted();
+ whenRollOverIsExecuted(EST_MAR_8_2300);
// then
- thenRemainingPartitionsAre(Arrays.asList(0, 1, 2));
+ thenTheRemainingPartitionsAre("2021-03-07", "2021-03-08",
"2021-03-09");
}
@Test
- public void testRollOverWhenOldPartitionsPresent() throws Exception {
+ public void testPositiveOffsetTimeZoneWhenOverlaps() throws Exception {
// given
givenTableIsCreated(dbContext);
- givenTableIsPopulated(givenMeasurementTimes(Arrays.asList(0, 1, 2, 3,
4)));
+ givenTableIsPopulated(SGT_MAR_4_1200, SGT_MAR_5_1200, SGT_MAR_6_1200,
SGT_MAR_7_1200, SGT_MAR_8_1200);
// when
- whenRollOverIsExecuted();
+ whenRollOverIsExecuted(SGT_MAR_8_1300);
// then
- thenRemainingPartitionsAre(Arrays.asList(0, 1, 2));
+ thenTheRemainingPartitionsAre("2021-03-06", "2021-03-07",
"2021-03-08");
}
@Test
- // This scenario might occurs when the NiFi was stopped and the persistens
storage remaing
- public void testRollOverWhenNonconsecutivePartitionsPresent() throws
Exception {
+ public void testPositiveOffsetTimeZoneWhenHangsOver() throws Exception {
// given
givenTableIsCreated(dbContext);
- givenTableIsPopulated(givenMeasurementTimes(Arrays.asList(0, 1, 7, 8,
9)));
+ givenTableIsPopulated(SGT_MAR_4_1200, SGT_MAR_5_1200, SGT_MAR_6_1200,
SGT_MAR_7_1200, SGT_MAR_8_1200);
// when
- whenRollOverIsExecuted();
+ whenRollOverIsExecuted("03/08/2021 23:00:00 SGT"); // UTC: 03/09/2021
04:00:00
Review comment:
I think the comment should be 3/8?
##########
File path:
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/status/history/EmbeddedQuestDbRolloverHandlerTest.java
##########
@@ -75,68 +95,146 @@ public void tearDown() throws Exception {
}
@Test
- public void testRollOverWhenWithEmptyDatabase() throws Exception {
+ public void testNoOffsetTimeZoneWhenPartitionNeedsToBeRolled() throws
Exception {
+ // given
+ givenTableIsCreated(dbContext);
+ givenTableIsPopulated(UTC_MAR_5_1200, UTC_MAR_6_1200, UTC_MAR_7_1200,
UTC_MAR_8_1200);
+
+ // when
+ whenRollOverIsExecuted(UTC_MAR_8_1700);
+
+ // then
+ thenTheRemainingPartitionsAre("2021-03-06", "2021-03-07",
"2021-03-08");
+ }
+
+ @Test
+ // This scenario might occur when the NiFi was stopped and the persistent
storage remains in place
+ public void testNoOffsetTimeZoneAndNonConsecutive() throws Exception {
+ // given
+ givenTableIsCreated(dbContext);
+ givenTableIsPopulated(UTC_MAR_1_1200, UTC_MAR_2_1200, UTC_MAR_7_1200,
UTC_MAR_8_1200);
+
+ // when
+ whenRollOverIsExecuted(UTC_MAR_8_1700);
+
+ // then
+ thenTheRemainingPartitionsAre("2021-03-07", "2021-03-08");
+ }
+
+ @Test
+ public void testNoOffsetTimeWhenNoPartitionsNeedToBeDropped() throws
Exception {
+ // given
+ givenTableIsCreated(dbContext);
+ givenTableIsPopulated(UTC_MAR_6_1200, UTC_MAR_7_1200, UTC_MAR_8_1200);
+
+ // when
+ whenRollOverIsExecuted(UTC_MAR_8_1700);
+
+ // then
+ thenTheRemainingPartitionsAre("2021-03-06", "2021-03-07",
"2021-03-08");
+ }
+
+ @Test
+ public void testNoOffsetTimeZoneAndLessPartitionThanNeeded() throws
Exception {
+ // given
+ givenTableIsCreated(dbContext);
+ givenTableIsPopulated(UTC_MAR_7_1200, UTC_MAR_8_1200);
+
+ // when
+ whenRollOverIsExecuted(UTC_MAR_8_1700);
+
+ // then
+ thenTheRemainingPartitionsAre("2021-03-07", "2021-03-08");
+ }
+
+ @Test
+ public void testNoOffsetTimeZoneAndOldPartitionsOnly() throws Exception {
// given
givenTableIsCreated(dbContext);
+ givenTableIsPopulated(UTC_MAR_1_1200, UTC_MAR_2_1200);
// when
- whenRollOverIsExecuted();
+ whenRollOverIsExecuted(UTC_MAR_8_1700);
+
+ // then - QuestDB will not remove the active partition if presents
+ thenTheRemainingPartitionsAre("2021-03-02");
Review comment:
Is this behavior documented? Should it be? It makes sense once thinking
about it but I think it could be surprising.
##########
File path:
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/status/history/EmbeddedQuestDbRolloverHandler.java
##########
@@ -48,16 +47,22 @@
// Distinct keyword is not recognized if the date mapping is not within an
inner query
static final String SELECTION_QUERY = "SELECT DISTINCT * FROM (SELECT
(to_str(capturedAt, 'yyyy-MM-dd')) AS partitionName FROM %s)";
- static final DateTimeFormatter DATE_FORMATTER =
DateTimeFormatter.ofPattern("yyyy-MM-dd").withZone(ZoneId.systemDefault());
+ DateTimeFormatter DATE_FORMATTER =
DateTimeFormatter.ofPattern("yyyy-MM-dd").withZone(ZoneId.of("UTC"));
- private final QuestDbContext dbContext;
+ private final Supplier<ZonedDateTime> timeSource;
private final List<String> tables = new ArrayList<>();
private final int daysToKeepData;
+ private final QuestDbContext dbContext;
- public EmbeddedQuestDbRolloverHandler(final Collection<String> tables,
final int daysToKeepData, final QuestDbContext dbContext) {
+ EmbeddedQuestDbRolloverHandler(final Supplier<ZonedDateTime> timeSource,
final Collection<String> tables, final int daysToKeepData, final QuestDbContext
dbContext) {
+ this.timeSource = timeSource;
this.tables.addAll(tables);
- this.dbContext = dbContext;
this.daysToKeepData = daysToKeepData;
+ this.dbContext = dbContext;
+ }
+
+ public EmbeddedQuestDbRolloverHandler(final Collection<String> tables,
final int daysToKeepData, final QuestDbContext dbContext) {
+ this(() -> ZonedDateTime.now(), tables, daysToKeepData, dbContext);
Review comment:
Is there a reason to leave `timeSource` in local and do the conversion
in `getOldestPartitionToKeep()` vs. having `timeSource` in UTC?
##########
File path:
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/status/history/EmbeddedQuestDbRolloverHandlerTest.java
##########
@@ -198,10 +292,14 @@ private void thenRemainingPartitionsAre(final
List<Integer> expectedDays) throws
existingPartitions.add(new
StringBuilder(record.getStr(0)).toString());
}
- Assert.assertEquals(expectedPartitions.size(),
existingPartitions.size());
+ Assert.assertEquals(expectedPartitions.length,
existingPartitions.size());
for (final String expectedPartition : expectedPartitions) {
Assert.assertTrue("Partition " + expectedPartition + " is
expected", existingPartitions.contains(expectedPartition));
}
}
+
+ private void thenNoPartitionsExpected() throws Exception {
+ thenTheRemainingPartitionsAre();
+ }
}
Review comment:
Missing newline
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
