github-advanced-security[bot] commented on code in PR #19884:
URL: https://github.com/apache/druid/pull/19884#discussion_r3717079780
##########
embedded-tests/src/test/java/org/apache/druid/testing/embedded/query/PartialProjectionLoadRuleQueryTest.java:
##########
@@ -319,6 +307,134 @@
);
}
+ @Test
+ void testRuleTransitionBetweenPartialAndFullLoad()
+ {
+ // Rule churn in both directions against an already-loaded segment.
+ //
+ // Observed through sys.servers.curr_size, which is driven by the
loadedBytes the historical announces: the
+ // partial footprint excludes the unmatched projection's bytes, while a
full-load announcement carries no profile
+ // at all and falls back to segment.getSize().
+ final long fullSize = queryFullSegmentSize();
+ Assertions.assertTrue(fullSize > 0, "sys.segments.size must be populated
for the ingested segment");
+
+ // Precondition from loadDataAndConfigureRule(): the segment is already
loaded under the partial rule.
+ MatcherAssert.assertThat(
+ "partial-load rule should already be in effect at the start of this
test",
+ queryHistoricalCurrSize(),
+ Matchers.lessThan(fullSize)
+ );
+
+ boolean restored = false;
+ try {
+ final LatchableEmitter coordinatorEmitter =
coordinator.latchableEmitter();
+ coordinatorEmitter.flush();
+
+ // Leg 1: partial -> full.
+ applyRule(new ForeverLoadRule(Map.of("_default_tier", 1), null));
+ coordinatorEmitter.waitForEvent(
+ event -> event.hasMetricName("segment/partial/ruleReverted/count")
+ .hasDimension(DruidMetrics.DATASOURCE, dataSource)
+ );
+ Assertions.assertEquals(
+ fullSize,
+ awaitHistoricalCurrSize(
+ size -> size == fullSize,
+ "full segment size after reverting to a regular load rule"
+ ),
+ "a reverted replica must announce with no partial-load profile, so
loadedBytes falls back to segment size"
+ );
+
+ // Leg 2: full -> partial. Already covered by the initial load, but
asserting it here proves the transition is
+ // symmetric on an already-loaded segment and restores the state the
other tests in this class depend on.
+ applyRule(partialProjectionRule());
+ restored = true;
+ MatcherAssert.assertThat(
+ "re-applying the partial rule must shrink the announced footprint
again",
+ awaitHistoricalCurrSize(size -> size < fullSize, "partial footprint
after re-applying the partial rule"),
+ Matchers.lessThan(fullSize)
+ );
+ }
+ finally {
+ // Best effort only, and deliberately silent: on the success path leg 2
already restored and verified the rule.
+ // This exists so a failure above can't leave the wrong rule behind for
the rest of the class, and it must never
+ // mask the original failure.
+ if (!restored) {
+ try {
+ applyRule(partialProjectionRule());
+ }
+ catch (Exception ignored) {
+ // fall through; the failure that got us here is the one worth
reporting
+ }
+ }
+ }
+ }
+
+ /**
+ * The partial-load rule this class runs under: selects only {@link
#PROJECTION_NAME}, leaving the base-table
+ * cluster-group bundles and {@link #UNMATCHED_PROJECTION_NAME} off the
historical's disk.
+ */
+ private static ForeverPartialLoadRule partialProjectionRule()
+ {
+ return new ForeverPartialLoadRule(
+ Map.of("_default_tier", 1),
+ null,
+ new WildcardProjectionPartialLoadMatcher(List.of(PROJECTION_NAME),
null),
+ CannotMatchBehavior.FALL_THROUGH
+ );
+ }
+
+ private void applyRule(Rule rule)
+ {
+ cluster.callApi().onLeaderCoordinator(c ->
c.updateRulesForDatasource(dataSource, List.of(rule)));
+ }
+
+ /**
+ * The ingested segment's full on-disk size, i.e. what the historical would
report if it held everything.
+ */
+ private long queryFullSegmentSize()
+ {
+ return Long.parseLong(
+ cluster.callApi().runSql(
+ "SELECT \"size\" FROM sys.segments WHERE datasource = '" +
dataSource + "'"
+ ).trim()
+ );
Review Comment:
## CodeQL / Missing catch of NumberFormatException
Potential uncaught 'java.lang.NumberFormatException'.
[Show more
details](https://github.com/apache/druid/security/code-scanning/11606)
##########
embedded-tests/src/test/java/org/apache/druid/testing/embedded/query/PartialProjectionLoadRuleQueryTest.java:
##########
@@ -319,6 +307,134 @@
);
}
+ @Test
+ void testRuleTransitionBetweenPartialAndFullLoad()
+ {
+ // Rule churn in both directions against an already-loaded segment.
+ //
+ // Observed through sys.servers.curr_size, which is driven by the
loadedBytes the historical announces: the
+ // partial footprint excludes the unmatched projection's bytes, while a
full-load announcement carries no profile
+ // at all and falls back to segment.getSize().
+ final long fullSize = queryFullSegmentSize();
+ Assertions.assertTrue(fullSize > 0, "sys.segments.size must be populated
for the ingested segment");
+
+ // Precondition from loadDataAndConfigureRule(): the segment is already
loaded under the partial rule.
+ MatcherAssert.assertThat(
+ "partial-load rule should already be in effect at the start of this
test",
+ queryHistoricalCurrSize(),
+ Matchers.lessThan(fullSize)
+ );
+
+ boolean restored = false;
+ try {
+ final LatchableEmitter coordinatorEmitter =
coordinator.latchableEmitter();
+ coordinatorEmitter.flush();
+
+ // Leg 1: partial -> full.
+ applyRule(new ForeverLoadRule(Map.of("_default_tier", 1), null));
+ coordinatorEmitter.waitForEvent(
+ event -> event.hasMetricName("segment/partial/ruleReverted/count")
+ .hasDimension(DruidMetrics.DATASOURCE, dataSource)
+ );
+ Assertions.assertEquals(
+ fullSize,
+ awaitHistoricalCurrSize(
+ size -> size == fullSize,
+ "full segment size after reverting to a regular load rule"
+ ),
+ "a reverted replica must announce with no partial-load profile, so
loadedBytes falls back to segment size"
+ );
+
+ // Leg 2: full -> partial. Already covered by the initial load, but
asserting it here proves the transition is
+ // symmetric on an already-loaded segment and restores the state the
other tests in this class depend on.
+ applyRule(partialProjectionRule());
+ restored = true;
+ MatcherAssert.assertThat(
+ "re-applying the partial rule must shrink the announced footprint
again",
+ awaitHistoricalCurrSize(size -> size < fullSize, "partial footprint
after re-applying the partial rule"),
+ Matchers.lessThan(fullSize)
+ );
+ }
+ finally {
+ // Best effort only, and deliberately silent: on the success path leg 2
already restored and verified the rule.
+ // This exists so a failure above can't leave the wrong rule behind for
the rest of the class, and it must never
+ // mask the original failure.
+ if (!restored) {
+ try {
+ applyRule(partialProjectionRule());
+ }
+ catch (Exception ignored) {
+ // fall through; the failure that got us here is the one worth
reporting
+ }
+ }
+ }
+ }
+
+ /**
+ * The partial-load rule this class runs under: selects only {@link
#PROJECTION_NAME}, leaving the base-table
+ * cluster-group bundles and {@link #UNMATCHED_PROJECTION_NAME} off the
historical's disk.
+ */
+ private static ForeverPartialLoadRule partialProjectionRule()
+ {
+ return new ForeverPartialLoadRule(
+ Map.of("_default_tier", 1),
+ null,
+ new WildcardProjectionPartialLoadMatcher(List.of(PROJECTION_NAME),
null),
+ CannotMatchBehavior.FALL_THROUGH
+ );
+ }
+
+ private void applyRule(Rule rule)
+ {
+ cluster.callApi().onLeaderCoordinator(c ->
c.updateRulesForDatasource(dataSource, List.of(rule)));
+ }
+
+ /**
+ * The ingested segment's full on-disk size, i.e. what the historical would
report if it held everything.
+ */
+ private long queryFullSegmentSize()
+ {
+ return Long.parseLong(
+ cluster.callApi().runSql(
+ "SELECT \"size\" FROM sys.segments WHERE datasource = '" +
dataSource + "'"
+ ).trim()
+ );
+ }
+
+ /**
+ * The footprint the coordinator currently attributes to the historical,
i.e. the {@code loadedBytes} carried by the
+ * segment's most recent announcement.
+ */
+ private long queryHistoricalCurrSize()
+ {
+ return Long.parseLong(
+ cluster.callApi().runSql(
+ "SELECT curr_size FROM sys.servers WHERE server_type =
'historical'"
+ ).trim()
+ );
Review Comment:
## CodeQL / Missing catch of NumberFormatException
Potential uncaught 'java.lang.NumberFormatException'.
[Show more
details](https://github.com/apache/druid/security/code-scanning/11607)
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
