jojochuang commented on code in PR #9954:
URL: https://github.com/apache/ozone/pull/9954#discussion_r3113479929
##########
hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeConfiguration.java:
##########
@@ -430,6 +441,39 @@ public class DatanodeConfiguration extends
ReconfigurableConfig {
)
private Duration diskCheckSlidingWindowTimeout =
DISK_CHECK_SLIDING_WINDOW_TIMEOUT_DEFAULT;
+ @Config(key = "hdds.datanode.disk.check.timeout.test.enabled",
+ defaultValue = "true",
+ type = ConfigType.BOOLEAN,
+ tags = { DATANODE },
+ description = "Enable or disable timeout-based volume failure tracking "
+ + "independently of the disk IO health checks."
+ )
+ private boolean isDiskCheckTimeoutTestEnabled =
+ DISK_CHECK_TIMEOUT_TEST_ENABLED_DEFAULT;
+
+ @Config(key = "hdds.datanode.disk.check.timeout.failures.tolerated",
+ defaultValue = "1",
+ type = ConfigType.INT,
+ tags = { DATANODE },
+ description = "The number of volume check timeouts allowed within the "
+ + "timeout sliding window before the volume is marked as failed."
+ )
+ private int diskCheckTimeoutFailureTolerance =
+ DISK_CHECK_TIMEOUT_FAILURES_TOLERATED_DEFAULT;
+
+ @Config(key = "hdds.datanode.disk.check.timeout.sliding.window.timeout",
+ defaultValue = "70m",
Review Comment:
i wonder why 70m instead of 1h. We should have default values that's
sensible.
##########
hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/volume/TestStorageVolumeChecker.java:
##########
@@ -301,6 +310,165 @@ public void testNumScansSkipped() throws Exception {
checker.shutdownAndWait(0, TimeUnit.SECONDS);
}
+ /**
+ * Explicitly captures the {@link Throwable} type that
+ * {@link Futures#withTimeout} delivers to {@code onFailure()} when the
+ * deadline fires.
+ *
+ * <p>{@link ThrottledAsyncChecker} uses {@code Futures.withTimeout()}
+ * internally; this test replicates that exact call to confirm that Guava
+ * 28+ (including the 33.5.0-jre version used by Ozone) delivers a
+ * {@link TimeoutException} — NOT a {@link
java.util.concurrent.CancellationException}.
+ * {@code CancellationException} in new Guava means external cancellation
+ * (e.g. executor shutdown), not a disk-check timeout, so
+ * {@link StorageVolumeChecker} {@code ResultHandler.onFailure()} only
+ * treats {@link TimeoutException} as a timeout.
+ */
+ @Test
+ public void testFuturesWithTimeoutExceptionType() throws Exception {
+ ScheduledExecutorService scheduler =
+ Executors.newSingleThreadScheduledExecutor();
+ AtomicReference<Throwable> captured = new AtomicReference<>();
+ CountDownLatch done = new CountDownLatch(1);
+
+ try {
+ // A future that never completes on its own — same situation as a
+ // check() thread blocked inside fsync().
+ SettableFuture<VolumeCheckResult> neverCompletes =
SettableFuture.create();
+
+ // Wrap with a real Futures.withTimeout(), identical to what
+ // ThrottledAsyncChecker does.
+ ListenableFuture<VolumeCheckResult> timedFuture =
+ Futures.withTimeout(neverCompletes, 100, TimeUnit.MILLISECONDS,
+ scheduler);
+
+ Futures.addCallback(timedFuture, new FutureCallback<VolumeCheckResult>()
{
+ @Override
+ public void onSuccess(VolumeCheckResult result) {
+ done.countDown();
+ }
+
+ @Override
+ public void onFailure(Throwable t) {
+ captured.set(t);
+ done.countDown();
+ }
+ }, MoreExecutors.directExecutor());
+
+ assertTrue(done.await(2, TimeUnit.SECONDS),
+ "Timeout should have fired within 2 seconds");
+
+ Throwable thrown = captured.get();
+ LOG.info("Futures.withTimeout() delivered to onFailure(): {}",
+ thrown.getClass().getName());
+
+ // Guava 28+ delivers TimeoutException for a timeout.
+ // CancellationException would mean external cancellation, not a timeout.
+ assertTrue(thrown instanceof TimeoutException,
+ "Expected TimeoutException from Futures.withTimeout() but got: "
+ + thrown.getClass().getName());
+ } finally {
+ scheduler.shutdownNow();
+ }
+ }
+
+ /**
+ * Verifies that when the per-check timeout inside {@link
ThrottledAsyncChecker}
+ * fires on {@link StorageVolumeChecker#checkVolume}, the first timeout is
+ * tolerated (callback not invoked, volume not failed) and the second timeout
+ * within the timeout window causes the volume to be failed.
+ *
+ * <p>This test uses the real {@link ThrottledAsyncChecker} (not
+ * {@link DummyChecker}) so that the actual {@code TimeoutException}
+ * path in {@code ResultHandler.onFailure()} fires.
+ */
+ @Test
+ public void testCheckVolumeTimeoutTolerance() throws Exception {
+ setup();
+ // Use a very short check timeout so the test completes quickly.
+ OzoneConfiguration timeoutConf = new OzoneConfiguration();
+ DatanodeConfiguration dnConf =
timeoutConf.getObject(DatanodeConfiguration.class);
+ dnConf.setDiskCheckTimeout(Duration.ofMillis(200));
+ dnConf.setDiskCheckMinGap(Duration.ZERO);
+ timeoutConf.setFromObject(dnConf);
+
+ // A latch-controlled mock volume: check() blocks until released.
+ HddsVolume volume = mock(HddsVolume.class);
+ CountDownLatch blockLatch = new CountDownLatch(1);
+ when(volume.check(any())).thenAnswer(inv -> {
+ blockLatch.await();
+ return VolumeCheckResult.HEALTHY;
+ });
+ // First timeout returns false (within tolerance), second returns true.
+
when(volume.recordTimeoutAndCheckFailure()).thenReturn(false).thenReturn(true);
+
+ AtomicLong callbackCount = new AtomicLong(0);
+ StorageVolumeChecker checker =
+ new StorageVolumeChecker(timeoutConf, new FakeTimer(), "test-");
+
+ // First checkVolume — should time out and be tolerated (callback NOT
fired).
+ checker.checkVolume(volume, (healthy, failed) ->
callbackCount.incrementAndGet());
+
+ // Wait long enough for the timeout to fire.
+ Thread.sleep(600);
Review Comment:
I'm not a fan of calling Tread.sleep() -- it's either too long or too short
and become flaky.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]