From 94658e488e942382185c078d71c00c1c1cf27b53 Mon Sep 17 00:00:00 2001 From: Russole <850905junior@gmail.com> Date: Wed, 12 Nov 2025 16:08:16 +0800 Subject: [PATCH 1/3] HDDS-12136. Allow StorageVolumeChecker to be disabled --- .../statemachine/DatanodeConfiguration.java | 21 ++++++++++++------- .../common/volume/StorageVolumeChecker.java | 5 +++++ 2 files changed, 19 insertions(+), 7 deletions(-) diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeConfiguration.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeConfiguration.java index 31caad4dce14..124456616d5f 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeConfiguration.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeConfiguration.java @@ -83,7 +83,7 @@ public class DatanodeConfiguration extends ReconfigurableConfig { static final boolean CHUNK_DATA_VALIDATION_CHECK_DEFAULT = false; - static final long PERIODIC_DISK_CHECK_INTERVAL_MINUTES_DEFAULT = 60; + static final long PERIODIC_DISK_CHECK_INTERVAL_MINUTES_DEFAULT = 1; static final int FAILED_VOLUMES_TOLERATED_DEFAULT = -1; @@ -602,13 +602,20 @@ public void validate() { containerCloseThreads = CONTAINER_CLOSE_THREADS_DEFAULT; } - if (periodicDiskCheckIntervalMinutes < 1) { - LOG.warn(PERIODIC_DISK_CHECK_INTERVAL_MINUTES_KEY + - " must be greater than zero and was set to {}. Defaulting to {}", - periodicDiskCheckIntervalMinutes, +// if (periodicDiskCheckIntervalMinutes < 1) { +// LOG.warn(PERIODIC_DISK_CHECK_INTERVAL_MINUTES_KEY + +// " must be greater than zero and was set to {}. Defaulting to {}", +// periodicDiskCheckIntervalMinutes, +// PERIODIC_DISK_CHECK_INTERVAL_MINUTES_DEFAULT); +// periodicDiskCheckIntervalMinutes = +// PERIODIC_DISK_CHECK_INTERVAL_MINUTES_DEFAULT; +// } + + if (periodicDiskCheckIntervalMinutes == 0) { + LOG.warn("{} must not be zero. Defaulting to {}", + PERIODIC_DISK_CHECK_INTERVAL_MINUTES_KEY, PERIODIC_DISK_CHECK_INTERVAL_MINUTES_DEFAULT); - periodicDiskCheckIntervalMinutes = - PERIODIC_DISK_CHECK_INTERVAL_MINUTES_DEFAULT; + periodicDiskCheckIntervalMinutes = PERIODIC_DISK_CHECK_INTERVAL_MINUTES_DEFAULT; } if (failedDataVolumesTolerated < -1) { diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/StorageVolumeChecker.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/StorageVolumeChecker.java index b48b0dac1180..1237b452334b 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/StorageVolumeChecker.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/StorageVolumeChecker.java @@ -150,6 +150,11 @@ public void start() { if (started.compareAndSet(false, true)) { long periodicDiskCheckIntervalMinutes = dnConf.getPeriodicDiskCheckIntervalMinutes(); + if (periodicDiskCheckIntervalMinutes < 0) { + LOG.warn("StorageVolumeChecker is DISABLED because " + + "hdds.datanode.periodic.disk.check.interval.minutes = {} (negative).", periodicDiskCheckIntervalMinutes); + return; + } periodicDiskChecker = diskCheckerservice.scheduleWithFixedDelay(this::checkAllVolumeSets, periodicDiskCheckIntervalMinutes, From 5cfc31c9d2387e146f8e16d145f116a2395a0319 Mon Sep 17 00:00:00 2001 From: Russole <850905junior@gmail.com> Date: Fri, 14 Nov 2025 21:46:05 +0800 Subject: [PATCH 2/3] =?UTF-8?q?Fix=20TestDatanodeConfiguration=20=E2=80=94?= =?UTF-8?q?=20invalid=20override=20now=20only=20applies=20to=20zero,=20neg?= =?UTF-8?q?ative=20values=20preserved?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../common/statemachine/DatanodeConfiguration.java | 11 +---------- .../statemachine/TestDatanodeConfiguration.java | 2 +- 2 files changed, 2 insertions(+), 11 deletions(-) diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeConfiguration.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeConfiguration.java index 124456616d5f..1e67fcef4d0f 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeConfiguration.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeConfiguration.java @@ -83,7 +83,7 @@ public class DatanodeConfiguration extends ReconfigurableConfig { static final boolean CHUNK_DATA_VALIDATION_CHECK_DEFAULT = false; - static final long PERIODIC_DISK_CHECK_INTERVAL_MINUTES_DEFAULT = 1; + static final long PERIODIC_DISK_CHECK_INTERVAL_MINUTES_DEFAULT = 60; static final int FAILED_VOLUMES_TOLERATED_DEFAULT = -1; @@ -602,15 +602,6 @@ public void validate() { containerCloseThreads = CONTAINER_CLOSE_THREADS_DEFAULT; } -// if (periodicDiskCheckIntervalMinutes < 1) { -// LOG.warn(PERIODIC_DISK_CHECK_INTERVAL_MINUTES_KEY + -// " must be greater than zero and was set to {}. Defaulting to {}", -// periodicDiskCheckIntervalMinutes, -// PERIODIC_DISK_CHECK_INTERVAL_MINUTES_DEFAULT); -// periodicDiskCheckIntervalMinutes = -// PERIODIC_DISK_CHECK_INTERVAL_MINUTES_DEFAULT; -// } - if (periodicDiskCheckIntervalMinutes == 0) { LOG.warn("{} must not be zero. Defaulting to {}", PERIODIC_DISK_CHECK_INTERVAL_MINUTES_KEY, diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/TestDatanodeConfiguration.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/TestDatanodeConfiguration.java index 5012526782aa..12b296522758 100644 --- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/TestDatanodeConfiguration.java +++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/TestDatanodeConfiguration.java @@ -108,7 +108,7 @@ public void acceptsValidValues() { public void overridesInvalidValues() { // GIVEN int invalidDeleteThreads = 0; - long invalidDiskCheckIntervalMinutes = -1; + long invalidDiskCheckIntervalMinutes = 0; int invalidFailedVolumesTolerated = -2; long invalidDiskCheckMinGap = -1; long invalidDiskCheckTimeout = -1; From 7dcd34f83bab9a7717f5cb3650d095d428d43d38 Mon Sep 17 00:00:00 2001 From: Russole <850905junior@gmail.com> Date: Mon, 29 Dec 2025 23:58:14 +0800 Subject: [PATCH 3/3] Clarify periodic disk check config behavior and add unit test --- .../statemachine/DatanodeConfiguration.java | 4 +++- .../common/volume/StorageVolumeChecker.java | 4 +++- .../volume/TestStorageVolumeChecker.java | 23 +++++++++++++++++++ 3 files changed, 29 insertions(+), 2 deletions(-) diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeConfiguration.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeConfiguration.java index a0312ffe8d6b..163858571dfb 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeConfiguration.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeConfiguration.java @@ -309,7 +309,9 @@ public class DatanodeConfiguration extends ReconfigurableConfig { defaultValue = "60", type = ConfigType.LONG, tags = { DATANODE }, - description = "Periodic disk check run interval in minutes." + description = "Periodic disk check run interval in minutes. " + + "A negative value disables periodic disk checks. " + + "A value of 0 is invalid and defaults to 60." ) private long periodicDiskCheckIntervalMinutes = PERIODIC_DISK_CHECK_INTERVAL_MINUTES_DEFAULT; diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/StorageVolumeChecker.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/StorageVolumeChecker.java index 1237b452334b..024a22389763 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/StorageVolumeChecker.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/StorageVolumeChecker.java @@ -420,7 +420,9 @@ private void invokeCallback() { */ public void shutdownAndWait(int gracePeriod, TimeUnit timeUnit) { if (started.compareAndSet(true, false)) { - periodicDiskChecker.cancel(true); + if (periodicDiskChecker != null) { + periodicDiskChecker.cancel(true); + } diskCheckerservice.shutdownNow(); checkVolumeResultHandlerExecutorService.shutdownNow(); metrics.unregister(); diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/volume/TestStorageVolumeChecker.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/volume/TestStorageVolumeChecker.java index 92287bc617a6..ba4ee3ccb23a 100644 --- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/volume/TestStorageVolumeChecker.java +++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/volume/TestStorageVolumeChecker.java @@ -31,6 +31,7 @@ import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import java.io.File; +import java.lang.reflect.Field; import java.nio.file.Path; import java.time.Duration; import java.util.ArrayList; @@ -299,6 +300,28 @@ public void testNumScansSkipped() throws Exception { checker.shutdownAndWait(0, TimeUnit.SECONDS); } + /** + * Verify that negative values disable the periodic disk checker. + */ + @Test + public void testNegativeIntervalDisablesPeriodicDiskChecker() throws Exception { + setup(); + conf.setLong("hdds.datanode.periodic.disk.check.interval.minutes", -1); + + StorageVolumeChecker checker = + new StorageVolumeChecker(conf, new FakeTimer(), "test-"); + + checker.start(); + + Field f = StorageVolumeChecker.class.getDeclaredField("periodicDiskChecker"); + f.setAccessible(true); + // Expect no ScheduledFuture to be created, as a negative interval disables + // the periodic disk checker entirely. + assertThat(f.get(checker)).isNull(); + // Verify shutdown is safe when no periodic task was scheduled. + checker.shutdownAndWait(0, TimeUnit.SECONDS); + } + /** * A checker to wraps the result of {@link HddsVolume#check} in * an ImmediateFuture.