diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMHAUtils.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMHAUtils.java index daa25648f329..939777bebd15 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMHAUtils.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMHAUtils.java @@ -17,19 +17,15 @@ package org.apache.hadoop.hdds.scm.ha; -import static org.apache.hadoop.hdds.HddsConfigKeys.OZONE_METADATA_DIRS; -import static org.apache.hadoop.ozone.OzoneConsts.OZONE_RATIS_SNAPSHOT_DIR; - import com.google.common.base.Strings; import com.google.common.collect.ImmutableList; -import java.io.File; import java.io.IOException; -import java.nio.file.Paths; import java.util.ArrayList; import java.util.List; import org.apache.hadoop.hdds.HddsUtils; import org.apache.hadoop.hdds.conf.ConfigurationSource; import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeType; import org.apache.hadoop.hdds.ratis.ServerNotLeaderException; import org.apache.hadoop.hdds.scm.ScmConfigKeys; import org.apache.hadoop.hdds.scm.container.ContainerNotFoundException; @@ -46,15 +42,11 @@ import org.apache.ratis.protocol.exceptions.ReconfigurationTimeoutException; import org.apache.ratis.protocol.exceptions.ResourceUnavailableException; import org.apache.ratis.protocol.exceptions.StateMachineException; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; /** * Utility class used by SCM HA. */ public final class SCMHAUtils { - private static final Logger LOG = - LoggerFactory.getLogger(SCMHAUtils.class); private static final ImmutableList> RETRIABLE_WITH_NO_FAILOVER_EXCEPTION_LIST = @@ -97,31 +89,18 @@ public static String getSCMRatisDirectory(ConfigurationSource conf) { conf.get(ScmConfigKeys.OZONE_SCM_HA_RATIS_STORAGE_DIR); if (Strings.isNullOrEmpty(scmRatisDirectory)) { - scmRatisDirectory = ServerUtils.getDefaultRatisDirectory(conf); + scmRatisDirectory = ServerUtils.getDefaultRatisDirectory(conf, NodeType.SCM); } return scmRatisDirectory; } - public static String getRatisStorageDir(final ConfigurationSource conf) { - String storageDir = conf.get(ScmConfigKeys.OZONE_SCM_HA_RATIS_STORAGE_DIR); - if (Strings.isNullOrEmpty(storageDir)) { - File metaDirPath = ServerUtils.getOzoneMetaDirPath(conf); - storageDir = (new File(metaDirPath, "scm-ha")).getPath(); - } - return storageDir; - } - public static String getSCMRatisSnapshotDirectory(ConfigurationSource conf) { String snapshotDir = conf.get(ScmConfigKeys.OZONE_SCM_HA_RATIS_SNAPSHOT_DIR); - // If ratis snapshot directory is not set, fall back to ozone.metadata.dir. + // If ratis snapshot directory is not set, fall back to ozone.metadata.dir with component-specific location. if (Strings.isNullOrEmpty(snapshotDir)) { - LOG.warn("SCM snapshot dir is not configured. Falling back to {} config", - OZONE_METADATA_DIRS); - File metaDirPath = ServerUtils.getOzoneMetaDirPath(conf); - snapshotDir = - Paths.get(metaDirPath.getPath(), OZONE_RATIS_SNAPSHOT_DIR).toString(); + snapshotDir = ServerUtils.getDefaultRatisSnapshotDirectory(conf, NodeType.SCM); } return snapshotDir; } diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/server/ServerUtils.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/server/ServerUtils.java index 01f271b26ef2..d6864d4f15db 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/server/ServerUtils.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/server/ServerUtils.java @@ -21,17 +21,20 @@ import java.net.InetSocketAddress; import java.nio.file.Files; import java.nio.file.Path; +import java.nio.file.Paths; import java.nio.file.attribute.PosixFilePermissions; import java.util.Collection; import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.hdds.HddsConfigKeys; import org.apache.hadoop.hdds.conf.ConfigurationSource; import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeType; import org.apache.hadoop.hdds.recon.ReconConfigKeys; import org.apache.hadoop.hdds.scm.ScmConfigKeys; import org.apache.hadoop.ipc_.RPC; import org.apache.hadoop.ipc_.Server; import org.apache.hadoop.ozone.OzoneConfigKeys; +import org.apache.hadoop.ozone.OzoneConsts; import org.apache.hadoop.security.UserGroupInformation; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -300,11 +303,136 @@ public static String getRemoteUserName() { return remoteUser != null ? remoteUser.getUserName() : null; } - public static String getDefaultRatisDirectory(ConfigurationSource conf) { + /** + * Get the default Ratis directory for a component when the specific + * configuration is not set. This creates a component-specific subdirectory + * under ozone.metadata.dirs to avoid conflicts when multiple components + * are colocated on the same host. + * + *

For backward compatibility during upgrades, this method checks for + * existing Ratis data in old locations before using the new component-specific + * location. See {@link #findExistingRatisDirectory} for details on old locations. + * + * @param conf Configuration source + * @param nodeType Type of the node component + * @return Path to the component-specific ratis directory + */ + public static String getDefaultRatisDirectory(ConfigurationSource conf, + NodeType nodeType) { LOG.warn("Storage directory for Ratis is not configured. It is a good " + "idea to map this to an SSD disk. Falling back to {}", HddsConfigKeys.OZONE_METADATA_DIRS); File metaDirPath = ServerUtils.getOzoneMetaDirPath(conf); - return (new File(metaDirPath, "ratis")).getPath(); + + // Check for existing Ratis data from old versions for backward compatibility + String existingDir = findExistingRatisDirectory(metaDirPath, nodeType); + if (existingDir != null) { + return existingDir; + } + + // Use new component-specific location for new installations + String componentName = getComponentName(nodeType); + return Paths.get(metaDirPath.getPath(), componentName + ".ratis").toString(); + } + + /** + * Checks for existing Ratis directories from previous versions for backward + * compatibility during upgrades. + * + *

Older versions of Ozone used different directory structures: + *

+ * + * @param metaDirPath The ozone metadata directory path + * @param nodeType Type of the node component + * @return Path to existing old Ratis directory if found, null otherwise + */ + private static String findExistingRatisDirectory(File metaDirPath, + NodeType nodeType) { + // Check component-specific old location (SCM used scm-ha in some versions) + if ("scm".equals(getComponentName(nodeType))) { + File oldScmRatisDir = new File(metaDirPath, "scm-ha"); + if (isNonEmptyDirectory(oldScmRatisDir)) { + LOG.info("Found existing SCM Ratis directory at old location: {}. " + + "Using it for backward compatibility during upgrade.", + oldScmRatisDir.getPath()); + return oldScmRatisDir.getPath(); + } + } + + // Check old shared Ratis location (used by version 2.0.0 and earlier) + // All components (OM, SCM) shared /data/metadata/ratis + File oldSharedRatisDir = new File(metaDirPath, "ratis"); + if (isNonEmptyDirectory(oldSharedRatisDir)) { + LOG.info("Found existing Ratis directory at old shared location: {}. " + + "Using it for backward compatibility during upgrade.", + oldSharedRatisDir.getPath()); + return oldSharedRatisDir.getPath(); + } + + return null; + } + + /** + * Converts NodeType enum to the component name string used for directory naming. + * + * @param nodeType Type of the node component + * @return Component name string (e.g., "om", "scm", "dn", "recon") + */ + private static String getComponentName(NodeType nodeType) { + switch (nodeType) { + case OM: + return "om"; + case SCM: + return "scm"; + case DATANODE: + return "dn"; + case RECON: + return "recon"; + default: + throw new IllegalArgumentException("Unknown NodeType: " + nodeType); + } + } + + /** + * Get the default Ratis snapshot directory for a component when the specific + * configuration is not set. This creates a component-specific subdirectory + * under ozone.metadata.dirs to avoid conflicts when multiple components + * are colocated on the same host. + * + * New path format: {ozone.metadata.dirs}/{NodeType}.ratis.snapshot + * eg: /data/metadata/om.ratis.snapshot + * /data/metadata/scm.ratis.snapshot + * + * @param conf Configuration source + * @param nodeType Type of the node component + * @return Path to the component-specific ratis snapshot directory + */ + public static String getDefaultRatisSnapshotDirectory(ConfigurationSource conf, + NodeType nodeType) { + LOG.warn("Snapshot directory for Ratis is not configured. Falling back to {}", + HddsConfigKeys.OZONE_METADATA_DIRS); + File metaDirPath = ServerUtils.getOzoneMetaDirPath(conf); + + // Use component-specific location + String componentName = getComponentName(nodeType); + return Paths.get(metaDirPath.getPath(), + componentName + ".ratis." + OzoneConsts.OZONE_RATIS_SNAPSHOT_DIR).toString(); + } + + /** + * Checks if a directory exists and is non-empty. + * + * @param dir Directory to check + * @return true if directory exists and contains at least one file + */ + private static boolean isNonEmptyDirectory(File dir) { + if (dir != null && dir.exists() && dir.isDirectory()) { + File[] files = dir.listFiles(); + return files != null && files.length > 0; + } + return false; } } diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/HddsServerUtil.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/HddsServerUtil.java index ad37bc7531e4..df4c31077aa6 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/HddsServerUtil.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/HddsServerUtil.java @@ -76,6 +76,7 @@ import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.protocol.SCMSecurityProtocol; import org.apache.hadoop.hdds.protocol.SecretKeyProtocolScm; +import org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeType; import org.apache.hadoop.hdds.protocolPB.SCMSecurityProtocolClientSideTranslatorPB; import org.apache.hadoop.hdds.protocolPB.SecretKeyProtocolClientSideTranslatorPB; import org.apache.hadoop.hdds.protocolPB.SecretKeyProtocolDatanodePB; @@ -424,7 +425,7 @@ public static Collection getOzoneDatanodeRatisDirectory( if (rawLocations.isEmpty()) { rawLocations = new ArrayList<>(1); - rawLocations.add(ServerUtils.getDefaultRatisDirectory(conf)); + rawLocations.add(ServerUtils.getDefaultRatisDirectory(conf, NodeType.DATANODE)); } return rawLocations; } diff --git a/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/server/TestServerUtils.java b/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/server/TestServerUtils.java index a7bbeccbd19e..1eb633f455ed 100644 --- a/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/server/TestServerUtils.java +++ b/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/server/TestServerUtils.java @@ -17,8 +17,12 @@ package org.apache.hadoop.hdds.server; +import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeType.DATANODE; +import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeType.OM; +import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeType.SCM; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertThrows; @@ -28,6 +32,7 @@ import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; +import java.nio.file.Paths; import java.nio.file.attribute.PosixFilePermission; import java.nio.file.attribute.PosixFilePermissions; import java.util.Set; @@ -263,4 +268,175 @@ public void ozoneMetadataDirRejectsList() { () -> ServerUtils.getOzoneMetaDirPath(conf)); } + /** + * Test that SCM, OM, and Datanode colocated on the same host with only + * ozone.metadata.dirs configured don't conflict with Ratis directories. + */ + @Test + public void testColocatedComponentsWithSharedMetadataDir() { + final File metaDir = new File(folder.toFile(), "sharedMetaDir"); + final OzoneConfiguration conf = new OzoneConfiguration(); + + // Only configure ozone.metadata.dirs (the fallback config) + conf.set(HddsConfigKeys.OZONE_METADATA_DIRS, metaDir.getPath()); + + try { + assertFalse(metaDir.exists()); + + // Test Ratis directories - each component should get its own with flat naming + String scmRatisDir = ServerUtils.getDefaultRatisDirectory(conf, SCM); + String omRatisDir = ServerUtils.getDefaultRatisDirectory(conf, OM); + String dnRatisDir = ServerUtils.getDefaultRatisDirectory(conf, DATANODE); + + // Verify Ratis directories use flat naming pattern (component.ratis) + assertEquals(new File(metaDir, "scm.ratis").getPath(), scmRatisDir); + assertEquals(new File(metaDir, "om.ratis").getPath(), omRatisDir); + assertEquals(new File(metaDir, "dn.ratis").getPath(), dnRatisDir); + + // Verify all Ratis directories are different + assertNotEquals(scmRatisDir, omRatisDir); + assertNotEquals(scmRatisDir, dnRatisDir); + assertNotEquals(omRatisDir, dnRatisDir); + + // Verify the base metadata dir exists + assertTrue(metaDir.exists()); + + } finally { + FileUtils.deleteQuietly(metaDir); + } + } + + @Test + public void testEmptyOldSharedRatisIgnored() throws IOException { + final File metaDir = new File(folder.toFile(), "upgradeMetaDir"); + final File oldSharedRatisDir = new File(metaDir, "ratis"); + final OzoneConfiguration conf = new OzoneConfiguration(); + conf.set(HddsConfigKeys.OZONE_METADATA_DIRS, metaDir.getPath()); + + try { + // Create old Ratis directory (empty) + assertTrue(oldSharedRatisDir.mkdirs()); + + // SCM should use new SCM path + String scmRatisDir = ServerUtils.getDefaultRatisDirectory(conf, SCM); + assertEquals(Paths.get(metaDir.getPath(), "scm.ratis").toString(), scmRatisDir); + + // OM should use new OM path + String omRatisDir = ServerUtils.getDefaultRatisDirectory(conf, OM); + assertEquals(Paths.get(metaDir.getPath(), "om.ratis").toString(), omRatisDir); + + } finally { + FileUtils.deleteQuietly(metaDir); + } + } + + /** + * Test backward compatibility: old shared /ratis directory should be used + * when it exists and is non-empty (simulating upgrade from version 2.0.0). + */ + @Test + public void testBackwardCompatibilityWithOldSharedRatisDir() throws IOException { + final File metaDir = new File(folder.toFile(), "upgradeMetaDir"); + final File oldSharedRatisDir = new File(metaDir, "ratis"); + final OzoneConfiguration conf = new OzoneConfiguration(); + conf.set(HddsConfigKeys.OZONE_METADATA_DIRS, metaDir.getPath()); + + try { + // Create old shared ratis directory with some files (simulating existing data) + assertTrue(oldSharedRatisDir.mkdirs()); + File testFile = new File(oldSharedRatisDir, "test-file"); + assertTrue(testFile.createNewFile()); + + // Test that all components use the old shared location + String scmRatisDir = ServerUtils.getDefaultRatisDirectory(conf, SCM); + String omRatisDir = ServerUtils.getDefaultRatisDirectory(conf, OM); + String dnRatisDir = ServerUtils.getDefaultRatisDirectory(conf, DATANODE); + + // All should use the old shared location + assertEquals(oldSharedRatisDir.getPath(), scmRatisDir); + assertEquals(oldSharedRatisDir.getPath(), omRatisDir); + assertEquals(oldSharedRatisDir.getPath(), dnRatisDir); + + } finally { + FileUtils.deleteQuietly(metaDir); + } + } + + /** + * Test backward compatibility: SCM-specific /scm-ha directory should be preferred + * over shared /ratis directory when both exist and are non-empty. + * OM and DATANODE should continue using /ratis even when /scm-ha exists. + */ + @Test + public void testBackwardCompatibilityWithScmHaDirectory() throws IOException { + final File metaDir = new File(folder.toFile(), "upgradeMetaDir"); + final File oldScmHaDir = new File(metaDir, "scm-ha"); + final File oldSharedRatisDir = new File(metaDir, "ratis"); + final OzoneConfiguration conf = new OzoneConfiguration(); + conf.set(HddsConfigKeys.OZONE_METADATA_DIRS, metaDir.getPath()); + + try { + // Create both scm-ha and ratis directories with files + assertTrue(oldScmHaDir.mkdirs()); + File scmHaTestFile = new File(oldScmHaDir, "scm-ha-file"); + assertTrue(scmHaTestFile.createNewFile()); + + assertTrue(oldSharedRatisDir.mkdirs()); + File ratisTestFile = new File(oldSharedRatisDir, "ratis-file"); + assertTrue(ratisTestFile.createNewFile()); + + // SCM should prefer scm-ha over ratis + String scmRatisDir = ServerUtils.getDefaultRatisDirectory(conf, SCM); + assertEquals(oldScmHaDir.getPath(), scmRatisDir); + assertNotEquals(oldSharedRatisDir.getPath(), scmRatisDir); + + // OM and DATANODE should still use ratis even when scm-ha exists + String omRatisDir = ServerUtils.getDefaultRatisDirectory(conf, OM); + String dnRatisDir = ServerUtils.getDefaultRatisDirectory(conf, DATANODE); + assertEquals(oldSharedRatisDir.getPath(), omRatisDir); + assertEquals(oldSharedRatisDir.getPath(), dnRatisDir); + + // Test that empty scm-ha directory is ignored (SCM should fall back to ratis) + FileUtils.deleteQuietly(scmHaTestFile); + String scmRatisDirWithEmptyScmHa = ServerUtils.getDefaultRatisDirectory(conf, SCM); + assertEquals(oldSharedRatisDir.getPath(), scmRatisDirWithEmptyScmHa); + + } finally { + FileUtils.deleteQuietly(metaDir); + } + } + + /** + * Test that SCM and OM colocated on the same host with only + * ozone.metadata.dirs configured get separate Ratis snapshot directories. + */ + @Test + public void testColocatedComponentsWithSharedMetadataDirForSnapshots() { + final File metaDir = new File(folder.toFile(), "sharedMetaDir"); + final OzoneConfiguration conf = new OzoneConfiguration(); + + // Only configure ozone.metadata.dirs (the fallback config) + conf.set(HddsConfigKeys.OZONE_METADATA_DIRS, metaDir.getPath()); + + try { + assertFalse(metaDir.exists()); + + // Test Ratis snapshot directories - OM and SCM should get their own + String scmSnapshotDir = ServerUtils.getDefaultRatisSnapshotDirectory(conf, SCM); + String omSnapshotDir = ServerUtils.getDefaultRatisSnapshotDirectory(conf, OM); + + // Verify snapshot directories use: /.ratis.snapshot + assertEquals(new File(metaDir, "scm.ratis.snapshot").getPath(), scmSnapshotDir); + assertEquals(new File(metaDir, "om.ratis.snapshot").getPath(), omSnapshotDir); + + // Verify snapshot directories are different + assertNotEquals(scmSnapshotDir, omSnapshotDir); + + // Verify the base metadata dir exists + assertTrue(metaDir.exists()); + + } finally { + FileUtils.deleteQuietly(metaDir); + } + } } diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/RatisUtil.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/RatisUtil.java index 5860d2523d49..f2900e38f405 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/RatisUtil.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/RatisUtil.java @@ -90,7 +90,7 @@ public static RaftProperties newRaftProperties( public static void setRaftStorageDir(final RaftProperties properties, final ConfigurationSource conf) { RaftServerConfigKeys.setStorageDir(properties, Collections - .singletonList(new File(SCMHAUtils.getRatisStorageDir(conf)))); + .singletonList(new File(SCMHAUtils.getSCMRatisDirectory(conf)))); } /** diff --git a/hadoop-ozone/dist/src/main/smoketest/omha/testOMHA.robot b/hadoop-ozone/dist/src/main/smoketest/omha/testOMHA.robot index 70fc66a228f2..df7c914273ed 100644 --- a/hadoop-ozone/dist/src/main/smoketest/omha/testOMHA.robot +++ b/hadoop-ozone/dist/src/main/smoketest/omha/testOMHA.robot @@ -28,7 +28,7 @@ ${USERNAME} hadoop ${PUBLIC_KEY} /opt/.ssh/id_rsa ${OM_SERVICE_ID} %{OM_SERVICE_ID} ${OZONE_LOG_DIR} /ozone/logs/ -${RATIS_DIR} /data/metadata/ratis +${RATIS_DIR} /data/metadata/om.ratis ${VOLUME} volume1 ${BUCKET} bucket1 ${TEST_FILE} NOTICE.txt diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestStorageContainerManager.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestStorageContainerManager.java index 7276dc871eac..8af00a8b0ede 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestStorageContainerManager.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestStorageContainerManager.java @@ -183,7 +183,7 @@ void test(@TempDir Path tempDir) throws Exception { StorageContainerManager scm = cluster.getStorageContainerManager(); List directories = Arrays.asList( - new File(SCMHAUtils.getRatisStorageDir(scm.getConfiguration())), + new File(SCMHAUtils.getSCMRatisDirectory(scm.getConfiguration())), scm.getScmMetadataStore().getStore().getDbLocation(), new File(scm.getScmStorageConfig().getStorageDir()) ); diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/parser/TestOzoneHARatisLogParser.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/parser/TestOzoneHARatisLogParser.java index c3c9b08d9963..02f518c65e88 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/parser/TestOzoneHARatisLogParser.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/parser/TestOzoneHARatisLogParser.java @@ -18,7 +18,6 @@ package org.apache.hadoop.ozone.parser; import static java.nio.charset.StandardCharsets.UTF_8; -import static org.apache.hadoop.hdds.HddsConfigKeys.OZONE_METADATA_DIRS; import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; @@ -40,6 +39,7 @@ import org.apache.hadoop.ozone.client.OzoneClientFactory; import org.apache.hadoop.ozone.debug.ratis.parse.RatisLogParser; import org.apache.hadoop.ozone.om.helpers.OMRatisHelper; +import org.apache.hadoop.ozone.om.ratis.utils.OzoneManagerRatisUtils; import org.apache.ozone.test.GenericTestUtils; import org.apache.ozone.test.tag.Flaky; import org.junit.jupiter.api.AfterEach; @@ -105,8 +105,9 @@ void testRatisLogParsing() throws Exception { cluster.stop(); - File omMetaDir = new File(ozoneConfiguration.get(OZONE_METADATA_DIRS), - "ratis"); + // Get the OM Ratis directory using the proper utility method + File omMetaDir = new File( + OzoneManagerRatisUtils.getOMRatisDirectory(ozoneConfiguration)); assertThat(omMetaDir).isDirectory(); String[] ratisDirs = omMetaDir.list(); @@ -134,7 +135,7 @@ void testRatisLogParsing() throws Exception { // Now check for SCM. File scmMetadataDir = - new File(SCMHAUtils.getRatisStorageDir(leaderSCMConfig)); + new File(SCMHAUtils.getSCMRatisDirectory(leaderSCMConfig)); assertThat(scmMetadataDir).isDirectory(); ratisDirs = scmMetadataDir.list(); diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java index 343c46158dc6..a51a0b56ecd0 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java @@ -184,7 +184,6 @@ import org.apache.hadoop.hdds.scm.ScmInfo; import org.apache.hadoop.hdds.scm.client.HddsClientUtils; import org.apache.hadoop.hdds.scm.client.ScmTopologyClient; -import org.apache.hadoop.hdds.scm.ha.SCMHAUtils; import org.apache.hadoop.hdds.scm.ha.SCMNodeInfo; import org.apache.hadoop.hdds.scm.net.NetworkTopology; import org.apache.hadoop.hdds.scm.protocol.ScmBlockLocationProtocol; @@ -1607,10 +1606,13 @@ private void initializeRatisDirs(OzoneConfiguration conf) throws IOException { } OmUtils.createOMDir(omRatisDirectory); - String scmStorageDir = SCMHAUtils.getRatisStorageDir(conf); - if (!Strings.isNullOrEmpty(omRatisDirectory) && !Strings - .isNullOrEmpty(scmStorageDir) && omRatisDirectory - .equals(scmStorageDir)) { + String omRatisConfigured = conf.get(OMConfigKeys.OZONE_OM_RATIS_STORAGE_DIR); + String scmRatisConfigured = conf.get(ScmConfigKeys.OZONE_SCM_HA_RATIS_STORAGE_DIR); + + // Only check co-location if BOTH are explicitly configured + if (!Strings.isNullOrEmpty(omRatisConfigured) && + !Strings.isNullOrEmpty(scmRatisConfigured) && + omRatisConfigured.equals(scmRatisConfigured)) { throw new IOException( "Path of " + OMConfigKeys.OZONE_OM_RATIS_STORAGE_DIR + " and " + ScmConfigKeys.OZONE_SCM_HA_RATIS_STORAGE_DIR diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/utils/OzoneManagerRatisUtils.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/utils/OzoneManagerRatisUtils.java index 5548be7bd8ba..1778de3520d9 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/utils/OzoneManagerRatisUtils.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/utils/OzoneManagerRatisUtils.java @@ -17,21 +17,18 @@ package org.apache.hadoop.ozone.om.ratis.utils; -import static org.apache.hadoop.hdds.HddsConfigKeys.OZONE_METADATA_DIRS; -import static org.apache.hadoop.ozone.OzoneConsts.OZONE_RATIS_SNAPSHOT_DIR; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_RATIS_SNAPSHOT_DIR; import static org.apache.hadoop.ozone.om.OzoneManagerUtils.getBucketLayout; import com.google.common.base.Preconditions; import com.google.common.base.Strings; import com.google.protobuf.ServiceException; -import java.io.File; import java.io.IOException; import java.nio.file.InvalidPathException; import java.nio.file.Path; -import java.nio.file.Paths; import org.apache.hadoop.hdds.conf.ConfigurationSource; import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeType; import org.apache.hadoop.hdds.security.SecurityConfig; import org.apache.hadoop.hdds.security.x509.certificate.client.CertificateClient; import org.apache.hadoop.hdds.server.ServerUtils; @@ -472,7 +469,7 @@ public static String getOMRatisDirectory(ConfigurationSource conf) { String storageDir = conf.get(OMConfigKeys.OZONE_OM_RATIS_STORAGE_DIR); if (Strings.isNullOrEmpty(storageDir)) { - storageDir = ServerUtils.getDefaultRatisDirectory(conf); + storageDir = ServerUtils.getDefaultRatisDirectory(conf, NodeType.OM); } return storageDir; } @@ -483,13 +480,9 @@ public static String getOMRatisDirectory(ConfigurationSource conf) { public static String getOMRatisSnapshotDirectory(ConfigurationSource conf) { String snapshotDir = conf.get(OZONE_OM_RATIS_SNAPSHOT_DIR); - // If ratis snapshot directory is not set, fall back to ozone.metadata.dir. + // If ratis snapshot directory is not set, fall back to default component-specific location. if (Strings.isNullOrEmpty(snapshotDir)) { - LOG.warn("{} is not configured. Falling back to {} config", - OZONE_OM_RATIS_SNAPSHOT_DIR, OZONE_METADATA_DIRS); - File metaDirPath = ServerUtils.getOzoneMetaDirPath(conf); - snapshotDir = Paths.get(metaDirPath.getPath(), - OZONE_RATIS_SNAPSHOT_DIR).toString(); + snapshotDir = ServerUtils.getDefaultRatisSnapshotDirectory(conf, NodeType.OM); } return snapshotDir; }