HDDS-13866. Use component-specific default directory for Ratis

hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMHAUtils.java

@@ -17,19 +17,15 @@
 
 package org.apache.hadoop.hdds.scm.ha;
 
-import static org.apache.hadoop.hdds.HddsConfigKeys.OZONE_METADATA_DIRS;
-import static org.apache.hadoop.ozone.OzoneConsts.OZONE_RATIS_SNAPSHOT_DIR;
-
 import com.google.common.base.Strings;
 import com.google.common.collect.ImmutableList;
-import java.io.File;
 import java.io.IOException;
-import java.nio.file.Paths;
 import java.util.ArrayList;
 import java.util.List;
 import org.apache.hadoop.hdds.HddsUtils;
 import org.apache.hadoop.hdds.conf.ConfigurationSource;
 import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeType;
 import org.apache.hadoop.hdds.ratis.ServerNotLeaderException;
 import org.apache.hadoop.hdds.scm.ScmConfigKeys;
 import org.apache.hadoop.hdds.scm.container.ContainerNotFoundException;
@@ -46,15 +42,11 @@
 import org.apache.ratis.protocol.exceptions.ReconfigurationTimeoutException;
 import org.apache.ratis.protocol.exceptions.ResourceUnavailableException;
 import org.apache.ratis.protocol.exceptions.StateMachineException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
 
 /**
  * Utility class used by SCM HA.
  */
 public final class SCMHAUtils {
-  private static final Logger LOG =
-      LoggerFactory.getLogger(SCMHAUtils.class);
 
   private static final ImmutableList<Class<? extends Exception>>
       RETRIABLE_WITH_NO_FAILOVER_EXCEPTION_LIST =
@@ -97,31 +89,18 @@ public static String getSCMRatisDirectory(ConfigurationSource conf) {
             conf.get(ScmConfigKeys.OZONE_SCM_HA_RATIS_STORAGE_DIR);
 
     if (Strings.isNullOrEmpty(scmRatisDirectory)) {
-      scmRatisDirectory = ServerUtils.getDefaultRatisDirectory(conf);
+      scmRatisDirectory = ServerUtils.getDefaultRatisDirectory(conf, NodeType.SCM);
     }
     return scmRatisDirectory;
   }
 
-  public static String getRatisStorageDir(final ConfigurationSource conf) {
-    String storageDir = conf.get(ScmConfigKeys.OZONE_SCM_HA_RATIS_STORAGE_DIR);
-    if (Strings.isNullOrEmpty(storageDir)) {
-      File metaDirPath = ServerUtils.getOzoneMetaDirPath(conf);
-      storageDir = (new File(metaDirPath, "scm-ha")).getPath();
-    }
-    return storageDir;
-  }
-
   public static String getSCMRatisSnapshotDirectory(ConfigurationSource conf) {
     String snapshotDir =
             conf.get(ScmConfigKeys.OZONE_SCM_HA_RATIS_SNAPSHOT_DIR);
 
-    // If ratis snapshot directory is not set, fall back to ozone.metadata.dir.
+    // If ratis snapshot directory is not set, fall back to ozone.metadata.dir with component-specific location.
     if (Strings.isNullOrEmpty(snapshotDir)) {
-      LOG.warn("SCM snapshot dir is not configured. Falling back to {} config",
-          OZONE_METADATA_DIRS);
-      File metaDirPath = ServerUtils.getOzoneMetaDirPath(conf);
-      snapshotDir =
-          Paths.get(metaDirPath.getPath(), OZONE_RATIS_SNAPSHOT_DIR).toString();
+      snapshotDir = ServerUtils.getDefaultRatisSnapshotDirectory(conf, NodeType.SCM);
     }
     return snapshotDir;
   }

hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/server/ServerUtils.java

@@ -21,17 +21,20 @@
 import java.net.InetSocketAddress;
 import java.nio.file.Files;
 import java.nio.file.Path;
+import java.nio.file.Paths;
 import java.nio.file.attribute.PosixFilePermissions;
 import java.util.Collection;
 import org.apache.hadoop.fs.permission.FsPermission;
 import org.apache.hadoop.hdds.HddsConfigKeys;
 import org.apache.hadoop.hdds.conf.ConfigurationSource;
 import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeType;
 import org.apache.hadoop.hdds.recon.ReconConfigKeys;
 import org.apache.hadoop.hdds.scm.ScmConfigKeys;
 import org.apache.hadoop.ipc_.RPC;
 import org.apache.hadoop.ipc_.Server;
 import org.apache.hadoop.ozone.OzoneConfigKeys;
+import org.apache.hadoop.ozone.OzoneConsts;
 import org.apache.hadoop.security.UserGroupInformation;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -300,11 +303,136 @@ public static String getRemoteUserName() {
     return remoteUser != null ? remoteUser.getUserName() : null;
   }
 
-  public static String getDefaultRatisDirectory(ConfigurationSource conf) {
+  /**
+   * Get the default Ratis directory for a component when the specific
+   * configuration is not set. This creates a component-specific subdirectory
+   * under ozone.metadata.dirs to avoid conflicts when multiple components
+   * are colocated on the same host.
+   *
+   * <p>For backward compatibility during upgrades, this method checks for
+   * existing Ratis data in old locations before using the new component-specific
+   * location. See {@link #findExistingRatisDirectory} for details on old locations.
+   *
+   * @param conf Configuration source
+   * @param nodeType Type of the node component
+   * @return Path to the component-specific ratis directory
+   */
+  public static String getDefaultRatisDirectory(ConfigurationSource conf,
+      NodeType nodeType) {
     LOG.warn("Storage directory for Ratis is not configured. It is a good " +
             "idea to map this to an SSD disk. Falling back to {}",
         HddsConfigKeys.OZONE_METADATA_DIRS);
     File metaDirPath = ServerUtils.getOzoneMetaDirPath(conf);
-    return (new File(metaDirPath, "ratis")).getPath();
+
+    // Check for existing Ratis data from old versions for backward compatibility
+    String existingDir = findExistingRatisDirectory(metaDirPath, nodeType);
+    if (existingDir != null) {
+      return existingDir;
+    }
+
+    // Use new component-specific location for new installations
+    String componentName = getComponentName(nodeType);
+    return Paths.get(metaDirPath.getPath(), componentName + ".ratis").toString();
+  }
+
+  /**
+   * Checks for existing Ratis directories from previous versions for backward
+   * compatibility during upgrades.
+   *
+   * <p>Older versions of Ozone used different directory structures:
+   * <ul>
+   *   <li>Versions up to 2.0.0: Shared {@code <ozone.metadata.dirs>/ratis} for all components</li>
+   *   <li>Some SCM versions: Used {@code <ozone.metadata.dirs>/scm-ha}</li>
+   * </ul>
+   *
+   * @param metaDirPath The ozone metadata directory path
+   * @param nodeType Type of the node component
+   * @return Path to existing old Ratis directory if found, null otherwise
+   */
+  private static String findExistingRatisDirectory(File metaDirPath,
+      NodeType nodeType) {
+    // Check component-specific old location (SCM used scm-ha in some versions)
+    if ("scm".equals(getComponentName(nodeType))) {
+      File oldScmRatisDir = new File(metaDirPath, "scm-ha");
+      if (isNonEmptyDirectory(oldScmRatisDir)) {
+        LOG.info("Found existing SCM Ratis directory at old location: {}. " +
+                "Using it for backward compatibility during upgrade.",
+            oldScmRatisDir.getPath());
+        return oldScmRatisDir.getPath();
+      }
+    }
+
+    // Check old shared Ratis location (used by version 2.0.0 and earlier)
+    // All components (OM, SCM) shared /data/metadata/ratis
+    File oldSharedRatisDir = new File(metaDirPath, "ratis");
+    if (isNonEmptyDirectory(oldSharedRatisDir)) {
+      LOG.info("Found existing Ratis directory at old shared location: {}. " +
+              "Using it for backward compatibility during upgrade.",
+          oldSharedRatisDir.getPath());
+      return oldSharedRatisDir.getPath();
+    }
+
+    return null;
+  }
+
+  /**
+   * Converts NodeType enum to the component name string used for directory naming.
+   *
+   * @param nodeType Type of the node component
+   * @return Component name string (e.g., "om", "scm", "dn", "recon")
+   */
+  private static String getComponentName(NodeType nodeType) {
+    switch (nodeType) {
+    case OM:
+      return "om";
+    case SCM:
+      return "scm";
+    case DATANODE:
+      return "dn";
+    case RECON:
+      return "recon";
+    default:
+      throw new IllegalArgumentException("Unknown NodeType: " + nodeType);
+    }
+  }
+
+  /**
+   * Get the default Ratis snapshot directory for a component when the specific
+   * configuration is not set. This creates a component-specific subdirectory
+   * under ozone.metadata.dirs to avoid conflicts when multiple components
+   * are colocated on the same host.
+   *
+   * New path format: {ozone.metadata.dirs}/{NodeType}.ratis.snapshot
+   * eg: /data/metadata/om.ratis.snapshot
+   *     /data/metadata/scm.ratis.snapshot
+   *
+   * @param conf Configuration source
+   * @param nodeType Type of the node component
+   * @return Path to the component-specific ratis snapshot directory
+   */
+  public static String getDefaultRatisSnapshotDirectory(ConfigurationSource conf,
+      NodeType nodeType) {
+    LOG.warn("Snapshot directory for Ratis is not configured. Falling back to {}",
+        HddsConfigKeys.OZONE_METADATA_DIRS);
+    File metaDirPath = ServerUtils.getOzoneMetaDirPath(conf);
+
+    // Use component-specific location
+    String componentName = getComponentName(nodeType);
+    return Paths.get(metaDirPath.getPath(),
+        componentName + ".ratis." + OzoneConsts.OZONE_RATIS_SNAPSHOT_DIR).toString();
+  }
+
+  /**
+   * Checks if a directory exists and is non-empty.
+   *
+   * @param dir Directory to check
+   * @return true if directory exists and contains at least one file
+   */
+  private static boolean isNonEmptyDirectory(File dir) {
+    if (dir != null && dir.exists() && dir.isDirectory()) {
+      File[] files = dir.listFiles();
+      return files != null && files.length > 0;
+    }
+    return false;
   }
 }

hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/HddsServerUtil.java

@@ -76,6 +76,7 @@
 import org.apache.hadoop.hdds.conf.OzoneConfiguration;
 import org.apache.hadoop.hdds.protocol.SCMSecurityProtocol;
 import org.apache.hadoop.hdds.protocol.SecretKeyProtocolScm;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeType;
 import org.apache.hadoop.hdds.protocolPB.SCMSecurityProtocolClientSideTranslatorPB;
 import org.apache.hadoop.hdds.protocolPB.SecretKeyProtocolClientSideTranslatorPB;
 import org.apache.hadoop.hdds.protocolPB.SecretKeyProtocolDatanodePB;
@@ -424,7 +425,7 @@ public static Collection<String> getOzoneDatanodeRatisDirectory(
 
     if (rawLocations.isEmpty()) {
       rawLocations = new ArrayList<>(1);
-      rawLocations.add(ServerUtils.getDefaultRatisDirectory(conf));
+      rawLocations.add(ServerUtils.getDefaultRatisDirectory(conf, NodeType.DATANODE));
     }
     return rawLocations;
   }