LBPs: rate limit various warnings

There were some more warnings that were not rate-limited, which could
flood the logs in case of misconfiguration. This commit adds rate
limiting there, too.

Author: wprzytula

scylla/src/policies/load_balancing/default.rs

@@ -20,6 +20,8 @@ use std::{fmt, sync::Arc, time::Duration};
 use tracing::debug;
 use uuid::Uuid;
 
+const SANITY_CHECKS_WARNING_INTERVAL: std::time::Duration = std::time::Duration::from_secs(1);
+
 #[derive(Clone, Copy)]
 enum NodeLocationCriteria<'a> {
     Any,
@@ -591,8 +593,6 @@ impl DefaultPolicy {
 
     /// Checks for misconfiguration and warns if any is discovered.
     fn pick_sanity_checks(&self, routing_info: &ProcessedRoutingInfo, cluster: &ClusterState) {
-        const SANITY_CHECKS_WARNING_INTERVAL: std::time::Duration =
-            std::time::Duration::from_secs(1);
         if let Some(preferred_dc) = self.preferences.datacenter() {
             // Preferred DC + no datacenter failover + SimpleStrategy is an anti-pattern.
             if let Some(ref token_with_strategy) = routing_info.token_with_strategy {
@@ -677,7 +677,8 @@ You won't be able to execute any requests! This is most likely a misconfiguratio
             {
                 nodes
             } else {
-                tracing::warn!(
+                warn_rate_limited!(
+                    SANITY_CHECKS_WARNING_INTERVAL,
                     "Datacenter specified as the preferred one ({}) does not exist!",
                     preferred_datacenter
                 );

scylla/src/policies/load_balancing/single_target.rs

@@ -4,6 +4,7 @@ use std::sync::Arc;
 use uuid::Uuid;
 
 use crate::cluster::{ClusterState, Node, NodeRef};
+use crate::observability::warn_rate_limited;
 use crate::routing::Shard;
 
 use super::{LoadBalancingPolicy, RoutingInfo};
@@ -69,7 +70,8 @@ impl LoadBalancingPolicy for SingleTargetLoadBalancingPolicy {
         match node {
             Some(node) => Some((node, self.shard)),
             None => {
-                tracing::warn!(
+                warn_rate_limited!(
+                    std::time::Duration::from_secs(1),
                     "SingleTargetLoadBalancingPolicy failed to find requested node {:?} in cluster metadata.",
                     self.node_identifier
                 );