From e69d17ffc88351c554a568b7d1240b2b32954079 Mon Sep 17 00:00:00 2001 From: szywilliam Date: Wed, 2 Aug 2023 16:38:20 +0800 Subject: [PATCH 1/5] add leader lease bound ratio --- ratis-docs/src/site/markdown/configurations.md | 8 ++++++++ .../org/apache/ratis/server/DivisionProperties.java | 8 ++++++++ .../apache/ratis/server/RaftServerConfigKeys.java | 12 ++++++++++++ .../ratis/server/impl/DivisionPropertiesImpl.java | 11 +++++++++++ 4 files changed, 39 insertions(+) diff --git a/ratis-docs/src/site/markdown/configurations.md b/ratis-docs/src/site/markdown/configurations.md index 4f6d215555..ba60864524 100644 --- a/ratis-docs/src/site/markdown/configurations.md +++ b/ratis-docs/src/site/markdown/configurations.md @@ -595,6 +595,14 @@ First election timeout is introduced to reduce unavailable time when a RaftGroup | **Type** | TimeDuration | | **Default** | 10s | + +| **Property** | `raft.server.leaderelection.leader.lease.timeout-bound.ratio` | +|:----------------|:--------------------------------------------------------------| +| **Description** | maximum bound ratio of leader lease timeout | +| **Type** | int, ranging from (0,100] | +| **Default** | 90 | + + | **Property** | `raft.server.leaderelection.pre-vote` | |:----------------|:--------------------------------------| | **Description** | enable pre-vote | diff --git a/ratis-server-api/src/main/java/org/apache/ratis/server/DivisionProperties.java b/ratis-server-api/src/main/java/org/apache/ratis/server/DivisionProperties.java index a7b8c9d6b6..dcba41a646 100644 --- a/ratis-server-api/src/main/java/org/apache/ratis/server/DivisionProperties.java +++ b/ratis-server-api/src/main/java/org/apache/ratis/server/DivisionProperties.java @@ -47,6 +47,14 @@ default int maxRpcTimeoutMs() { return maxRpcTimeout().toIntExact(TimeUnit.MILLISECONDS); } + /** @return the bound of leader lease timeout */ + TimeDuration leaderLeaseTimeout(); + + /** @return the bound of leader lease timeout in milliseconds */ + default int leaderLeaseTimeoutMs() { + return leaderLeaseTimeout().toIntExact(TimeUnit.MILLISECONDS); + } + /** @return the rpc sleep time period. */ TimeDuration rpcSleepTime(); diff --git a/ratis-server-api/src/main/java/org/apache/ratis/server/RaftServerConfigKeys.java b/ratis-server-api/src/main/java/org/apache/ratis/server/RaftServerConfigKeys.java index 211edd7960..2e7fed418b 100644 --- a/ratis-server-api/src/main/java/org/apache/ratis/server/RaftServerConfigKeys.java +++ b/ratis-server-api/src/main/java/org/apache/ratis/server/RaftServerConfigKeys.java @@ -770,6 +770,18 @@ static TimeDuration slownessTimeout(RaftProperties properties) { static void setSlownessTimeout(RaftProperties properties, TimeDuration expiryTime) { setTimeDuration(properties::setTimeDuration, SLOWNESS_TIMEOUT_KEY, expiryTime); } + + String LEADER_LEASE_TIMEOUT_BOUND_RATIO_KEY = PREFIX + ".leader.lease.timeout-bound.ratio"; + int LEADER_LEASE_TIMEOUT_BOUND_RATIO_DEFAULT = 90; + static int getLeaderLeaseTimeoutRatio(RaftProperties properties) { + return getInt(properties::getInt, LEADER_LEASE_TIMEOUT_BOUND_RATIO_KEY, + LEADER_LEASE_TIMEOUT_BOUND_RATIO_DEFAULT, getDefaultLog(), + requireMin(0), requireMax(100)); + } + + static void setLeaderLeaseTimeoutRatio(RaftProperties properties, int ratio) { + setInt(properties::setInt, LEADER_LEASE_TIMEOUT_BOUND_RATIO_KEY, ratio); + } } /** server retry cache related */ diff --git a/ratis-server/src/main/java/org/apache/ratis/server/impl/DivisionPropertiesImpl.java b/ratis-server/src/main/java/org/apache/ratis/server/impl/DivisionPropertiesImpl.java index 63cbc02ed6..b109d9d676 100644 --- a/ratis-server/src/main/java/org/apache/ratis/server/impl/DivisionPropertiesImpl.java +++ b/ratis-server/src/main/java/org/apache/ratis/server/impl/DivisionPropertiesImpl.java @@ -28,6 +28,7 @@ class DivisionPropertiesImpl implements DivisionProperties { private final TimeDuration rpcTimeoutMax; private final TimeDuration rpcSleepTime; private final TimeDuration rpcSlownessTimeout; + private final TimeDuration leaderLeaseTimeout; DivisionPropertiesImpl(RaftProperties properties) { this.rpcTimeoutMin = RaftServerConfigKeys.Rpc.timeoutMin(properties); @@ -35,6 +36,11 @@ class DivisionPropertiesImpl implements DivisionProperties { Preconditions.assertTrue(rpcTimeoutMax.compareTo(rpcTimeoutMin) >= 0, "rpcTimeoutMax = %s < rpcTimeoutMin = %s", rpcTimeoutMax, rpcTimeoutMin); + final double leaderLeaseBoundRatio = RaftServerConfigKeys.Rpc.getLeaderLeaseTimeoutRatio(properties) * 1.0 / 100; + this.leaderLeaseTimeout = this.rpcTimeoutMin.multiply(leaderLeaseBoundRatio); + Preconditions.assertTrue(rpcTimeoutMin.compareTo(leaderLeaseTimeout) >= 0, + "rpcTimeoutMin = %s < leaderLeaseTimeout = %s", rpcTimeoutMin, leaderLeaseTimeout); + this.rpcSleepTime = RaftServerConfigKeys.Rpc.sleepTime(properties); this.rpcSlownessTimeout = RaftServerConfigKeys.Rpc.slownessTimeout(properties); } @@ -49,6 +55,11 @@ public TimeDuration maxRpcTimeout() { return rpcTimeoutMax; } + @Override + public TimeDuration leaderLeaseTimeout() { + return leaderLeaseTimeout; + } + @Override public TimeDuration rpcSleepTime() { return rpcSleepTime; From e9fb730782a4b90b96826d1ba8dc1c21c8b97036 Mon Sep 17 00:00:00 2001 From: szywilliam Date: Wed, 2 Aug 2023 17:21:02 +0800 Subject: [PATCH 2/5] rename getting method --- .../main/java/org/apache/ratis/server/RaftServerConfigKeys.java | 2 +- .../org/apache/ratis/server/impl/DivisionPropertiesImpl.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ratis-server-api/src/main/java/org/apache/ratis/server/RaftServerConfigKeys.java b/ratis-server-api/src/main/java/org/apache/ratis/server/RaftServerConfigKeys.java index 2e7fed418b..32ca5cd5a1 100644 --- a/ratis-server-api/src/main/java/org/apache/ratis/server/RaftServerConfigKeys.java +++ b/ratis-server-api/src/main/java/org/apache/ratis/server/RaftServerConfigKeys.java @@ -773,7 +773,7 @@ static void setSlownessTimeout(RaftProperties properties, TimeDuration expiryTim String LEADER_LEASE_TIMEOUT_BOUND_RATIO_KEY = PREFIX + ".leader.lease.timeout-bound.ratio"; int LEADER_LEASE_TIMEOUT_BOUND_RATIO_DEFAULT = 90; - static int getLeaderLeaseTimeoutRatio(RaftProperties properties) { + static int leaderLeaseTimeoutRatio(RaftProperties properties) { return getInt(properties::getInt, LEADER_LEASE_TIMEOUT_BOUND_RATIO_KEY, LEADER_LEASE_TIMEOUT_BOUND_RATIO_DEFAULT, getDefaultLog(), requireMin(0), requireMax(100)); diff --git a/ratis-server/src/main/java/org/apache/ratis/server/impl/DivisionPropertiesImpl.java b/ratis-server/src/main/java/org/apache/ratis/server/impl/DivisionPropertiesImpl.java index b109d9d676..21907b82bb 100644 --- a/ratis-server/src/main/java/org/apache/ratis/server/impl/DivisionPropertiesImpl.java +++ b/ratis-server/src/main/java/org/apache/ratis/server/impl/DivisionPropertiesImpl.java @@ -36,7 +36,7 @@ class DivisionPropertiesImpl implements DivisionProperties { Preconditions.assertTrue(rpcTimeoutMax.compareTo(rpcTimeoutMin) >= 0, "rpcTimeoutMax = %s < rpcTimeoutMin = %s", rpcTimeoutMax, rpcTimeoutMin); - final double leaderLeaseBoundRatio = RaftServerConfigKeys.Rpc.getLeaderLeaseTimeoutRatio(properties) * 1.0 / 100; + final double leaderLeaseBoundRatio = RaftServerConfigKeys.Rpc.leaderLeaseTimeoutRatio(properties) * 1.0 / 100; this.leaderLeaseTimeout = this.rpcTimeoutMin.multiply(leaderLeaseBoundRatio); Preconditions.assertTrue(rpcTimeoutMin.compareTo(leaderLeaseTimeout) >= 0, "rpcTimeoutMin = %s < leaderLeaseTimeout = %s", rpcTimeoutMin, leaderLeaseTimeout); From 650724c7fea39f6a4c3e077133e3d24a1c6b3d29 Mon Sep 17 00:00:00 2001 From: szywilliam Date: Wed, 2 Aug 2023 17:39:59 +0800 Subject: [PATCH 3/5] rename getting method --- .../java/org/apache/ratis/server/RaftServerConfigKeys.java | 4 ++-- .../org/apache/ratis/server/impl/DivisionPropertiesImpl.java | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ratis-server-api/src/main/java/org/apache/ratis/server/RaftServerConfigKeys.java b/ratis-server-api/src/main/java/org/apache/ratis/server/RaftServerConfigKeys.java index 32ca5cd5a1..f4ae435d3a 100644 --- a/ratis-server-api/src/main/java/org/apache/ratis/server/RaftServerConfigKeys.java +++ b/ratis-server-api/src/main/java/org/apache/ratis/server/RaftServerConfigKeys.java @@ -773,13 +773,13 @@ static void setSlownessTimeout(RaftProperties properties, TimeDuration expiryTim String LEADER_LEASE_TIMEOUT_BOUND_RATIO_KEY = PREFIX + ".leader.lease.timeout-bound.ratio"; int LEADER_LEASE_TIMEOUT_BOUND_RATIO_DEFAULT = 90; - static int leaderLeaseTimeoutRatio(RaftProperties properties) { + static int leaderLeaseTimeoutBoundRatio(RaftProperties properties) { return getInt(properties::getInt, LEADER_LEASE_TIMEOUT_BOUND_RATIO_KEY, LEADER_LEASE_TIMEOUT_BOUND_RATIO_DEFAULT, getDefaultLog(), requireMin(0), requireMax(100)); } - static void setLeaderLeaseTimeoutRatio(RaftProperties properties, int ratio) { + static void setLeaderLeaseTimeoutBoundRatio(RaftProperties properties, int ratio) { setInt(properties::setInt, LEADER_LEASE_TIMEOUT_BOUND_RATIO_KEY, ratio); } } diff --git a/ratis-server/src/main/java/org/apache/ratis/server/impl/DivisionPropertiesImpl.java b/ratis-server/src/main/java/org/apache/ratis/server/impl/DivisionPropertiesImpl.java index 21907b82bb..903b9f91ca 100644 --- a/ratis-server/src/main/java/org/apache/ratis/server/impl/DivisionPropertiesImpl.java +++ b/ratis-server/src/main/java/org/apache/ratis/server/impl/DivisionPropertiesImpl.java @@ -36,7 +36,7 @@ class DivisionPropertiesImpl implements DivisionProperties { Preconditions.assertTrue(rpcTimeoutMax.compareTo(rpcTimeoutMin) >= 0, "rpcTimeoutMax = %s < rpcTimeoutMin = %s", rpcTimeoutMax, rpcTimeoutMin); - final double leaderLeaseBoundRatio = RaftServerConfigKeys.Rpc.leaderLeaseTimeoutRatio(properties) * 1.0 / 100; + final double leaderLeaseBoundRatio = RaftServerConfigKeys.Rpc.leaderLeaseTimeoutBoundRatio(properties) * 1.0 / 100; this.leaderLeaseTimeout = this.rpcTimeoutMin.multiply(leaderLeaseBoundRatio); Preconditions.assertTrue(rpcTimeoutMin.compareTo(leaderLeaseTimeout) >= 0, "rpcTimeoutMin = %s < leaderLeaseTimeout = %s", rpcTimeoutMin, leaderLeaseTimeout); From de898a719e01acc257f90c4d94e5e91ec85db58e Mon Sep 17 00:00:00 2001 From: szywilliam Date: Thu, 3 Aug 2023 23:46:49 +0800 Subject: [PATCH 4/5] address review issues --- .../java/org/apache/ratis/conf/ConfUtils.java | 9 +++++++ .../src/site/markdown/configurations.md | 13 +++++----- .../ratis/server/RaftServerConfigKeys.java | 24 +++++++++---------- .../server/impl/DivisionPropertiesImpl.java | 4 ++-- 4 files changed, 30 insertions(+), 20 deletions(-) diff --git a/ratis-common/src/main/java/org/apache/ratis/conf/ConfUtils.java b/ratis-common/src/main/java/org/apache/ratis/conf/ConfUtils.java index 629d5a3899..a642ebb9d2 100644 --- a/ratis-common/src/main/java/org/apache/ratis/conf/ConfUtils.java +++ b/ratis-common/src/main/java/org/apache/ratis/conf/ConfUtils.java @@ -76,6 +76,15 @@ static BiConsumer requireMax(int max) { }; } + static BiConsumer requireMin(double min) { + return (key, value) -> { + if (value < min) { + throw new IllegalArgumentException( + key + " = " + value + " < min = " + min); + } + }; + } + static BiConsumer requireMax(double max) { return (key, value) -> { if (value > max) { diff --git a/ratis-docs/src/site/markdown/configurations.md b/ratis-docs/src/site/markdown/configurations.md index ba60864524..468eea31a9 100644 --- a/ratis-docs/src/site/markdown/configurations.md +++ b/ratis-docs/src/site/markdown/configurations.md @@ -185,6 +185,13 @@ treat the peer as caught-up. Increase this number when write throughput is high. -------------------------------------------------------------------------------- +| **Property** | `raft.server.read.leader.lease.timeout.ratio` | +|:----------------|:----------------------------------------------| +| **Description** | maximum bound ratio of leader lease timeout | +| **Type** | double, ranging from (0.0,1.0) | +| **Default** | 0.9 | + + ### Write - Configurations related to write requests. * Limits on pending write requests @@ -596,12 +603,6 @@ First election timeout is introduced to reduce unavailable time when a RaftGroup | **Default** | 10s | -| **Property** | `raft.server.leaderelection.leader.lease.timeout-bound.ratio` | -|:----------------|:--------------------------------------------------------------| -| **Description** | maximum bound ratio of leader lease timeout | -| **Type** | int, ranging from (0,100] | -| **Default** | 90 | - | **Property** | `raft.server.leaderelection.pre-vote` | |:----------------|:--------------------------------------| diff --git a/ratis-server-api/src/main/java/org/apache/ratis/server/RaftServerConfigKeys.java b/ratis-server-api/src/main/java/org/apache/ratis/server/RaftServerConfigKeys.java index f4ae435d3a..469bceba00 100644 --- a/ratis-server-api/src/main/java/org/apache/ratis/server/RaftServerConfigKeys.java +++ b/ratis-server-api/src/main/java/org/apache/ratis/server/RaftServerConfigKeys.java @@ -191,6 +191,18 @@ static Option option(RaftProperties properties) { static void setOption(RaftProperties properties, Option option) { set(properties::setEnum, OPTION_KEY, option); } + + String LEADER_LEASE_TIMEOUT_RATIO_KEY = PREFIX + ".leader.lease.timeout.ratio"; + double LEADER_LEASE_TIMEOUT_RATIO_DEFAULT = 0.9; + static double leaderLeaseTimeoutRatio(RaftProperties properties) { + return getDouble(properties::getDouble, LEADER_LEASE_TIMEOUT_RATIO_KEY, + LEADER_LEASE_TIMEOUT_RATIO_DEFAULT, getDefaultLog(), + requireMin(0.0), requireMax(1.0)); + } + + static void setLeaderLeaseTimeoutRatio(RaftProperties properties, double ratio) { + setDouble(properties::setDouble, LEADER_LEASE_TIMEOUT_RATIO_KEY, ratio); + } } interface Write { @@ -770,18 +782,6 @@ static TimeDuration slownessTimeout(RaftProperties properties) { static void setSlownessTimeout(RaftProperties properties, TimeDuration expiryTime) { setTimeDuration(properties::setTimeDuration, SLOWNESS_TIMEOUT_KEY, expiryTime); } - - String LEADER_LEASE_TIMEOUT_BOUND_RATIO_KEY = PREFIX + ".leader.lease.timeout-bound.ratio"; - int LEADER_LEASE_TIMEOUT_BOUND_RATIO_DEFAULT = 90; - static int leaderLeaseTimeoutBoundRatio(RaftProperties properties) { - return getInt(properties::getInt, LEADER_LEASE_TIMEOUT_BOUND_RATIO_KEY, - LEADER_LEASE_TIMEOUT_BOUND_RATIO_DEFAULT, getDefaultLog(), - requireMin(0), requireMax(100)); - } - - static void setLeaderLeaseTimeoutBoundRatio(RaftProperties properties, int ratio) { - setInt(properties::setInt, LEADER_LEASE_TIMEOUT_BOUND_RATIO_KEY, ratio); - } } /** server retry cache related */ diff --git a/ratis-server/src/main/java/org/apache/ratis/server/impl/DivisionPropertiesImpl.java b/ratis-server/src/main/java/org/apache/ratis/server/impl/DivisionPropertiesImpl.java index 903b9f91ca..bc5a279296 100644 --- a/ratis-server/src/main/java/org/apache/ratis/server/impl/DivisionPropertiesImpl.java +++ b/ratis-server/src/main/java/org/apache/ratis/server/impl/DivisionPropertiesImpl.java @@ -36,8 +36,8 @@ class DivisionPropertiesImpl implements DivisionProperties { Preconditions.assertTrue(rpcTimeoutMax.compareTo(rpcTimeoutMin) >= 0, "rpcTimeoutMax = %s < rpcTimeoutMin = %s", rpcTimeoutMax, rpcTimeoutMin); - final double leaderLeaseBoundRatio = RaftServerConfigKeys.Rpc.leaderLeaseTimeoutBoundRatio(properties) * 1.0 / 100; - this.leaderLeaseTimeout = this.rpcTimeoutMin.multiply(leaderLeaseBoundRatio); + final double leaderLeaseTimeoutRatio = RaftServerConfigKeys.Read.leaderLeaseTimeoutRatio(properties); + this.leaderLeaseTimeout = this.rpcTimeoutMin.multiply(leaderLeaseTimeoutRatio); Preconditions.assertTrue(rpcTimeoutMin.compareTo(leaderLeaseTimeout) >= 0, "rpcTimeoutMin = %s < leaderLeaseTimeout = %s", rpcTimeoutMin, leaderLeaseTimeout); From 35daca0b33523a5995d13d9fae9c86f50a984de4 Mon Sep 17 00:00:00 2001 From: szywilliam Date: Thu, 3 Aug 2023 23:50:06 +0800 Subject: [PATCH 5/5] address review issues --- ratis-docs/src/site/markdown/configurations.md | 4 +--- .../java/org/apache/ratis/server/DivisionProperties.java | 8 -------- .../apache/ratis/server/impl/DivisionPropertiesImpl.java | 2 +- 3 files changed, 2 insertions(+), 12 deletions(-) diff --git a/ratis-docs/src/site/markdown/configurations.md b/ratis-docs/src/site/markdown/configurations.md index 468eea31a9..d688a7f9ec 100644 --- a/ratis-docs/src/site/markdown/configurations.md +++ b/ratis-docs/src/site/markdown/configurations.md @@ -187,7 +187,7 @@ treat the peer as caught-up. Increase this number when write throughput is high. | **Property** | `raft.server.read.leader.lease.timeout.ratio` | |:----------------|:----------------------------------------------| -| **Description** | maximum bound ratio of leader lease timeout | +| **Description** | maximum timeout ratio of leader lease | | **Type** | double, ranging from (0.0,1.0) | | **Default** | 0.9 | @@ -602,8 +602,6 @@ First election timeout is introduced to reduce unavailable time when a RaftGroup | **Type** | TimeDuration | | **Default** | 10s | - - | **Property** | `raft.server.leaderelection.pre-vote` | |:----------------|:--------------------------------------| | **Description** | enable pre-vote | diff --git a/ratis-server-api/src/main/java/org/apache/ratis/server/DivisionProperties.java b/ratis-server-api/src/main/java/org/apache/ratis/server/DivisionProperties.java index dcba41a646..a7b8c9d6b6 100644 --- a/ratis-server-api/src/main/java/org/apache/ratis/server/DivisionProperties.java +++ b/ratis-server-api/src/main/java/org/apache/ratis/server/DivisionProperties.java @@ -47,14 +47,6 @@ default int maxRpcTimeoutMs() { return maxRpcTimeout().toIntExact(TimeUnit.MILLISECONDS); } - /** @return the bound of leader lease timeout */ - TimeDuration leaderLeaseTimeout(); - - /** @return the bound of leader lease timeout in milliseconds */ - default int leaderLeaseTimeoutMs() { - return leaderLeaseTimeout().toIntExact(TimeUnit.MILLISECONDS); - } - /** @return the rpc sleep time period. */ TimeDuration rpcSleepTime(); diff --git a/ratis-server/src/main/java/org/apache/ratis/server/impl/DivisionPropertiesImpl.java b/ratis-server/src/main/java/org/apache/ratis/server/impl/DivisionPropertiesImpl.java index bc5a279296..b3e8dd94d0 100644 --- a/ratis-server/src/main/java/org/apache/ratis/server/impl/DivisionPropertiesImpl.java +++ b/ratis-server/src/main/java/org/apache/ratis/server/impl/DivisionPropertiesImpl.java @@ -55,7 +55,7 @@ public TimeDuration maxRpcTimeout() { return rpcTimeoutMax; } - @Override + /** @return the ratio of leader lease timeout */ public TimeDuration leaderLeaseTimeout() { return leaderLeaseTimeout; }