Merge pull request #330 from ava-labs/update-health-checks

updated health check limits

Author: StephenButtolph

main/params.go

@@ -180,9 +180,9 @@ func avalancheFlagSet() *flag.FlagSet {
 	fs.Duration(networkHealthMaxTimeSinceMsgReceivedKey, time.Minute, "Network layer returns unhealthy if haven't received a message for at least this much time")
 	fs.Float64(networkHealthMaxPortionSendQueueFillKey, 0.9, "Network layer returns unhealthy if more than this portion of the pending send queue is full")
 	fs.Uint(networkHealthMinPeersKey, 1, "Network layer returns unhealthy if connected to less than this many peers")
-	fs.Float64(networkHealthMaxSendFailRateKey, .25, "Network layer reports unhealthy if more than this portion of attempted message sends fail")
+	fs.Float64(networkHealthMaxSendFailRateKey, .9, "Network layer reports unhealthy if more than this portion of attempted message sends fail")
 	// Router Health
-	fs.Float64(routerHealthMaxDropRateKey, 0.25, "Node reports unhealthy if the router drops more than this portion of messages.")
+	fs.Float64(routerHealthMaxDropRateKey, 1, "Node reports unhealthy if the router drops more than this portion of messages.")
 	fs.Uint(routerHealthMaxOutstandingRequestsKey, 1024, "Node reports unhealthy if there are more than this many outstanding consensus requests (Get, PullQuery, etc.) over all chains")
 	fs.Duration(networkHealthMaxTimeSinceNoReqsKey, 5*time.Minute, "Node reports unhealthy if there is at least 1 outstanding request continuously for this duration")
 
@@ -226,7 +226,7 @@ func avalancheFlagSet() *flag.FlagSet {
 	fs.Int(snowConcurrentRepollsKey, 4, "Minimum number of concurrent polls for finalizing consensus")
 	fs.Int(snowOptimalProcessingKey, 50, "Optimal number of processing vertices in consensus")
 	fs.Int(snowMaxProcessingKey, 1024, "Maximum number of processing items to be considered healthy")
-	fs.Duration(snowMaxTimeProcessingKey, 10*time.Second, "Maximum amount of time an item should be processing and still be healthy")
+	fs.Duration(snowMaxTimeProcessingKey, 2*time.Minute, "Maximum amount of time an item should be processing and still be healthy")
 	fs.Int64(snowEpochFirstTransition, 1607626800, "Unix timestamp of the first epoch transaction, in seconds. Defaults to 12/10/2020 @ 7:00pm (UTC)")
 	fs.Duration(snowEpochDuration, 6*time.Hour, "Duration of each epoch")
 

network/network.go

@@ -1471,27 +1471,23 @@ func (n *network) restartOnDisconnect() {
 // 2) An error if the health check reports unhealthy
 // Assumes [n.stateLock] is not held
 func (n *network) HealthCheck() (interface{}, error) {
-	details := map[string]interface{}{}
-
 	// Get some data with the state lock held
 	connectedTo := 0
 	n.stateLock.RLock()
 	for _, peer := range n.peers {
 		if peer != nil && peer.connected.GetValue() {
 			connectedTo++
-			if connectedTo > int(n.healthConfig.MinConnectedPeers) {
-				break
-			}
 		}
 	}
 	pendingSendBytes := n.pendingBytes
 	sendFailRate := n.sendFailRateCalculator.Read()
 	n.stateLock.RUnlock()
 
 	// Make sure we're connected to at least the minimum number of peers
-	isSufficientlyConnected := connectedTo >= int(n.healthConfig.MinConnectedPeers)
-	healthy := isSufficientlyConnected
-	details["connectedToMinPeers"] = isSufficientlyConnected
+	healthy := connectedTo >= int(n.healthConfig.MinConnectedPeers)
+	details := map[string]interface{}{
+		"connectedPeers": connectedTo,
+	}
 
 	// Make sure we've received an incoming message within the threshold
 	now := n.clock.Time()