Merge pull request #312 from ava-labs/fix-health-checks

fixed router health check

Author: StephenButtolph

api/health/service.go

@@ -7,13 +7,16 @@ import (
 	"net/http"
 	"time"
 
+	stdjson "encoding/json"
+
 	"github.com/gorilla/rpc/v2"
 
 	"github.com/ava-labs/avalanchego/health"
-	healthlib "github.com/ava-labs/avalanchego/health"
 	"github.com/ava-labs/avalanchego/snow/engine/common"
 	"github.com/ava-labs/avalanchego/utils/json"
 	"github.com/ava-labs/avalanchego/utils/logging"
+
+	healthlib "github.com/ava-labs/avalanchego/health"
 )
 
 // Service wraps a [healthlib.Service]. Handler() returns a handler
@@ -74,15 +77,25 @@ type APIHealthReply struct {
 func (as *apiServer) Health(_ *http.Request, _ *APIHealthArgs, reply *APIHealthReply) error {
 	as.log.Info("Health.health called")
 	reply.Checks, reply.Healthy = as.Results()
-	return nil
+	if reply.Healthy {
+		return nil
+	}
+	replyStr, err := stdjson.Marshal(reply.Checks)
+	as.log.Warn("Health.health is returning an error: %s", string(replyStr))
+	return err
 }
 
 // GetLiveness returns a summation of the health of the node
 // Deprecated: in favor of Health
 func (as *apiServer) GetLiveness(_ *http.Request, _ *APIHealthArgs, reply *APIHealthReply) error {
-	as.log.Info("Health: GetLiveness called")
+	as.log.Info("Health.getLiveness called")
 	reply.Checks, reply.Healthy = as.Results()
-	return nil
+	if reply.Healthy {
+		return nil
+	}
+	replyStr, err := stdjson.Marshal(reply.Checks)
+	as.log.Warn("Health.getLiveness is returning an error: %s", string(replyStr))
+	return err
 }
 
 type noOp struct{}

snow/networking/router/chain_router.go

@@ -120,9 +120,6 @@ func (cr *ChainRouter) Initialize(
 // Assumes [cr.lock] is held
 func (cr *ChainRouter) removeRequest(id ids.ID) {
 	delete(cr.requests, id)
-	if len(cr.requests) == 0 {
-		cr.lastTimeNoOutstanding = cr.clock.Time()
-	}
 	cr.metrics.outstandingRequests.Set(float64(len(cr.requests)))
 }
 
@@ -140,8 +137,12 @@ func (cr *ChainRouter) RegisterRequest(
 ) {
 	uniqueRequestID := createRequestID(validatorID, chainID, requestID)
 	cr.lock.Lock()
+	if len(cr.requests) == 0 {
+		cr.lastTimeNoOutstanding = cr.clock.Time()
+	}
 	// Add to the set of unfulfilled requests
 	cr.requests[uniqueRequestID] = request{Time: cr.clock.Time(), MsgType: msgType}
+	cr.metrics.outstandingRequests.Set(float64(len(cr.requests)))
 	cr.lock.Unlock()
 	// Register a timeout to fire if we don't get a reply in time.
 	var timeoutHandler func() // Called upon timeout
@@ -760,7 +761,11 @@ func (cr *ChainRouter) HealthCheck() (interface{}, error) {
 	healthy = healthy && numOutstandingReqs <= cr.healthConfig.MaxOutstandingRequests
 	details["outstandingRequests"] = numOutstandingReqs
 
-	timeSinceNoOutstandingRequests := cr.clock.Time().Sub(cr.lastTimeNoOutstanding)
+	now := cr.clock.Time()
+	if numOutstandingReqs == 0 {
+		cr.lastTimeNoOutstanding = now
+	}
+	timeSinceNoOutstandingRequests := now.Sub(cr.lastTimeNoOutstanding)
 	healthy = healthy && timeSinceNoOutstandingRequests <= cr.healthConfig.MaxTimeSinceNoOutstandingRequests
 	details["timeSinceNoOutstandingRequests"] = timeSinceNoOutstandingRequests.String()