Merge pull request #3339 from YahiaBadr/threshold-by-perc

Adding threshold by percentile for podStartupLatency

Author: k8s-ci-robot

clusterloader2/pkg/measurement/common/slos/pod_startup_latency.go

@@ -79,6 +79,10 @@ type podStartupLatencyMeasurement struct {
 	podStartupEntries *measurementutil.ObjectTransitionTimes
 	podMetadata       *measurementutil.PodsMetadata
 	threshold         time.Duration
+	// Threshold for pod startup latency by percentile. The default value is threshold.
+	perc50Threshold time.Duration
+	perc90Threshold time.Duration
+	perc99Threshold time.Duration
 }
 
 // Execute supports two actions:
@@ -101,6 +105,18 @@ func (p *podStartupLatencyMeasurement) Execute(config *measurement.Config) ([]me
 		if err != nil {
 			return nil, err
 		}
+		p.perc50Threshold, err = util.GetDurationOrDefault(config.Params, "perc50Threshold", p.threshold)
+		if err != nil {
+			return nil, err
+		}
+		p.perc90Threshold, err = util.GetDurationOrDefault(config.Params, "perc90Threshold", p.threshold)
+		if err != nil {
+			return nil, err
+		}
+		p.perc99Threshold, err = util.GetDurationOrDefault(config.Params, "perc99Threshold", p.threshold)
+		if err != nil {
+			return nil, err
+		}
 		return nil, p.start(config.ClusterFramework.GetClientSets().GetClient())
 	case "gather":
 		schedulerName, err := util.GetStringOrDefault(config.Params, "schedulerName", defaultSchedulerName)
@@ -255,7 +271,7 @@ func (p *podStartupLatencyMeasurement) gather(c clientset.Interface, identifier
 		transitions := podStartupTransitionsWithThreshold(p.threshold)
 		podStartupLatency := p.podStartupEntries.CalculateTransitionsLatency(transitions, check.filter)
 
-		if slosErr := podStartupLatency["pod_startup"].VerifyThreshold(p.threshold); slosErr != nil {
+		if slosErr := podStartupLatency["pod_startup"].VerifyThresholdByPercentile(p.perc50Threshold, p.perc90Threshold, p.perc99Threshold); slosErr != nil {
 			err = errors.NewMetricViolationError("pod startup", slosErr.Error())
 			klog.Errorf("%s%s: %v", check.namePrefix, p, err)
 		}

clusterloader2/pkg/measurement/common/slos/slo_measurement_test.go

@@ -32,8 +32,11 @@ func Test_getMeasurementConfig(t *testing.T) {
 					"threshold": 200,
 				},
 				"PodStartupLatency": map[string]interface{}{
-					"threshold": 5,
-					"latency":   "10s",
+					"threshold":       5,
+					"perc50Threshold": 5,
+					"perc90Threshold": 10,
+					"perc99Threshold": 15,
+					"latency":         "10s",
 				},
 			},
 		},
@@ -53,6 +56,15 @@ func Test_getMeasurementConfig(t *testing.T) {
 	if result := params["threshold"]; result != 5 {
 		t.Errorf("want %v, got %v", 5, result)
 	}
+	if result := params["perc50Threshold"]; result != 5 {
+		t.Errorf("want %v, got %v", 5, result)
+	}
+	if result := params["perc90Threshold"]; result != 10 {
+		t.Errorf("want %v, got %v", 5, result)
+	}
+	if result := params["perc99Threshold"]; result != 15 {
+		t.Errorf("want %v, got %v", 5, result)
+	}
 	if result := params["latency"]; result != "10s" {
 		t.Errorf("want %v, got %v", "10s", result)
 	}

clusterloader2/pkg/measurement/util/latency_metric.go

@@ -59,6 +59,20 @@ func (metric *LatencyMetric) VerifyThreshold(threshold time.Duration) error {
 	return nil
 }
 
+// VerifyThreshold verifies latency metric against given percentile thresholds.
+func (metric *LatencyMetric) VerifyThresholdByPercentile(perc50Threshold, perc90Threshold, perc99Threshold time.Duration) error {
+	if metric.Perc50 > perc50Threshold {
+		return fmt.Errorf("too high latency 50th percentile: got %v expected: %v", metric.Perc50, perc50Threshold)
+	}
+	if metric.Perc90 > perc90Threshold {
+		return fmt.Errorf("too high latency 90th percentile: got %v expected: %v", metric.Perc90, perc90Threshold)
+	}
+	if metric.Perc99 > perc99Threshold {
+		return fmt.Errorf("too high latency 99th percentile: got %v expected: %v", metric.Perc99, perc99Threshold)
+	}
+	return nil
+}
+
 // ToPerfData converts latency metric to PerfData.
 func (metric *LatencyMetric) ToPerfData(name string) DataItem {
 	return DataItem{