Adding threshold by percentile for podStartupLatency

Intorducing threshold by percentile 50,90,and 99 for the
podStartupLatency only. The default values of the new thresholds are the
"threshold" value. So this change should be no-op unless the params are
used in the test configuration.

Author: YahiaBadr

clusterloader2/pkg/measurement/common/slos/pod_startup_latency.go

@@ -79,6 +79,10 @@ type podStartupLatencyMeasurement struct {
 	podStartupEntries *measurementutil.ObjectTransitionTimes
 	podMetadata       *measurementutil.PodsMetadata
 	threshold         time.Duration
+	// Threshold for pod startup latency by percentile. The default value is threshold.
+	perc50Threshold time.Duration
+	perc90Threshold time.Duration
+	perc99Threshold time.Duration
 }
 
 // Execute supports two actions:
@@ -101,6 +105,18 @@ func (p *podStartupLatencyMeasurement) Execute(config *measurement.Config) ([]me
 		if err != nil {
 			return nil, err
 		}
+		p.perc50Threshold, err = util.GetDurationOrDefault(config.Params, "perc50Threshold", p.threshold)
+		if err != nil {
+			return nil, err
+		}
+		p.perc90Threshold, err = util.GetDurationOrDefault(config.Params, "perc90Threshold", p.threshold)
+		if err != nil {
+			return nil, err
+		}
+		p.perc99Threshold, err = util.GetDurationOrDefault(config.Params, "perc99Threshold", p.threshold)
+		if err != nil {
+			return nil, err
+		}
 		return nil, p.start(config.ClusterFramework.GetClientSets().GetClient())
 	case "gather":
 		schedulerName, err := util.GetStringOrDefault(config.Params, "schedulerName", defaultSchedulerName)
@@ -255,7 +271,7 @@ func (p *podStartupLatencyMeasurement) gather(c clientset.Interface, identifier
 		transitions := podStartupTransitionsWithThreshold(p.threshold)
 		podStartupLatency := p.podStartupEntries.CalculateTransitionsLatency(transitions, check.filter)
 
-		if slosErr := podStartupLatency["pod_startup"].VerifyThreshold(p.threshold); slosErr != nil {
+		if slosErr := podStartupLatency["pod_startup"].VerifyThresholdByPercentile(p.perc50Threshold, p.perc90Threshold, p.perc99Threshold); slosErr != nil {
 			err = errors.NewMetricViolationError("pod startup", slosErr.Error())
 			klog.Errorf("%s%s: %v", check.namePrefix, p, err)
 		}

clusterloader2/pkg/measurement/common/slos/slo_measurement_test.go

@@ -32,8 +32,11 @@ func Test_getMeasurementConfig(t *testing.T) {
 					"threshold": 200,
 				},
 				"PodStartupLatency": map[string]interface{}{
-					"threshold": 5,
-					"latency":   "10s",
+					"threshold":       5,
+					"perc50Threshold": 5,
+					"perc90Threshold": 10,
+					"perc99Threshold": 15,
+					"latency":         "10s",
 				},
 			},
 		},
@@ -53,6 +56,15 @@ func Test_getMeasurementConfig(t *testing.T) {
 	if result := params["threshold"]; result != 5 {
 		t.Errorf("want %v, got %v", 5, result)
 	}
+	if result := params["perc50Threshold"]; result != 5 {
+		t.Errorf("want %v, got %v", 5, result)
+	}
+	if result := params["perc90Threshold"]; result != 10 {
+		t.Errorf("want %v, got %v", 5, result)
+	}
+	if result := params["perc99Threshold"]; result != 15 {
+		t.Errorf("want %v, got %v", 5, result)
+	}
 	if result := params["latency"]; result != "10s" {
 		t.Errorf("want %v, got %v", "10s", result)
 	}

clusterloader2/pkg/measurement/util/latency_metric.go

@@ -59,6 +59,20 @@ func (metric *LatencyMetric) VerifyThreshold(threshold time.Duration) error {
 	return nil
 }
 
+// VerifyThreshold verifies latency metric against given percentile thresholds.
+func (metric *LatencyMetric) VerifyThresholdByPercentile(perc50Threshold, perc90Threshold, perc99Threshold time.Duration) error {
+	if metric.Perc50 > perc50Threshold {
+		return fmt.Errorf("too high latency 50th percentile: got %v expected: %v", metric.Perc50, perc50Threshold)
+	}
+	if metric.Perc90 > perc90Threshold {
+		return fmt.Errorf("too high latency 90th percentile: got %v expected: %v", metric.Perc90, perc90Threshold)
+	}
+	if metric.Perc99 > perc99Threshold {
+		return fmt.Errorf("too high latency 99th percentile: got %v expected: %v", metric.Perc99, perc99Threshold)
+	}
+	return nil
+}
+
 // ToPerfData converts latency metric to PerfData.
 func (metric *LatencyMetric) ToPerfData(name string) DataItem {
 	return DataItem{