From 402d29505a057fb0e4e5c52a1579f649ad830809 Mon Sep 17 00:00:00 2001 From: Priti Desai Date: Mon, 21 Jul 2025 15:21:12 -0700 Subject: [PATCH] adding efficient polling to waitForStepsToFinish The current waitForStepsToFinish implementation is a classic busy-wait. It checks for file existence without any sleep, resulting in a high CPU usage. Adding a profile with a unit test to show that almost all time is spent in system calls with a high total sample count. This led to execssive CPU usage by the sidecar even when just waiting. The function now sleeps 100ms between checks, drastically reducing the frequency. The sidecar now uses minimal CPU while waiting. Signed-off-by: Priti Desai --- config/config-defaults.yaml | 8 +++ docs/additional-configs.md | 19 ++++++ .../sidecarlogresults/sidecarlogresults.go | 26 +++++++- .../sidecarlogresults_test.go | 62 +++++++++++++++++++ pkg/apis/config/default.go | 17 +++++ pkg/apis/config/default_test.go | 54 ++++++++++++++++ pkg/pod/pod.go | 12 +++- pkg/pod/pod_test.go | 6 ++ test/e2e-tests.sh | 8 +++ 9 files changed, 207 insertions(+), 5 deletions(-) diff --git a/config/config-defaults.yaml b/config/config-defaults.yaml index e089a378fc7..120b19320e0 100644 --- a/config/config-defaults.yaml +++ b/config/config-defaults.yaml @@ -149,3 +149,11 @@ data: # limits: # memory: "256Mi" # cpu: "500m" + + # default-sidecar-log-polling-interval specifies the polling interval for the Tekton sidecar log results container. + # This controls how frequently the sidecar checks for step completion files written by steps in a TaskRun. + # Lower values (e.g., "10ms") make the sidecar more responsive but may increase CPU usage; higher values (e.g., "1s") + # reduce resource usage but may delay result collection. + # This value is used by the sidecar-tekton-log-results container and can be tuned for performance or test scenarios. + # Example values: "100ms", "500ms", "1s" + default-sidecar-log-polling-interval: "100ms" diff --git a/docs/additional-configs.md b/docs/additional-configs.md index 6729db8cbb9..f7d1ac54830 100644 --- a/docs/additional-configs.md +++ b/docs/additional-configs.md @@ -243,6 +243,7 @@ The example below customizes the following: - the default maximum combinations of `Parameters` in a `Matrix` that can be used to fan out a `PipelineTask`. For more information, see [`Matrix`](matrix.md). - the default resolver type to `git`. +- the default polling interval for the sidecar log results container via `default-sidecar-log-polling-interval`. ```yaml apiVersion: v1 @@ -260,8 +261,26 @@ data: emptyDir: {} default-max-matrix-combinations-count: "1024" default-resolver-type: "git" + default-sidecar-log-polling-interval: "100ms" ``` +### `default-sidecar-log-polling-interval` + +The `default-sidecar-log-polling-interval` key in the `config-defaults` ConfigMap specifies how frequently the Tekton +sidecar log results container polls for step completion files written by steps in a TaskRun. Lower values (e.g., `10ms`) +make the sidecar more responsive but may increase CPU usage; higher values (e.g., `1s`) reduce resource usage but may +delay result collection. This value is used by the `sidecar-tekton-log-results` container and can be tuned for performance +or test scenarios. + +**Example values:** +- `100ms` (default) +- `500ms` +- `1s` +- `10ms` (for fast polling in tests) + +**Note:** The `default-sidecar-log-polling-interval` setting is only applicable when results are created using the +[sidecar approach](#enabling-larger-results-using-sidecar-logs). + **Note:** The `_example` key in the provided [config-defaults.yaml](./../config/config-defaults.yaml) file lists the keys you can customize along with their default values. diff --git a/internal/sidecarlogresults/sidecarlogresults.go b/internal/sidecarlogresults/sidecarlogresults.go index 823fe448075..445734c9dad 100644 --- a/internal/sidecarlogresults/sidecarlogresults.go +++ b/internal/sidecarlogresults/sidecarlogresults.go @@ -26,6 +26,7 @@ import ( "os" "path/filepath" "strings" + "time" "github.com/tektoncd/pipeline/pkg/apis/config" "github.com/tektoncd/pipeline/pkg/apis/pipeline" @@ -74,7 +75,7 @@ func encode(w io.Writer, v any) error { return json.NewEncoder(w).Encode(v) } -func waitForStepsToFinish(runDir string) error { +func waitForStepsToFinish(runDir string, sleepInterval time.Duration) error { steps := make(map[string]bool) files, err := os.ReadDir(runDir) if err != nil { @@ -103,6 +104,9 @@ func waitForStepsToFinish(runDir string) error { return err } } + if sleepInterval > 0 { + time.Sleep(sleepInterval) + } } return nil } @@ -143,7 +147,15 @@ func readResults(resultsDir, resultFile, stepName string, resultType SidecarLogR // in their results path and prints them in a structured way to its // stdout so that the reconciler can parse those logs. func LookForResults(w io.Writer, runDir string, resultsDir string, resultNames []string, stepResultsDir string, stepResults map[string][]string) error { - if err := waitForStepsToFinish(runDir); err != nil { + intervalStr := os.Getenv("SIDECAR_LOG_POLLING_INTERVAL") + if intervalStr == "" { + intervalStr = "100ms" + } + interval, err := time.ParseDuration(intervalStr) + if err != nil { + interval = 100 * time.Millisecond + } + if err := waitForStepsToFinish(runDir, interval); err != nil { return fmt.Errorf("error while waiting for the steps to finish %w", err) } results := make(chan SidecarLogResult) @@ -205,7 +217,15 @@ func LookForResults(w io.Writer, runDir string, resultsDir string, resultNames [ // If the provenance file exists, the function extracts artifact information, formats it into a // JSON string, and encodes it for output alongside relevant metadata (step name, artifact type). func LookForArtifacts(w io.Writer, names []string, runDir string) error { - if err := waitForStepsToFinish(runDir); err != nil { + intervalStr := os.Getenv("SIDECAR_LOG_POLLING_INTERVAL") + if intervalStr == "" { + intervalStr = "100ms" + } + interval, err := time.ParseDuration(intervalStr) + if err != nil { + interval = 100 * time.Millisecond + } + if err := waitForStepsToFinish(runDir, interval); err != nil { return err } diff --git a/internal/sidecarlogresults/sidecarlogresults_test.go b/internal/sidecarlogresults/sidecarlogresults_test.go index ed3d7ba1dbf..c31e6760529 100644 --- a/internal/sidecarlogresults/sidecarlogresults_test.go +++ b/internal/sidecarlogresults/sidecarlogresults_test.go @@ -24,9 +24,11 @@ import ( "fmt" "os" "path/filepath" + "runtime/pprof" "sort" "strings" "testing" + "time" "github.com/google/go-cmp/cmp" v1 "github.com/tektoncd/pipeline/pkg/apis/pipeline/v1" @@ -609,6 +611,66 @@ func TestExtractStepAndResultFromSidecarResultName_Error(t *testing.T) { } } +// TestWaitForStepsToFinish_Profile ensures that waitForStepsToFinish correctly waits for all step output files to appear before returning +// The test creates a file called cpu.prof and starts Go's CPU profiler +// A temporary directory is created to simulate the Tekton step run directory. +// The test creates a large number of subdirectories e.g. step0, step1, ..., each representing a step in a TaskRun +// A goroutine is started that, one by one, writes an out file in each step directory, with a small delay between each +// The test calls the function and waits for it to complete and the profile is saved for later analysis +// This is helpful to compare the impact of code changes, provides a reproducible way to profile and optimize the function waitForStepsToFinish +func TestWaitForStepsToFinish_Profile(t *testing.T) { + f, err := os.Create("cpu.prof") + if err != nil { + t.Fatalf("could not create CPU profile: %v", err) + } + defer func(f *os.File) { + err := f.Close() + if err != nil { + return + } + }(f) + err = pprof.StartCPUProfile(f) + if err != nil { + return + } + defer pprof.StopCPUProfile() + + // Setup: create a temp runDir with many fake step files + runDir := t.TempDir() + stepCount := 100 + for i := range stepCount { + dir := filepath.Join(runDir, fmt.Sprintf("step%d", i)) + err := os.MkdirAll(dir, 0755) + if err != nil { + return + } + } + + // Simulate steps finishing one by one with a delay + go func() { + for i := range stepCount { + file := filepath.Join(runDir, fmt.Sprintf("step%d", i), "out") + err := os.WriteFile(file, []byte("done"), 0644) + if err != nil { + return + } + time.Sleep(10 * time.Millisecond) + } + }() + + intervalStr := os.Getenv("SIDECAR_LOG_POLLING_INTERVAL") + if intervalStr == "" { + intervalStr = "100ms" + } + interval, err := time.ParseDuration(intervalStr) + if err != nil { + interval = 100 * time.Millisecond + } + if err := waitForStepsToFinish(runDir, interval); err != nil { + t.Fatalf("waitForStepsToFinish failed: %v", err) + } +} + func TestLookForArtifacts(t *testing.T) { base := basicArtifacts() modified := base.DeepCopy() diff --git a/pkg/apis/config/default.go b/pkg/apis/config/default.go index 3bb5e02ab55..435f33f32ca 100644 --- a/pkg/apis/config/default.go +++ b/pkg/apis/config/default.go @@ -54,6 +54,8 @@ const ( // Default maximum resolution timeout used by the resolution controller before timing out when exceeded DefaultMaximumResolutionTimeout = 1 * time.Minute + DefaultSidecarLogPollingInterval = 100 * time.Millisecond + defaultTimeoutMinutesKey = "default-timeout-minutes" defaultServiceAccountKey = "default-service-account" defaultManagedByLabelValueKey = "default-managed-by-label-value" @@ -67,6 +69,7 @@ const ( defaultContainerResourceRequirementsKey = "default-container-resource-requirements" defaultImagePullBackOffTimeout = "default-imagepullbackoff-timeout" defaultMaximumResolutionTimeout = "default-maximum-resolution-timeout" + defaultSidecarLogPollingIntervalKey = "default-sidecar-log-polling-interval" ) // DefaultConfig holds all the default configurations for the config. @@ -88,6 +91,10 @@ type Defaults struct { DefaultContainerResourceRequirements map[string]corev1.ResourceRequirements DefaultImagePullBackOffTimeout time.Duration DefaultMaximumResolutionTimeout time.Duration + // DefaultSidecarLogPollingInterval specifies how frequently (as a time.Duration) the Tekton sidecar log results container polls for step completion files. + // This value is loaded from the 'sidecar-log-polling-interval' key in the config-defaults ConfigMap. + // It is used to control the responsiveness and resource usage of the sidecar in both production and test environments. + DefaultSidecarLogPollingInterval time.Duration } // GetDefaultsConfigName returns the name of the configmap containing all @@ -120,6 +127,7 @@ func (cfg *Defaults) Equals(other *Defaults) bool { other.DefaultResolverType == cfg.DefaultResolverType && other.DefaultImagePullBackOffTimeout == cfg.DefaultImagePullBackOffTimeout && other.DefaultMaximumResolutionTimeout == cfg.DefaultMaximumResolutionTimeout && + other.DefaultSidecarLogPollingInterval == cfg.DefaultSidecarLogPollingInterval && reflect.DeepEqual(other.DefaultForbiddenEnv, cfg.DefaultForbiddenEnv) } @@ -134,6 +142,7 @@ func NewDefaultsFromMap(cfgMap map[string]string) (*Defaults, error) { DefaultResolverType: DefaultResolverTypeValue, DefaultImagePullBackOffTimeout: DefaultImagePullBackOffTimeout, DefaultMaximumResolutionTimeout: DefaultMaximumResolutionTimeout, + DefaultSidecarLogPollingInterval: DefaultSidecarLogPollingInterval, } if defaultTimeoutMin, ok := cfgMap[defaultTimeoutMinutesKey]; ok { @@ -220,6 +229,14 @@ func NewDefaultsFromMap(cfgMap map[string]string) (*Defaults, error) { tc.DefaultMaximumResolutionTimeout = timeout } + if defaultSidecarPollingInterval, ok := cfgMap[defaultSidecarLogPollingIntervalKey]; ok { + interval, err := time.ParseDuration(defaultSidecarPollingInterval) + if err != nil { + return nil, fmt.Errorf("failed parsing default config %q", defaultSidecarPollingInterval) + } + tc.DefaultSidecarLogPollingInterval = interval + } + return &tc, nil } diff --git a/pkg/apis/config/default_test.go b/pkg/apis/config/default_test.go index fdc8ba92378..f6de590a7eb 100644 --- a/pkg/apis/config/default_test.go +++ b/pkg/apis/config/default_test.go @@ -46,6 +46,7 @@ func TestNewDefaultsFromConfigMap(t *testing.T) { DefaultResolverType: "git", DefaultImagePullBackOffTimeout: time.Duration(5) * time.Second, DefaultMaximumResolutionTimeout: 1 * time.Minute, + DefaultSidecarLogPollingInterval: 100 * time.Millisecond, }, fileName: config.GetDefaultsConfigName(), }, @@ -67,6 +68,7 @@ func TestNewDefaultsFromConfigMap(t *testing.T) { DefaultMaxMatrixCombinationsCount: 256, DefaultImagePullBackOffTimeout: 0, DefaultMaximumResolutionTimeout: 1 * time.Minute, + DefaultSidecarLogPollingInterval: 100 * time.Millisecond, }, fileName: "config-defaults-with-pod-template", }, @@ -91,6 +93,7 @@ func TestNewDefaultsFromConfigMap(t *testing.T) { DefaultMaxMatrixCombinationsCount: 256, DefaultImagePullBackOffTimeout: 0, DefaultMaximumResolutionTimeout: 1 * time.Minute, + DefaultSidecarLogPollingInterval: 100 * time.Millisecond, }, }, { @@ -104,6 +107,7 @@ func TestNewDefaultsFromConfigMap(t *testing.T) { DefaultMaxMatrixCombinationsCount: 256, DefaultImagePullBackOffTimeout: 0, DefaultMaximumResolutionTimeout: 1 * time.Minute, + DefaultSidecarLogPollingInterval: 100 * time.Millisecond, }, }, { @@ -120,6 +124,7 @@ func TestNewDefaultsFromConfigMap(t *testing.T) { DefaultManagedByLabelValue: config.DefaultManagedByLabelValue, DefaultImagePullBackOffTimeout: 0, DefaultMaximumResolutionTimeout: 1 * time.Minute, + DefaultSidecarLogPollingInterval: 100 * time.Millisecond, }, }, { @@ -133,6 +138,7 @@ func TestNewDefaultsFromConfigMap(t *testing.T) { DefaultForbiddenEnv: []string{"TEKTON_POWER_MODE", "TEST_ENV", "TEST_TEKTON"}, DefaultImagePullBackOffTimeout: time.Duration(15) * time.Second, DefaultMaximumResolutionTimeout: 1 * time.Minute, + DefaultSidecarLogPollingInterval: 100 * time.Millisecond, }, }, { @@ -146,6 +152,7 @@ func TestNewDefaultsFromConfigMap(t *testing.T) { DefaultContainerResourceRequirements: map[string]corev1.ResourceRequirements{}, DefaultImagePullBackOffTimeout: 0, DefaultMaximumResolutionTimeout: 1 * time.Minute, + DefaultSidecarLogPollingInterval: 100 * time.Millisecond, }, }, { @@ -162,6 +169,7 @@ func TestNewDefaultsFromConfigMap(t *testing.T) { DefaultMaxMatrixCombinationsCount: 256, DefaultImagePullBackOffTimeout: 0, DefaultMaximumResolutionTimeout: 1 * time.Minute, + DefaultSidecarLogPollingInterval: 100 * time.Millisecond, DefaultContainerResourceRequirements: map[string]corev1.ResourceRequirements{ config.ResourceRequirementDefaultContainerKey: { Requests: corev1.ResourceList{ @@ -219,6 +227,7 @@ func TestNewDefaultsFromEmptyConfigMap(t *testing.T) { DefaultMaxMatrixCombinationsCount: 256, DefaultImagePullBackOffTimeout: 0, DefaultMaximumResolutionTimeout: 1 * time.Minute, + DefaultSidecarLogPollingInterval: 100 * time.Millisecond, } verifyConfigFileWithExpectedConfig(t, DefaultsConfigEmptyName, expectedConfig) } @@ -417,6 +426,51 @@ func TestEquals(t *testing.T) { } } +func TestSidecarLogPollingIntervalParsing(t *testing.T) { + cases := []struct { + name string + data map[string]string + expected time.Duration + wantErr bool + }{ + { + name: "valid interval", + data: map[string]string{"default-sidecar-log-polling-interval": "42ms"}, + expected: 42 * time.Millisecond, + wantErr: false, + }, + { + name: "invalid interval", + data: map[string]string{"default-sidecar-log-polling-interval": "notaduration"}, + expected: 0, + wantErr: true, + }, + { + name: "not set (default)", + data: map[string]string{}, + expected: 100 * time.Millisecond, + wantErr: false, + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + cfg, err := config.NewDefaultsFromMap(tc.data) + if tc.wantErr { + if err == nil { + t.Errorf("expected error, got nil") + } + return + } + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if cfg.DefaultSidecarLogPollingInterval != tc.expected { + t.Errorf("got %v, want %v", cfg.DefaultSidecarLogPollingInterval, tc.expected) + } + }) + } +} + func verifyConfigFileWithExpectedConfig(t *testing.T, fileName string, expectedConfig *config.Defaults) { t.Helper() cm := test.ConfigMapFromTestFile(t, fileName) diff --git a/pkg/pod/pod.go b/pkg/pod/pod.go index daa7e72d594..9d86568a5e4 100644 --- a/pkg/pod/pod.go +++ b/pkg/pod/pod.go @@ -25,6 +25,7 @@ import ( "path/filepath" "strconv" "strings" + "time" "github.com/tektoncd/pipeline/internal/artifactref" "github.com/tektoncd/pipeline/pkg/apis/config" @@ -200,10 +201,11 @@ func (b *Builder) Build(ctx context.Context, taskRun *v1.TaskRun, taskSpec v1.Ta } windows := usesWindows(taskRun) + pollingInterval := config.FromContextOrDefaults(ctx).Defaults.DefaultSidecarLogPollingInterval if sidecarLogsResultsEnabled { if taskSpec.Results != nil || artifactsPathReferenced(steps) { // create a results sidecar - resultsSidecar, err := createResultsSidecar(taskSpec, b.Images.SidecarLogResultsImage, securityContextConfig, windows) + resultsSidecar, err := createResultsSidecar(taskSpec, b.Images.SidecarLogResultsImage, securityContextConfig, windows, pollingInterval) if err != nil { return nil, err } @@ -613,7 +615,7 @@ func entrypointInitContainer(image string, steps []v1.Step, securityContext Secu // whether it will run on a windows node, and whether the sidecar should include a security context // that will allow it to run in namespaces with "restricted" pod security admission. // It will also provide arguments to the binary that allow it to surface the step results. -func createResultsSidecar(taskSpec v1.TaskSpec, image string, securityContext SecurityContextConfig, windows bool) (v1.Sidecar, error) { +func createResultsSidecar(taskSpec v1.TaskSpec, image string, securityContext SecurityContextConfig, windows bool, pollingInterval time.Duration) (v1.Sidecar, error) { names := make([]string, 0, len(taskSpec.Results)) for _, r := range taskSpec.Results { names = append(names, r.Name) @@ -655,6 +657,12 @@ func createResultsSidecar(taskSpec v1.TaskSpec, image string, securityContext Se Name: pipeline.ReservedResultsSidecarName, Image: image, Command: command, + Env: []corev1.EnvVar{ + { + Name: "SIDECAR_LOG_POLLING_INTERVAL", + Value: pollingInterval.String(), + }, + }, } if securityContext.SetSecurityContext { diff --git a/pkg/pod/pod_test.go b/pkg/pod/pod_test.go index 9f7ce8a17a6..3391ec33b4f 100644 --- a/pkg/pod/pod_test.go +++ b/pkg/pod/pod_test.go @@ -2006,6 +2006,7 @@ _EOF_ {Name: "tekton-internal-bin", ReadOnly: true, MountPath: "/tekton/bin"}, {Name: "tekton-internal-run-0", ReadOnly: true, MountPath: "/tekton/run/0"}, }, implicitVolumeMounts...), + Env: []corev1.EnvVar{{Name: "SIDECAR_LOG_POLLING_INTERVAL", Value: "100ms"}}, }}, Volumes: append(implicitVolumes, binVolume, runVolume(0), downwardVolume, corev1.Volume{ Name: "tekton-creds-init-home-0", @@ -2087,6 +2088,7 @@ _EOF_ {Name: "tekton-internal-bin", ReadOnly: true, MountPath: "/tekton/bin"}, {Name: "tekton-internal-run-0", ReadOnly: true, MountPath: "/tekton/run/0"}, }, implicitVolumeMounts...), + Env: []corev1.EnvVar{{Name: "SIDECAR_LOG_POLLING_INTERVAL", Value: "100ms"}}, }}, Volumes: append(implicitVolumes, binVolume, runVolume(0), downwardVolume, corev1.Volume{ Name: "tekton-creds-init-home-0", @@ -2163,6 +2165,7 @@ _EOF_ {Name: "tekton-internal-run-0", ReadOnly: true, MountPath: "/tekton/run/0"}, }, implicitVolumeMounts...), SecurityContext: SecurityContextConfig{SetSecurityContext: true, SetReadOnlyRootFilesystem: true}.GetSecurityContext(false), + Env: []corev1.EnvVar{{Name: "SIDECAR_LOG_POLLING_INTERVAL", Value: "100ms"}}, }}, Volumes: append(implicitVolumes, binVolume, runVolume(0), downwardVolume, corev1.Volume{ Name: "tekton-creds-init-home-0", @@ -2241,6 +2244,7 @@ _EOF_ {Name: "tekton-internal-bin", ReadOnly: true, MountPath: "/tekton/bin"}, {Name: "tekton-internal-run-0", ReadOnly: true, MountPath: "/tekton/run/0"}, }, implicitVolumeMounts...), + Env: []corev1.EnvVar{{Name: "SIDECAR_LOG_POLLING_INTERVAL", Value: "100ms"}}, }}, Volumes: append(implicitVolumes, binVolume, runVolume(0), downwardVolume, corev1.Volume{ Name: "tekton-creds-init-home-0", @@ -2325,6 +2329,7 @@ _EOF_ {Name: "tekton-internal-bin", ReadOnly: true, MountPath: "/tekton/bin"}, {Name: "tekton-internal-run-0", ReadOnly: true, MountPath: "/tekton/run/0"}, }, implicitVolumeMounts...), + Env: []corev1.EnvVar{{Name: "SIDECAR_LOG_POLLING_INTERVAL", Value: "100ms"}}, }}, Volumes: append(implicitVolumes, binVolume, runVolume(0), downwardVolume, corev1.Volume{ Name: "tekton-creds-init-home-0", @@ -2404,6 +2409,7 @@ _EOF_ {Name: "tekton-internal-run-0", ReadOnly: true, MountPath: "/tekton/run/0"}, }, implicitVolumeMounts...), SecurityContext: SecurityContextConfig{SetSecurityContext: true, SetReadOnlyRootFilesystem: true}.GetSecurityContext(false), + Env: []corev1.EnvVar{{Name: "SIDECAR_LOG_POLLING_INTERVAL", Value: "100ms"}}, }}, Volumes: append(implicitVolumes, binVolume, runVolume(0), downwardVolume, corev1.Volume{ Name: "tekton-creds-init-home-0", diff --git a/test/e2e-tests.sh b/test/e2e-tests.sh index d67138c27c2..4ddd15ac88d 100755 --- a/test/e2e-tests.sh +++ b/test/e2e-tests.sh @@ -158,6 +158,13 @@ function set_enable_kubernetes_sidecar() { kubectl patch configmap feature-flags -n tekton-pipelines -p "$jsonpatch" } +function set_default_sidecar_log_polling_interval() { + # Sets the default-sidecar-log-polling-interval in the config-defaults ConfigMap to 0ms for e2e tests + echo "Patching config-defaults ConfigMap: setting default-sidecar-log-polling-interval to 0ms" + jsonpatch='{"data": {"default-sidecar-log-polling-interval": "0ms"}}' + kubectl patch configmap config-defaults -n tekton-pipelines -p "$jsonpatch" +} + function run_e2e() { # Run the integration tests header "Running Go e2e tests" @@ -187,6 +194,7 @@ set_enable_param_enum "$ENABLE_PARAM_ENUM" set_enable_artifacts "$ENABLE_ARTIFACTS" set_enable_concise_resolver_syntax "$ENABLE_CONCISE_RESOLVER_SYNTAX" set_enable_kubernetes_sidecar "$ENABLE_KUBERNETES_SIDECAR" +set_default_sidecar_log_polling_interval run_e2e (( failed )) && fail_test