Merge pull request #1440 from stlaz/deads2k_featuresets_412_psa_enable

[4.12] OCPBUGS-6789: enable pod security admission for techpreview

Author: openshift-merge-robot

bindata/assets/config/defaultconfig.yaml

@@ -14,12 +14,12 @@ admission:
         kind: PodSecurityConfiguration
         apiVersion: pod-security.admission.config.k8s.io/v1beta1
         defaults:
-          enforce: "privileged"
-          enforce-version: "latest"
-          audit: "restricted"
-          audit-version: "latest"
-          warn: "restricted"
-          warn-version: "latest"
+          enforce: "invalid-to-force-substitution"
+          enforce-version: "invalid-to-force-substitution"
+          audit: "invalid-to-force-substitution"
+          audit-version: "invalid-to-force-substitution"
+          warn: "invalid-to-force-substitution"
+          warn-version: "invalid-to-force-substitution"
         exemptions:
           usernames:
           # The build controller creates pods that are likely to be privileged

go.mod

@@ -12,10 +12,10 @@ require (
 	github.com/google/go-cmp v0.5.6
 	github.com/imdario/mergo v0.3.8
 	github.com/miekg/dns v1.1.25
-	github.com/openshift/api v0.0.0-20221013123531-622889ac07cf
+	github.com/openshift/api v0.0.0-20230213134911-7ba313770556
 	github.com/openshift/build-machinery-go v0.0.0-20220913142420-e25cf57ea46d
 	github.com/openshift/client-go v0.0.0-20220831193253-4950ae70c8ea
-	github.com/openshift/library-go v0.0.0-20230215170552-9fdba43d77f8
+	github.com/openshift/library-go v0.0.0-20230301092340-c13b89190a26
 	github.com/pkg/profile v1.5.0 // indirect
 	github.com/prometheus-operator/prometheus-operator/pkg/client v0.45.0
 	github.com/prometheus/client_golang v1.12.1

go.sum

@@ -417,14 +417,14 @@ github.com/onsi/ginkgo/v2 v2.1.4 h1:GNapqRSid3zijZ9H77KrgVG4/8KqiyRsxcSxe+7ApXY=
 github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA=
 github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
 github.com/onsi/gomega v1.19.0 h1:4ieX6qQjPP/BfC3mpsAtIGGlxTWPeA3Inl/7DtXw1tw=
-github.com/openshift/api v0.0.0-20221013123531-622889ac07cf h1:GmMn1c1Y/G9bB8qCDbbBLEjYBfYBaZy4SQqGFgxzB6w=
-github.com/openshift/api v0.0.0-20221013123531-622889ac07cf/go.mod h1:JRz+ZvTqu9u7t6suhhPTacbFl5K65Y6rJbNM7HjWA3g=
+github.com/openshift/api v0.0.0-20230213134911-7ba313770556 h1:7W2fOhJicyEff24VaF7ASNzPtYvr+iSCVft4SIBAzaE=
+github.com/openshift/api v0.0.0-20230213134911-7ba313770556/go.mod h1:aQ6LDasvHMvHZXqLHnX2GRmnfTWCF/iIwz8EMTTIE9A=
 github.com/openshift/build-machinery-go v0.0.0-20220913142420-e25cf57ea46d h1:RR4ah7FfaPR1WePizm0jlrsbmPu91xQZnAsVVreQV1k=
 github.com/openshift/build-machinery-go v0.0.0-20220913142420-e25cf57ea46d/go.mod h1:b1BuldmJlbA/xYtdZvKi+7j5YGB44qJUJDZ9zwiNCfE=
 github.com/openshift/client-go v0.0.0-20220831193253-4950ae70c8ea h1:7JbjIzWt3Q75ErY1PAZ+gCA+bErI6HSlpffHFmMMzqM=
 github.com/openshift/client-go v0.0.0-20220831193253-4950ae70c8ea/go.mod h1:+J8DqZC60acCdpYkwVy/KH4cudgWiFZRNOBeghCzdGA=
-github.com/openshift/library-go v0.0.0-20230215170552-9fdba43d77f8 h1:4NAEsSYjERaGPAwDnD8cTPUQ3lQkuA3FS3quztT/+Co=
-github.com/openshift/library-go v0.0.0-20230215170552-9fdba43d77f8/go.mod h1:KPBAXGaq7pPmA+1wUVtKr5Axg3R68IomWDkzaOxIhxM=
+github.com/openshift/library-go v0.0.0-20230301092340-c13b89190a26 h1:vXYT3dX03Fm5FCX1284aTGoa5qBZFp3zMnIVaV9WOdg=
+github.com/openshift/library-go v0.0.0-20230301092340-c13b89190a26/go.mod h1:KPBAXGaq7pPmA+1wUVtKr5Axg3R68IomWDkzaOxIhxM=
 github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k=
 github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
 github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU=

pkg/cmd/render/render.go

@@ -15,21 +15,21 @@ import (
 	"path/filepath"
 
 	"github.com/ghodss/yaml"
+	configv1 "github.com/openshift/api/config/v1"
+	kubecontrolplanev1 "github.com/openshift/api/kubecontrolplane/v1"
+	"github.com/openshift/cluster-kube-apiserver-operator/bindata"
+	"github.com/openshift/cluster-kube-apiserver-operator/pkg/operator/configobservation/auth"
+	libgoaudit "github.com/openshift/library-go/pkg/operator/apiserver/audit"
+	genericrender "github.com/openshift/library-go/pkg/operator/render"
+	genericrenderoptions "github.com/openshift/library-go/pkg/operator/render/options"
 	"github.com/spf13/cobra"
 	"github.com/spf13/pflag"
-
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/util/sets"
 	kyaml "k8s.io/apimachinery/pkg/util/yaml"
 	auditv1 "k8s.io/apiserver/pkg/apis/audit/v1"
 	"k8s.io/klog/v2"
-
-	configv1 "github.com/openshift/api/config/v1"
-	kubecontrolplanev1 "github.com/openshift/api/kubecontrolplane/v1"
-	"github.com/openshift/cluster-kube-apiserver-operator/bindata"
-	libgoaudit "github.com/openshift/library-go/pkg/operator/apiserver/audit"
-	genericrender "github.com/openshift/library-go/pkg/operator/render"
-	genericrenderoptions "github.com/openshift/library-go/pkg/operator/render/options"
 )
 
 // renderOpts holds values to drive the render command.
@@ -260,7 +260,7 @@ func (r *renderOpts) Run() error {
 		return err
 	}
 
-	defaultConfig, err := bootstrapDefaultConfig()
+	defaultConfig, err := bootstrapDefaultConfig(configv1.FeatureSet(r.generic.FeatureSet))
 	if err != nil {
 		return fmt.Errorf("failed to get default config with audit policy - %s", err)
 	}
@@ -278,7 +278,7 @@ func (r *renderOpts) Run() error {
 	return genericrender.WriteFiles(&r.generic, &renderConfig.FileConfig, renderConfig)
 }
 
-func bootstrapDefaultConfig() ([]byte, error) {
+func bootstrapDefaultConfig(featureSet configv1.FeatureSet) ([]byte, error) {
 	asset := filepath.Join("assets", "config", "defaultconfig.yaml")
 	raw, err := bindata.Asset(asset)
 	if err != nil {
@@ -303,6 +303,17 @@ func bootstrapDefaultConfig() ([]byte, error) {
 		return nil, fmt.Errorf("failed to add audit policy into default config - %s", err)
 	}
 
+	// modify config for TechPreviewNoUpgrade here.
+	if sets.NewString(configv1.FeatureSets[featureSet].Enabled...).Has("OpenShiftPodSecurityAdmission") {
+		if err := auth.SetPodSecurityAdmissionToEnforceRestricted(defaultConfig); err != nil {
+			return nil, err
+		}
+	} else {
+		if err := auth.SetPodSecurityAdmissionToEnforcePrivileged(defaultConfig); err != nil {
+			return nil, err
+		}
+	}
+
 	defaultConfigRaw, err := json.Marshal(defaultConfig)
 	if err != nil {
 		return nil, fmt.Errorf("failed to marshal default config - %s", err)

pkg/cmd/render/render_test.go

@@ -624,7 +624,7 @@ spec:
 }
 
 func TestGetDefaultConfigWithAuditPolicy(t *testing.T) {
-	raw, err := bootstrapDefaultConfig()
+	raw, err := bootstrapDefaultConfig(configv1.Default)
 	require.NoError(t, err)
 	require.True(t, len(raw) > 0)
 

pkg/operator/configobservation/auth/podsecurityadmission.go

@@ -0,0 +1,118 @@
+package auth
+
+import (
+	"fmt"
+
+	configv1 "github.com/openshift/api/config/v1"
+	configlistersv1 "github.com/openshift/client-go/config/listers/config/v1"
+	"github.com/openshift/library-go/pkg/operator/configobserver"
+	"github.com/openshift/library-go/pkg/operator/configobserver/featuregates"
+	"github.com/openshift/library-go/pkg/operator/events"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"k8s.io/apimachinery/pkg/util/sets"
+)
+
+type FeatureGateLister interface {
+	FeatureGateLister() configlistersv1.FeatureGateLister
+}
+
+var configPath = []string{"admission", "pluginConfig", "PodSecurity", "configuration", "defaults"}
+
+// We want this:
+/*
+	admission:
+		pluginConfig:
+			PodSecurity:
+				configuration:
+					kind: PodSecurityConfiguration
+					apiVersion: pod-security.admission.config.k8s.io/v1beta1
+					defaults:
+						enforce: "restricted"
+						enforce-version: "latest"
+*/
+func SetPodSecurityAdmissionToEnforceRestricted(config map[string]interface{}) error {
+	psaEnforceRestricted := map[string]interface{}{
+		"enforce":         "restricted",
+		"enforce-version": "latest",
+		"audit":           "restricted",
+		"audit-version":   "latest",
+		"warn":            "restricted",
+		"warn-version":    "latest",
+	}
+
+	unstructured.RemoveNestedField(config, configPath...)
+	if err := unstructured.SetNestedMap(config, psaEnforceRestricted, configPath...); err != nil {
+		return fmt.Errorf("failed to set PodSecurity to enforce restricted: %w", err)
+	}
+
+	return nil
+}
+
+func SetPodSecurityAdmissionToEnforcePrivileged(config map[string]interface{}) error {
+	psaEnforceRestricted := map[string]interface{}{
+		"enforce":         "privileged",
+		"enforce-version": "latest",
+		"audit":           "restricted",
+		"audit-version":   "latest",
+		"warn":            "restricted",
+		"warn-version":    "latest",
+	}
+
+	unstructured.RemoveNestedField(config, configPath...)
+	if err := unstructured.SetNestedMap(config, psaEnforceRestricted, configPath...); err != nil {
+		return fmt.Errorf("failed to set PodSecurity to enforce restricted: %w", err)
+	}
+
+	return nil
+}
+
+// ObserveFeatureFlags fills in --feature-flags for the kube-apiserver
+func ObservePodSecurityAdmissionEnforcement(genericListers configobserver.Listers, recorder events.Recorder, existingConfig map[string]interface{}) (ret map[string]interface{}, _ []error) {
+	listers := genericListers.(FeatureGateLister)
+	errs := []error{}
+
+	featureGate, err := listers.FeatureGateLister().Get("cluster")
+	// if we have no featuregate, then the installer and MCO probably still have way to reconcile certain custom resources
+	// we will assume that this means the same as default and hope for the best
+	if apierrors.IsNotFound(err) {
+		featureGate = &configv1.FeatureGate{
+			Spec: configv1.FeatureGateSpec{
+				FeatureGateSelection: configv1.FeatureGateSelection{
+					FeatureSet: configv1.Default,
+				},
+			},
+		}
+	} else if err != nil {
+		return existingConfig, append(errs, err)
+	}
+
+	return observePodSecurityAdmissionEnforcement(featureGate, recorder, existingConfig)
+}
+
+func observePodSecurityAdmissionEnforcement(featureGate *configv1.FeatureGate, recorder events.Recorder, existingConfig map[string]interface{}) (ret map[string]interface{}, _ []error) {
+	defer func() {
+		ret = configobserver.Pruned(ret, configPath)
+	}()
+
+	errs := []error{}
+
+	enabled, _, err := featuregates.FeaturesGatesFromFeatureSets(featureGate)
+	if err != nil {
+		return existingConfig, append(errs, err)
+	}
+
+	observedConfig := map[string]interface{}{}
+	switch {
+	case sets.NewString(enabled...).Has("OpenShiftPodSecurityAdmission"):
+		if err := SetPodSecurityAdmissionToEnforceRestricted(observedConfig); err != nil {
+			return existingConfig, append(errs, err)
+		}
+	default:
+		if err := SetPodSecurityAdmissionToEnforcePrivileged(observedConfig); err != nil {
+			return existingConfig, append(errs, err)
+		}
+	}
+
+	return observedConfig, errs
+}

pkg/operator/configobservation/auth/podsecurityadmission_test.go

@@ -0,0 +1,118 @@
+package auth
+
+import (
+	"encoding/json"
+	"strings"
+	"testing"
+
+	utilerrors "k8s.io/apimachinery/pkg/util/errors"
+
+	"github.com/google/go-cmp/cmp"
+	"github.com/stretchr/testify/require"
+
+	configv1 "github.com/openshift/api/config/v1"
+	"github.com/openshift/library-go/pkg/operator/events"
+)
+
+func TestObservePodSecurityAdmissionEnforcement(t *testing.T) {
+	privilegedMap := map[string]interface{}{}
+	require.NoError(t, SetPodSecurityAdmissionToEnforcePrivileged(privilegedMap))
+	privilegedJSON, err := json.Marshal(privilegedMap)
+	require.NoError(t, err)
+
+	restrictedMap := map[string]interface{}{}
+	require.NoError(t, SetPodSecurityAdmissionToEnforceRestricted(restrictedMap))
+	restrictedJSON, err := json.Marshal(restrictedMap)
+	require.NoError(t, err)
+
+	defaultFeatureSet := &configv1.FeatureGate{
+		Spec: configv1.FeatureGateSpec{
+			FeatureGateSelection: configv1.FeatureGateSelection{
+				FeatureSet:      "",
+				CustomNoUpgrade: nil,
+			},
+		},
+	}
+
+	corruptFeatureSet := &configv1.FeatureGate{
+		Spec: configv1.FeatureGateSpec{
+			FeatureGateSelection: configv1.FeatureGateSelection{
+				FeatureSet:      "Bad",
+				CustomNoUpgrade: nil,
+			},
+		},
+	}
+
+	disabledFeatureSet := &configv1.FeatureGate{
+		Spec: configv1.FeatureGateSpec{
+			FeatureGateSelection: configv1.FeatureGateSelection{
+				FeatureSet: "CustomNoUpgrade",
+				CustomNoUpgrade: &configv1.CustomFeatureGates{
+					Enabled:  []string{"OpenShiftPodSecurityAdmission"},
+					Disabled: nil,
+				},
+			},
+		},
+	}
+
+	for _, tc := range []struct {
+		name         string
+		existingJSON string
+		featureGate  *configv1.FeatureGate
+		expectedErr  string
+		expectedJSON string
+	}{
+		{
+			name:         "not enabled",
+			existingJSON: string(privilegedJSON),
+			featureGate:  defaultFeatureSet,
+			expectedErr:  "",
+			expectedJSON: string(privilegedJSON),
+		},
+		{
+			name:         "corrupt-1",
+			existingJSON: string(privilegedJSON),
+			featureGate:  corruptFeatureSet,
+			expectedErr:  "not found",
+			expectedJSON: string(privilegedJSON),
+		},
+		{
+			name:         "corrupt-2",
+			existingJSON: string(restrictedJSON),
+			featureGate:  corruptFeatureSet,
+			expectedErr:  "not found",
+			expectedJSON: string(restrictedJSON),
+		},
+		{
+			name:         "enabled",
+			existingJSON: string(restrictedJSON),
+			featureGate:  disabledFeatureSet,
+			expectedErr:  "",
+			expectedJSON: string(restrictedJSON),
+		},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			testRecorder := events.NewInMemoryRecorder("SAIssuerTest")
+			existingConfig := map[string]interface{}{}
+			require.NoError(t, json.Unmarshal([]byte(tc.existingJSON), &existingConfig))
+
+			actual, errs := observePodSecurityAdmissionEnforcement(tc.featureGate, testRecorder, existingConfig)
+
+			switch {
+			case len(errs) == 0 && len(tc.expectedErr) == 0:
+			case len(errs) == 0 && len(tc.expectedErr) > 0:
+				t.Fatalf("missing err: %v", tc.expectedErr)
+
+			case len(errs) > 0 && len(tc.expectedErr) == 0:
+				t.Fatal(errs)
+			case len(errs) > 0 && len(tc.expectedErr) > 0 && !strings.Contains(utilerrors.NewAggregate(errs).Error(), tc.expectedErr):
+				t.Fatalf("missing err: %v in \n%v", tc.expectedErr, errs)
+			}
+
+			actualJSON, err := json.Marshal(actual)
+			require.NoError(t, err)
+
+			require.Equal(t, tc.expectedJSON, string(actualJSON), cmp.Diff(tc.expectedJSON, string(actualJSON)))
+		})
+	}
+}

pkg/operator/configobservation/configobservercontroller/observe_config_controller.go

@@ -129,6 +129,7 @@ func NewConfigObserver(
 			auth.ObserveAuthMetadata,
 			auth.ObserveServiceAccountIssuer,
 			auth.ObserveWebhookTokenAuthenticator,
+			auth.ObservePodSecurityAdmissionEnforcement,
 			encryption.NewEncryptionConfigObserver(
 				operatorclient.TargetNamespace,
 				// static path at which we expect to find the encryption config secret