diff --git a/.gitignore b/.gitignore index 9622a0204..9464ae16c 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,5 @@ /openshift-router + +# OTE (OpenShift Tests Extension) generated files +tests-extension/bin/ +tests-extension/test/testdata/bindata.go diff --git a/tests-extension/Makefile b/tests-extension/Makefile new file mode 100644 index 000000000..5011f8b0f --- /dev/null +++ b/tests-extension/Makefile @@ -0,0 +1,26 @@ +# Include bindata targets +include bindata.mk + +# Binary name and output directory +BINARY := bin/router-tests-ext + +# Build extension binary +.PHONY: build +build: bindata + @echo "Building extension binary..." + @mkdir -p bin + GOTOOLCHAIN=auto GOSUMDB=sum.golang.org go build -o $(BINARY) ./cmd + @echo "Binary built successfully at $(BINARY)" + +# Clean generated files +.PHONY: clean +clean: + @echo "Cleaning binaries..." + @rm -f $(BINARY) + +.PHONY: help +help: + @echo "Available targets:" + @echo " bindata - Generate bindata.go from test/testdata" + @echo " build - Build extension binary (includes bindata)" + @echo " clean - Remove extension binary" diff --git a/tests-extension/bindata.mk b/tests-extension/bindata.mk new file mode 100644 index 000000000..804314b1f --- /dev/null +++ b/tests-extension/bindata.mk @@ -0,0 +1,29 @@ +# Bindata generation for testdata files + +# Testdata path +TESTDATA_PATH := test/testdata + +# go-bindata tool path +GOPATH ?= $(shell go env GOPATH) +GO_BINDATA := $(GOPATH)/bin/go-bindata + +# Install go-bindata if not present +$(GO_BINDATA): + @echo "Installing go-bindata to $(GO_BINDATA)..." + @go install github.com/go-bindata/go-bindata/v3/go-bindata@latest + @echo "go-bindata installed successfully" + +# Generate bindata.go from testdata directory +.PHONY: bindata +bindata: clean-bindata $(GO_BINDATA) + @echo "Generating bindata from $(TESTDATA_PATH)..." + @mkdir -p $(TESTDATA_PATH) + $(GO_BINDATA) -nocompress -nometadata \ + -pkg testdata -o $(TESTDATA_PATH)/bindata.go -prefix "test" $(TESTDATA_PATH)/... + @gofmt -s -w $(TESTDATA_PATH)/bindata.go + @echo "Bindata generated successfully at $(TESTDATA_PATH)/bindata.go" + +.PHONY: clean-bindata +clean-bindata: + @echo "Cleaning bindata..." + @rm -f $(TESTDATA_PATH)/bindata.go diff --git a/tests-extension/cmd/main.go b/tests-extension/cmd/main.go new file mode 100644 index 000000000..da3b920a6 --- /dev/null +++ b/tests-extension/cmd/main.go @@ -0,0 +1,108 @@ +package main + +import ( + "context" + "fmt" + "os" + "regexp" + "strings" + + "github.com/spf13/cobra" + + "github.com/openshift-eng/openshift-tests-extension/pkg/cmd" + e "github.com/openshift-eng/openshift-tests-extension/pkg/extension" + et "github.com/openshift-eng/openshift-tests-extension/pkg/extension/extensiontests" + g "github.com/openshift-eng/openshift-tests-extension/pkg/ginkgo" + + // Import test framework packages for initialization + "github.com/openshift/origin/test/extended/util" + "k8s.io/kubernetes/test/e2e/framework" + + // Import test packages + _ "github.com/openshift/router-tests-extension/test/e2e" +) + +func main() { + // Initialize test framework + // This sets TestContext.KubeConfig from KUBECONFIG env var and initializes the cloud provider + util.InitStandardFlags() + if err := util.InitTest(false); err != nil { + panic(fmt.Sprintf("couldn't initialize test framework: %+v", err.Error())) + } + framework.AfterReadingAllFlags(&framework.TestContext) + + registry := e.NewRegistry() + ext := e.NewExtension("openshift", "payload", "router") + + // Add main test suite + ext.AddSuite(e.Suite{ + Name: "openshift/router/tests", + Parents: []string{"openshift/conformance/parallel"}, + }) + + // Build test specs from Ginkgo + allSpecs, err := g.BuildExtensionTestSpecsFromOpenShiftGinkgoSuite() + if err != nil { + panic(fmt.Sprintf("couldn't build extension test specs from ginkgo: %+v", err.Error())) + } + + // IMPORTANT: Filter out upstream Kubernetes tests + // Only keep tests that have "Author:" in the name (component-specific tests) + // This excludes upstream K8s feature tests like [Feature:BootstrapTokens], [Feature:DRA], etc. + var filteredSpecs []*et.ExtensionTestSpec + allSpecs.Walk(func(spec *et.ExtensionTestSpec) { + if strings.Contains(spec.Name, "Author:") { + filteredSpecs = append(filteredSpecs, spec) + } + }) + + // Create new specs collection from filtered list + specs := et.ExtensionTestSpecs(filteredSpecs) + + // Apply platform filters based on Platform: labels + specs.Walk(func(spec *et.ExtensionTestSpec) { + for label := range spec.Labels { + if strings.HasPrefix(label, "Platform:") { + platformName := strings.TrimPrefix(label, "Platform:") + spec.Include(et.PlatformEquals(platformName)) + } + } + }) + + // Apply platform filters based on [platform:xxx] in test names + specs.Walk(func(spec *et.ExtensionTestSpec) { + re := regexp.MustCompile(`\[platform:([a-z]+)\]`) + if match := re.FindStringSubmatch(spec.Name); match != nil { + platform := match[1] + spec.Include(et.PlatformEquals(platform)) + } + }) + + // Wrap test execution with cleanup handler + // This marks tests as started and ensures proper cleanup + specs.Walk(func(spec *et.ExtensionTestSpec) { + originalRun := spec.Run + spec.Run = func(ctx context.Context) *et.ExtensionTestResult { + var result *et.ExtensionTestResult + util.WithCleanup(func() { + result = originalRun(ctx) + }) + return result + } + }) + + ext.AddSpecs(specs) + registry.Register(ext) + + root := &cobra.Command{ + Long: "Router Tests", + } + + root.AddCommand(cmd.DefaultExtensionCommands(registry)...) + + if err := func() error { + return root.Execute() + }(); err != nil { + os.Exit(1) + } +} diff --git a/tests-extension/go.mod b/tests-extension/go.mod new file mode 100644 index 000000000..2ac7ca473 --- /dev/null +++ b/tests-extension/go.mod @@ -0,0 +1,364 @@ +module github.com/openshift/router-tests-extension + +go 1.24.6 + +toolchain go1.24.11 + +require ( + github.com/aws/aws-sdk-go v1.50.25 + github.com/aws/aws-sdk-go-v2 v1.41.0 + github.com/aws/aws-sdk-go-v2/config v1.27.24 + github.com/aws/aws-sdk-go-v2/service/iam v1.53.1 + github.com/aws/aws-sdk-go-v2/service/sts v1.30.1 + github.com/onsi/ginkgo/v2 v2.27.3 + github.com/onsi/gomega v1.38.3 + github.com/openshift-eng/openshift-tests-extension v0.0.0-20251218142942-7ecc8801b9df + github.com/openshift/origin v1.5.0-alpha.3.0.20260105230330-0eae81eb0bd3 + github.com/spf13/cobra v1.10.1 + github.com/tidwall/gjson v1.18.0 + k8s.io/apimachinery v0.34.1 + k8s.io/kubernetes v1.34.1 + k8s.io/utils v0.0.0-20251002143259-bc988d571ff4 +) + +require ( + cel.dev/expr v0.24.0 // indirect + cloud.google.com/go v0.121.6 // indirect + cloud.google.com/go/auth v0.16.5 // indirect + cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect + cloud.google.com/go/compute/metadata v0.8.0 // indirect + cloud.google.com/go/iam v1.5.2 // indirect + cloud.google.com/go/monitoring v1.24.2 // indirect + cloud.google.com/go/storage v1.56.0 // indirect + github.com/Azure/azure-pipeline-go v0.2.3 // indirect + github.com/Azure/azure-sdk-for-go v68.0.0+incompatible // indirect + github.com/Azure/azure-sdk-for-go/sdk/azcore v1.18.1 // indirect + github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.8.2 // indirect + github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.0 // indirect + github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v5 v5.7.0 // indirect + github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/keyvault/armkeyvault v1.4.0 // indirect + github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armfeatures v1.2.0 // indirect + github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources v1.2.0 // indirect + github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.6.0 // indirect + github.com/Azure/azure-storage-blob-go v0.15.0 // indirect + github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 // indirect + github.com/Azure/go-autorest v14.2.0+incompatible // indirect + github.com/Azure/go-autorest/autorest v0.11.29 // indirect + github.com/Azure/go-autorest/autorest/adal v0.9.24 // indirect + github.com/Azure/go-autorest/autorest/azure/auth v0.5.13 // indirect + github.com/Azure/go-autorest/autorest/azure/cli v0.4.6 // indirect + github.com/Azure/go-autorest/autorest/date v0.3.0 // indirect + github.com/Azure/go-autorest/autorest/to v0.4.0 // indirect + github.com/Azure/go-autorest/autorest/validation v0.3.1 // indirect + github.com/Azure/go-autorest/logger v0.2.1 // indirect + github.com/Azure/go-autorest/tracing v0.6.0 // indirect + github.com/AzureAD/microsoft-authentication-library-for-go v1.4.2 // indirect + github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.29.0 // indirect + github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.53.0 // indirect + github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.53.0 // indirect + github.com/IBM-Cloud/power-go-client v1.12.0 // indirect + github.com/IBM/go-sdk-core/v5 v5.21.0 // indirect + github.com/IBM/platform-services-go-sdk v0.81.0 // indirect + github.com/IBM/vpc-go-sdk v0.70.1 // indirect + github.com/JeffAshton/win_pdh v0.0.0-20161109143554-76bb4ee9f0ab // indirect + github.com/MakeNowJust/heredoc v1.0.0 // indirect + github.com/Masterminds/semver/v3 v3.4.0 // indirect + github.com/Microsoft/go-winio v0.6.2 // indirect + github.com/Microsoft/hnslib v0.1.1 // indirect + github.com/NYTimes/gziphandler v1.1.1 // indirect + github.com/ProtonMail/go-crypto v1.1.6 // indirect + github.com/antlr4-go/antlr/v4 v4.13.1 // indirect + github.com/apparentlymart/go-textseg/v15 v15.0.0 // indirect + github.com/armon/circbuf v0.0.0-20190214190532-5111143e8da2 // indirect + github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect + github.com/aws/aws-sdk-go-v2/credentials v1.17.24 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.9 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.16 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.16 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.3 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.15 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.22.1 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.26.2 // indirect + github.com/aws/smithy-go v1.24.0 // indirect + github.com/beorn7/perks v1.0.1 // indirect + github.com/blang/semver/v4 v4.0.0 // indirect + github.com/cenkalti/backoff/v4 v4.3.0 // indirect + github.com/cenkalti/backoff/v5 v5.0.2 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/chai2010/gettext-go v1.0.2 // indirect + github.com/cloudflare/circl v1.6.0 // indirect + github.com/cncf/xds/go v0.0.0-20250501225837-2ac532fd4443 // indirect + github.com/container-storage-interface/spec v1.9.0 // indirect + github.com/containerd/containerd/api v1.8.0 // indirect + github.com/containerd/errdefs v1.0.0 // indirect + github.com/containerd/errdefs/pkg v0.3.0 // indirect + github.com/containerd/log v0.1.0 // indirect + github.com/containerd/ttrpc v1.2.6 // indirect + github.com/containerd/typeurl/v2 v2.2.2 // indirect + github.com/coreos/go-semver v0.3.1 // indirect + github.com/coreos/go-systemd/v22 v22.5.0 // indirect + github.com/coreos/stream-metadata-go v0.4.9 // indirect + github.com/cyphar/filepath-securejoin v0.4.1 // indirect + github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect + github.com/dimchansky/utfbom v1.1.1 // indirect + github.com/distribution/reference v0.6.0 // indirect + github.com/docker/docker v27.3.1+incompatible // indirect + github.com/docker/go-connections v0.5.0 // indirect + github.com/docker/go-units v0.5.0 // indirect + github.com/emicklei/go-restful/v3 v3.13.0 // indirect + github.com/envoyproxy/go-control-plane/envoy v1.32.4 // indirect + github.com/envoyproxy/protoc-gen-validate v1.2.1 // indirect + github.com/euank/go-kmsg-parser v2.0.0+incompatible // indirect + github.com/exponent-io/jsonpath v0.0.0-20210407135951-1de76d718b3f // indirect + github.com/fatih/camelcase v1.0.0 // indirect + github.com/felixge/httpsnoop v1.0.4 // indirect + github.com/fsnotify/fsnotify v1.9.0 // indirect + github.com/fsouza/go-dockerclient v1.12.0 // indirect + github.com/fxamacker/cbor/v2 v2.9.0 // indirect + github.com/gabriel-vasile/mimetype v1.4.9 // indirect + github.com/gebn/bmc v0.0.0-20250519231546-bf709e03fe3c // indirect + github.com/ghodss/yaml v1.0.1-0.20190212211648-25d852aebe32 // indirect + github.com/go-errors/errors v1.4.2 // indirect + github.com/go-jose/go-jose/v4 v4.1.1 // indirect + github.com/go-logr/logr v1.4.3 // indirect + github.com/go-logr/stdr v1.2.2 // indirect + github.com/go-openapi/analysis v0.23.0 // indirect + github.com/go-openapi/errors v0.22.1 // indirect + github.com/go-openapi/jsonpointer v0.21.2 // indirect + github.com/go-openapi/jsonreference v0.21.0 // indirect + github.com/go-openapi/loads v0.22.0 // indirect + github.com/go-openapi/runtime v0.28.0 // indirect + github.com/go-openapi/spec v0.21.0 // indirect + github.com/go-openapi/strfmt v0.23.0 // indirect + github.com/go-openapi/swag v0.23.1 // indirect + github.com/go-openapi/validate v0.24.0 // indirect + github.com/go-playground/locales v0.14.1 // indirect + github.com/go-playground/universal-translator v0.18.1 // indirect + github.com/go-playground/validator/v10 v10.26.0 // indirect + github.com/go-task/slim-sprig/v3 v3.0.0 // indirect + github.com/godbus/dbus/v5 v5.1.0 // indirect + github.com/gofrs/uuid v4.4.0+incompatible // indirect + github.com/gogo/protobuf v1.3.2 // indirect + github.com/golang-jwt/jwt/v4 v4.5.2 // indirect + github.com/golang-jwt/jwt/v5 v5.2.2 // indirect + github.com/golang/protobuf v1.5.4 // indirect + github.com/google/btree v1.1.3 // indirect + github.com/google/cadvisor v0.52.1 // indirect + github.com/google/cel-go v0.26.1 // indirect + github.com/google/gnostic-models v0.7.0 // indirect + github.com/google/go-cmp v0.7.0 // indirect + github.com/google/gopacket v1.1.19 // indirect + github.com/google/pprof v0.0.0-20250403155104-27863c87afa6 // indirect + github.com/google/s2a-go v0.1.9 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/googleapis/enterprise-certificate-proxy v0.3.6 // indirect + github.com/googleapis/gax-go/v2 v2.15.0 // indirect + github.com/gophercloud/gophercloud v1.14.1 // indirect + github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674 // indirect + github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79 // indirect + github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.0 // indirect + github.com/hashicorp/go-checkpoint v0.5.0 // indirect + github.com/hashicorp/go-cleanhttp v0.5.2 // indirect + github.com/hashicorp/go-retryablehttp v0.7.7 // indirect + github.com/hashicorp/go-uuid v1.0.3 // indirect + github.com/hashicorp/go-version v1.7.0 // indirect + github.com/hashicorp/hc-install v0.9.2 // indirect + github.com/hashicorp/terraform-exec v0.23.0 // indirect + github.com/hashicorp/terraform-json v0.25.0 // indirect + github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/jmespath/go-jmespath v0.4.0 // indirect + github.com/josharian/intern v1.0.0 // indirect + github.com/json-iterator/go v1.1.12 // indirect + github.com/karrick/godirwalk v1.17.0 // indirect + github.com/klauspost/compress v1.18.0 // indirect + github.com/kylelemons/godebug v1.1.0 // indirect + github.com/leodido/go-urn v1.4.0 // indirect + github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de // indirect + github.com/mailru/easyjson v0.9.0 // indirect + github.com/mattn/go-colorable v0.1.13 // indirect + github.com/mattn/go-ieproxy v0.0.11 // indirect + github.com/mattn/go-isatty v0.0.20 // indirect + github.com/microsoft/kiota-abstractions-go v1.9.3 // indirect + github.com/microsoft/kiota-authentication-azure-go v1.3.0 // indirect + github.com/microsoft/kiota-http-go v1.5.2 // indirect + github.com/microsoft/kiota-serialization-form-go v1.1.2 // indirect + github.com/microsoft/kiota-serialization-json-go v1.1.2 // indirect + github.com/microsoft/kiota-serialization-multipart-go v1.1.2 // indirect + github.com/microsoft/kiota-serialization-text-go v1.1.2 // indirect + github.com/microsoftgraph/msgraph-sdk-go v1.81.0 // indirect + github.com/microsoftgraph/msgraph-sdk-go-core v1.3.2 // indirect + github.com/mistifyio/go-zfs v2.1.2-0.20190413222219-f784269be439+incompatible // indirect + github.com/mitchellh/go-homedir v1.1.0 // indirect + github.com/mitchellh/go-wordwrap v1.0.1 // indirect + github.com/mitchellh/mapstructure v1.5.0 // indirect + github.com/moby/docker-image-spec v1.3.1 // indirect + github.com/moby/patternmatcher v0.6.0 // indirect + github.com/moby/spdystream v0.5.0 // indirect + github.com/moby/sys/mountinfo v0.7.2 // indirect + github.com/moby/sys/sequential v0.6.0 // indirect + github.com/moby/sys/user v0.4.0 // indirect + github.com/moby/sys/userns v0.1.0 // indirect + github.com/moby/term v0.5.0 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect + github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 // indirect + github.com/morikuni/aec v1.0.0 // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f // indirect + github.com/oklog/ulid v1.3.1 // indirect + github.com/opencontainers/cgroups v0.0.3 // indirect + github.com/opencontainers/go-digest v1.0.0 // indirect + github.com/opencontainers/image-spec v1.1.1 // indirect + github.com/opencontainers/runc v1.2.5 // indirect + github.com/opencontainers/runtime-spec v1.2.0 // indirect + github.com/opencontainers/selinux v1.11.1 // indirect + github.com/openshift/api v0.0.0-20251015095338-264e80a2b6e7 // indirect + github.com/openshift/client-go v0.0.0-20251015124057-db0dee36e235 // indirect + github.com/openshift/library-go v0.0.0-20251015151611-6fc7a74b67c5 // indirect + github.com/opentracing/opentracing-go v1.2.0 // indirect + github.com/pborman/uuid v1.2.0 // indirect + github.com/peterbourgon/diskv v2.0.1+incompatible // indirect + github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect + github.com/pkg/errors v0.9.1 // indirect + github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect + github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect + github.com/prometheus/client_golang v1.23.0 // indirect + github.com/prometheus/client_model v0.6.2 // indirect + github.com/prometheus/common v0.65.0 // indirect + github.com/prometheus/procfs v0.17.0 // indirect + github.com/rs/zerolog v1.34.0 // indirect + github.com/russross/blackfriday/v2 v2.1.0 // indirect + github.com/sirupsen/logrus v1.9.3 // indirect + github.com/spf13/pflag v1.0.9 // indirect + github.com/spiffe/go-spiffe/v2 v2.5.0 // indirect + github.com/std-uritemplate/std-uritemplate/go/v2 v2.0.3 // indirect + github.com/stoewer/go-strcase v1.3.1 // indirect + github.com/stretchr/objx v0.5.2 // indirect + github.com/stretchr/testify v1.11.1 // indirect + github.com/tecbiz-ch/nutanix-go-sdk v0.1.15 // indirect + github.com/tidwall/match v1.1.1 // indirect + github.com/tidwall/pretty v1.2.1 // indirect + github.com/tidwall/sjson v1.2.5 // indirect + github.com/vmware/govmomi v0.51.0 // indirect + github.com/x448/float16 v0.8.4 // indirect + github.com/xlab/treeprint v1.2.0 // indirect + github.com/zclconf/go-cty v1.16.2 // indirect + github.com/zeebo/errs v1.4.0 // indirect + go.etcd.io/etcd/api/v3 v3.6.4 // indirect + go.etcd.io/etcd/client/pkg/v3 v3.6.4 // indirect + go.etcd.io/etcd/client/v3 v3.6.4 // indirect + go.mongodb.org/mongo-driver v1.17.3 // indirect + go.opentelemetry.io/auto/sdk v1.1.0 // indirect + go.opentelemetry.io/contrib/detectors/gcp v1.36.0 // indirect + go.opentelemetry.io/contrib/instrumentation/github.com/emicklei/go-restful/otelrestful v0.44.0 // indirect + go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.61.0 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0 // indirect + go.opentelemetry.io/otel v1.37.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.36.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.36.0 // indirect + go.opentelemetry.io/otel/metric v1.37.0 // indirect + go.opentelemetry.io/otel/sdk v1.37.0 // indirect + go.opentelemetry.io/otel/sdk/metric v1.37.0 // indirect + go.opentelemetry.io/otel/trace v1.37.0 // indirect + go.opentelemetry.io/proto/otlp v1.7.0 // indirect + go.uber.org/atomic v1.11.0 // indirect + go.uber.org/multierr v1.11.0 // indirect + go.uber.org/zap v1.27.0 // indirect + go.yaml.in/yaml/v2 v2.4.2 // indirect + go.yaml.in/yaml/v3 v3.0.4 // indirect + golang.org/x/crypto v0.42.0 // indirect + golang.org/x/exp v0.0.0-20250620022241-b7579e27df2b // indirect + golang.org/x/mod v0.27.0 // indirect + golang.org/x/net v0.43.0 // indirect + golang.org/x/oauth2 v0.30.0 // indirect + golang.org/x/sync v0.17.0 // indirect + golang.org/x/sys v0.36.0 // indirect + golang.org/x/term v0.35.0 // indirect + golang.org/x/text v0.29.0 // indirect + golang.org/x/time v0.12.0 // indirect + golang.org/x/tools v0.36.0 // indirect + google.golang.org/api v0.247.0 // indirect + google.golang.org/genproto v0.0.0-20250603155806-513f23925822 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20250818200422-3122310a409c // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20250826171959-ef028d996bc1 // indirect + google.golang.org/grpc v1.75.1 // indirect + google.golang.org/protobuf v1.36.8 // indirect + gopkg.in/evanphx/json-patch.v4 v4.13.0 // indirect + gopkg.in/go-jose/go-jose.v2 v2.6.3 // indirect + gopkg.in/inf.v0 v0.9.1 // indirect + gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect + gopkg.in/yaml.v2 v2.4.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect + k8s.io/api v0.34.1 // indirect + k8s.io/apiextensions-apiserver v0.34.1 // indirect + k8s.io/apiserver v0.34.1 // indirect + k8s.io/cli-runtime v0.33.4 // indirect + k8s.io/client-go v0.34.1 // indirect + k8s.io/cloud-provider v0.31.1 // indirect + k8s.io/cluster-bootstrap v0.0.0 // indirect + k8s.io/component-base v0.34.1 // indirect + k8s.io/component-helpers v0.34.1 // indirect + k8s.io/controller-manager v0.32.1 // indirect + k8s.io/cri-api v0.27.1 // indirect + k8s.io/cri-client v0.0.0 // indirect + k8s.io/csi-translation-lib v0.0.0 // indirect + k8s.io/dynamic-resource-allocation v0.0.0 // indirect + k8s.io/klog/v2 v2.130.1 // indirect + k8s.io/kms v0.34.1 // indirect + k8s.io/kube-aggregator v0.34.1 // indirect + k8s.io/kube-openapi v0.0.0-20250814151709-d7b6acb124c3 // indirect + k8s.io/kube-scheduler v0.0.0 // indirect + k8s.io/kubectl v0.34.1 // indirect + k8s.io/kubelet v0.31.1 // indirect + k8s.io/mount-utils v0.0.0 // indirect + k8s.io/pod-security-admission v0.34.1 // indirect + k8s.io/sample-apiserver v0.0.0 // indirect + sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.33.0 // indirect + sigs.k8s.io/gateway-api v1.4.0 // indirect + sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 // indirect + sigs.k8s.io/kube-storage-version-migrator v0.0.6-0.20230721195810-5c8923c5ff96 // indirect + sigs.k8s.io/kustomize/api v0.20.1 // indirect + sigs.k8s.io/kustomize/kyaml v0.20.1 // indirect + sigs.k8s.io/randfill v1.0.0 // indirect + sigs.k8s.io/structured-merge-diff/v6 v6.3.0 // indirect + sigs.k8s.io/yaml v1.6.0 // indirect +) + +replace ( + bitbucket.org/ww/goautoneg => github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d + github.com/jteeuwen/go-bindata => github.com/jteeuwen/go-bindata v3.0.8-0.20151023091102-a0ff2567cfb7+incompatible + github.com/onsi/ginkgo/v2 => github.com/openshift/onsi-ginkgo/v2 v2.6.1-0.20251120221002-696928a6a0d7 + k8s.io/api => github.com/openshift/kubernetes/staging/src/k8s.io/api v0.0.0-20251215180958-76cb1db555ed + k8s.io/apiextensions-apiserver => github.com/openshift/kubernetes/staging/src/k8s.io/apiextensions-apiserver v0.0.0-20251215180958-76cb1db555ed + k8s.io/apimachinery => github.com/openshift/kubernetes/staging/src/k8s.io/apimachinery v0.0.0-20251215180958-76cb1db555ed + k8s.io/apiserver => github.com/openshift/kubernetes/staging/src/k8s.io/apiserver v0.0.0-20251215180958-76cb1db555ed + k8s.io/cli-runtime => github.com/openshift/kubernetes/staging/src/k8s.io/cli-runtime v0.0.0-20251215180958-76cb1db555ed + k8s.io/client-go => github.com/openshift/kubernetes/staging/src/k8s.io/client-go v0.0.0-20251215180958-76cb1db555ed + k8s.io/cloud-provider => github.com/openshift/kubernetes/staging/src/k8s.io/cloud-provider v0.0.0-20251215180958-76cb1db555ed + k8s.io/cluster-bootstrap => github.com/openshift/kubernetes/staging/src/k8s.io/cluster-bootstrap v0.0.0-20251215180958-76cb1db555ed + k8s.io/code-generator => github.com/openshift/kubernetes/staging/src/k8s.io/code-generator v0.0.0-20251215180958-76cb1db555ed + k8s.io/component-base => github.com/openshift/kubernetes/staging/src/k8s.io/component-base v0.0.0-20251215180958-76cb1db555ed + k8s.io/component-helpers => github.com/openshift/kubernetes/staging/src/k8s.io/component-helpers v0.0.0-20251215180958-76cb1db555ed + k8s.io/controller-manager => github.com/openshift/kubernetes/staging/src/k8s.io/controller-manager v0.0.0-20251215180958-76cb1db555ed + k8s.io/cri-api => github.com/openshift/kubernetes/staging/src/k8s.io/cri-api v0.0.0-20251215180958-76cb1db555ed + k8s.io/cri-client => github.com/openshift/kubernetes/staging/src/k8s.io/cri-client v0.0.0-20251215180958-76cb1db555ed + k8s.io/csi-translation-lib => github.com/openshift/kubernetes/staging/src/k8s.io/csi-translation-lib v0.0.0-20251215180958-76cb1db555ed + k8s.io/dynamic-resource-allocation => github.com/openshift/kubernetes/staging/src/k8s.io/dynamic-resource-allocation v0.0.0-20251215180958-76cb1db555ed + k8s.io/endpointslice => github.com/openshift/kubernetes/staging/src/k8s.io/endpointslice v0.0.0-20251215180958-76cb1db555ed + k8s.io/kube-aggregator => github.com/openshift/kubernetes/staging/src/k8s.io/kube-aggregator v0.0.0-20251215180958-76cb1db555ed + k8s.io/kube-controller-manager => github.com/openshift/kubernetes/staging/src/k8s.io/kube-controller-manager v0.0.0-20251215180958-76cb1db555ed + k8s.io/kube-proxy => github.com/openshift/kubernetes/staging/src/k8s.io/kube-proxy v0.0.0-20251215180958-76cb1db555ed + k8s.io/kube-scheduler => github.com/openshift/kubernetes/staging/src/k8s.io/kube-scheduler v0.0.0-20251215180958-76cb1db555ed + k8s.io/kubectl => github.com/openshift/kubernetes/staging/src/k8s.io/kubectl v0.0.0-20251215180958-76cb1db555ed + k8s.io/kubelet => github.com/openshift/kubernetes/staging/src/k8s.io/kubelet v0.0.0-20251215180958-76cb1db555ed + k8s.io/kubernetes => github.com/openshift/kubernetes v1.30.1-0.20251215180958-76cb1db555ed + k8s.io/legacy-cloud-providers => github.com/openshift/kubernetes/staging/src/k8s.io/legacy-cloud-providers v0.0.0-20251215180958-76cb1db555ed + k8s.io/metrics => github.com/openshift/kubernetes/staging/src/k8s.io/metrics v0.0.0-20251215180958-76cb1db555ed + k8s.io/mount-utils => github.com/openshift/kubernetes/staging/src/k8s.io/mount-utils v0.0.0-20251215180958-76cb1db555ed + k8s.io/pod-security-admission => github.com/openshift/kubernetes/staging/src/k8s.io/pod-security-admission v0.0.0-20251215180958-76cb1db555ed + k8s.io/sample-apiserver => github.com/openshift/kubernetes/staging/src/k8s.io/sample-apiserver v0.0.0-20251215180958-76cb1db555ed + k8s.io/sample-cli-plugin => github.com/openshift/kubernetes/staging/src/k8s.io/sample-cli-plugin v0.0.0-20251215180958-76cb1db555ed + k8s.io/sample-controller => github.com/openshift/kubernetes/staging/src/k8s.io/sample-controller v0.0.0-20251215180958-76cb1db555ed +) diff --git a/tests-extension/go.sum b/tests-extension/go.sum new file mode 100644 index 000000000..5ce3bc89e --- /dev/null +++ b/tests-extension/go.sum @@ -0,0 +1,902 @@ +cel.dev/expr v0.24.0 h1:56OvJKSH3hDGL0ml5uSxZmz3/3Pq4tJ+fb1unVLAFcY= +cel.dev/expr v0.24.0/go.mod h1:hLPLo1W4QUmuYdA72RBX06QTs6MXw941piREPl3Yfiw= +cloud.google.com/go v0.121.6 h1:waZiuajrI28iAf40cWgycWNgaXPO06dupuS+sgibK6c= +cloud.google.com/go v0.121.6/go.mod h1:coChdst4Ea5vUpiALcYKXEpR1S9ZgXbhEzzMcMR66vI= +cloud.google.com/go/auth v0.16.5 h1:mFWNQ2FEVWAliEQWpAdH80omXFokmrnbDhUS9cBywsI= +cloud.google.com/go/auth v0.16.5/go.mod h1:utzRfHMP+Vv0mpOkTRQoWD2q3BatTOoWbA7gCc2dUhQ= +cloud.google.com/go/auth/oauth2adapt v0.2.8 h1:keo8NaayQZ6wimpNSmW5OPc283g65QNIiLpZnkHRbnc= +cloud.google.com/go/auth/oauth2adapt v0.2.8/go.mod h1:XQ9y31RkqZCcwJWNSx2Xvric3RrU88hAYYbjDWYDL+c= +cloud.google.com/go/compute/metadata v0.8.0 h1:HxMRIbao8w17ZX6wBnjhcDkW6lTFpgcaobyVfZWqRLA= +cloud.google.com/go/compute/metadata v0.8.0/go.mod h1:sYOGTp851OV9bOFJ9CH7elVvyzopvWQFNNghtDQ/Biw= +cloud.google.com/go/iam v1.5.2 h1:qgFRAGEmd8z6dJ/qyEchAuL9jpswyODjA2lS+w234g8= +cloud.google.com/go/iam v1.5.2/go.mod h1:SE1vg0N81zQqLzQEwxL2WI6yhetBdbNQuTvIKCSkUHE= +cloud.google.com/go/logging v1.13.0 h1:7j0HgAp0B94o1YRDqiqm26w4q1rDMH7XNRU34lJXHYc= +cloud.google.com/go/logging v1.13.0/go.mod h1:36CoKh6KA/M0PbhPKMq6/qety2DCAErbhXT62TuXALA= +cloud.google.com/go/longrunning v0.6.7 h1:IGtfDWHhQCgCjwQjV9iiLnUta9LBCo8R9QmAFsS/PrE= +cloud.google.com/go/longrunning v0.6.7/go.mod h1:EAFV3IZAKmM56TyiE6VAP3VoTzhZzySwI/YI1s/nRsY= +cloud.google.com/go/monitoring v1.24.2 h1:5OTsoJ1dXYIiMiuL+sYscLc9BumrL3CarVLL7dd7lHM= +cloud.google.com/go/monitoring v1.24.2/go.mod h1:x7yzPWcgDRnPEv3sI+jJGBkwl5qINf+6qY4eq0I9B4U= +cloud.google.com/go/storage v1.56.0 h1:iixmq2Fse2tqxMbWhLWC9HfBj1qdxqAmiK8/eqtsLxI= +cloud.google.com/go/storage v1.56.0/go.mod h1:Tpuj6t4NweCLzlNbw9Z9iwxEkrSem20AetIeH/shgVU= +cloud.google.com/go/trace v1.11.6 h1:2O2zjPzqPYAHrn3OKl029qlqG6W8ZdYaOWRyr8NgMT4= +cloud.google.com/go/trace v1.11.6/go.mod h1:GA855OeDEBiBMzcckLPE2kDunIpC72N+Pq8WFieFjnI= +dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk= +dario.cat/mergo v1.0.0/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk= +github.com/AdaLogics/go-fuzz-headers v0.0.0-20230811130428-ced1acdcaa24 h1:bvDV9vkmnHYOMsOr4WLk+Vo07yKIzd94sVoIqshQ4bU= +github.com/AdaLogics/go-fuzz-headers v0.0.0-20230811130428-ced1acdcaa24/go.mod h1:8o94RPi1/7XTJvwPpRSzSUedZrtlirdB3r9Z20bi2f8= +github.com/Azure/azure-pipeline-go v0.2.3 h1:7U9HBg1JFK3jHl5qmo4CTZKFTVgMwdFHMVtCdfBE21U= +github.com/Azure/azure-pipeline-go v0.2.3/go.mod h1:x841ezTBIMG6O3lAcl8ATHnsOPVl2bqk7S3ta6S6u4k= +github.com/Azure/azure-sdk-for-go v68.0.0+incompatible h1:fcYLmCpyNYRnvJbPerq7U0hS+6+I79yEDJBqVNcqUzU= +github.com/Azure/azure-sdk-for-go v68.0.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.18.1 h1:Wc1ml6QlJs2BHQ/9Bqu1jiyggbsSjramq2oUmp5WeIo= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.18.1/go.mod h1:Ot/6aikWnKWi4l9QB7qVSwa8iMphQNqkWALMoNT3rzM= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.8.2 h1:F0gBpfdPLGsw+nsgk6aqqkZS1jiixa5WwFe3fk/T3Ys= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.8.2/go.mod h1:SqINnQ9lVVdRlyC8cd1lCI0SdX4n2paeABd2K8ggfnE= +github.com/Azure/azure-sdk-for-go/sdk/azidentity/cache v0.3.2 h1:yz1bePFlP5Vws5+8ez6T3HWXPmwOK7Yvq8QxDBD3SKY= +github.com/Azure/azure-sdk-for-go/sdk/azidentity/cache v0.3.2/go.mod h1:Pa9ZNPuoNu/GztvBSKk9J1cDJW6vk/n0zLtV4mgd8N8= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.0 h1:Bg8m3nq/X1DeePkAbCfb6ml6F3F0IunEhE8TMh+lY48= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.0/go.mod h1:j2chePtV91HrC22tGoRX3sGY42uF13WzmmV80/OdVAA= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v5 v5.7.0 h1:LkHbJbgF3YyvC53aqYGR+wWQDn2Rdp9AQdGndf9QvY4= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v5 v5.7.0/go.mod h1:QyiQdW4f4/BIfB8ZutZ2s+28RAgfa/pT+zS++ZHyM1I= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/internal/v2 v2.0.0 h1:PTFGRSlMKCQelWwxUyYVEUqseBJVemLyqWJjvMyt0do= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/internal/v2 v2.0.0/go.mod h1:LRr2FzBTQlONPPa5HREE5+RjSCTXl7BwOvYOaWTqCaI= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/internal/v3 v3.0.0 h1:Kb8eVvjdP6kZqYnER5w/PiGCFp91yVgaxve3d7kCEpY= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/internal/v3 v3.0.0/go.mod h1:lYq15QkJyEsNegz5EhI/0SXQ6spvGfgwBH/Qyzkoc/s= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/keyvault/armkeyvault v1.4.0 h1:HlZMUZW8S4P9oob1nCHxCCKrytxyLc+24nUJGssoEto= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/keyvault/armkeyvault v1.4.0/go.mod h1:StGsLbuJh06Bd8IBfnAlIFV3fLb+gkczONWf15hpX2E= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/managementgroups/armmanagementgroups v1.0.0 h1:pPvTJ1dY0sA35JOeFq6TsY2xj6Z85Yo23Pj4wCCvu4o= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/managementgroups/armmanagementgroups v1.0.0/go.mod h1:mLfWfj8v3jfWKsL9G4eoBoXVcsqcIUTapmdKy7uGOp0= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armfeatures v1.2.0 h1:wIDqH4WA5uJ6irRqjzodeSw6Pmp0tu3oIbwzBZEdMfQ= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armfeatures v1.2.0/go.mod h1:g8mnARUMaYRsg80mxm3PxjF7+oUotB/lneDbwYbGNxg= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources v1.2.0 h1:Dd+RhdJn0OTtVGaeDLZpcumkIVCtA/3/Fo42+eoYvVM= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources v1.2.0/go.mod h1:5kakwfW5CjC9KK+Q4wjXAg+ShuIm2mBMua0ZFj2C8PE= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.6.0 h1:PiSrjRPpkQNjrM8H0WwKMnZUdu1RGMtd/LdGKUrOo+c= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.6.0/go.mod h1:oDrbWx4ewMylP7xHivfgixbfGBT6APAwsSoHRKotnIc= +github.com/Azure/azure-storage-blob-go v0.15.0 h1:rXtgp8tN1p29GvpGgfJetavIG0V7OgcSXPpwp3tx6qk= +github.com/Azure/azure-storage-blob-go v0.15.0/go.mod h1:vbjsVbX0dlxnRc4FFMPsS9BsJWPcne7GB7onqlPvz58= +github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 h1:L/gRVlceqvL25UVaW/CKtUDjefjrs0SPonmDGUVOYP0= +github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= +github.com/Azure/go-autorest v14.2.0+incompatible h1:V5VMDjClD3GiElqLWO7mz2MxNAK/vTfRHdAubSIPRgs= +github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= +github.com/Azure/go-autorest/autorest v0.11.28/go.mod h1:MrkzG3Y3AH668QyF9KRk5neJnGgmhQ6krbhR8Q5eMvA= +github.com/Azure/go-autorest/autorest v0.11.29 h1:I4+HL/JDvErx2LjyzaVxllw2lRDB5/BT2Bm4g20iqYw= +github.com/Azure/go-autorest/autorest v0.11.29/go.mod h1:ZtEzC4Jy2JDrZLxvWs8LrBWEBycl1hbT1eknI8MtfAs= +github.com/Azure/go-autorest/autorest/adal v0.9.13/go.mod h1:W/MM4U6nLxnIskrw4UwWzlHfGjwUS50aOsc/I3yuU8M= +github.com/Azure/go-autorest/autorest/adal v0.9.18/go.mod h1:XVVeme+LZwABT8K5Lc3hA4nAe8LDBVle26gTrguhhPQ= +github.com/Azure/go-autorest/autorest/adal v0.9.22/go.mod h1:XuAbAEUv2Tta//+voMI038TrJBqjKam0me7qR+L8Cmk= +github.com/Azure/go-autorest/autorest/adal v0.9.24 h1:BHZfgGsGwdkHDyZdtQRQk1WeUdW0m2WPAwuHZwUi5i4= +github.com/Azure/go-autorest/autorest/adal v0.9.24/go.mod h1:7T1+g0PYFmACYW5LlG2fcoPiPlFHjClyRGL7dRlP5c8= +github.com/Azure/go-autorest/autorest/azure/auth v0.5.13 h1:Ov8avRZi2vmrE2JcXw+tu5K/yB41r7xK9GZDiBF7NdM= +github.com/Azure/go-autorest/autorest/azure/auth v0.5.13/go.mod h1:5BAVfWLWXihP47vYrPuBKKf4cS0bXI+KM9Qx6ETDJYo= +github.com/Azure/go-autorest/autorest/azure/cli v0.4.6 h1:w77/uPk80ZET2F+AfQExZyEWtn+0Rk/uw17m9fv5Ajc= +github.com/Azure/go-autorest/autorest/azure/cli v0.4.6/go.mod h1:piCfgPho7BiIDdEQ1+g4VmKyD5y+p/XtSNqE6Hc4QD0= +github.com/Azure/go-autorest/autorest/date v0.3.0 h1:7gUk1U5M/CQbp9WoqinNzJar+8KY+LPI6wiWrP/myHw= +github.com/Azure/go-autorest/autorest/date v0.3.0/go.mod h1:BI0uouVdmngYNUzGWeSYnokU+TrmwEsOqdt8Y6sso74= +github.com/Azure/go-autorest/autorest/mocks v0.4.1/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k= +github.com/Azure/go-autorest/autorest/mocks v0.4.2 h1:PGN4EDXnuQbojHbU0UWoNvmu9AGVwYHG9/fkDYhtAfw= +github.com/Azure/go-autorest/autorest/mocks v0.4.2/go.mod h1:Vy7OitM9Kei0i1Oj+LvyAWMXJHeKH1MVlzFugfVrmyU= +github.com/Azure/go-autorest/autorest/to v0.4.0 h1:oXVqrxakqqV1UZdSazDOPOLvOIz+XA683u8EctwboHk= +github.com/Azure/go-autorest/autorest/to v0.4.0/go.mod h1:fE8iZBn7LQR7zH/9XU2NcPR4o9jEImooCeWJcYV/zLE= +github.com/Azure/go-autorest/autorest/validation v0.3.1 h1:AgyqjAd94fwNAoTjl/WQXg4VvFeRFpO+UhNyRXqF1ac= +github.com/Azure/go-autorest/autorest/validation v0.3.1/go.mod h1:yhLgjC0Wda5DYXl6JAsWyUe4KVNffhoDhG0zVzUMo3E= +github.com/Azure/go-autorest/logger v0.2.1 h1:IG7i4p/mDa2Ce4TRyAO8IHnVhAVF3RFU+ZtXWSmf4Tg= +github.com/Azure/go-autorest/logger v0.2.1/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8= +github.com/Azure/go-autorest/tracing v0.6.0 h1:TYi4+3m5t6K48TGI9AUdb+IzbnSxvnvUMfuitfgcfuo= +github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= +github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1 h1:WJTmL004Abzc5wDB5VtZG2PJk5ndYDgVacGqfirKxjM= +github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1/go.mod h1:tCcJZ0uHAmvjsVYzEFivsRTN00oz5BEsRgQHu5JZ9WE= +github.com/AzureAD/microsoft-authentication-library-for-go v1.4.2 h1:oygO0locgZJe7PpYPXT5A29ZkwJaPqcva7BVeemZOZs= +github.com/AzureAD/microsoft-authentication-library-for-go v1.4.2/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.29.0 h1:UQUsRi8WTzhZntp5313l+CHIAT95ojUI2lpP/ExlZa4= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.29.0/go.mod h1:Cz6ft6Dkn3Et6l2v2a9/RpN7epQ1GtDlO6lj8bEcOvw= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.53.0 h1:owcC2UnmsZycprQ5RfRgjydWhuoxg71LUfyiQdijZuM= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.53.0/go.mod h1:ZPpqegjbE99EPKsu3iUWV22A04wzGPcAY/ziSIQEEgs= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.53.0 h1:4LP6hvB4I5ouTbGgWtixJhgED6xdf67twf9PoY96Tbg= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.53.0/go.mod h1:jUZ5LYlw40WMd07qxcQJD5M40aUxrfwqQX1g7zxYnrQ= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.53.0 h1:Ron4zCA/yk6U7WOBXhTJcDpsUBG9npumK6xw2auFltQ= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.53.0/go.mod h1:cSgYe11MCNYunTnRXrKiR/tHc0eoKjICUuWpNZoVCOo= +github.com/IBM-Cloud/power-go-client v1.12.0 h1:tF9Mq5GLYHebpzQT6IYB89lIxEST1E9teuchjxSAaw0= +github.com/IBM-Cloud/power-go-client v1.12.0/go.mod h1:SpTK1ttW8bfMNUVQS8qOEuWn2KOkzaCLyzfze8MG1JE= +github.com/IBM/go-sdk-core/v5 v5.21.0 h1:DUnYhvC4SoC8T84rx5omnhY3+xcQg/Whyoa3mDPIMkk= +github.com/IBM/go-sdk-core/v5 v5.21.0/go.mod h1:Q3BYO6iDA2zweQPDGbNTtqft5tDcEpm6RTuqMlPcvbw= +github.com/IBM/platform-services-go-sdk v0.81.0 h1:jyVQYobEeC+l770YmKVtETFwPpD8DoapZfKY005f/jQ= +github.com/IBM/platform-services-go-sdk v0.81.0/go.mod h1:XOowH+JnIih3FA7uilLVM/9VH7XgCmJ4T/i6eZi7gkw= +github.com/IBM/vpc-go-sdk v0.70.1 h1:6NsbRkiA5gDNxe7cjNx8Pi1j9s0PlhwNQj29wsKZxAo= +github.com/IBM/vpc-go-sdk v0.70.1/go.mod h1:K3vVlje72PYE3ZRt1iouE+jSIq+vCyYzT1HiFC06hUA= +github.com/JeffAshton/win_pdh v0.0.0-20161109143554-76bb4ee9f0ab h1:UKkYhof1njT1/xq4SEg5z+VpTgjmNeHwPGRQl7takDI= +github.com/JeffAshton/win_pdh v0.0.0-20161109143554-76bb4ee9f0ab/go.mod h1:3VYc5hodBMJ5+l/7J4xAyMeuM2PNuepvHlGs8yilUCA= +github.com/MakeNowJust/heredoc v1.0.0 h1:cXCdzVdstXyiTqTvfqk9SDHpKNjxuom+DOlyEeQ4pzQ= +github.com/MakeNowJust/heredoc v1.0.0/go.mod h1:mG5amYoWBHf8vpLOuehzbGGw0EHxpZZ6lCpQ4fNJ8LE= +github.com/Masterminds/semver/v3 v3.4.0 h1:Zog+i5UMtVoCU8oKka5P7i9q9HgrJeGzI9SA1Xbatp0= +github.com/Masterminds/semver/v3 v3.4.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM= +github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= +github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= +github.com/Microsoft/hnslib v0.1.1 h1:JsZy681SnvSOUAfCZVAxkX4LgQGp+CZZwPbLV0/pdF8= +github.com/Microsoft/hnslib v0.1.1/go.mod h1:DRQR4IjLae6WHYVhW7uqe44hmFUiNhmaWA+jwMbz5tM= +github.com/NYTimes/gziphandler v1.1.1 h1:ZUDjpQae29j0ryrS0u/B8HZfJBtBQHjqw2rQ2cqUQ3I= +github.com/NYTimes/gziphandler v1.1.1/go.mod h1:n/CVRwUEOgIxrgPvAQhUUr9oeUtvrhMomdKFjzJNB0c= +github.com/ProtonMail/go-crypto v1.1.6 h1:ZcV+Ropw6Qn0AX9brlQLAUXfqLBc7Bl+f/DmNxpLfdw= +github.com/ProtonMail/go-crypto v1.1.6/go.mod h1:rA3QumHc/FZ8pAHreoekgiAbzpNsfQAosU5td4SnOrE= +github.com/antlr4-go/antlr/v4 v4.13.1 h1:SqQKkuVZ+zWkMMNkjy5FZe5mr5WURWnlpmOuzYWrPrQ= +github.com/antlr4-go/antlr/v4 v4.13.1/go.mod h1:GKmUxMtwp6ZgGwZSva4eWPC5mS6vUAmOABFgjdkM7Nw= +github.com/apparentlymart/go-cidr v1.1.0 h1:2mAhrMoF+nhXqxTzSZMUzDHkLjmIHC+Zzn4tdgBZjnU= +github.com/apparentlymart/go-cidr v1.1.0/go.mod h1:EBcsNrHc3zQeuaeCeCtQruQm+n9/YjEn/vI25Lg7Gwc= +github.com/apparentlymart/go-textseg/v15 v15.0.0 h1:uYvfpb3DyLSCGWnctWKGj857c6ew1u1fNQOlOtuGxQY= +github.com/apparentlymart/go-textseg/v15 v15.0.0/go.mod h1:K8XmNZdhEBkdlyDdvbmmsvpAG721bKi0joRfFdHIWJ4= +github.com/armon/circbuf v0.0.0-20190214190532-5111143e8da2 h1:7Ip0wMmLHLRJdrloDxZfhMm0xrLXZS8+COSu2bXmEQs= +github.com/armon/circbuf v0.0.0-20190214190532-5111143e8da2/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= +github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= +github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= +github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 h1:DklsrG3dyBCFEj5IhUbnKptjxatkF07cF2ak3yi77so= +github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= +github.com/aws/aws-sdk-go v1.50.25 h1:vhiHtLYybv1Nhx3Kv18BBC6L0aPJHaG9aeEsr92W99c= +github.com/aws/aws-sdk-go v1.50.25/go.mod h1:LF8svs817+Nz+DmiMQKTO3ubZ/6IaTpq3TjupRn3Eqk= +github.com/aws/aws-sdk-go-v2 v1.41.0 h1:tNvqh1s+v0vFYdA1xq0aOJH+Y5cRyZ5upu6roPgPKd4= +github.com/aws/aws-sdk-go-v2 v1.41.0/go.mod h1:MayyLB8y+buD9hZqkCW3kX1AKq07Y5pXxtgB+rRFhz0= +github.com/aws/aws-sdk-go-v2/config v1.27.24 h1:NM9XicZ5o1CBU/MZaHwFtimRpWx9ohAUAqkG6AqSqPo= +github.com/aws/aws-sdk-go-v2/config v1.27.24/go.mod h1:aXzi6QJTuQRVVusAO8/NxpdTeTyr/wRcybdDtfUwJSs= +github.com/aws/aws-sdk-go-v2/credentials v1.17.24 h1:YclAsrnb1/GTQNt2nzv+756Iw4mF8AOzcDfweWwwm/M= +github.com/aws/aws-sdk-go-v2/credentials v1.17.24/go.mod h1:Hld7tmnAkoBQdTMNYZGzztzKRdA4fCdn9L83LOoigac= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.9 h1:Aznqksmd6Rfv2HQN9cpqIV/lQRMaIpJkLLaJ1ZI76no= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.9/go.mod h1:WQr3MY7AxGNxaqAtsDWn+fBxmd4XvLkzeqQ8P1VM0/w= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.16 h1:rgGwPzb82iBYSvHMHXc8h9mRoOUBZIGFgKb9qniaZZc= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.16/go.mod h1:L/UxsGeKpGoIj6DxfhOWHWQ/kGKcd4I1VncE4++IyKA= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.16 h1:1jtGzuV7c82xnqOVfx2F0xmJcOw5374L7N6juGW6x6U= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.16/go.mod h1:M2E5OQf+XLe+SZGmmpaI2yy+J326aFf6/+54PoxSANc= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 h1:hT8rVHwugYE2lEfdFE0QWVo81lF7jMrYJVDWI+f+VxU= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0/go.mod h1:8tu/lYfQfFe6IGnaOdrpVgEL2IrrDOf6/m9RQum4NkY= +github.com/aws/aws-sdk-go-v2/service/iam v1.53.1 h1:xNCUk9XN6Pa9PyzbEfzgRpvEIVlqtth402yjaWvNMu4= +github.com/aws/aws-sdk-go-v2/service/iam v1.53.1/go.mod h1:GNQZL4JRSGH6L0/SNGOtffaB1vmlToYp3KtcUIB0NhI= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.3 h1:dT3MqvGhSoaIhRseqw2I0yH81l7wiR2vjs57O51EAm8= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.3/go.mod h1:GlAeCkHwugxdHaueRr4nhPuY+WW+gR8UjlcqzPr1SPI= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.15 h1:I9zMeF107l0rJrpnHpjEiiTSCKYAIw8mALiXcPsGBiA= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.15/go.mod h1:9xWJ3Q/S6Ojusz1UIkfycgD1mGirJfLLKqq3LPT7WN8= +github.com/aws/aws-sdk-go-v2/service/sso v1.22.1 h1:p1GahKIjyMDZtiKoIn0/jAj/TkMzfzndDv5+zi2Mhgc= +github.com/aws/aws-sdk-go-v2/service/sso v1.22.1/go.mod h1:/vWdhoIoYA5hYoPZ6fm7Sv4d8701PiG5VKe8/pPJL60= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.26.2 h1:ORnrOK0C4WmYV/uYt3koHEWBLYsRDwk2Np+eEoyV4Z0= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.26.2/go.mod h1:xyFHA4zGxgYkdD73VeezHt3vSKEG9EmFnGwoKlP00u4= +github.com/aws/aws-sdk-go-v2/service/sts v1.30.1 h1:+woJ607dllHJQtsnJLi52ycuqHMwlW+Wqm2Ppsfp4nQ= +github.com/aws/aws-sdk-go-v2/service/sts v1.30.1/go.mod h1:jiNR3JqT15Dm+QWq2SRgh0x0bCNSRP2L25+CqPNpJlQ= +github.com/aws/smithy-go v1.24.0 h1:LpilSUItNPFr1eY85RYgTIg5eIEPtvFbskaFcmmIUnk= +github.com/aws/smithy-go v1.24.0/go.mod h1:LEj2LM3rBRQJxPZTB4KuzZkaZYnZPnvgIhb4pu07mx0= +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM= +github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= +github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= +github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= +github.com/cenkalti/backoff/v5 v5.0.2 h1:rIfFVxEf1QsI7E1ZHfp/B4DF/6QBAUhmgkxc0H7Zss8= +github.com/cenkalti/backoff/v5 v5.0.2/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/chai2010/gettext-go v1.0.2 h1:1Lwwip6Q2QGsAdl/ZKPCwTe9fe0CjlUbqj5bFNSjIRk= +github.com/chai2010/gettext-go v1.0.2/go.mod h1:y+wnP2cHYaVj19NZhYKAwEMH2CI1gNHeQQ+5AjwawxA= +github.com/cloudflare/circl v1.6.0 h1:cr5JKic4HI+LkINy2lg3W2jF8sHCVTBncJr5gIIq7qk= +github.com/cloudflare/circl v1.6.0/go.mod h1:uddAzsPgqdMAYatqJ0lsjX1oECcQLIlRpzZh3pJrofs= +github.com/cncf/xds/go v0.0.0-20250501225837-2ac532fd4443 h1:aQ3y1lwWyqYPiWZThqv1aFbZMiM9vblcSArJRf2Irls= +github.com/cncf/xds/go v0.0.0-20250501225837-2ac532fd4443/go.mod h1:W+zGtBO5Y1IgJhy4+A9GOqVhqLpfZi+vwmdNXUehLA8= +github.com/container-storage-interface/spec v1.9.0 h1:zKtX4STsq31Knz3gciCYCi1SXtO2HJDecIjDVboYavY= +github.com/container-storage-interface/spec v1.9.0/go.mod h1:ZfDu+3ZRyeVqxZM0Ds19MVLkN2d1XJ5MAfi1L3VjlT0= +github.com/containerd/containerd/api v1.8.0 h1:hVTNJKR8fMc/2Tiw60ZRijntNMd1U+JVMyTRdsD2bS0= +github.com/containerd/containerd/api v1.8.0/go.mod h1:dFv4lt6S20wTu/hMcP4350RL87qPWLVa/OHOwmmdnYc= +github.com/containerd/errdefs v1.0.0 h1:tg5yIfIlQIrxYtu9ajqY42W3lpS19XqdxRQeEwYG8PI= +github.com/containerd/errdefs v1.0.0/go.mod h1:+YBYIdtsnF4Iw6nWZhJcqGSg/dwvV7tyJ/kCkyJ2k+M= +github.com/containerd/errdefs/pkg v0.3.0 h1:9IKJ06FvyNlexW690DXuQNx2KA2cUJXx151Xdx3ZPPE= +github.com/containerd/errdefs/pkg v0.3.0/go.mod h1:NJw6s9HwNuRhnjJhM7pylWwMyAkmCQvQ4GpJHEqRLVk= +github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I= +github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo= +github.com/containerd/ttrpc v1.2.6 h1:zG+Kn5EZ6MUYCS1t2Hmt2J4tMVaLSFEJVOraDQwNPC4= +github.com/containerd/ttrpc v1.2.6/go.mod h1:YCXHsb32f+Sq5/72xHubdiJRQY9inL4a4ZQrAbN1q9o= +github.com/containerd/typeurl/v2 v2.2.2 h1:3jN/k2ysKuPCsln5Qv8bzR9cxal8XjkxPogJfSNO31k= +github.com/containerd/typeurl/v2 v2.2.2/go.mod h1:95ljDnPfD3bAbDJRugOiShd/DlAAsxGtUBhJxIn7SCk= +github.com/coreos/go-semver v0.3.1 h1:yi21YpKnrx1gt5R+la8n5WgS0kCrsPp33dmEyHReZr4= +github.com/coreos/go-semver v0.3.1/go.mod h1:irMmmIw/7yzSRPWryHsK7EYSg09caPQL03VsM8rvUec= +github.com/coreos/go-systemd/v22 v22.5.0 h1:RrqgGjYQKalulkV8NGVIfkXQf6YYmOyiJKk8iXXhfZs= +github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= +github.com/coreos/stream-metadata-go v0.4.9 h1:7EHsEYr0/oEJZumWc4b7+2KxD5PXy43esipOII2+JVk= +github.com/coreos/stream-metadata-go v0.4.9/go.mod h1:fMObQqQm8Ku91G04btKzEH3AsdP1mrAb986z9aaK0tE= +github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= +github.com/creack/pty v1.1.18 h1:n56/Zwd5o6whRC5PMGretI4IdRLlmBXYNjScPaBgsbY= +github.com/creack/pty v1.1.18/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4= +github.com/cyphar/filepath-securejoin v0.4.1 h1:JyxxyPEaktOD+GAnqIqTf9A8tHyAG22rowi7HkoSU1s= +github.com/cyphar/filepath-securejoin v0.4.1/go.mod h1:Sdj7gXlvMcPZsbhwhQ33GguGLDGQL7h7bg04C/+u9jI= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= +github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= +github.com/dimchansky/utfbom v1.1.1 h1:vV6w1AhK4VMnhBno/TPVCoK9U/LP0PkLCS9tbxHdi/U= +github.com/dimchansky/utfbom v1.1.1/go.mod h1:SxdoEBH5qIqFocHMyGOXVAybYJdr71b1Q/j0mACtrfE= +github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk= +github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= +github.com/docker/docker v27.3.1+incompatible h1:KttF0XoteNTicmUtBO0L2tP+J7FGRFTjaEF4k6WdhfI= +github.com/docker/docker v27.3.1+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/go-connections v0.5.0 h1:USnMq7hx7gwdVZq1L49hLXaFtUdTADjXGp+uj1Br63c= +github.com/docker/go-connections v0.5.0/go.mod h1:ov60Kzw0kKElRwhNs9UlUHAE/F9Fe6GLaXnqyDdmEXc= +github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= +github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/emicklei/go-restful/v3 v3.13.0 h1:C4Bl2xDndpU6nJ4bc1jXd+uTmYPVUwkD6bFY/oTyCes= +github.com/emicklei/go-restful/v3 v3.13.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= +github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc= +github.com/emirpasic/gods v1.18.1/go.mod h1:8tpGGwCnJ5H4r6BWwaV6OrWmMoPhUl5jm/FMNAnJvWQ= +github.com/envoyproxy/go-control-plane v0.13.4 h1:zEqyPVyku6IvWCFwux4x9RxkLOMUL+1vC9xUFv5l2/M= +github.com/envoyproxy/go-control-plane v0.13.4/go.mod h1:kDfuBlDVsSj2MjrLEtRWtHlsWIFcGyB2RMO44Dc5GZA= +github.com/envoyproxy/go-control-plane/envoy v1.32.4 h1:jb83lalDRZSpPWW2Z7Mck/8kXZ5CQAFYVjQcdVIr83A= +github.com/envoyproxy/go-control-plane/envoy v1.32.4/go.mod h1:Gzjc5k8JcJswLjAx1Zm+wSYE20UrLtt7JZMWiWQXQEw= +github.com/envoyproxy/go-control-plane/ratelimit v0.1.0 h1:/G9QYbddjL25KvtKTv3an9lx6VBE2cnb8wp1vEGNYGI= +github.com/envoyproxy/go-control-plane/ratelimit v0.1.0/go.mod h1:Wk+tMFAFbCXaJPzVVHnPgRKdUdwW/KdbRt94AzgRee4= +github.com/envoyproxy/protoc-gen-validate v1.2.1 h1:DEo3O99U8j4hBFwbJfrz9VtgcDfUKS7KJ7spH3d86P8= +github.com/envoyproxy/protoc-gen-validate v1.2.1/go.mod h1:d/C80l/jxXLdfEIhX1W2TmLfsJ31lvEjwamM4DxlWXU= +github.com/euank/go-kmsg-parser v2.0.0+incompatible h1:cHD53+PLQuuQyLZeriD1V/esuG4MuU0Pjs5y6iknohY= +github.com/euank/go-kmsg-parser v2.0.0+incompatible/go.mod h1:MhmAMZ8V4CYH4ybgdRwPr2TU5ThnS43puaKEMpja1uw= +github.com/exponent-io/jsonpath v0.0.0-20210407135951-1de76d718b3f h1:Wl78ApPPB2Wvf/TIe2xdyJxTlb6obmF18d8QdkxNDu4= +github.com/exponent-io/jsonpath v0.0.0-20210407135951-1de76d718b3f/go.mod h1:OSYXu++VVOHnXeitef/D8n/6y4QV8uLHSFXX4NeXMGc= +github.com/fatih/camelcase v1.0.0 h1:hxNvNX/xYBp0ovncs8WyWZrOrpBNub/JfaMvbURyft8= +github.com/fatih/camelcase v1.0.0/go.mod h1:yN2Sb0lFhZJUdVvtELVWefmrXpuZESvPmqwoZc+/fpc= +github.com/fatih/color v1.18.0 h1:S8gINlzdQ840/4pfAwic/ZE0djQEH3wM94VfqLTZcOM= +github.com/fatih/color v1.18.0/go.mod h1:4FelSpRwEGDpQ12mAdzqdOukCy4u8WUtOY6lkT/6HfU= +github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= +github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= +github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= +github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k= +github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= +github.com/fsouza/go-dockerclient v1.12.0 h1:S2f2crEUbBNCFiF06kR/GvioEB8EMsb3Td/bpawD+aU= +github.com/fsouza/go-dockerclient v1.12.0/go.mod h1:YWUtjg8japrqD/80L98nTtCoxQFp5B5wrSsnyeB5lFo= +github.com/fxamacker/cbor/v2 v2.9.0 h1:NpKPmjDBgUfBms6tr6JZkTHtfFGcMKsw3eGcmD/sapM= +github.com/fxamacker/cbor/v2 v2.9.0/go.mod h1:vM4b+DJCtHn+zz7h3FFp/hDAI9WNWCsZj23V5ytsSxQ= +github.com/gabriel-vasile/mimetype v1.4.9 h1:5k+WDwEsD9eTLL8Tz3L0VnmVh9QxGjRmjBvAG7U/oYY= +github.com/gabriel-vasile/mimetype v1.4.9/go.mod h1:WnSQhFKJuBlRyLiKohA/2DtIlPFAbguNaG7QCHcyGok= +github.com/gebn/bmc v0.0.0-20250519231546-bf709e03fe3c h1:EIa2nx4hmRi1tayvnbw7++VVKN/pIIirmdvSh6DGR68= +github.com/gebn/bmc v0.0.0-20250519231546-bf709e03fe3c/go.mod h1:7FfuX+OqHI+MyF1eUL5/HBoDhzHBfp6qfKMb87PHPtQ= +github.com/ghodss/yaml v1.0.1-0.20190212211648-25d852aebe32 h1:Mn26/9ZMNWSw9C9ERFA1PUxfmGpolnw2v0bKOREu5ew= +github.com/ghodss/yaml v1.0.1-0.20190212211648-25d852aebe32/go.mod h1:GIjDIg/heH5DOkXY3YJ/wNhfHsQHoXGjl8G8amsYQ1I= +github.com/gkampitakis/ciinfo v0.3.2 h1:JcuOPk8ZU7nZQjdUhctuhQofk7BGHuIy0c9Ez8BNhXs= +github.com/gkampitakis/ciinfo v0.3.2/go.mod h1:1NIwaOcFChN4fa/B0hEBdAb6npDlFL8Bwx4dfRLRqAo= +github.com/gkampitakis/go-diff v1.3.2 h1:Qyn0J9XJSDTgnsgHRdz9Zp24RaJeKMUHg2+PDZZdC4M= +github.com/gkampitakis/go-diff v1.3.2/go.mod h1:LLgOrpqleQe26cte8s36HTWcTmMEur6OPYerdAAS9tk= +github.com/gkampitakis/go-snaps v0.5.15 h1:amyJrvM1D33cPHwVrjo9jQxX8g/7E2wYdZ+01KS3zGE= +github.com/gkampitakis/go-snaps v0.5.15/go.mod h1:HNpx/9GoKisdhw9AFOBT1N7DBs9DiHo/hGheFGBZ+mc= +github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA= +github.com/go-errors/errors v1.4.2/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og= +github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 h1:+zs/tPmkDkHx3U66DAb0lQFJrpS6731Oaa12ikc+DiI= +github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376/go.mod h1:an3vInlBmSxCcxctByoQdvwPiA7DTK7jaaFDBTtu0ic= +github.com/go-git/go-billy/v5 v5.6.2 h1:6Q86EsPXMa7c3YZ3aLAQsMA0VlWmy43r6FHqa/UNbRM= +github.com/go-git/go-billy/v5 v5.6.2/go.mod h1:rcFC2rAsp/erv7CMz9GczHcuD0D32fWzH+MJAU+jaUU= +github.com/go-git/go-git/v5 v5.14.0 h1:/MD3lCrGjCen5WfEAzKg00MJJffKhC8gzS80ycmCi60= +github.com/go-git/go-git/v5 v5.14.0/go.mod h1:Z5Xhoia5PcWA3NF8vRLURn9E5FRhSl7dGj9ItW3Wk5k= +github.com/go-jose/go-jose/v4 v4.1.1 h1:JYhSgy4mXXzAdF3nUx3ygx347LRXJRrpgyU3adRmkAI= +github.com/go-jose/go-jose/v4 v4.1.1/go.mod h1:BdsZGqgdO3b6tTc6LSE56wcDbMMLuPsw5d4ZD5f94kA= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= +github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/go-logr/zapr v1.3.0 h1:XGdV8XW8zdwFiwOA2Dryh1gj2KRQyOOoNmBy4EplIcQ= +github.com/go-logr/zapr v1.3.0/go.mod h1:YKepepNBd1u/oyhd/yQmtjVXmm9uML4IXUgMOwR8/Gg= +github.com/go-openapi/analysis v0.23.0 h1:aGday7OWupfMs+LbmLZG4k0MYXIANxcuBTYUC03zFCU= +github.com/go-openapi/analysis v0.23.0/go.mod h1:9mz9ZWaSlV8TvjQHLl2mUW2PbZtemkE8yA5v22ohupo= +github.com/go-openapi/errors v0.22.1 h1:kslMRRnK7NCb/CvR1q1VWuEQCEIsBGn5GgKD9e+HYhU= +github.com/go-openapi/errors v0.22.1/go.mod h1:+n/5UdIqdVnLIJ6Q9Se8HNGUXYaY6CN8ImWzfi/Gzp0= +github.com/go-openapi/jsonpointer v0.21.2 h1:AqQaNADVwq/VnkCmQg6ogE+M3FOsKTytwges0JdwVuA= +github.com/go-openapi/jsonpointer v0.21.2/go.mod h1:50I1STOfbY1ycR8jGz8DaMeLCdXiI6aDteEdRNNzpdk= +github.com/go-openapi/jsonreference v0.21.0 h1:Rs+Y7hSXT83Jacb7kFyjn4ijOuVGSvOdF2+tg1TRrwQ= +github.com/go-openapi/jsonreference v0.21.0/go.mod h1:LmZmgsrTkVg9LG4EaHeY8cBDslNPMo06cago5JNLkm4= +github.com/go-openapi/loads v0.22.0 h1:ECPGd4jX1U6NApCGG1We+uEozOAvXvJSF4nnwHZ8Aco= +github.com/go-openapi/loads v0.22.0/go.mod h1:yLsaTCS92mnSAZX5WWoxszLj0u+Ojl+Zs5Stn1oF+rs= +github.com/go-openapi/runtime v0.28.0 h1:gpPPmWSNGo214l6n8hzdXYhPuJcGtziTOgUpvsFWGIQ= +github.com/go-openapi/runtime v0.28.0/go.mod h1:QN7OzcS+XuYmkQLw05akXk0jRH/eZ3kb18+1KwW9gyc= +github.com/go-openapi/spec v0.21.0 h1:LTVzPc3p/RzRnkQqLRndbAzjY0d0BCL72A6j3CdL9ZY= +github.com/go-openapi/spec v0.21.0/go.mod h1:78u6VdPw81XU44qEWGhtr982gJ5BWg2c0I5XwVMotYk= +github.com/go-openapi/strfmt v0.23.0 h1:nlUS6BCqcnAk0pyhi9Y+kdDVZdZMHfEKQiS4HaMgO/c= +github.com/go-openapi/strfmt v0.23.0/go.mod h1:NrtIpfKtWIygRkKVsxh7XQMDQW5HKQl6S5ik2elW+K4= +github.com/go-openapi/swag v0.23.1 h1:lpsStH0n2ittzTnbaSloVZLuB5+fvSY/+hnagBjSNZU= +github.com/go-openapi/swag v0.23.1/go.mod h1:STZs8TbRvEQQKUA+JZNAm3EWlgaOBGpyFDqQnDHMef0= +github.com/go-openapi/validate v0.24.0 h1:LdfDKwNbpB6Vn40xhTdNZAnfLECL81w+VX3BumrGD58= +github.com/go-openapi/validate v0.24.0/go.mod h1:iyeX1sEufmv3nPbBdX3ieNviWnOZaJ1+zquzJEf2BAQ= +github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s= +github.com/go-playground/assert/v2 v2.2.0/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= +github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA= +github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY= +github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY= +github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= +github.com/go-playground/validator/v10 v10.26.0 h1:SP05Nqhjcvz81uJaRfEV0YBSSSGMc/iMaVtFbr3Sw2k= +github.com/go-playground/validator/v10 v10.26.0/go.mod h1:I5QpIEbmr8On7W0TktmJAumgzX4CA1XNl4ZmDuVHKKo= +github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= +github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= +github.com/goccy/go-yaml v1.18.0 h1:8W7wMFS12Pcas7KU+VVkaiCng+kG8QiFeFwzFb+rwuw= +github.com/goccy/go-yaml v1.18.0/go.mod h1:XBurs7gK8ATbW4ZPGKgcbrY1Br56PdM69F7LkFRi1kA= +github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= +github.com/godbus/dbus/v5 v5.1.0 h1:4KLkAxT3aOY8Li4FRJe/KvhoNFFxo0m6fNuFUO8QJUk= +github.com/godbus/dbus/v5 v5.1.0/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= +github.com/gofrs/uuid v4.4.0+incompatible h1:3qXRTX8/NbyulANqlc0lchS1gqAVxRgsuW1YrTJupqA= +github.com/gofrs/uuid v4.4.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang-jwt/jwt/v4 v4.0.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg= +github.com/golang-jwt/jwt/v4 v4.2.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg= +github.com/golang-jwt/jwt/v4 v4.5.0/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= +github.com/golang-jwt/jwt/v4 v4.5.2 h1:YtQM7lnr8iZ+j5q71MGKkNw9Mn7AjHM68uc9g5fXeUI= +github.com/golang-jwt/jwt/v4 v4.5.2/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= +github.com/golang-jwt/jwt/v5 v5.2.2 h1:Rl4B7itRWVtYIHFrSNd7vhTiz9UpLdi6gZhZ3wEeDy8= +github.com/golang-jwt/jwt/v5 v5.2.2/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= +github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 h1:f+oWsMOmNPc8JmEHVZIycC7hBoQxHH9pNKQORJNozsQ= +github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8/go.mod h1:wcDNUvekVysuuOpQKo3191zZyTpiI6se1N1ULghS0sw= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/google/btree v1.1.3 h1:CVpQJjYgC4VbzxeGVHfvZrv1ctoYCAI8vbl07Fcxlyg= +github.com/google/btree v1.1.3/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4= +github.com/google/cadvisor v0.52.1 h1:sC8SZ6jio9ds+P2dk51bgbeYeufxo55n0X3tmrpA9as= +github.com/google/cadvisor v0.52.1/go.mod h1:OAhPcx1nOm5YwMh/JhpUOMKyv1YKLRtS9KgzWPndHmA= +github.com/google/cel-go v0.26.1 h1:iPbVVEdkhTX++hpe3lzSk7D3G3QSYqLGoHOcEio+UXQ= +github.com/google/cel-go v0.26.1/go.mod h1:A9O8OU9rdvrK5MQyrqfIxo1a0u4g3sF8KB6PUIaryMM= +github.com/google/gnostic-models v0.7.0 h1:qwTtogB15McXDaNqTZdzPJRHvaVJlAl+HVQnLmJEJxo= +github.com/google/gnostic-models v0.7.0/go.mod h1:whL5G0m6dmc5cPxKc5bdKdEN3UjI7OUGxBlw57miDrQ= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/gopacket v1.1.19 h1:ves8RnFZPGiFnTS0uPQStjwru6uO6h+nlr9j6fL7kF8= +github.com/google/gopacket v1.1.19/go.mod h1:iJ8V8n6KS+z2U1A8pUwu8bW5SyEMkXJB8Yo/Vo+TKTo= +github.com/google/martian/v3 v3.3.3 h1:DIhPTQrbPkgs2yJYdXU/eNACCG5DVQjySNRNlflZ9Fc= +github.com/google/martian/v3 v3.3.3/go.mod h1:iEPrYcgCF7jA9OtScMFQyAlZZ4YXTKEtJ1E6RWzmBA0= +github.com/google/pprof v0.0.0-20250403155104-27863c87afa6 h1:BHT72Gu3keYf3ZEu2J0b1vyeLSOYI8bm5wbJM/8yDe8= +github.com/google/pprof v0.0.0-20250403155104-27863c87afa6/go.mod h1:boTsfXsheKC2y+lKOCMpSfarhxDeIzfZG1jqGcPl3cA= +github.com/google/s2a-go v0.1.9 h1:LGD7gtMgezd8a/Xak7mEWL0PjoTQFvpRudN895yqKW0= +github.com/google/s2a-go v0.1.9/go.mod h1:YA0Ei2ZQL3acow2O62kdp9UlnvMmU7kA6Eutn0dXayM= +github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/enterprise-certificate-proxy v0.3.6 h1:GW/XbdyBFQ8Qe+YAmFU9uHLo7OnF5tL52HFAgMmyrf4= +github.com/googleapis/enterprise-certificate-proxy v0.3.6/go.mod h1:MkHOF77EYAE7qfSuSS9PU6g4Nt4e11cnsDUowfwewLA= +github.com/googleapis/gax-go/v2 v2.15.0 h1:SyjDc1mGgZU5LncH8gimWo9lW1DtIfPibOG81vgd/bo= +github.com/googleapis/gax-go/v2 v2.15.0/go.mod h1:zVVkkxAQHa1RQpg9z2AUCMnKhi0Qld9rcmyfL1OZhoc= +github.com/gophercloud/gophercloud v1.14.1 h1:DTCNaTVGl8/cFu58O1JwWgis9gtISAFONqpMKNg/Vpw= +github.com/gophercloud/gophercloud v1.14.1/go.mod h1:aAVqcocTSXh2vYFZ1JTvx4EQmfgzxRcNupUfxZbBNDM= +github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674 h1:JeSE6pjso5THxAzdVpqr6/geYxZytqFMBCOtn/ujyeo= +github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674/go.mod h1:r4w70xmWCQKmi1ONH4KIaBptdivuRPyosB9RmPlGEwA= +github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79 h1:+ngKgrYPPJrOjhax5N+uePQ0Fh1Z7PheYoUI/0nzkPA= +github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= +github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.0.1 h1:qnpSQwGEnkcRpTqNOIR6bJbR0gAorgP9CSALpRcKoAA= +github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.0.1/go.mod h1:lXGCsh6c22WGtjr+qGHj1otzZpV/1kwTMAqkwZsnWRU= +github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.3.0 h1:FbSCl+KggFl+Ocym490i/EyXF4lPgLoUtcSWquBM0Rs= +github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.3.0/go.mod h1:qOchhhIlmRcqk/O9uCo/puJlyo07YINaIqdZfZG3Jkc= +github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 h1:Ovs26xHkKqVztRpIrF/92BcuyuQ/YW4NSIpoGtfXNho= +github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.0 h1:+epNPbD5EqgpEMm5wrl4Hqts3jZt8+kYaqUisuuIGTk= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.0/go.mod h1:Zanoh4+gvIgluNqcfMVTJueD4wSS5hT7zTt4Mrutd90= +github.com/hashicorp/go-checkpoint v0.5.0 h1:MFYpPZCnQqQTE18jFwSII6eUQrD/oxMFp3mlgcqk5mU= +github.com/hashicorp/go-checkpoint v0.5.0/go.mod h1:7nfLNL10NsxqO4iWuW6tWW0HjZuDrwkBuEQsVcpCOgg= +github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= +github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ= +github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= +github.com/hashicorp/go-hclog v1.6.3 h1:Qr2kF+eVWjTiYmU7Y31tYlP1h0q/X3Nl3tPGdaB11/k= +github.com/hashicorp/go-hclog v1.6.3/go.mod h1:W4Qnvbt70Wk/zYJryRzDRU/4r0kIg0PVHBcfoyhpF5M= +github.com/hashicorp/go-retryablehttp v0.7.7 h1:C8hUCYzor8PIfXHa4UrZkU4VvK8o9ISHxT2Q8+VepXU= +github.com/hashicorp/go-retryablehttp v0.7.7/go.mod h1:pkQpWZeYWskR+D1tR2O5OcBFOxfA7DoAO6xtkuQnHTk= +github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go-uuid v1.0.3 h1:2gKiV6YVmrJ1i2CKKa9obLvRieoRGviZFL26PcT/Co8= +github.com/hashicorp/go-uuid v1.0.3/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go-version v1.7.0 h1:5tqGy27NaOTB8yJKUZELlFAS/LTKJkrmONwQKeRZfjY= +github.com/hashicorp/go-version v1.7.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= +github.com/hashicorp/hc-install v0.9.2 h1:v80EtNX4fCVHqzL9Lg/2xkp62bbvQMnvPQ0G+OmtO24= +github.com/hashicorp/hc-install v0.9.2/go.mod h1:XUqBQNnuT4RsxoxiM9ZaUk0NX8hi2h+Lb6/c0OZnC/I= +github.com/hashicorp/terraform-exec v0.23.0 h1:MUiBM1s0CNlRFsCLJuM5wXZrzA3MnPYEsiXmzATMW/I= +github.com/hashicorp/terraform-exec v0.23.0/go.mod h1:mA+qnx1R8eePycfwKkCRk3Wy65mwInvlpAeOwmA7vlY= +github.com/hashicorp/terraform-json v0.25.0 h1:rmNqc/CIfcWawGiwXmRuiXJKEiJu1ntGoxseG1hLhoQ= +github.com/hashicorp/terraform-json v0.25.0/go.mod h1:sMKS8fiRDX4rVlR6EJUMudg1WcanxCMoWwTLkgZP/vc= +github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= +github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A= +github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo= +github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= +github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= +github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= +github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= +github.com/jonboulle/clockwork v0.5.0 h1:Hyh9A8u51kptdkR+cqRpT1EebBwTn1oK9YfGYbdFz6I= +github.com/jonboulle/clockwork v0.5.0/go.mod h1:3mZlmanh0g2NDKO5TWZVJAfofYk64M7XN3SzBPjZF60= +github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= +github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= +github.com/joshdk/go-junit v1.0.0 h1:S86cUKIdwBHWwA6xCmFlf3RTLfVXYQfvanM5Uh+K6GE= +github.com/joshdk/go-junit v1.0.0/go.mod h1:TiiV0PqkaNfFXjEiyjWM3XXrhVyCa1K4Zfga6W52ung= +github.com/jpillora/backoff v1.0.0 h1:uvFg412JmmHBHw7iwprIxkPMI+sGQ4kzOWsMeHnm2EA= +github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/karrick/godirwalk v1.17.0 h1:b4kY7nqDdioR/6qnbHQyDvmA17u5G1cZ6J+CZXwSWoI= +github.com/karrick/godirwalk v1.17.0/go.mod h1:j4mkqPuvaLI8mp1DroR3P6ad7cyYd4c1qeJ3RV7ULlk= +github.com/kevinburke/ssh_config v1.2.0 h1:x584FjTGwHzMwvHx18PXxbBVzfnxogHaAReU4gf13a4= +github.com/kevinburke/ssh_config v1.2.0/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM= +github.com/keybase/go-keychain v0.0.0-20231219164618-57a3676c3af6 h1:IsMZxCuZqKuao2vNdfD82fjjgPLfyHLpR41Z88viRWs= +github.com/keybase/go-keychain v0.0.0-20231219164618-57a3676c3af6/go.mod h1:3VeWNIJaW+O5xpRQbPp0Ybqu1vJd/pm7s2F473HRrkw= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= +github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= +github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ= +github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI= +github.com/libopenstorage/openstorage v1.0.0 h1:GLPam7/0mpdP8ZZtKjbfcXJBTIA/T1O6CBErVEFEyIM= +github.com/libopenstorage/openstorage v1.0.0/go.mod h1:Sp1sIObHjat1BeXhfMqLZ14wnOzEhNx2YQedreMcUyc= +github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de h1:9TO3cAIGXtEhnIaL+V+BEER86oLrvS+kWobKpbJuye0= +github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de/go.mod h1:zAbeS9B/r2mtpb6U+EI2rYA5OAXxsYw6wTamcNW+zcE= +github.com/lithammer/dedent v1.1.0 h1:VNzHMVCBNG1j0fh3OrsFRkVUwStdDArbgBWoPAffktY= +github.com/lithammer/dedent v1.1.0/go.mod h1:jrXYCQtgg0nJiN+StA2KgR7w6CiQNv9Fd/Z9BP0jIOc= +github.com/mailru/easyjson v0.9.0 h1:PrnmzHw7262yW8sTBwxi1PdJA3Iw/EKBa8psRf7d9a4= +github.com/mailru/easyjson v0.9.0/go.mod h1:1+xMtQp2MRNVL/V1bOzuP3aP8VNwRW55fQUto+XFtTU= +github.com/maruel/natural v1.1.1 h1:Hja7XhhmvEFhcByqDoHz9QZbkWey+COd9xWfCfn1ioo= +github.com/maruel/natural v1.1.1/go.mod h1:v+Rfd79xlw1AgVBjbO0BEQmptqb5HvL/k9GRHB7ZKEg= +github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= +github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= +github.com/mattn/go-ieproxy v0.0.1/go.mod h1:pYabZ6IHcRpFh7vIaLfK7rdcWgFEb3SFJ6/gNWuh88E= +github.com/mattn/go-ieproxy v0.0.11 h1:MQ/5BuGSgDAHZOJe6YY80IF2UVCfGkwfo6AeD7HtHYo= +github.com/mattn/go-ieproxy v0.0.11/go.mod h1:/NsJd+kxZBmjMc5hrJCKMbP57B84rvq9BiDRbtO9AS0= +github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mfridman/tparse v0.18.0 h1:wh6dzOKaIwkUGyKgOntDW4liXSo37qg5AXbIhkMV3vE= +github.com/mfridman/tparse v0.18.0/go.mod h1:gEvqZTuCgEhPbYk/2lS3Kcxg1GmTxxU7kTC8DvP0i/A= +github.com/microsoft/kiota-abstractions-go v1.9.3 h1:cqhbqro+VynJ7kObmo7850h3WN2SbvoyhypPn8uJ1SE= +github.com/microsoft/kiota-abstractions-go v1.9.3/go.mod h1:f06pl3qSyvUHEfVNkiRpXPkafx7khZqQEb71hN/pmuU= +github.com/microsoft/kiota-authentication-azure-go v1.3.0 h1:PWH6PgtzhJjnmvR6N1CFjriwX09Kv7S5K3vL6VbPVrg= +github.com/microsoft/kiota-authentication-azure-go v1.3.0/go.mod h1:l/MPGUVvD7xfQ+MYSdZaFPv0CsLDqgSOp8mXwVgArIs= +github.com/microsoft/kiota-http-go v1.5.2 h1:xqvo4ssWwSvCJw2yuRocKFTxm3Y1iN+a4rrhuTYtBWg= +github.com/microsoft/kiota-http-go v1.5.2/go.mod h1:L+5Ri+SzwELnUcNA0cpbFKp/pBbvypLh3Cd1PR6sjx0= +github.com/microsoft/kiota-serialization-form-go v1.1.2 h1:SD6MATqNw+Dc5beILlsb/D87C36HKC/Zw7l+N9+HY2A= +github.com/microsoft/kiota-serialization-form-go v1.1.2/go.mod h1:m4tY2JT42jAZmgbqFwPy3zGDF+NPJACuyzmjNXeuHio= +github.com/microsoft/kiota-serialization-json-go v1.1.2 h1:eJrPWeQ665nbjO0gsHWJ0Bw6V/ZHHU1OfFPaYfRG39k= +github.com/microsoft/kiota-serialization-json-go v1.1.2/go.mod h1:deaGt7fjZarywyp7TOTiRsjfYiyWxwJJPQZytXwYQn8= +github.com/microsoft/kiota-serialization-multipart-go v1.1.2 h1:1pUyA1QgIeKslQwbk7/ox1TehjlCUUT3r1f8cNlkvn4= +github.com/microsoft/kiota-serialization-multipart-go v1.1.2/go.mod h1:j2K7ZyYErloDu7Kuuk993DsvfoP7LPWvAo7rfDpdPio= +github.com/microsoft/kiota-serialization-text-go v1.1.2 h1:7OfKFlzdjpPygca/+OtqafkEqCWR7+94efUFGC28cLw= +github.com/microsoft/kiota-serialization-text-go v1.1.2/go.mod h1:QNTcswkBPFY3QVBFmzfk00UMNViKQtV0AQKCrRw5ibM= +github.com/microsoftgraph/msgraph-sdk-go v1.81.0 h1:TZ+YbXGCOyRU2A5IWJLOIIKMECMyeRQBr6mcExLne80= +github.com/microsoftgraph/msgraph-sdk-go v1.81.0/go.mod h1:1V9jKcRL+Czs3u8gI2XjUn7xJCAWRKGizA7l14Bg9zQ= +github.com/microsoftgraph/msgraph-sdk-go-core v1.3.2 h1:5jCUSosTKaINzPPQXsz7wsHWwknyBmJSu8+ZWxx3kdQ= +github.com/microsoftgraph/msgraph-sdk-go-core v1.3.2/go.mod h1:iD75MK3LX8EuwjDYCmh0hkojKXK6VKME33u4daCo3cE= +github.com/mistifyio/go-zfs v2.1.2-0.20190413222219-f784269be439+incompatible h1:aKW/4cBs+yK6gpqU3K/oIwk9Q/XICqd3zOX/UFuvqmk= +github.com/mistifyio/go-zfs v2.1.2-0.20190413222219-f784269be439+incompatible/go.mod h1:8AuVvqP/mXw1px98n46wfvcGfQ4ci2FwoAjKYxuo3Z4= +github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= +github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/go-wordwrap v1.0.1 h1:TLuKupo69TCn6TQSyGxwI1EblZZEsQ0vMlAFQflz0v0= +github.com/mitchellh/go-wordwrap v1.0.1/go.mod h1:R62XHJLzvMFRBbcrT7m7WgmE1eOyTSsCt+hzestvNj0= +github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= +github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0= +github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo= +github.com/moby/patternmatcher v0.6.0 h1:GmP9lR19aU5GqSSFko+5pRqHi+Ohk1O69aFiKkVGiPk= +github.com/moby/patternmatcher v0.6.0/go.mod h1:hDPoyOpDY7OrrMDLaYoY3hf52gNCR/YOUYxkhApJIxc= +github.com/moby/spdystream v0.5.0 h1:7r0J1Si3QO/kjRitvSLVVFUjxMEb/YLj6S9FF62JBCU= +github.com/moby/spdystream v0.5.0/go.mod h1:xBAYlnt/ay+11ShkdFKNAG7LsyK/tmNBVvVOwrfMgdI= +github.com/moby/sys/mountinfo v0.7.2 h1:1shs6aH5s4o5H2zQLn796ADW1wMrIwHsyJ2v9KouLrg= +github.com/moby/sys/mountinfo v0.7.2/go.mod h1:1YOa8w8Ih7uW0wALDUgT1dTTSBrZ+HiBLGws92L2RU4= +github.com/moby/sys/sequential v0.6.0 h1:qrx7XFUd/5DxtqcoH1h438hF5TmOvzC/lspjy7zgvCU= +github.com/moby/sys/sequential v0.6.0/go.mod h1:uyv8EUTrca5PnDsdMGXhZe6CCe8U/UiTWd+lL+7b/Ko= +github.com/moby/sys/user v0.4.0 h1:jhcMKit7SA80hivmFJcbB1vqmw//wU61Zdui2eQXuMs= +github.com/moby/sys/user v0.4.0/go.mod h1:bG+tYYYJgaMtRKgEmuueC0hJEAZWwtIbZTB+85uoHjs= +github.com/moby/sys/userns v0.1.0 h1:tVLXkFOxVu9A64/yh59slHVv9ahO9UIev4JZusOLG/g= +github.com/moby/sys/userns v0.1.0/go.mod h1:IHUYgu/kao6N8YZlp9Cf444ySSvCmDlmzUcYfDHOl28= +github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0= +github.com/moby/term v0.5.0/go.mod h1:8FzsFHVUBGZdbDsJw/ot+X+d5HLUbvklYLJ9uGfcI3Y= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee h1:W5t00kpgFdJifH4BDsTlE89Zl93FEloxaWZfGcifgq8= +github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 h1:RWengNIwukTxcDr9M+97sNutRR1RKhG96O6jWumTTnw= +github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826/go.mod h1:TaXosZuwdSHYgviHp1DAtfrULt5eUgsSMsZf+YrPgl8= +github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 h1:n6/2gBQ3RWajuToeY6ZtZTIKv2v7ThUy5KKusIT0yc0= +github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00/go.mod h1:Pm3mSP3c5uWn86xMLZ5Sa7JB9GsEZySvHYXCTK4E9q4= +github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= +github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f h1:KUppIJq7/+SVif2QVs3tOP0zanoHgBEVAwHxUSIzRqU= +github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f h1:y5//uYreIhSUg3J1GEMiLbxo1LJaP8RfCpH6pymGZus= +github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= +github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= +github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= +github.com/oklog/ulid v1.3.1 h1:EGfNDEx6MqHz8B3uNV6QAib1UR2Lm97sHi3ocA6ESJ4= +github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= +github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= +github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= +github.com/onsi/gomega v1.38.3 h1:eTX+W6dobAYfFeGC2PV6RwXRu/MyT+cQguijutvkpSM= +github.com/onsi/gomega v1.38.3/go.mod h1:ZCU1pkQcXDO5Sl9/VVEGlDyp+zm0m1cmeG5TOzLgdh4= +github.com/opencontainers/cgroups v0.0.3 h1:Jc9dWh/0YLGjdy6J/9Ln8NM5BfTA4W2BY0GMozy3aDU= +github.com/opencontainers/cgroups v0.0.3/go.mod h1:s8lktyhlGUqM7OSRL5P7eAW6Wb+kWPNvt4qvVfzA5vs= +github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= +github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= +github.com/opencontainers/image-spec v1.1.1 h1:y0fUlFfIZhPF1W537XOLg0/fcx6zcHCJwooC2xJA040= +github.com/opencontainers/image-spec v1.1.1/go.mod h1:qpqAh3Dmcf36wStyyWU+kCeDgrGnAve2nCC8+7h8Q0M= +github.com/opencontainers/runc v1.2.5 h1:8KAkq3Wrem8bApgOHyhRI/8IeLXIfmZ6Qaw6DNSLnA4= +github.com/opencontainers/runc v1.2.5/go.mod h1:dOQeFo29xZKBNeRBI0B19mJtfHv68YgCTh1X+YphA+4= +github.com/opencontainers/runtime-spec v1.2.0 h1:z97+pHb3uELt/yiAWD691HNHQIF07bE7dzrbT927iTk= +github.com/opencontainers/runtime-spec v1.2.0/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= +github.com/opencontainers/selinux v1.11.1 h1:nHFvthhM0qY8/m+vfhJylliSshm8G1jJ2jDMcgULaH8= +github.com/opencontainers/selinux v1.11.1/go.mod h1:E5dMC3VPuVvVHDYmi78qvhJp8+M586T4DlDRYpFkyec= +github.com/openshift-eng/openshift-tests-extension v0.0.0-20251218142942-7ecc8801b9df h1:/KiCxPFpkZN4HErfAX5tyhn6G3ziPFbkGswHVAZKY5Q= +github.com/openshift-eng/openshift-tests-extension v0.0.0-20251218142942-7ecc8801b9df/go.mod h1:6gkP5f2HL0meusT0Aim8icAspcD1cG055xxBZ9yC68M= +github.com/openshift/api v0.0.0-20251015095338-264e80a2b6e7 h1:Ot2fbEEPmF3WlPQkyEW/bUCV38GMugH/UmZvxpWceNc= +github.com/openshift/api v0.0.0-20251015095338-264e80a2b6e7/go.mod h1:d5uzF0YN2nQQFA0jIEWzzOZ+edmo6wzlGLvx5Fhz4uY= +github.com/openshift/client-go v0.0.0-20251015124057-db0dee36e235 h1:9JBeIXmnHlpXTQPi7LPmu1jdxznBhAE7bb1K+3D8gxY= +github.com/openshift/client-go v0.0.0-20251015124057-db0dee36e235/go.mod h1:L49W6pfrZkfOE5iC1PqEkuLkXG4W0BX4w8b+L2Bv7fM= +github.com/openshift/kubernetes v1.30.1-0.20251215180958-76cb1db555ed h1:AGjAGNig2gMfHQHZdxtC4YKAeUZgbvl4qvJrJS+iu80= +github.com/openshift/kubernetes v1.30.1-0.20251215180958-76cb1db555ed/go.mod h1:HLQNQ0r0fv5kDp8jOA6KkbN0LZGzXchMvOuai95o1QM= +github.com/openshift/kubernetes/staging/src/k8s.io/api v0.0.0-20251215180958-76cb1db555ed h1:APJ/ctBnAaq+U/b6eJZFzf8/fFLLibmkh2LtzBcT+h4= +github.com/openshift/kubernetes/staging/src/k8s.io/api v0.0.0-20251215180958-76cb1db555ed/go.mod h1:sRDdfB9W3pU52PnpjJ9RuMVsg/UQ5iLNlVfbRpb250o= +github.com/openshift/kubernetes/staging/src/k8s.io/apiextensions-apiserver v0.0.0-20251215180958-76cb1db555ed h1:iZ7MwdJ195AyECWP/crpvG5PrEByAm9tntrLeGiAB1k= +github.com/openshift/kubernetes/staging/src/k8s.io/apiextensions-apiserver v0.0.0-20251215180958-76cb1db555ed/go.mod h1:ZxysqjDkqvJUamd823zDHJXKD4X19Q1HFmkLG63o9eU= +github.com/openshift/kubernetes/staging/src/k8s.io/apimachinery v0.0.0-20251215180958-76cb1db555ed h1:QYoSkjcWJOeXSzFwnqTyELs+XVs1kXi4RTtg9dYJ5sM= +github.com/openshift/kubernetes/staging/src/k8s.io/apimachinery v0.0.0-20251215180958-76cb1db555ed/go.mod h1:c6W+CrhzWKfUpUBjoSx/88x7wmaGRznQEcR6jN1H3Tg= +github.com/openshift/kubernetes/staging/src/k8s.io/apiserver v0.0.0-20251215180958-76cb1db555ed h1:yXWuI9K2NuaGPMPMQrBxt22Tf2fSKPPTTiYNZnSHqho= +github.com/openshift/kubernetes/staging/src/k8s.io/apiserver v0.0.0-20251215180958-76cb1db555ed/go.mod h1:TRSSqgXggJaDK5vtVtlQ9wEYOk32Pl+9tf0ROf3ljiM= +github.com/openshift/kubernetes/staging/src/k8s.io/cli-runtime v0.0.0-20251215180958-76cb1db555ed h1:x9VGwEpt/zOM8UpA6SMax/VCANU98gUSufW02tcdybQ= +github.com/openshift/kubernetes/staging/src/k8s.io/cli-runtime v0.0.0-20251215180958-76cb1db555ed/go.mod h1:Y/3QBingpsDLPy704y1u6CCThOBJTNxITcOAjVWZcKg= +github.com/openshift/kubernetes/staging/src/k8s.io/client-go v0.0.0-20251215180958-76cb1db555ed h1:lN+J0XiPJK+3ZUpUvmVVHmKn+2IdaKxTjU2lRCPb7MI= +github.com/openshift/kubernetes/staging/src/k8s.io/client-go v0.0.0-20251215180958-76cb1db555ed/go.mod h1:pqajivnjOqvKyXx5bPYITDe/uBLBA+Tk6f8E01CGcA4= +github.com/openshift/kubernetes/staging/src/k8s.io/cloud-provider v0.0.0-20251215180958-76cb1db555ed h1:St9lGMPLatbTbBAkjz4mH3LrBoJEtneqvqeS+xD1oxg= +github.com/openshift/kubernetes/staging/src/k8s.io/cloud-provider v0.0.0-20251215180958-76cb1db555ed/go.mod h1:46jYZR2jZ3bmcRXZPZzHfJLFD7qR44/AcZ72oiAGVsQ= +github.com/openshift/kubernetes/staging/src/k8s.io/cluster-bootstrap v0.0.0-20251215180958-76cb1db555ed h1:Ae1AJbuS/dqygKDFow06eY47r8qediub13VBhGkMghY= +github.com/openshift/kubernetes/staging/src/k8s.io/cluster-bootstrap v0.0.0-20251215180958-76cb1db555ed/go.mod h1:3jCpXJTFASgtHmv0Ax+rFB6BVpe3DOxJKe/v8wTf/iE= +github.com/openshift/kubernetes/staging/src/k8s.io/component-base v0.0.0-20251215180958-76cb1db555ed h1:F3on4zKoB05JuoZDA21xnAKw0ahqZG6mH3NrF4Bs5Qo= +github.com/openshift/kubernetes/staging/src/k8s.io/component-base v0.0.0-20251215180958-76cb1db555ed/go.mod h1:TYThr4NC8GXH90tsn+yCMH6LiXHj7pGNijDwBN6ZsG0= +github.com/openshift/kubernetes/staging/src/k8s.io/component-helpers v0.0.0-20251215180958-76cb1db555ed h1:pB+JaSeK7Zt2aVsPDLlFrbtjGnqQRykQpV96LMJd00k= +github.com/openshift/kubernetes/staging/src/k8s.io/component-helpers v0.0.0-20251215180958-76cb1db555ed/go.mod h1:/yyEEP5EdBUI2dmEyMKzS9XDXrKQBD1Q3G/UFGyBIy0= +github.com/openshift/kubernetes/staging/src/k8s.io/controller-manager v0.0.0-20251215180958-76cb1db555ed h1:6RbjM2q0y9bTrMyZQCWWQ4lbNOJCBp3YSkTwtJRw6jE= +github.com/openshift/kubernetes/staging/src/k8s.io/controller-manager v0.0.0-20251215180958-76cb1db555ed/go.mod h1:uIPPF88dUOgzUajix3EMCWGA4YChCoOo8ikkPyhwDnI= +github.com/openshift/kubernetes/staging/src/k8s.io/cri-api v0.0.0-20251215180958-76cb1db555ed h1:UoUpndJ8WGJ7WjWhncrJ8NIu/poetH3F+S5h9Q7+CF0= +github.com/openshift/kubernetes/staging/src/k8s.io/cri-api v0.0.0-20251215180958-76cb1db555ed/go.mod h1:SrD2bRkLK0Fra2C8qzzuRWciVrAkVq6qKgQZqY+psvs= +github.com/openshift/kubernetes/staging/src/k8s.io/cri-client v0.0.0-20251215180958-76cb1db555ed h1:Qqcn+0MCbk5Zu3M/4IeZ20/HBR/JHVYM/50QYweOSLI= +github.com/openshift/kubernetes/staging/src/k8s.io/cri-client v0.0.0-20251215180958-76cb1db555ed/go.mod h1:oiryEAfmSayRHtdki0nmpAjQfku0aP4Y+0NIqaqRn3E= +github.com/openshift/kubernetes/staging/src/k8s.io/csi-translation-lib v0.0.0-20251215180958-76cb1db555ed h1:xDojQBinSlSIC1t+Yalyt3pZhEZ2kb2/CAhGs3KJtuw= +github.com/openshift/kubernetes/staging/src/k8s.io/csi-translation-lib v0.0.0-20251215180958-76cb1db555ed/go.mod h1:jYAZWKz2s5rQTVDh35tpx466iVAoZO+JvuNkt8h2um4= +github.com/openshift/kubernetes/staging/src/k8s.io/dynamic-resource-allocation v0.0.0-20251215180958-76cb1db555ed h1:y2HRDvyfrxTNZd+U4RzgjStmbDG2JxQQfQTLi/yMb/A= +github.com/openshift/kubernetes/staging/src/k8s.io/dynamic-resource-allocation v0.0.0-20251215180958-76cb1db555ed/go.mod h1:DdewGEPN49xRm+9KnI5T8nFsDKjSVAfyWtLO7H6Mlsc= +github.com/openshift/kubernetes/staging/src/k8s.io/kube-aggregator v0.0.0-20251215180958-76cb1db555ed h1:KsvAGAxjGHIGZq0dwpURXwfjHPSQEecuo4XFMguighQ= +github.com/openshift/kubernetes/staging/src/k8s.io/kube-aggregator v0.0.0-20251215180958-76cb1db555ed/go.mod h1:e9QMt2iRFn39p0C6B/6qirIs9hj8p3y4DaGrdEGXuY8= +github.com/openshift/kubernetes/staging/src/k8s.io/kube-scheduler v0.0.0-20251215180958-76cb1db555ed h1:YcI+R/lfewd9ZZOkerZs+PVtW4LJLGTQQyD9LIIf8DU= +github.com/openshift/kubernetes/staging/src/k8s.io/kube-scheduler v0.0.0-20251215180958-76cb1db555ed/go.mod h1:pTSelVB5l12qziWSDxi77oc4P2t5N0dxYhhj4uxjxiM= +github.com/openshift/kubernetes/staging/src/k8s.io/kubectl v0.0.0-20251215180958-76cb1db555ed h1:e6TDlFcRxemWAAz8zymX0Qc9AL/zTgTnbwcX7Xw/LL4= +github.com/openshift/kubernetes/staging/src/k8s.io/kubectl v0.0.0-20251215180958-76cb1db555ed/go.mod h1:EU/sHfUc/w62dGZ1VmEysozxDAFvARFrIcQmHEmObaY= +github.com/openshift/kubernetes/staging/src/k8s.io/kubelet v0.0.0-20251215180958-76cb1db555ed h1:yhAlWx8PNC/RviPaa1+r6OVimpolkfy3dbAzaUcrQGI= +github.com/openshift/kubernetes/staging/src/k8s.io/kubelet v0.0.0-20251215180958-76cb1db555ed/go.mod h1:+bTwPbT5dZB8j6eKQHBZRMfNmY6MEryba1wQljr9VWw= +github.com/openshift/kubernetes/staging/src/k8s.io/mount-utils v0.0.0-20251215180958-76cb1db555ed h1:FpVjMX95CtuyoEk3JDNlmvVSjQm+erHzSSfInVv5Wtk= +github.com/openshift/kubernetes/staging/src/k8s.io/mount-utils v0.0.0-20251215180958-76cb1db555ed/go.mod h1:T7oGB72dQtHfSIJWZmeNU4Xo5QvIpzIuJ8X20TWu628= +github.com/openshift/kubernetes/staging/src/k8s.io/pod-security-admission v0.0.0-20251215180958-76cb1db555ed h1:MrHWBRSo9Pk8SHUJbSCcSvqBUhX8A04zNCohN9UB9Tg= +github.com/openshift/kubernetes/staging/src/k8s.io/pod-security-admission v0.0.0-20251215180958-76cb1db555ed/go.mod h1:yuCdx9wLndqpNhmsYZh48wtbgrqc8ql1191ke9zIOfg= +github.com/openshift/kubernetes/staging/src/k8s.io/sample-apiserver v0.0.0-20251215180958-76cb1db555ed h1:yMe9p9kn7c3wxDdPWNBd6KZEMKwrUBTDp5WQslxz89Y= +github.com/openshift/kubernetes/staging/src/k8s.io/sample-apiserver v0.0.0-20251215180958-76cb1db555ed/go.mod h1:7JLAj6I7UWR3Akqvb3hwGRBdV3dgTASNQJhMqdowC0s= +github.com/openshift/library-go v0.0.0-20251015151611-6fc7a74b67c5 h1:bANtDc8SgetSK4nQehf59x3+H9FqVJCprgjs49/OTg0= +github.com/openshift/library-go v0.0.0-20251015151611-6fc7a74b67c5/go.mod h1:OlFFws1AO51uzfc48MsStGE4SFMWlMZD0+f5a/zCtKI= +github.com/openshift/onsi-ginkgo/v2 v2.6.1-0.20251120221002-696928a6a0d7 h1:02E4Ttpu+7yCQLQxtY42JfcfHU7TBGnje6uB2ytBSdU= +github.com/openshift/onsi-ginkgo/v2 v2.6.1-0.20251120221002-696928a6a0d7/go.mod h1:ArE1D/XhNXBXCBkKOLkbsb2c81dQHCRcF5zwn/ykDRo= +github.com/openshift/origin v1.5.0-alpha.3.0.20260105230330-0eae81eb0bd3 h1:OFVFCQxPLjcOQFAmvL6p0+dnm5rDiZFqHJrIhUUOTK8= +github.com/openshift/origin v1.5.0-alpha.3.0.20260105230330-0eae81eb0bd3/go.mod h1:1+nO5IhP6ZTOMMf3TzCsLZZYM/9KfEBRT6uQxc38WXI= +github.com/opentracing/opentracing-go v1.2.0 h1:uEJPy/1a5RIPAJ0Ov+OIO8OxWu77jEv+1B0VhjKrZUs= +github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc= +github.com/pborman/uuid v1.2.0 h1:J7Q5mO4ysT1dv8hyrUGHb9+ooztCXu1D8MY8DZYsu3g= +github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= +github.com/peterbourgon/diskv v2.0.1+incompatible h1:UBdAOUP5p4RWqPBg048CAvpKN+vxiaj6gdUUzhl4XmI= +github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= +github.com/pjbgf/sha1cd v0.3.2 h1:a9wb0bp1oC2TGwStyn0Umc/IGKQnEgF0vVaZ8QF8eo4= +github.com/pjbgf/sha1cd v0.3.2/go.mod h1:zQWigSxVmsHEZow5qaLtPYxpcKMMQpa09ixqBxuCS6A= +github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ= +github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjLxUqIJNnCWiEdr3bn6IUYi15bNlnbCCU= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 h1:GFCKgmp0tecUJ0sJuv4pzYCqS9+RGSn52M3FUwPs+uo= +github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10/go.mod h1:t/avpk3KcrXxUnYOhZhMXJlSEyie6gQbtLq5NM3loB8= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_golang v1.23.0 h1:ust4zpdl9r4trLY/gSjlm07PuiBq2ynaXXlptpfy8Uc= +github.com/prometheus/client_golang v1.23.0/go.mod h1:i/o0R9ByOnHX0McrTMTyhYvKE4haaf2mW08I+jGAjEE= +github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk= +github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE= +github.com/prometheus/common v0.65.0 h1:QDwzd+G1twt//Kwj/Ww6E9FQq1iVMmODnILtW1t2VzE= +github.com/prometheus/common v0.65.0/go.mod h1:0gZns+BLRQ3V6NdaerOhMbwwRbNh9hkGINtQAsP5GS8= +github.com/prometheus/procfs v0.17.0 h1:FuLQ+05u4ZI+SS/w9+BWEM2TXiHKsUQ9TADiRH7DuK0= +github.com/prometheus/procfs v0.17.0/go.mod h1:oPQLaDAMRbA+u8H5Pbfq+dl3VDAvHxMUOVhe0wYB2zw= +github.com/redis/go-redis/v9 v9.7.0 h1:HhLSs+B6O021gwzl+locl0zEDnyNkxMtf/Z3NNBMa9E= +github.com/redis/go-redis/v9 v9.7.0/go.mod h1:f6zhXITC7JUJIlPEiBOTXxJgPLdZcA93GewI7inzyWw= +github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= +github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= +github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0= +github.com/rs/zerolog v1.34.0 h1:k43nTLIwcTVQAncfCw4KZ2VY6ukYoZaBPNOE8txlOeY= +github.com/rs/zerolog v1.34.0/go.mod h1:bJsvje4Z08ROH4Nhs5iH600c3IkWhwp44iRc54W6wYQ= +github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 h1:n661drycOFuPLCN3Uc8sB6B/s6Z4t2xvBgU1htSHuq8= +github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4= +github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= +github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/skeema/knownhosts v1.3.1 h1:X2osQ+RAjK76shCbvhHHHVl3ZlgDm8apHEHFqRjnBY8= +github.com/skeema/knownhosts v1.3.1/go.mod h1:r7KTdC8l4uxWRyK2TpQZ/1o5HaSzh06ePQNxPwTcfiY= +github.com/soheilhy/cmux v0.1.5 h1:jjzc5WVemNEDTLwv9tlmemhC73tI08BNOIGwBOo10Js= +github.com/soheilhy/cmux v0.1.5/go.mod h1:T7TcVDs9LWfQgPlPsdngu6I6QIoyIFZDDC6sNE1GqG0= +github.com/spf13/cobra v1.10.1 h1:lJeBwCfmrnXthfAupyUTzJ/J4Nc1RsHC/mSRU2dll/s= +github.com/spf13/cobra v1.10.1/go.mod h1:7SmJGaTHFVBY0jW4NXGluQoLvhqFQM+6XSKD+P4XaB0= +github.com/spf13/pflag v1.0.9 h1:9exaQaMOCwffKiiiYk6/BndUBv+iRViNW+4lEMi0PvY= +github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spiffe/go-spiffe/v2 v2.5.0 h1:N2I01KCUkv1FAjZXJMwh95KK1ZIQLYbPfhaxw8WS0hE= +github.com/spiffe/go-spiffe/v2 v2.5.0/go.mod h1:P+NxobPc6wXhVtINNtFjNWGBTreew1GBUCwT2wPmb7g= +github.com/std-uritemplate/std-uritemplate/go/v2 v2.0.3 h1:7hth9376EoQEd1hH4lAp3vnaLP2UMyxuMMghLKzDHyU= +github.com/std-uritemplate/std-uritemplate/go/v2 v2.0.3/go.mod h1:Z5KcoM0YLC7INlNhEezeIZ0TZNYf7WSNO0Lvah4DSeQ= +github.com/stoewer/go-strcase v1.3.1 h1:iS0MdW+kVTxgMoE1LAZyMiYJFKlOzLooE4MxjirtkAs= +github.com/stoewer/go-strcase v1.3.1/go.mod h1:fAH5hQ5pehh+j3nZfvwdk2RgEgQjAoM8wodgtPmh1xo= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/tecbiz-ch/nutanix-go-sdk v0.1.15 h1:ZT5I6OFGswvMceujUE10ZXPNnT5UQIW9gAX4FEFK6Ds= +github.com/tecbiz-ch/nutanix-go-sdk v0.1.15/go.mod h1:wpqz3KCR/I3t/IGzZiFOy6ZRcz1IcR0hzAVbj0UJ388= +github.com/tidwall/gjson v1.14.2/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= +github.com/tidwall/gjson v1.18.0 h1:FIDeeyB800efLX89e5a8Y0BNH+LOngJyGrIWxG2FKQY= +github.com/tidwall/gjson v1.18.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= +github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= +github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= +github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= +github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4= +github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= +github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY= +github.com/tidwall/sjson v1.2.5/go.mod h1:Fvgq9kS/6ociJEDnK0Fk1cpYF4FIW6ZF7LAe+6jwd28= +github.com/tmc/grpc-websocket-proxy v0.0.0-20220101234140-673ab2c3ae75 h1:6fotK7otjonDflCTK0BCfls4SPy3NcCVb5dqqmbRknE= +github.com/tmc/grpc-websocket-proxy v0.0.0-20220101234140-673ab2c3ae75/go.mod h1:KO6IkyS8Y3j8OdNO85qEYBsRPuteD+YciPomcXdrMnk= +github.com/vmware/govmomi v0.51.0 h1:n3RLS9aw/irTOKbiIyJzAb6rOat4YOVv/uDoRsNTSQI= +github.com/vmware/govmomi v0.51.0/go.mod h1:3ywivawGRfMP2SDCeyKqxTl2xNIHTXF0ilvp72dot5A= +github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= +github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= +github.com/xanzy/ssh-agent v0.3.3 h1:+/15pJfg/RsTxqYcX6fHqOXZwwMP+2VyYWJeWM2qQFM= +github.com/xanzy/ssh-agent v0.3.3/go.mod h1:6dzNDKs0J9rVPHPhaGCukekBHKqfl+L3KghI1Bc68Uw= +github.com/xiang90/probing v0.0.0-20221125231312-a49e3df8f510 h1:S2dVYn90KE98chqDkyE9Z4N61UnQd+KOfgp5Iu53llk= +github.com/xiang90/probing v0.0.0-20221125231312-a49e3df8f510/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= +github.com/xlab/treeprint v1.2.0 h1:HzHnuAF1plUN2zGlAFHbSQP2qJ0ZAD3XF5XD7OesXRQ= +github.com/xlab/treeprint v1.2.0/go.mod h1:gj5Gd3gPdKtR1ikdDK6fnFLdmIS0X30kTTuNd/WEJu0= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +github.com/zclconf/go-cty v1.16.2 h1:LAJSwc3v81IRBZyUVQDUdZ7hs3SYs9jv0eZJDWHD/70= +github.com/zclconf/go-cty v1.16.2/go.mod h1:VvMs5i0vgZdhYawQNq5kePSpLAoz8u1xvZgrPIxfnZE= +github.com/zeebo/errs v1.4.0 h1:XNdoD/RRMKP7HD0UhJnIzUy74ISdGGxURlYG8HSWSfM= +github.com/zeebo/errs v1.4.0/go.mod h1:sgbWHsvVuTPHcqJJGQ1WhI5KbWlHYz+2+2C/LSEtCw4= +go.etcd.io/bbolt v1.4.2 h1:IrUHp260R8c+zYx/Tm8QZr04CX+qWS5PGfPdevhdm1I= +go.etcd.io/bbolt v1.4.2/go.mod h1:Is8rSHO/b4f3XigBC0lL0+4FwAQv3HXEEIgFMuKHceM= +go.etcd.io/etcd/api/v3 v3.6.4 h1:7F6N7toCKcV72QmoUKa23yYLiiljMrT4xCeBL9BmXdo= +go.etcd.io/etcd/api/v3 v3.6.4/go.mod h1:eFhhvfR8Px1P6SEuLT600v+vrhdDTdcfMzmnxVXXSbk= +go.etcd.io/etcd/client/pkg/v3 v3.6.4 h1:9HBYrjppeOfFjBjaMTRxT3R7xT0GLK8EJMVC4xg6ok0= +go.etcd.io/etcd/client/pkg/v3 v3.6.4/go.mod h1:sbdzr2cl3HzVmxNw//PH7aLGVtY4QySjQFuaCgcRFAI= +go.etcd.io/etcd/client/v3 v3.6.4 h1:YOMrCfMhRzY8NgtzUsHl8hC2EBSnuqbR3dh84Uryl7A= +go.etcd.io/etcd/client/v3 v3.6.4/go.mod h1:jaNNHCyg2FdALyKWnd7hxZXZxZANb0+KGY+YQaEMISo= +go.etcd.io/etcd/pkg/v3 v3.6.4 h1:fy8bmXIec1Q35/jRZ0KOes8vuFxbvdN0aAFqmEfJZWA= +go.etcd.io/etcd/pkg/v3 v3.6.4/go.mod h1:kKcYWP8gHuBRcteyv6MXWSN0+bVMnfgqiHueIZnKMtE= +go.etcd.io/etcd/server/v3 v3.6.4 h1:LsCA7CzjVt+8WGrdsnh6RhC0XqCsLkBly3ve5rTxMAU= +go.etcd.io/etcd/server/v3 v3.6.4/go.mod h1:aYCL/h43yiONOv0QIR82kH/2xZ7m+IWYjzRmyQfnCAg= +go.etcd.io/raft/v3 v3.6.0 h1:5NtvbDVYpnfZWcIHgGRk9DyzkBIXOi8j+DDp1IcnUWQ= +go.etcd.io/raft/v3 v3.6.0/go.mod h1:nLvLevg6+xrVtHUmVaTcTz603gQPHfh7kUAwV6YpfGo= +go.mongodb.org/mongo-driver v1.17.3 h1:TQyXhnsWfWtgAhMtOgtYHMTkZIfBTpMTsMnd9ZBeHxQ= +go.mongodb.org/mongo-driver v1.17.3/go.mod h1:Hy04i7O2kC4RS06ZrhPRqj/u4DTYkFDAAccj+rVKqgQ= +go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= +go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= +go.opentelemetry.io/contrib/detectors/gcp v1.36.0 h1:F7q2tNlCaHY9nMKHR6XH9/qkp8FktLnIcy6jJNyOCQw= +go.opentelemetry.io/contrib/detectors/gcp v1.36.0/go.mod h1:IbBN8uAIIx734PTonTPxAxnjc2pQTxWNkwfstZ+6H2k= +go.opentelemetry.io/contrib/instrumentation/github.com/emicklei/go-restful/otelrestful v0.44.0 h1:KemlMZlVwBSEGaO91WKgp41BBFsnWqqj9sKRwmOqC40= +go.opentelemetry.io/contrib/instrumentation/github.com/emicklei/go-restful/otelrestful v0.44.0/go.mod h1:uq8DrRaen3suIWTpdR/JNHCGpurSvMv9D5Nr5CU5TXc= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.61.0 h1:q4XOmH/0opmeuJtPsbFNivyl7bCt7yRBbeEm2sC/XtQ= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.61.0/go.mod h1:snMWehoOh2wsEwnvvwtDyFCxVeDAODenXHtn5vzrKjo= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0 h1:F7Jx+6hwnZ41NSFTO5q4LYDtJRXBf2PD0rNBkeB/lus= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0/go.mod h1:UHB22Z8QsdRDrnAtX4PntOl36ajSxcdUMt1sF7Y6E7Q= +go.opentelemetry.io/contrib/propagators/b3 v1.19.0 h1:ulz44cpm6V5oAeg5Aw9HyqGFMS6XM7untlMEhD7YzzA= +go.opentelemetry.io/contrib/propagators/b3 v1.19.0/go.mod h1:OzCmE2IVS+asTI+odXQstRGVfXQ4bXv9nMBRK0nNyqQ= +go.opentelemetry.io/otel v1.37.0 h1:9zhNfelUvx0KBfu/gb+ZgeAfAgtWrfHJZcAqFC228wQ= +go.opentelemetry.io/otel v1.37.0/go.mod h1:ehE/umFRLnuLa/vSccNq9oS1ErUlkkK71gMcN34UG8I= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.36.0 h1:dNzwXjZKpMpE2JhmO+9HsPl42NIXFIFSUSSs0fiqra0= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.36.0/go.mod h1:90PoxvaEB5n6AOdZvi+yWJQoE95U8Dhhw2bSyRqnTD0= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.36.0 h1:JgtbA0xkWHnTmYk7YusopJFX6uleBmAuZ8n05NEh8nQ= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.36.0/go.mod h1:179AK5aar5R3eS9FucPy6rggvU0g52cvKId8pv4+v0c= +go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.36.0 h1:rixTyDGXFxRy1xzhKrotaHy3/KXdPhlWARrCgK+eqUY= +go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.36.0/go.mod h1:dowW6UsM9MKbJq5JTz2AMVp3/5iW5I/TStsk8S+CfHw= +go.opentelemetry.io/otel/metric v1.37.0 h1:mvwbQS5m0tbmqML4NqK+e3aDiO02vsf/WgbsdpcPoZE= +go.opentelemetry.io/otel/metric v1.37.0/go.mod h1:04wGrZurHYKOc+RKeye86GwKiTb9FKm1WHtO+4EVr2E= +go.opentelemetry.io/otel/sdk v1.37.0 h1:ItB0QUqnjesGRvNcmAcU0LyvkVyGJ2xftD29bWdDvKI= +go.opentelemetry.io/otel/sdk v1.37.0/go.mod h1:VredYzxUvuo2q3WRcDnKDjbdvmO0sCzOvVAiY+yUkAg= +go.opentelemetry.io/otel/sdk/metric v1.37.0 h1:90lI228XrB9jCMuSdA0673aubgRobVZFhbjxHHspCPc= +go.opentelemetry.io/otel/sdk/metric v1.37.0/go.mod h1:cNen4ZWfiD37l5NhS+Keb5RXVWZWpRE+9WyVCpbo5ps= +go.opentelemetry.io/otel/trace v1.37.0 h1:HLdcFNbRQBE2imdSEgm/kwqmQj1Or1l/7bW6mxVK7z4= +go.opentelemetry.io/otel/trace v1.37.0/go.mod h1:TlgrlQ+PtQO5XFerSPUYG0JSgGyryXewPGyayAWSBS0= +go.opentelemetry.io/proto/otlp v1.7.0 h1:jX1VolD6nHuFzOYso2E73H85i92Mv8JQYk0K9vz09os= +go.opentelemetry.io/proto/otlp v1.7.0/go.mod h1:fSKjH6YJ7HDlwzltzyMj036AJ3ejJLCgCSHGj4efDDo= +go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= +go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= +go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= +go.yaml.in/yaml/v2 v2.4.2 h1:DzmwEr2rDGHl7lsFgAHxmNz/1NlQ7xLIrlN2h5d1eGI= +go.yaml.in/yaml/v2 v2.4.2/go.mod h1:081UH+NErpNdqlCXm3TtEran0rJZGxAYx9hb/ELlsPU= +go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= +go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20201016220609-9e8e0b390897/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.0.0-20220829220503-c86fa9a7ed90/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= +golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= +golang.org/x/crypto v0.42.0 h1:chiH31gIWm57EkTXpwnqf8qeuMUi0yekh6mT2AvFlqI= +golang.org/x/crypto v0.42.0/go.mod h1:4+rDnOTJhQCx2q7/j6rAN5XDw8kPjeaXEUR2eL94ix8= +golang.org/x/exp v0.0.0-20250620022241-b7579e27df2b h1:M2rDM6z3Fhozi9O7NWsxAkg/yqS/lQJ6PmkyIV3YP+o= +golang.org/x/exp v0.0.0-20250620022241-b7579e27df2b/go.mod h1:3//PLf8L/X+8b4vuAfHzxeRUl04Adcb341+IGKfnqS8= +golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.27.0 h1:kb+q2PyFnEADO2IEF935ehFUXlWiNjJWtRNgBLSfbxQ= +golang.org/x/mod v0.27.0/go.mod h1:rWI627Fq0DEoudcK+MBkNkCe0EetEaDSwJJkCcjpazc= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20191112182307-2180aed22343/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210610132358-84b48f89b13b/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= +golang.org/x/net v0.43.0 h1:lat02VYK2j4aLzMzecihNvTlJNQUq316m2Mr9rnM6YE= +golang.org/x/net v0.43.0/go.mod h1:vhO1fvI4dGsIjh73sWfUVjj3N7CA9WkKJNQm2svM6Jg= +golang.org/x/oauth2 v0.30.0 h1:dnDm7JmhM45NNpd8FDDeLhK6FwqbOf4MLCM9zb1BOHI= +golang.org/x/oauth2 v0.30.0/go.mod h1:B++QgG3ZKulg6sRPGD/mqlHQs5rB3Ml9erfeDY7xKlU= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.17.0 h1:l60nONMj9l5drqw6jlhIELNv9I0A4OFgRsG9k2oT9Ug= +golang.org/x/sync v0.17.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191112214154-59a1497f0cea/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.36.0 h1:KVRy2GtZBrk1cBYA7MKu5bEZFxQk4NIDV6RLVcC8o0k= +golang.org/x/sys v0.36.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= +golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= +golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0= +golang.org/x/term v0.35.0 h1:bZBVKBudEyhRcajGcNc3jIfWPqV4y/Kt2XcoigOWtDQ= +golang.org/x/term v0.35.0/go.mod h1:TPGtkTLesOwf2DE8CgVYiZinHAOuy5AYUYT1lENIZnA= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.29.0 h1:1neNs90w9YzJ9BocxfsQNHKuAT4pkghyXc4nhZ6sJvk= +golang.org/x/text v0.29.0/go.mod h1:7MhJOA9CD2qZyOKYazxdYMF85OwPdEr9jTtBpO7ydH4= +golang.org/x/time v0.12.0 h1:ScB/8o8olJvc+CQPWrK3fPZNfh7qgwCrY0zJmoEQLSE= +golang.org/x/time v0.12.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= +golang.org/x/tools v0.36.0 h1:kWS0uv/zsvHEle1LbV5LE8QujrxB3wfQyxHfhOk0Qkg= +golang.org/x/tools v0.36.0/go.mod h1:WBDiHKJK8YgLHlcQPYQzNCkUxUypCaa5ZegCVutKm+s= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk= +gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E= +google.golang.org/api v0.247.0 h1:tSd/e0QrUlLsrwMKmkbQhYVa109qIintOls2Wh6bngc= +google.golang.org/api v0.247.0/go.mod h1:r1qZOPmxXffXg6xS5uhx16Fa/UFY8QU/K4bfKrnvovM= +google.golang.org/genproto v0.0.0-20250603155806-513f23925822 h1:rHWScKit0gvAPuOnu87KpaYtjK5zBMLcULh7gxkCXu4= +google.golang.org/genproto v0.0.0-20250603155806-513f23925822/go.mod h1:HubltRL7rMh0LfnQPkMH4NPDFEWp0jw3vixw7jEM53s= +google.golang.org/genproto/googleapis/api v0.0.0-20250818200422-3122310a409c h1:AtEkQdl5b6zsybXcbz00j1LwNodDuH6hVifIaNqk7NQ= +google.golang.org/genproto/googleapis/api v0.0.0-20250818200422-3122310a409c/go.mod h1:ea2MjsO70ssTfCjiwHgI0ZFqcw45Ksuk2ckf9G468GA= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250826171959-ef028d996bc1 h1:pmJpJEvT846VzausCQ5d7KreSROcDqmO388w5YbnltA= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250826171959-ef028d996bc1/go.mod h1:GmFNa4BdJZ2a8G+wCe9Bg3wwThLrJun751XstdJt5Og= +google.golang.org/grpc v1.75.1 h1:/ODCNEuf9VghjgO3rqLcfg8fiOP0nSluljWFlDxELLI= +google.golang.org/grpc v1.75.1/go.mod h1:JtPAzKiq4v1xcAB2hydNlWI2RnF85XXcV0mhKXr2ecQ= +google.golang.org/protobuf v1.36.8 h1:xHScyCOEuuwZEc6UtSOvPbAT4zRh0xcNRYekJwfqyMc= +google.golang.org/protobuf v1.36.8/go.mod h1:fuxRtAxBytpl4zzqUh6/eyUujkJdNiuEkXntxiD/uRU= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/evanphx/json-patch.v4 v4.13.0 h1:czT3CmqEaQ1aanPc5SdlgQrrEIb8w/wwCvWWnfEbYzo= +gopkg.in/evanphx/json-patch.v4 v4.13.0/go.mod h1:p8EYWUEYMpynmqDbY58zCKCFZw8pRWMG4EsWvDvM72M= +gopkg.in/go-jose/go-jose.v2 v2.6.3 h1:nt80fvSDlhKWQgSWyHyy5CfmlQr+asih51R8PTWNKKs= +gopkg.in/go-jose/go-jose.v2 v2.6.3/go.mod h1:zzZDPkNNw/c9IE7Z9jr11mBZQhKQTMzoEEIoEdZlFBI= +gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= +gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= +gopkg.in/natefinch/lumberjack.v2 v2.2.1 h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST95x9zc= +gopkg.in/natefinch/lumberjack.v2 v2.2.1/go.mod h1:YD8tP3GAjkrDg1eZH7EGmyESg/lsYskCTPBJVb9jqSc= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/warnings.v0 v0.1.2 h1:wFXVbFY8DY5/xOe1ECiWdKCzZlxgshcYVNkBHstARME= +gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gotest.tools/v3 v3.5.0 h1:Ljk6PdHdOhAb5aDMWXjDLMMhph+BpztA4v1QdqEW2eY= +gotest.tools/v3 v3.5.0/go.mod h1:isy3WKz7GK6uNw/sbHzfKBLvlvXwUyV06n6brMxxopU= +k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk= +k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= +k8s.io/kms v0.34.1 h1:iCFOvewDPzWM9fMTfyIPO+4MeuZ0tcZbugxLNSHFG4w= +k8s.io/kms v0.34.1/go.mod h1:s1CFkLG7w9eaTYvctOxosx88fl4spqmixnNpys0JAtM= +k8s.io/kube-openapi v0.0.0-20250814151709-d7b6acb124c3 h1:liMHz39T5dJO1aOKHLvwaCjDbf07wVh6yaUlTpunnkE= +k8s.io/kube-openapi v0.0.0-20250814151709-d7b6acb124c3/go.mod h1:UZ2yyWbFTpuhSbFhv24aGNOdoRdJZgsIObGBUaYVsts= +k8s.io/utils v0.0.0-20251002143259-bc988d571ff4 h1:SjGebBtkBqHFOli+05xYbK8YF1Dzkbzn+gDM4X9T4Ck= +k8s.io/utils v0.0.0-20251002143259-bc988d571ff4/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.33.0 h1:qPrZsv1cwQiFeieFlRqT627fVZ+tyfou/+S5S0H5ua0= +sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.33.0/go.mod h1:Ve9uj1L+deCXFrPOk1LpFXqTg7LCFzFso6PA48q/XZw= +sigs.k8s.io/gateway-api v1.4.0 h1:ZwlNM6zOHq0h3WUX2gfByPs2yAEsy/EenYJB78jpQfQ= +sigs.k8s.io/gateway-api v1.4.0/go.mod h1:AR5RSqciWP98OPckEjOjh2XJhAe2Na4LHyXD2FUY7Qk= +sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 h1:IpInykpT6ceI+QxKBbEflcR5EXP7sU1kvOlxwZh5txg= +sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730/go.mod h1:mdzfpAEoE6DHQEN0uh9ZbOCuHbLK5wOm7dK4ctXE9Tg= +sigs.k8s.io/kube-storage-version-migrator v0.0.6-0.20230721195810-5c8923c5ff96 h1:PFWFSkpArPNJxFX4ZKWAk9NSeRoZaXschn+ULa4xVek= +sigs.k8s.io/kube-storage-version-migrator v0.0.6-0.20230721195810-5c8923c5ff96/go.mod h1:EOBQyBowOUsd7U4CJnMHNE0ri+zCXyouGdLwC/jZU+I= +sigs.k8s.io/kustomize/api v0.20.1 h1:iWP1Ydh3/lmldBnH/S5RXgT98vWYMaTUL1ADcr+Sv7I= +sigs.k8s.io/kustomize/api v0.20.1/go.mod h1:t6hUFxO+Ph0VxIk1sKp1WS0dOjbPCtLJ4p8aADLwqjM= +sigs.k8s.io/kustomize/kyaml v0.20.1 h1:PCMnA2mrVbRP3NIB6v9kYCAc38uvFLVs8j/CD567A78= +sigs.k8s.io/kustomize/kyaml v0.20.1/go.mod h1:0EmkQHRUsJxY8Ug9Niig1pUMSCGHxQ5RklbpV/Ri6po= +sigs.k8s.io/randfill v1.0.0 h1:JfjMILfT8A6RbawdsK2JXGBR5AQVfd+9TbzrlneTyrU= +sigs.k8s.io/randfill v1.0.0/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY= +sigs.k8s.io/structured-merge-diff/v6 v6.3.0 h1:jTijUJbW353oVOd9oTlifJqOGEkUw2jB/fXCbTiQEco= +sigs.k8s.io/structured-merge-diff/v6 v6.3.0/go.mod h1:M3W8sfWvn2HhQDIbGWj3S099YozAsymCo/wrT5ohRUE= +sigs.k8s.io/yaml v1.6.0 h1:G8fkbMSAFqgEFgh4b1wmtzDnioxFCUgTZhlbj5P9QYs= +sigs.k8s.io/yaml v1.6.0/go.mod h1:796bPqUfzR/0jLAl6XjHl3Ck7MiyVv8dbTdyT3/pMf4= diff --git a/tests-extension/test/e2e/OWNERS b/tests-extension/test/e2e/OWNERS new file mode 100644 index 000000000..a7308c70f --- /dev/null +++ b/tests-extension/test/e2e/OWNERS @@ -0,0 +1,10 @@ +reviewers: + - lihongan + - melvinjoseph86 + - ShudiLi + - rhamini3 +approvers: + - lihongan + - melvinjoseph86 + - ShudiLi + - rhamini3 diff --git a/tests-extension/test/e2e/aws_util.go b/tests-extension/test/e2e/aws_util.go new file mode 100644 index 000000000..a20be2815 --- /dev/null +++ b/tests-extension/test/e2e/aws_util.go @@ -0,0 +1,308 @@ +package router + +import ( + "context" + "fmt" + "os" + "path/filepath" + "time" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/service/iam" + "github.com/aws/aws-sdk-go-v2/service/sts" + "github.com/aws/aws-sdk-go/aws/awserr" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/ec2" + g "github.com/onsi/ginkgo/v2" + o "github.com/onsi/gomega" + exutil "github.com/openshift/origin/test/extended/util" + compat_otp "github.com/openshift/origin/test/extended/util/compat_otp" + "github.com/openshift/router-tests-extension/test/testdata" + clusterinfra "github.com/openshift/origin/test/extended/util/compat_otp/clusterinfra" + "k8s.io/apimachinery/pkg/util/wait" + e2e "k8s.io/kubernetes/test/e2e/framework" +) + +// check if AWS Credentials exist +func checkAwsCredentials() { + envAws, present := os.LookupEnv("AWS_SHARED_CREDENTIALS_FILE") + if present { + e2e.Logf("The env AWS_SHARED_CREDENTIALS_FILE has been set: %v", envAws) + } else { + e2e.Logf("The env AWS_SHARED_CREDENTIALS_FILE is not set") + envDir, present := os.LookupEnv("CLUSTER_PROFILE_DIR") + if present { + e2e.Logf("But the env CLUSTER_PROFILE_DIR has been set: %v", envDir) + awsCredFile := filepath.Join(envDir, ".awscred") + if _, err := os.Stat(awsCredFile); err == nil { + e2e.Logf("And the .awscred file exists, export env AWS_SHARED_CREDENTIALS_FILE") + err := os.Setenv("AWS_SHARED_CREDENTIALS_FILE", awsCredFile) + o.Expect(err).NotTo(o.HaveOccurred()) + } else { + e2e.Logf("Error: %v", err) + g.Skip("Skip since .awscred file does not exist\n") + } + } else { + g.Skip("Skip since env CLUSTER_PROFILE_DIR is not set either, no valid aws credential") + } + } +} + +// new AWS STS client +func newStsClient() *sts.Client { + checkAwsCredentials() + cfg, err := config.LoadDefaultConfig(context.TODO(), + config.WithRegion("us-west-2"), + ) + if err != nil { + e2e.Logf("failed to load AWS configuration, %v", err) + } + o.Expect(err).NotTo(o.HaveOccurred()) + + return sts.NewFromConfig(cfg) +} + +// get AWS Account +func getAwsAccount(stsClient *sts.Client) string { + result, err := stsClient.GetCallerIdentity(context.TODO(), &sts.GetCallerIdentityInput{}) + if err != nil { + e2e.Logf("Couldn't get AWS caller identity. Here's why: %v\n", err) + } + o.Expect(err).NotTo(o.HaveOccurred()) + awsAccount := aws.ToString(result.Account) + e2e.Logf("The current AWS account is: %v", awsAccount) + return awsAccount +} + +// new AWS IAM client +func newIamClient() *iam.Client { + checkAwsCredentials() + sdkConfig, err := config.LoadDefaultConfig(context.TODO(), + config.WithRegion("us-west-2"), + ) + if err != nil { + e2e.Logf("failed to load AWS configuration, %v", err) + } + o.Expect(err).NotTo(o.HaveOccurred()) + + return iam.NewFromConfig(sdkConfig) +} + +// AWS IAM CreateRole (== aws iam create-role) +func iamCreateRole(iamClient *iam.Client, trustPolicy string, roleName string) string { + result, err := iamClient.CreateRole(context.TODO(), &iam.CreateRoleInput{ + AssumeRolePolicyDocument: aws.String(string(trustPolicy)), + RoleName: aws.String(roleName), + }) + if err != nil { + e2e.Logf("Couldn't create role %v. Here's why: %v\n", roleName, err) + } + o.Expect(err).NotTo(o.HaveOccurred()) + roleArn := aws.ToString(result.Role.Arn) + e2e.Logf("The created role ARN is: %v", roleArn) + return roleArn +} + +// AWS IAM PutRolePolicy (== aws iam put-role-policy) +func iamPutRolePolicy(iamClient *iam.Client, permissionPolicy string, policyName string, roleName string) { + e2e.Logf("To put/attach role policy %v to the role %v......", policyName, roleName) + _, err := iamClient.PutRolePolicy(context.TODO(), &iam.PutRolePolicyInput{ + PolicyDocument: aws.String(string(permissionPolicy)), + PolicyName: aws.String(policyName), + RoleName: aws.String(roleName), + }) + + if err != nil { + e2e.Logf("Couldn't attach policy to role %v. Here's why: %v\n", roleName, err) + } + o.Expect(err).NotTo(o.HaveOccurred()) +} + +// AWS IAM DeleteRole (== aws iam delete-role) +// Before attempting to delete a role, remove the attached items: Inline policies ( DeleteRolePolicy ) +func iamDeleteRole(iamClient *iam.Client, roleName string) { + e2e.Logf("To delete the role %v......", roleName) + _, err := iamClient.DeleteRole(context.TODO(), &iam.DeleteRoleInput{ + RoleName: aws.String(roleName), + }) + if err != nil { + e2e.Logf("Couldn't delete role %v. Here's why: %v\n", roleName, err) + } + // it is used for clear up, won't fail the case even err != nil +} + +// AWS IAM DeleteRolePolicy (== aws iam delete-role-policy) +func iamDeleteRolePolicy(iamClient *iam.Client, policyName string, roleName string) { + e2e.Logf("To delete the inline policy %v from role %v......", policyName, roleName) + _, err := iamClient.DeleteRolePolicy(context.TODO(), &iam.DeleteRolePolicyInput{ + PolicyName: aws.String(policyName), + RoleName: aws.String(roleName), + }) + + if err != nil { + e2e.Logf("Couldn't delete inline policy %v from role %v. Here's why: %v\n", policyName, roleName, err) + } + // it is used for clear up, won't fail the case even err != nil +} + +// Create ALB Operator Role and inline Policy +func createAlboRolePolicy(iamClient *iam.Client, infraID string, oidcArnPrefix string, oidcName string) string { + buildPruningBaseDir := testdata.FixturePath("router/awslb") + alboPermissionPolicyFile := filepath.Join(buildPruningBaseDir, "sts-albo-perms-policy.json") + + alboRoleName := infraID + "-albo-role" + alboPolicyName := infraID + "-albo-perms-policy" + + alboTrustPolicy := `{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Principal": { + "Federated": "%s" + }, + "Action": "sts:AssumeRoleWithWebIdentity", + "Condition": { + "StringEquals": { + "%s:sub": "system:serviceaccount:aws-load-balancer-operator:aws-load-balancer-operator-controller-manager" + } + } + } + ] +}` + oidcArn := oidcArnPrefix + oidcName + alboTrustPolicy = fmt.Sprintf(alboTrustPolicy, oidcArn, oidcName) + alboRoleArn := iamCreateRole(iamClient, alboTrustPolicy, alboRoleName) + + alboPermissionPolicy, err := os.ReadFile(alboPermissionPolicyFile) + o.Expect(err).NotTo(o.HaveOccurred()) + iamPutRolePolicy(iamClient, string(alboPermissionPolicy), alboPolicyName, alboRoleName) + + return alboRoleArn +} + +// Create ALB Controller (operand) Role and inline policy +func createAlbcRolePolicy(iamClient *iam.Client, infraID string, oidcArnPrefix string, oidcName string) string { + buildPruningBaseDir := testdata.FixturePath("router/awslb") + albcPermissionPolicyFile := filepath.Join(buildPruningBaseDir, "sts-albc-perms-policy.json") + albcRoleName := infraID + "-albc-role" + albcPolicyName := infraID + "-albc-perms-policy" + + albcTrustPolicy := `{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Principal": { + "Federated": "%s" + }, + "Action": "sts:AssumeRoleWithWebIdentity", + "Condition": { + "StringEquals": { + "%s:sub": "system:serviceaccount:aws-load-balancer-operator:aws-load-balancer-controller-cluster" + } + } + } + ] +}` + oidcArn := oidcArnPrefix + oidcName + albcTrustPolicy = fmt.Sprintf(albcTrustPolicy, oidcArn, oidcName) + albcRoleArn := iamCreateRole(iamClient, albcTrustPolicy, albcRoleName) + + albcPermissionPolicy, err := os.ReadFile(albcPermissionPolicyFile) + o.Expect(err).NotTo(o.HaveOccurred()) + iamPutRolePolicy(iamClient, string(albcPermissionPolicy), albcPolicyName, albcRoleName) + return albcRoleArn +} + +// Remove ALB Operator role on STS cluster +func deleteAlboRolePolicy(iamClient *iam.Client, infraID string) { + alboRoleName := infraID + "-albo-role" + alboPolicyName := infraID + "-albo-perms-policy" + iamDeleteRolePolicy(iamClient, alboPolicyName, alboRoleName) + iamDeleteRole(iamClient, alboRoleName) +} + +// Remove ALB Controller role on STS cluster +func deleteAlbcRolePolicy(iamClient *iam.Client, infraID string) { + albcRoleName := infraID + "-albc-role" + albcPolicyName := infraID + "-albc-perms-policy" + iamDeleteRolePolicy(iamClient, albcPolicyName, albcRoleName) + iamDeleteRole(iamClient, albcRoleName) +} + +// Prepare all roles and secrets for STS cluster +func prepareAllForStsCluster(oc *exutil.CLI) { + infraID, _ := compat_otp.GetInfraID(oc) + oidcName := getOidc(oc) + iamClient := newIamClient() + stsClient := newStsClient() + account := getAwsAccount(stsClient) + oidcArnPrefix := "arn:aws:iam::" + account + ":oidc-provider/" + + alboRoleArn := createAlboRolePolicy(iamClient, infraID, oidcArnPrefix, oidcName) + err := os.Setenv("ALBO_ROLE_ARN", alboRoleArn) + o.Expect(err).NotTo(o.HaveOccurred()) + albcRoleArn := createAlbcRolePolicy(iamClient, infraID, oidcArnPrefix, oidcName) + err = os.Setenv("ALBC_ROLE_ARN", albcRoleArn) + o.Expect(err).NotTo(o.HaveOccurred()) +} + +// Clear up all roles for STS cluster and namespace aws-load-balancer-operator +func clearUpAlbOnStsCluster(oc *exutil.CLI) { + ns := "aws-load-balancer-operator" + infraID, _ := compat_otp.GetInfraID(oc) + iamClient := newIamClient() + deleteAlboRolePolicy(iamClient, infraID) + deleteAlbcRolePolicy(iamClient, infraID) + + // delete all resources of aws-load-balancer-operator (only for STS cluster) + deleteNamespace(oc, ns) + // delete the credentialsrequest for ALB operator + oc.AsAdmin().WithoutNamespace().Run("delete").Args("credentialsrequest", "aws-load-balancer-operator", "-n", "openshift-cloud-credential-operator", "--ignore-not-found").Execute() +} + +// Create as many as Elatic IPs as number of subnets that are attached to the load balancer +func allocateElaticIP(oc *exutil.CLI, num int) []string { + var eipAllocationsList []string + // get the aws region + clusterinfra.GetAwsCredentialFromCluster(oc) + mySession := session.Must(session.NewSession()) + // Create an EC2 service client. + svc := ec2.New(mySession) + for i := 0; i < num; i++ { + // Attempt to allocate the Elastic IP address. + allocRes, err := svc.AllocateAddress(&ec2.AllocateAddressInput{Domain: aws.String("vpc")}) + o.Expect(err).NotTo(o.HaveOccurred()) + e2e.Logf("The eip allocation details for the %v element is %v", i, allocRes) + eipAllocationsList = append(eipAllocationsList, *allocRes.AllocationId) + } + return eipAllocationsList +} + +// Delete the Elastic IP +func ensureReleaseElaticIP(oc *exutil.CLI, eipList *[]string) { + var eipAllocationsList []string = *eipList + clusterinfra.GetAwsCredentialFromCluster(oc) + awsSession := session.Must(session.NewSession()) + // Create an EC2 service client. + svc := ec2.New(awsSession) + for i := range eipAllocationsList { + waitErr := wait.Poll(10*time.Second, 150*time.Second, func() (bool, error) { + // Attempt to release the Elastic IP address. + _, err := svc.ReleaseAddress(&ec2.ReleaseAddressInput{AllocationId: aws.String(eipAllocationsList[i])}) + if err != nil { + if aerr, ok := err.(awserr.Error); ok && aerr.Code() == "AuthFailure" { + e2e.Logf("Try again as EIP allocation %s is not yet released", eipAllocationsList[i]) + } else { + e2e.Logf("Error releasing EIP %s: %v, keep polling", eipAllocationsList[i], err) + } + return false, nil + } + return true, nil + }) + compat_otp.AssertWaitPollNoErr(waitErr, fmt.Sprintf("reached max time, but unable to delete the EIP %s", eipAllocationsList[i])) + e2e.Logf("EIP allocation %s is released", eipAllocationsList[i]) + } +} diff --git a/tests-extension/test/e2e/awslb-operator.go b/tests-extension/test/e2e/awslb-operator.go new file mode 100644 index 000000000..8848be557 --- /dev/null +++ b/tests-extension/test/e2e/awslb-operator.go @@ -0,0 +1,131 @@ +package router + +import ( + "path/filepath" + "strings" + + "github.com/openshift/origin/test/extended/util/compat_otp/architecture" + + g "github.com/onsi/ginkgo/v2" + o "github.com/onsi/gomega" + compat_otp "github.com/openshift/origin/test/extended/util/compat_otp" + "github.com/openshift/router-tests-extension/test/testdata" + clusterinfra "github.com/openshift/origin/test/extended/util/compat_otp/clusterinfra" + e2e "k8s.io/kubernetes/test/e2e/framework" +) + +var _ = g.Describe("[sig-network-edge] Network_Edge Component_ALBO", func() { + defer g.GinkgoRecover() + + var ( + oc = compat_otp.NewCLI("awslb", compat_otp.KubeConfigPath()) + operatorNamespace = "aws-load-balancer-operator" + operatorPodLabel = "control-plane=controller-manager" + ) + + g.BeforeEach(func() { + // skip ARM64 arch + architecture.SkipNonAmd64SingleArch(oc) + compat_otp.SkipIfPlatformTypeNot(oc, "AWS") + // skip if us-gov region + region, err := compat_otp.GetAWSClusterRegion(oc) + o.Expect(err).NotTo(o.HaveOccurred()) + if strings.Contains(region, "us-gov") { + g.Skip("Skipping for the aws cluster in us-gov region.") + } + // skip if OLM capability is disabled + compat_otp.SkipNoOLMCore(oc) + + compat_otp.By("Deploy AWS Load Balancer konflux FBC") + createAWSLoadBalancerCatalogSource(oc) + + output, _ := oc.AsAdmin().WithoutNamespace().Run("get").Args("-n", operatorNamespace, "pod", "-l", operatorPodLabel).Output() + if !strings.Contains(output, "Running") { + createAWSLoadBalancerOperator(oc) + } + }) + + g.AfterEach(func() { + if compat_otp.IsSTSCluster(oc) { + e2e.Logf("This is STS cluster so clear up AWS IAM resources as well as albo namespace") + clearUpAlbOnStsCluster(oc) + } + }) + + // author: hongli@redhat.com + g.It("Author:hongli-ROSA-OSD_CCS-ConnectedOnly-LEVEL0-High-51189-Install aws-load-balancer-operator and controller [Serial]", func() { + var ( + buildPruningBaseDir = testdata.FixturePath("router/awslb") + AWSLBController = filepath.Join(buildPruningBaseDir, "awslbcontroller.yaml") + operandCRName = "cluster" + operandPodLabel = "app.kubernetes.io/name=aws-load-balancer-operator" + ) + + compat_otp.By("Ensure the operartor pod is ready") + ensurePodWithLabelReady(oc, operatorNamespace, operatorPodLabel) + + compat_otp.By("Create CR AWSLoadBalancerController") + defer oc.AsAdmin().WithoutNamespace().Run("delete").Args("awsloadbalancercontroller", operandCRName).Output() + _, err := oc.AsAdmin().WithoutNamespace().Run("create").Args("-f", AWSLBController).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + if compat_otp.IsSTSCluster(oc) { + patchAlbControllerWithRoleArn(oc, operatorNamespace) + } + ensurePodWithLabelReady(oc, operatorNamespace, operandPodLabel) + }) + + // author: hongli@redhat.com + g.It("Author:hongli-ROSA-OSD_CCS-ConnectedOnly-Medium-51191-Provision ALB by creating an ingress [Serial]", func() { + var ( + buildPruningBaseDir = testdata.FixturePath("router/awslb") + AWSLBController = filepath.Join(buildPruningBaseDir, "awslbcontroller.yaml") + podsvc = filepath.Join(buildPruningBaseDir, "podsvc.yaml") + ingress = filepath.Join(buildPruningBaseDir, "ingress-test.yaml") + operandCRName = "cluster" + operandPodLabel = "app.kubernetes.io/name=aws-load-balancer-operator" + ) + + compat_otp.By("Ensure the operartor pod is ready") + ensurePodWithLabelReady(oc, operatorNamespace, operatorPodLabel) + + compat_otp.By("Create CR AWSLoadBalancerController") + defer oc.AsAdmin().WithoutNamespace().Run("delete").Args("awsloadbalancercontroller", operandCRName).Output() + _, err := oc.AsAdmin().WithoutNamespace().Run("create").Args("-f", AWSLBController).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + if compat_otp.IsSTSCluster(oc) { + patchAlbControllerWithRoleArn(oc, operatorNamespace) + } + ensurePodWithLabelReady(oc, operatorNamespace, operandPodLabel) + + compat_otp.By("Create user project, pod and NodePort service") + oc.SetupProject() + createResourceFromFile(oc, oc.Namespace(), podsvc) + ensurePodWithLabelReady(oc, oc.Namespace(), "name=web-server") + + compat_otp.By("create ingress with alb annotation in the project and ensure the alb is provsioned") + // need to ensure the ingress is deleted before deleting the CR AWSLoadBalancerController + // otherwise the lb resources cannot be cleared + defer oc.AsAdmin().WithoutNamespace().Run("delete").Args("ingress/ingress-test", "-n", oc.Namespace()).Output() + createResourceFromFile(oc, oc.Namespace(), ingress) + // if outpost cluster we need to add annotation to ingress + if clusterinfra.IsAwsOutpostCluster(oc) { + annotation := "alb.ingress.kubernetes.io/subnets=" + getOutpostSubnetId(oc) + _, err := oc.AsAdmin().WithoutNamespace().Run("annotate").Args("ingress", "ingress-test", annotation, "-n", oc.Namespace()).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + } + output, err := oc.Run("get").Args("ingress").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(output).To(o.ContainSubstring("ingress-test")) + // ALB provision relies on the number of subnets (should >=2) + internalSubnets, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("awsloadbalancercontroller/cluster", "-ojsonpath={.status.subnets.internal}").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + e2e.Logf("The discovered subnets are: %v", internalSubnets) + if len(internalSubnets) > 1 { + waitForLoadBalancerProvision(oc, oc.Namespace(), "ingress-test") + } else { + output, err := oc.Run("describe").Args("ingress", "ingress-test").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + e2e.Logf("Number of subnets doesn't meet the requirement, see event of ingress: %v", output) + } + }) +}) diff --git a/tests-extension/test/e2e/cloud_load_balancer.go b/tests-extension/test/e2e/cloud_load_balancer.go new file mode 100644 index 000000000..5057a31e6 --- /dev/null +++ b/tests-extension/test/e2e/cloud_load_balancer.go @@ -0,0 +1,518 @@ +package router + +import ( + "fmt" + "os/exec" + "path/filepath" + "regexp" + "strings" + "time" + + g "github.com/onsi/ginkgo/v2" + o "github.com/onsi/gomega" + compat_otp "github.com/openshift/origin/test/extended/util/compat_otp" + "github.com/openshift/router-tests-extension/test/testdata" + "k8s.io/apimachinery/pkg/util/wait" + e2e "k8s.io/kubernetes/test/e2e/framework" +) + +var _ = g.Describe("[sig-network-edge] Network_Edge Component_Router", func() { + defer g.GinkgoRecover() + + var oc = compat_otp.NewCLI("load-balancer", compat_otp.KubeConfigPath()) + + // incorporate OCP-21599 and 29204 into one + // OCP-21599:NetworkEdge ingresscontroller can set proper endpointPublishingStrategy in cloud platform + // OCP-29204:NetworkEdge ingresscontroller can set proper endpointPublishingStrategy in non-cloud platform + // author: hongli@redhat.com + g.It("Author:hongli-ROSA-OSD_CCS-ARO-Critical-21599-ingresscontroller can set proper endpointPublishingStrategy in all platforms", func() { + compat_otp.By("Get the platform type and check the endpointPublishingStrategy type") + platformtype := compat_otp.CheckPlatform(oc) + platforms := map[string]bool{ + "aws": true, + "azure": true, + "gcp": true, + "alibabacloud": true, + "ibmcloud": true, + "powervs": true, + } + + output, _ := oc.AsAdmin().WithoutNamespace().Run("get").Args("-n", "openshift-ingress-operator", "ingresscontroller/default", "-o=jsonpath={.status.endpointPublishingStrategy.type}").Output() + if platforms[platformtype] { + o.Expect(output).To(o.ContainSubstring("LoadBalancerService")) + } else { + o.Expect(output).To(o.ContainSubstring("HostNetwork")) + } + }) + + // incorporate OCP-24504 and 36891 into one case + // OCP-24504:NetworkEdge the load balancer scope can be set to Internal when creating ingresscontroller + // OCP-36891:NetworkEdge ingress operator supports mutating load balancer scope + // author: hongli@redhat.com + g.It("Author:hongli-ROSA-OSD_CCS-ARO-Critical-36891-ingress operator supports mutating load balancer scope", func() { + // skip on non-cloud platform + // ibmcloud/powervs has bug https://issues.redhat.com/browse/OCPBUGS-32776 + platformtype := compat_otp.CheckPlatform(oc) + platforms := map[string]bool{ + "aws": true, + "azure": true, + "gcp": true, + "alibabacloud": true, + } + if !platforms[platformtype] { + g.Skip("Skip for non-cloud platforms and ibmcloud/powervs due to OCPBUGS-32776") + } + // skip if private cluster in 4.19+ + if isInternalLBScopeInDefaultIngresscontroller(oc) { + g.Skip("Skip for private cluster since Internal LB scope in default ingresscontroller") + } + + buildPruningBaseDir := testdata.FixturePath("router") + customTemp := filepath.Join(buildPruningBaseDir, "ingresscontroller-external.yaml") + var ( + ingctrl = ingressControllerDescription{ + name: "ocp36891", + namespace: "openshift-ingress-operator", + domain: "", + template: customTemp, + } + ns = "openshift-ingress" + dnsRecordName = ingctrl.name + "-wildcard" + ) + + compat_otp.By("Create custom ingresscontroller with Internal scope") + baseDomain := getBaseDomain(oc) + ingctrl.domain = ingctrl.name + "." + baseDomain + // Updating LB scope `External` to `Internal` in the yaml file + sedCmd := fmt.Sprintf(`sed -i'' -e 's|External|Internal|g' %s`, customTemp) + _, err := exec.Command("bash", "-c", sedCmd).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + defer ingctrl.delete(oc) + ingctrl.create(oc) + err = waitForCustomIngressControllerAvailable(oc, ingctrl.name) + // check the LB service event if any error before exit + if err != nil { + output, _ := oc.AsAdmin().WithoutNamespace().Run("describe").Args("-n", ns, "service", "router-"+ingctrl.name).Output() + e2e.Logf("The output of describe LB service: %v", output) + } + compat_otp.AssertWaitPollNoErr(err, fmt.Sprintf("ingresscontroller %s conditions not available", ingctrl.name)) + + compat_otp.By("Get the Interanl LB ingress ip or hostname") + // AWS, IBMCloud use hostname, other cloud platforms use ip + internalLB, _ := oc.AsAdmin().WithoutNamespace().Run("get").Args("-n", ns, "service", "router-"+ingctrl.name, "-o=jsonpath={.status.loadBalancer.ingress}").Output() + e2e.Logf("the internal LB is %v", internalLB) + if platformtype == "aws" { + o.Expect(internalLB).To(o.MatchRegexp(`"hostname":.*elb.*amazonaws.com`)) + } else { + o.Expect(internalLB).To(o.MatchRegexp(`"ip":"10\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}"`)) + } + + compat_otp.By("Updating scope from Internal to External") + patchScope := `{"spec":{"endpointPublishingStrategy":{"loadBalancer":{"scope":"External"}}}}` + patchResourceAsAdmin(oc, ingctrl.namespace, "ingresscontroller/"+ingctrl.name, patchScope) + // AWS needs user to delete the LoadBalancer service manually + if platformtype == "aws" { + waitForOutputContains(oc, "default", "co/ingress", `{.status.conditions[?(@.type == "Progressing")].message}`, "To effectuate this change, you must delete the service") + oc.AsAdmin().WithoutNamespace().Run("delete").Args("-n", ns, "service", "router-"+ingctrl.name).Execute() + } + waitForOutputEquals(oc, "openshift-ingress-operator", "dnsrecords/"+dnsRecordName, "{.metadata.generation}", "2") + + compat_otp.By("Ensure the ingress LB is updated and the IP is not private") + externalLB, _ := oc.AsAdmin().WithoutNamespace().Run("get").Args("-n", ns, "service", "router-"+ingctrl.name, "-o=jsonpath={.status.loadBalancer.ingress}").Output() + e2e.Logf("the external LB is %v", externalLB) + if platformtype == "aws" { + o.Expect(externalLB).To(o.MatchRegexp(`"hostname":.*elb.*amazonaws.com`)) + } else { + o.Expect(externalLB).NotTo(o.MatchRegexp(`"ip":"10\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}"`)) + o.Expect(externalLB).To(o.MatchRegexp(`"ip":"[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}"`)) + } + + compat_otp.By("Ensure the dnsrecord with new LB IP/hostname are published") + publishStatus, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("-n", "openshift-ingress-operator", "dnsrecord", dnsRecordName, `-o=jsonpath={.status.zones[*].conditions[?(@.type == "Published")].status})`).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(publishStatus).NotTo(o.ContainSubstring("False")) + }) + + // author: hongli@redhat.com + g.It("Author:hongli-ROSA-OSD_CCS-High-52837-switching of AWS CLB to NLB without deletion of ingresscontroller", func() { + // skip if platform is not AWS + compat_otp.SkipIfPlatformTypeNot(oc, "AWS") + // skip if private cluster in 4.19+ + if isInternalLBScopeInDefaultIngresscontroller(oc) { + g.Skip("Skip for private cluster since Internal LB scope in default ingresscontroller") + } + + buildPruningBaseDir := testdata.FixturePath("router") + customTemp := filepath.Join(buildPruningBaseDir, "ingresscontroller-clb.yaml") + ns := "openshift-ingress" + var ( + ingctrl = ingressControllerDescription{ + name: "ocp52837", + namespace: "openshift-ingress-operator", + domain: "", + template: customTemp, + } + ) + + compat_otp.By("Create one custom ingresscontroller") + baseDomain := getBaseDomain(oc) + ingctrl.domain = ingctrl.name + "." + baseDomain + defer ingctrl.delete(oc) + ingctrl.create(oc) + ensureCustomIngressControllerAvailable(oc, ingctrl.name) + + compat_otp.By("patch the existing custom ingress controller with NLB") + patchResourceAsAdmin(oc, ingctrl.namespace, "ingresscontroller/ocp52837", "{\"spec\":{\"endpointPublishingStrategy\":{\"loadBalancer\":{\"providerParameters\":{\"aws\":{\"type\":\"NLB\"}}}}}}") + // the LB svc keep the same but just annotation is updated + waitForOutputContains(oc, ns, "service/router-"+ingctrl.name, `{.metadata.annotations}`, "aws-load-balancer-type") + + compat_otp.By("check the LB service and ensure the annotations are updated") + findAnnotation := getAnnotation(oc, ns, "service", "router-"+ingctrl.name) + e2e.Logf("all annotations are: %v", findAnnotation) + o.Expect(findAnnotation).To(o.ContainSubstring(`"service.beta.kubernetes.io/aws-load-balancer-type":"nlb"`)) + o.Expect(findAnnotation).NotTo(o.ContainSubstring("aws-load-balancer-proxy-protocol")) + + compat_otp.By("patch the existing custom ingress controller with CLB") + patchResourceAsAdmin(oc, ingctrl.namespace, "ingresscontroller/ocp52837", "{\"spec\":{\"endpointPublishingStrategy\":{\"loadBalancer\":{\"providerParameters\":{\"aws\":{\"type\":\"Classic\"}}}}}}") + waitForOutputContains(oc, ns, "service/router-"+ingctrl.name, `{.metadata.annotations}`, "aws-load-balancer-proxy-protocol") + + // Classic LB doesn't has explicit "classic" annotation but it needs proxy-protocol annotation + // so we use "aws-load-balancer-proxy-protocol" to check if using CLB + compat_otp.By("check the LB service and ensure the annotations are updated") + findAnnotation = getAnnotation(oc, ns, "service", "router-"+ingctrl.name) + e2e.Logf("all annotations are: %v", findAnnotation) + o.Expect(findAnnotation).To(o.ContainSubstring("aws-load-balancer-proxy-protocol")) + o.Expect(findAnnotation).NotTo(o.ContainSubstring(`"service.beta.kubernetes.io/aws-load-balancer-type":"nlb"`)) + }) + + // author: hongli@redhat.com + g.It("Author:hongli-High-72126-Support multiple cidr blocks for one NSG rule in the IngressController", func() { + g.By("Pre-flight check for the platform type") + compat_otp.SkipIfPlatformTypeNot(oc, "Azure") + + buildPruningBaseDir := testdata.FixturePath("router") + customTemp := filepath.Join(buildPruningBaseDir, "ingresscontroller-azure-cidr.yaml") + ns := "openshift-ingress" + var ( + ingctrl = ingressControllerDescription{ + name: "ocp72126", + namespace: "openshift-ingress-operator", + domain: "", + template: customTemp, + } + ) + + compat_otp.By("1. Create the custom ingresscontroller with 3995 CIDRs, by default 2 CIDRs are occupied on non private cluster, and 3 more are occupied on profile with windows node") + baseDomain := getBaseDomain(oc) + ingctrl.domain = ingctrl.name + "." + baseDomain + defer ingctrl.delete(oc) + ingctrl.create(oc) + err := waitForCustomIngressControllerAvailable(oc, ingctrl.name) + // check the LB service event if any error before exit + if err != nil { + output, _ := oc.AsAdmin().WithoutNamespace().Run("describe").Args("-n", ns, "service", "router-"+ingctrl.name).Output() + e2e.Logf("The output of describe LB service: %v", output) + } + compat_otp.AssertWaitPollNoErr(err, fmt.Sprintf("ingresscontroller %s conditions not available", ingctrl.name)) + + compat_otp.By("2. Check the LB service event to ensure no exceeds maximum number error") + output1, _ := oc.AsAdmin().WithoutNamespace().Run("describe").Args("-n", ns, "service", "router-"+ingctrl.name).Output() + o.Expect(output1).To(o.ContainSubstring(`Ensured load balancer`)) + o.Expect(output1).NotTo(o.ContainSubstring(`exceeds the maximum number of source IP addresses`)) + + compat_otp.By("3. Patch the custom ingress controller and add 6 more IPs to allowedSourceRanges") + jsonPatch := `[{"op":"add", "path": "/spec/endpointPublishingStrategy/loadBalancer/allowedSourceRanges/-", "value":"1.1.32.118/32"},{"op":"add", "path": "/spec/endpointPublishingStrategy/loadBalancer/allowedSourceRanges/-", "value":"1.1.32.120/32"},{"op":"add", "path": "/spec/endpointPublishingStrategy/loadBalancer/allowedSourceRanges/-", "value":"1.1.32.122/32"},{"op":"add", "path": "/spec/endpointPublishingStrategy/loadBalancer/allowedSourceRanges/-", "value":"1.1.32.124/32"},{"op":"add", "path": "/spec/endpointPublishingStrategy/loadBalancer/allowedSourceRanges/-", "value":"1.1.32.126/32"},{"op":"add", "path": "/spec/endpointPublishingStrategy/loadBalancer/allowedSourceRanges/-", "value":"1.1.32.128/32"}]` + _, err = oc.AsAdmin().WithoutNamespace().Run("patch").Args("-n", ingctrl.namespace, "ingresscontroller", ingctrl.name, "--type=json", "-p", jsonPatch).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + + compat_otp.By("4. Check the error message in event of the Load Balancer service") + // on some profiles it needs more than 6 seconds until the message appears + expectedMessage := `exceeds the maximum number of source IP addresses \(400[1-9] > 4000\)` + err = wait.PollImmediate(3*time.Second, 30*time.Second, func() (bool, error) { + output2, _ := oc.AsAdmin().WithoutNamespace().Run("describe").Args("-n", ns, "service", "router-"+ingctrl.name).Output() + if matched, _ := regexp.MatchString(expectedMessage, output2); matched { + return true, nil + } + return false, nil + }) + compat_otp.AssertWaitPollNoErr(err, fmt.Sprintf("reached max time allowed but the error message doesn't appear", err)) + }) + + // author: hongli@redhat.com + g.It("Author:hongli-NonHyperShiftHOST-ROSA-OSD_CCS-High-75439-AWS CLB supports to choose subnets", func() { + g.By("Pre-flight check for the platform type") + compat_otp.SkipIfPlatformTypeNot(oc, "AWS") + + buildPruningBaseDir := testdata.FixturePath("router") + customTemp := filepath.Join(buildPruningBaseDir, "ingresscontroller-clb.yaml") + ns := "openshift-ingress" + var ( + ingctrl = ingressControllerDescription{ + name: "ocp75439", + namespace: "openshift-ingress-operator", + domain: "", + template: customTemp, + } + ) + + compat_otp.By("1. Get public subnets and skip conditionally") + publicSubnetList := getPublicSubnetList(oc) + if len(publicSubnetList) < 1 { + g.Skip("Skipping since no public subnet found") + } + + compat_otp.By("2. Create the custom ingresscontroller with CLB") + baseDomain := getBaseDomain(oc) + ingctrl.domain = ingctrl.name + "." + baseDomain + defer ingctrl.delete(oc) + ingctrl.create(oc) + ensureCustomIngressControllerAvailable(oc, ingctrl.name) + + compat_otp.By("3. Patch the custom ingresscontroller, then delete the LB svc manually") + jsonPatch := fmt.Sprintf(`{"spec":{"endpointPublishingStrategy":{"type":"LoadBalancerService","loadBalancer":{"providerParameters":{"type":"AWS","aws":{"type":"Classic","classicLoadBalancer":{"subnets":{"ids":null,"names":[%s]}}}}}}}}`, strings.Join(publicSubnetList, ",")) + patchResourceAsAdmin(oc, ingctrl.namespace, "ingresscontroller/"+ingctrl.name, jsonPatch) + waitForOutputContains(oc, ingctrl.namespace, "ingresscontroller/"+ingctrl.name, `{.status.conditions[?(@.type == "Progressing")].message}`, "To effectuate this change, you must delete the service") + oc.AsAdmin().WithoutNamespace().Run("delete").Args("-n", ns, "service", "router-"+ingctrl.name).Execute() + ensureCustomIngressControllerAvailable(oc, ingctrl.name) + + compat_otp.By("4. Ensure the ingress LB svc is provisioned and the subnets annotation is added") + externalLB := getByJsonPath(oc, ns, "service/router-"+ingctrl.name, "{.status.loadBalancer.ingress}") + o.Expect(externalLB).To(o.MatchRegexp(`"hostname":.*elb.*amazonaws.com`)) + findAnnotation := getAnnotation(oc, ns, "service", "router-"+ingctrl.name) + e2e.Logf("all annotations are: %v", findAnnotation) + o.Expect(findAnnotation).To(o.ContainSubstring("service.beta.kubernetes.io/aws-load-balancer-subnets\":\"" + strings.Replace(strings.Join(publicSubnetList, ","), "\"", "", -1))) + }) + + // author: hongli@redhat.com + g.It("Author:hongli-NonHyperShiftHOST-ROSA-OSD_CCS-High-75440-AWS NLB supports to choose subnets", func() { + g.By("Pre-flight check for the platform type") + compat_otp.SkipIfPlatformTypeNot(oc, "AWS") + + buildPruningBaseDir := testdata.FixturePath("router") + customTemp := filepath.Join(buildPruningBaseDir, "ingresscontroller-clb.yaml") + ns := "openshift-ingress" + var ( + ingctrl = ingressControllerDescription{ + name: "ocp75440", + namespace: "openshift-ingress-operator", + domain: "", + template: customTemp, + } + ) + + compat_otp.By("1. Get public subnets and skip conditionally") + publicSubnetList := getPublicSubnetList(oc) + if len(publicSubnetList) < 1 { + g.Skip("Skipping since no public subnet found") + } + + compat_otp.By("2. Create the custom ingresscontroller with NLB") + baseDomain := getBaseDomain(oc) + ingctrl.domain = ingctrl.name + "." + baseDomain + // Updating LB type from `Classic` to `NLB` in the yaml file + sedCmd := fmt.Sprintf(`sed -i'' -e 's|Classic|NLB|g' %s`, customTemp) + _, err := exec.Command("bash", "-c", sedCmd).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + defer ingctrl.delete(oc) + ingctrl.create(oc) + ensureCustomIngressControllerAvailable(oc, ingctrl.name) + + compat_otp.By("3. Annotate and patch the custom ingresscontroller, enable LB svc can be auto deleted") + annotation := `ingress.operator.openshift.io/auto-delete-load-balancer=` + err = oc.AsAdmin().WithoutNamespace().Run("annotate").Args("-n", ingctrl.namespace, "ingresscontroller/"+ingctrl.name, annotation, "--overwrite").Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + jsonPatch := fmt.Sprintf(`{"spec":{"endpointPublishingStrategy":{"type":"LoadBalancerService","loadBalancer":{"providerParameters":{"type":"AWS","aws":{"type":"NLB","networkLoadBalancer":{"subnets":{"ids":null,"names":[%s]}}}}}}}}`, strings.Join(publicSubnetList, ",")) + patchResourceAsAdmin(oc, ingctrl.namespace, "ingresscontroller/"+ingctrl.name, jsonPatch) + // the old svc should be auto deleted in a few seconds and wait the new one has the annotation + waitForOutputContains(oc, ns, "service/router-"+ingctrl.name, `{.metadata.annotations}`, "service.beta.kubernetes.io/aws-load-balancer-subnets") + ensureCustomIngressControllerAvailable(oc, ingctrl.name) + + compat_otp.By("4. Ensure the ingress LB svc is provisioned and subnets annotation is added") + externalLB := getByJsonPath(oc, ns, "service/router-"+ingctrl.name, "{.status.loadBalancer.ingress}") + o.Expect(externalLB).To(o.MatchRegexp(`"hostname":.*elb.*amazonaws.com`)) + findAnnotation := getAnnotation(oc, ns, "service", "router-"+ingctrl.name) + e2e.Logf("all annotations are: %v", findAnnotation) + o.Expect(findAnnotation).To(o.ContainSubstring("service.beta.kubernetes.io/aws-load-balancer-subnets\":\"" + strings.Replace(strings.Join(publicSubnetList, ","), "\"", "", -1))) + }) + + g.It("Author:mjoseph-NonHyperShiftHOST-ROSA-OSD_CCS-High-75499-Allocating and updating EIPs on AWS NLB cluster", func() { + g.By("Pre-flight check for the platform type") + compat_otp.SkipIfPlatformTypeNot(oc, "AWS") + // Number of EIPs should be same to number of public subnets + num := len(getPublicSubnetList(oc)) + if num < 1 { + g.Skip("Skipping since no public subnet found") + } + + buildPruningBaseDir := testdata.FixturePath("router") + customTemp := filepath.Join(buildPruningBaseDir, "ingresscontroller-clb.yaml") + ns := "openshift-ingress" + var ( + ingctrl = ingressControllerDescription{ + name: "ocp75499", + namespace: "openshift-ingress-operator", + domain: "", + template: customTemp, + } + ) + + compat_otp.By("1. Create the required elastic address") + var eipAllocationsList []string + defer ensureReleaseElaticIP(oc, &eipAllocationsList) + eipAllocationsList = allocateElaticIP(oc, num) + e2e.Logf("The allocated eip list is %s ", eipAllocationsList) + + compat_otp.By("2. Create another set of elastic address") + var eipAllocationsList1 []string + defer ensureReleaseElaticIP(oc, &eipAllocationsList1) + eipAllocationsList1 = allocateElaticIP(oc, num) + e2e.Logf("The allocated second eip list is %s ", eipAllocationsList1) + + compat_otp.By("3. Create the custom ingresscontroller with NLB") + baseDomain := getBaseDomain(oc) + ingctrl.domain = ingctrl.name + "." + baseDomain + // Updating LB type from `Classic` to `NLB` in the yaml file + sedCmd := fmt.Sprintf(`sed -i'' -e 's|Classic|NLB|g' %s`, customTemp) + _, err := exec.Command("bash", "-c", sedCmd).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + defer ingctrl.delete(oc) + ingctrl.create(oc) + ensureCustomIngressControllerAvailable(oc, ingctrl.name) + + compat_otp.By("4. Patch the custom ingresscontroller with the EIPs") + jsonPatch := fmt.Sprintf(`{"spec":{"endpointPublishingStrategy":{"type":"LoadBalancerService","loadBalancer":{"providerParameters":{"type":"AWS","aws":{"type":"NLB","networkLoadBalancer":{"eipAllocations":["%s"]}}}}}}}`, strings.Join(eipAllocationsList, "\",\"")) + patchResourceAsAdmin(oc, ingctrl.namespace, "ingresscontroller/"+ingctrl.name, jsonPatch) + + compat_otp.By("5: Ensure the status of ingresscontroller which is in Progressing with the below message") + jsonPath := `{.status.conditions[?(@.type=="Available")].status}{.status.conditions[?(@.type=="Progressing")].status}{.status.conditions[?(@.type=="Degraded")].status}` + waitForOutputEquals(oc, ingctrl.namespace, "ingresscontroller/"+ingctrl.name, jsonPath, "TrueTrueFalse") + waitForOutputContains(oc, ingctrl.namespace, "ingresscontroller/"+ingctrl.name, `{.status.conditions[?(@.type == "Progressing")].message}`, "To effectuate this change, you must delete the service") + + compat_otp.By("6. Delete the LB svc manually and check controller availability") + oc.AsAdmin().WithoutNamespace().Run("delete").Args("-n", ns, "service", "router-"+ingctrl.name).Execute() + ensureCustomIngressControllerAvailable(oc, ingctrl.name) + + compat_otp.By("7. Ensure the eip annotation is added and ingress LB svc is provisioned") + findAnnotation := getAnnotation(oc, ns, "service", "router-"+ingctrl.name) + o.Expect(findAnnotation).To(o.ContainSubstring("service.beta.kubernetes.io/aws-load-balancer-eip-allocations\":\"" + strings.Replace(strings.Join(eipAllocationsList, ","), "\"", "", -1))) + externalLB := getByJsonPath(oc, ns, "service/router-"+ingctrl.name, "{.status.loadBalancer.ingress}") + o.Expect(externalLB).To(o.MatchRegexp(`"hostname":.*elb.*amazonaws.com`)) + + compat_otp.By("8: Ensure the status of ingresscontroller is not degraded") + waitForOutputEquals(oc, ingctrl.namespace, "ingresscontroller/"+ingctrl.name, jsonPath, "TrueFalseFalse") + + compat_otp.By("9:Patch the controller with a new set of EIP and observer the status of ingresscontroller is again in Progressing with the below message") + jsonPatch1 := fmt.Sprintf(`{"spec":{"endpointPublishingStrategy":{"type":"LoadBalancerService","loadBalancer":{"providerParameters":{"type":"AWS","aws":{"type":"NLB","networkLoadBalancer":{"eipAllocations":["%s"]}}}}}}}`, strings.Join(eipAllocationsList1, "\",\"")) + patchResourceAsAdmin(oc, ingctrl.namespace, "ingresscontroller/"+ingctrl.name, jsonPatch1) + waitForOutputEquals(oc, ingctrl.namespace, "ingresscontroller/"+ingctrl.name, jsonPath, "TrueTrueFalse") + waitForOutputContains(oc, ingctrl.namespace, "ingresscontroller/"+ingctrl.name, `{.status.conditions[?(@.type == "Progressing")].message}`, "To effectuate this change, you must delete the service") + + compat_otp.By("10. Again delete the LB svc manually and check controller availability") + oc.AsAdmin().WithoutNamespace().Run("delete").Args("-n", ns, "service", "router-"+ingctrl.name).Execute() + ensureCustomIngressControllerAvailable(oc, ingctrl.name) + + compat_otp.By("11. Ensure the new eip annotation is added and ingress LB svc is provisioned") + findAnnotation1 := getAnnotation(oc, ns, "service", "router-"+ingctrl.name) + o.Expect(findAnnotation1).To(o.ContainSubstring("service.beta.kubernetes.io/aws-load-balancer-eip-allocations\":\"" + strings.Replace(strings.Join(eipAllocationsList1, ","), "\"", "", -1))) + newExternalLB := getByJsonPath(oc, ns, "service/router-"+ingctrl.name, "{.status.loadBalancer.ingress}") + o.Expect(newExternalLB).To(o.MatchRegexp(`"hostname":.*elb.*amazonaws.com`)) + + compat_otp.By("12: Ensure the status of ingresscontroller is now working fine") + waitForOutputEquals(oc, ingctrl.namespace, "ingresscontroller/"+ingctrl.name, jsonPath, "TrueFalseFalse") + }) + + g.It("Author:mjoseph-NonHyperShiftHOST-ROSA-OSD_CCS-High-75617-Update with 'auto-delete-loadbalancer' annotation and unmanaged EIP allocation annotation on AWS NLB cluster", func() { + g.By("Pre-flight check for the platform type") + compat_otp.SkipIfPlatformTypeNot(oc, "AWS") + // The number of EIPs should be same to number of public subnets + num := len(getPublicSubnetList(oc)) + if num < 1 { + g.Skip("Skipping since no public subnet found") + } + + buildPruningBaseDir := testdata.FixturePath("router") + customTemp := filepath.Join(buildPruningBaseDir, "ingresscontroller-clb.yaml") + ns := "openshift-ingress" + + var ( + ingctrl = ingressControllerDescription{ + name: "ocp75617one", + namespace: "openshift-ingress-operator", + domain: "", + template: customTemp, + } + ingctrl2 = ingressControllerDescription{ + name: "ocp75617two", + namespace: "openshift-ingress-operator", + domain: "", + template: customTemp, + } + ) + + compat_otp.By("1. Create the required elastic address") + var eipAllocationsList []string + defer ensureReleaseElaticIP(oc, &eipAllocationsList) + eipAllocationsList = allocateElaticIP(oc, num) + e2e.Logf("The allocated eip list is %s ", eipAllocationsList) + + compat_otp.By("2. Create the custom ingresscontroller with NLB") + baseDomain := getBaseDomain(oc) + ingctrl.domain = ingctrl.name + "." + baseDomain + // Updating LB type from `Classic` to `NLB` in the yaml file + sedCmd := fmt.Sprintf(`sed -i'' -e 's|Classic|NLB|g' %s`, customTemp) + _, err := exec.Command("bash", "-c", sedCmd).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + defer ingctrl.delete(oc) + ingctrl.create(oc) + ensureCustomIngressControllerAvailable(oc, ingctrl.name) + + compat_otp.By("3. Annotate and patch the custom ingresscontroller, enabling LB svc to be auto deleted") + setAnnotationAsAdmin(oc, ingctrl.namespace, "ingresscontroller/"+ingctrl.name, `ingress.operator.openshift.io/auto-delete-load-balancer=`) + jsonPatch := fmt.Sprintf(`{"spec":{"endpointPublishingStrategy":{"type":"LoadBalancerService","loadBalancer":{"providerParameters":{"type":"AWS","aws":{"type":"NLB","networkLoadBalancer":{"eipAllocations":["%s"]}}}}}}}`, strings.Join(eipAllocationsList, "\",\"")) + patchResourceAsAdmin(oc, ingctrl.namespace, "ingresscontroller/"+ingctrl.name, jsonPatch) + ensureCustomIngressControllerAvailable(oc, ingctrl.name) + + compat_otp.By("4. Ensure the ingress LB svc is provisioned and eip annotation is added") + externalLB := getByJsonPath(oc, ns, "service/router-"+ingctrl.name, "{.status.loadBalancer.ingress}") + o.Expect(externalLB).To(o.MatchRegexp(`"hostname":.*elb.*amazonaws.com`)) + findAnnotation := getAnnotation(oc, ns, "service", "router-"+ingctrl.name) + o.Expect(findAnnotation).To(o.ContainSubstring("service.beta.kubernetes.io/aws-load-balancer-eip-allocations\":\"" + strings.Replace(strings.Join(eipAllocationsList, ","), "\"", "", -1))) + + compat_otp.By("5: Ensure the status of ingresscontroller is not degraded") + jsonPath := `{.status.conditions[?(@.type=="Available")].status}{.status.conditions[?(@.type=="Progressing")].status}{.status.conditions[?(@.type=="Degraded")].status}` + waitForOutputEquals(oc, ingctrl.namespace, "ingresscontroller/"+ingctrl.name, jsonPath, "TrueFalseFalse") + + compat_otp.By("6. Create a new set of elastic address") + var eipAllocationsList1 []string + defer ensureReleaseElaticIP(oc, &eipAllocationsList1) + eipAllocationsList1 = allocateElaticIP(oc, num) + e2e.Logf("The allocated second eip list is %s ", eipAllocationsList1) + + compat_otp.By("7. Create another custom ingresscontroller with NLB") + ingctrl2.domain = ingctrl2.name + "." + baseDomain + defer ingctrl2.delete(oc) + ingctrl2.create(oc) + ensureCustomIngressControllerAvailable(oc, ingctrl2.name) + + compat_otp.By("8. Set an EIP annotation on the LB service directly") + setAnnotationAsAdmin(oc, ns, "svc/router-"+ingctrl2.name, `service.beta.kubernetes.io/aws-load-balancer-eip-allocations=`+strings.Join(eipAllocationsList1, ",")) + findAnnotation1 := getAnnotation(oc, ns, "service", "router-"+ingctrl2.name) + o.Expect(findAnnotation1).To(o.ContainSubstring(`"service.beta.kubernetes.io/aws-load-balancer-eip-allocations":"` + strings.Join(eipAllocationsList1, ",") + "\"")) + + compat_otp.By("9: Ensure the status of ingresscontroller is Progressing with the below message") + waitForOutputEquals(oc, ingctrl2.namespace, "ingresscontroller/"+ingctrl2.name, jsonPath, "TrueTrueFalse") + waitForOutputContains(oc, ingctrl2.namespace, "ingresscontroller/"+ingctrl2.name, `{.status.conditions[?(@.type == "Progressing")].message}`, "To effectuate this change, you must delete the service") + + compat_otp.By("10. Patch the custom ingresscontroller with same EIP values") + jsonPatch2 := fmt.Sprintf(`{"spec":{"endpointPublishingStrategy":{"type":"LoadBalancerService","loadBalancer":{"providerParameters":{"type":"AWS","aws":{"type":"NLB","networkLoadBalancer":{"eipAllocations":["%s"]}}}}}}}`, strings.Join(eipAllocationsList1, "\",\"")) + patchResourceAsAdmin(oc, ingctrl2.namespace, "ingresscontroller/"+ingctrl2.name, jsonPatch2) + ensureCustomIngressControllerAvailable(oc, ingctrl2.name) + + compat_otp.By("11. Ensure the ingress LB svc is provisioned") + externalLB2 := getByJsonPath(oc, ns, "service/router-"+ingctrl2.name, "{.status.loadBalancer.ingress}") + o.Expect(externalLB2).To(o.MatchRegexp(`"hostname":.*elb.*amazonaws.com`)) + + compat_otp.By("12: Ensure the status of ingresscontroller is not degraded after the patching") + waitForOutputEquals(oc, ingctrl2.namespace, "ingresscontroller/"+ingctrl2.name, jsonPath, "TrueFalseFalse") + }) +}) diff --git a/tests-extension/test/e2e/cookies.go b/tests-extension/test/e2e/cookies.go new file mode 100644 index 000000000..7bfd8847a --- /dev/null +++ b/tests-extension/test/e2e/cookies.go @@ -0,0 +1,304 @@ +package router + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + + g "github.com/onsi/ginkgo/v2" + o "github.com/onsi/gomega" + + compat_otp "github.com/openshift/origin/test/extended/util/compat_otp" + "github.com/openshift/router-tests-extension/test/testdata" +) + +var _ = g.Describe("[sig-network-edge] Network_Edge Component_Router", func() { + defer g.GinkgoRecover() + + var oc = compat_otp.NewCLI("route-cookies", compat_otp.KubeConfigPath()) + + // includes: OCP-11903 haproxy cookies based sticky session for unsecure routes + // OCP-11679 Disable haproxy hash based sticky session for unsecure routes + // author: hongli@redhat.com + g.It("Author:shudili-ROSA-OSD_CCS-ARO-Critical-11903-haproxy cookies based sticky session for unsecure routes", func() { + var ( + buildPruningBaseDir = testdata.FixturePath("router") + customTemp = filepath.Join(buildPruningBaseDir, "ingresscontroller-np.yaml") + clientPod = filepath.Join(buildPruningBaseDir, "test-client-pod.yaml") + clientPodName = "hello-pod" + clientPodLabel = "app=hello-pod" + testPodSvc = filepath.Join(buildPruningBaseDir, "web-server-deploy.yaml") + srvrcInfo = "web-server-deploy" + unsecSvcName = "service-unsecure" + cookie = "/data/OCP-11903-cookie" + + ingctrl = ingressControllerDescription{ + name: "11903", + namespace: "openshift-ingress-operator", + domain: "", + template: customTemp, + } + ) + + compat_otp.By("1.0 Create a custom ingresscontroller") + baseDomain := getBaseDomain(oc) + ingctrl.domain = ingctrl.name + "." + baseDomain + defer ingctrl.delete(oc) + ingctrl.create(oc) + ensureRouterDeployGenerationIs(oc, ingctrl.name, "1") + + compat_otp.By("2.0: Prepare file for testing") + updateFilebySedCmd(testPodSvc, "replicas: 1", "replicas: 2") + + compat_otp.By("3.0: Create a client pod and two server pods and the service") + ns := oc.Namespace() + err := oc.AsAdmin().WithoutNamespace().Run("create").Args("-n", ns, "-f", clientPod).Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + ensurePodWithLabelReady(oc, ns, clientPodLabel) + srvPodList := createResourceFromWebServer(oc, ns, testPodSvc, srvrcInfo) + + compat_otp.By("4.0: Create a plain HTTP route") + routehost := "unsecure11903" + "." + ingctrl.domain + createRoute(oc, ns, "http", unsecSvcName, unsecSvcName, []string{"--hostname=" + routehost}) + ensureRouteIsAdmittedByIngressController(oc, ns, unsecSvcName, ingctrl.name) + + compat_otp.By("5.0: Curl the http route, make sure the second server is hit") + routerpod := getOneNewRouterPodFromRollingUpdate(oc, ingctrl.name) + podIP := getPodv4Address(oc, routerpod, "openshift-ingress") + toDst := routehost + ":80:" + podIP + curlCmd := []string{"-n", ns, clientPodName, "--", "curl", "http://" + routehost, "-s", "-c", cookie, "--resolve", toDst, "--connect-timeout", "10"} + expectOutput := []string{"Hello-OpenShift " + srvPodList[1] + " http-8080"} + repeatCmdOnClient(oc, curlCmd, expectOutput, 60, 1) + + compat_otp.By("6.0: Curl the http route again, make sure the first server is hit") + curlCmd = []string{"-n", ns, clientPodName, "--", "curl", "http://" + routehost, "-s", "--resolve", toDst, "--connect-timeout", "10"} + expectOutput = []string{"Hello-OpenShift " + srvPodList[0] + " http-8080"} + repeatCmdOnClient(oc, curlCmd, expectOutput, 60, 1) + + compat_otp.By("7.0: Curl the http route with the specified cookie for 10 times, expect all are forwarded to the second server") + curlCmd = []string{"-n", ns, clientPodName, "--", "curl", "http://" + routehost, "-s", "-b", cookie, "--resolve", toDst, "--connect-timeout", "10"} + expectOutput = []string{"Hello-OpenShift " + srvPodList[0] + " http-8080", "Hello-OpenShift " + srvPodList[1] + " http-8080"} + _, result := repeatCmdOnClient(oc, curlCmd, expectOutput, 120, 10) + o.Expect(result[1]).To(o.Equal(10)) + + // OCP-11679 + compat_otp.By(`8.0: Disable haproxy hash based sticky session for the route by adding 'disable cookies' annotation to it`) + setAnnotation(oc, ns, "route/"+unsecSvcName, "haproxy.router.openshift.io/disable_cookies=true") + + compat_otp.By("9.0: Check the disable cookies configuration in haproxy") + backendStart := fmt.Sprintf(`backend be_http:%s:%s`, ns, unsecSvcName) + ensureHaproxyBlockConfigNotMatchRegexp(oc, routerpod, backendStart, []string{`cookie .+httponly`}) + + compat_otp.By("10.0: Curl the http route with the specified cookie for 10 times again, expect the requests are forwarded to the two servers") + _, result = repeatCmdOnClient(oc, curlCmd, expectOutput, 120, 10) + o.Expect(result[0] + result[1]).To(o.Equal(10)) + o.Expect(result[0]*result[1] > 0).To(o.BeTrue()) + }) + + // author: hongli@redhat.com + g.It("Author:shudili-ROSA-OSD_CCS-ARO-High-12566-Cookie name should not use openshift prefix", func() { + // if the ingress canary route isn't accessable from outside, skip it + if !isCanaryRouteAvailable(oc) { + g.Skip("Skip for the ingress canary route could not be available to the outside") + } + + var ( + buildPruningBaseDir = testdata.FixturePath("router") + testPodSvc = filepath.Join(buildPruningBaseDir, "web-server-deploy.yaml") + srvrcInfo = "web-server-deploy" + unsecSvcName = "service-unsecure" + fileDir = "/tmp/OCP-12566-cookie" + cookie = fileDir + "/cookie12566" + routeName = "unsecureroute2" + ) + + compat_otp.By("1.0: Prepare file folder and file for testing") + defer os.RemoveAll(fileDir) + err := os.MkdirAll(fileDir, 0755) + o.Expect(err).NotTo(o.HaveOccurred()) + + compat_otp.By("2.0: Create two server pods and the service") + ns := oc.Namespace() + updateFilebySedCmd(testPodSvc, "replicas: 1", "replicas: 2") + createResourceFromWebServer(oc, ns, testPodSvc, srvrcInfo) + + compat_otp.By("3.0: Create a plain HTTP route") + routehost := "unsecure12566" + ".apps." + getBaseDomain(oc) + createRoute(oc, ns, "http", routeName, unsecSvcName, []string{"--hostname=" + routehost}) + ensureRouteIsAdmittedByIngressController(oc, ns, routeName, "default") + + compat_otp.By("4.0: Curl the http route, make sure one server is hit") + curlCmd := fmt.Sprintf(`curl http://%s -s -c %s --connect-timeout 10`, routehost, cookie) + expectOutput := []string{"Hello-OpenShift"} + repeatCmdOnClient(oc, curlCmd, expectOutput, 60, 1) + + compat_otp.By("5.0: Check the cookies which should not contain a OPENSHIFT prefix or a md5 hash") + cmd := fmt.Sprintf(`cat %s`, cookie) + cookiesOutput, err := exec.Command("bash", "-c", cmd).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(string(cookiesOutput)).NotTo(o.ContainSubstring("OPENSHIFT")) + o.Expect(string(cookiesOutput)).NotTo(o.MatchRegexp(`\d+\.\d+\.\d+\.\d+`)) + }) + + // author: hongli@redhat.com + g.It("Author:shudili-ROSA-OSD_CCS-ARO-Critical-15872-can set cookie name for unsecure routes by annotation", func() { + // if the ingress canary route isn't accessable from outside, skip it + if !isCanaryRouteAvailable(oc) { + g.Skip("Skip for the ingress canary route could not be available to the outside") + } + + var ( + buildPruningBaseDir = testdata.FixturePath("router") + testPodSvc = filepath.Join(buildPruningBaseDir, "web-server-deploy.yaml") + srvrcInfo = "web-server-deploy" + unsecSvcName = "service-unsecure" + ) + + compat_otp.By("1.0: Create two server pods and the service") + updateFilebySedCmd(testPodSvc, "replicas: 1", "replicas: 2") + ns := oc.Namespace() + srvPodList := createResourceFromWebServer(oc, ns, testPodSvc, srvrcInfo) + + compat_otp.By("2.0: Create a plain HTTP route") + routehost := "unsecure15872" + ".apps." + getBaseDomain(oc) + createRoute(oc, ns, "http", unsecSvcName, unsecSvcName, []string{"--hostname=" + routehost}) + ensureRouteIsAdmittedByIngressController(oc, ns, unsecSvcName, "default") + + compat_otp.By("3.0: Set a cookie name to the route by the annotation, and then ensure the change in haproxy") + setAnnotation(oc, ns, "route/"+unsecSvcName, "router.openshift.io/cookie_name=unsecure-cookie_1") + routerPod := getOneRouterPodNameByIC(oc, "default") + ensureHaproxyBlockConfigContains(oc, routerPod, "be_http:"+ns+":"+unsecSvcName, []string{"unsecure-cookie_1"}) + + compat_otp.By("4.0: Curl the http route, make sure the second server is hit and the cookie is set in the client side") + curlCmd := fmt.Sprintf(`curl http://%s -sv --connect-timeout 10`, routehost) + expectOutput := []string{"Hello-OpenShift " + srvPodList[1] + " http-8080"} + output, _ := repeatCmdOnClient(oc, curlCmd, expectOutput, 60, 1) + o.Expect(output).To(o.MatchRegexp(`(s|S)et-(c|C)ookie: unsecure-cookie_1=[0-9a-zA-Z]+`)) + }) + + // author: mjoseph@redhat.com + g.It("Author:shudili-ROSA-OSD_CCS-ARO-Critical-35547-router.openshift.io/cookie-same-site route annotation accepts None Lax or Strict attribute for edge routes", func() { + // if the ingress canary route isn't accessable from outside, skip it + if !isCanaryRouteAvailable(oc) { + g.Skip("Skip for the ingress canary route could not be available to the outside") + } + + var ( + buildPruningBaseDir = testdata.FixturePath("router") + testPodSvc = filepath.Join(buildPruningBaseDir, "web-server-deploy.yaml") + srvrcInfo = "web-server-deploy" + unsecSvcName = "service-unsecure" + ) + + compat_otp.By("1.0: Create two server pods and the service") + ns := oc.Namespace() + createResourceFromWebServer(oc, ns, testPodSvc, srvrcInfo) + + compat_otp.By("2.0: Create an edge route") + routehost := "edge35547" + ".apps." + getBaseDomain(oc) + createRoute(oc, ns, "edge", "edge35547", unsecSvcName, []string{"--hostname=" + routehost}) + ensureRouteIsAdmittedByIngressController(oc, ns, "edge35547", "default") + + compat_otp.By("3.0: Curl the edge route, and check set-cookie header, expect getting SameSite=None") + curlCmd := fmt.Sprintf(`curl https://%s -sSkI --connect-timeout 10`, routehost) + expectOutput := []string{"set-cookie:"} + result, _ := repeatCmdOnClient(oc, curlCmd, expectOutput, 60, 1) + o.Expect(result).To(o.ContainSubstring(`Secure; SameSite=None`)) + + compat_otp.By("4.0: Add Strict annotation to the edge route, and then ensure the change in haproxy") + setAnnotation(oc, ns, "route/edge35547", "router.openshift.io/cookie-same-site=Strict") + routerPod := getOneRouterPodNameByIC(oc, "default") + ensureHaproxyBlockConfigContains(oc, routerPod, "be_edge_http:"+ns+":edge35547", []string{"SameSite=Strict"}) + + compat_otp.By("5.0: Curl the edge route again, and check set-cookie header, expect getting SameSite=Strict") + result, _ = repeatCmdOnClient(oc, curlCmd, expectOutput, 60, 1) + o.Expect(result).To(o.ContainSubstring(`Secure; SameSite=Strict`)) + }) + + // author: mjoseph@redhat.com + g.It("Author:shudili-ROSA-OSD_CCS-ARO-Critical-35548-router.openshift.io/cookie-same-site route annotation accepts None Lax or Strict attribute for Reencrypt routes", func() { + // if the ingress canary route isn't accessable from outside, skip it + if !isCanaryRouteAvailable(oc) { + g.Skip("Skip for the ingress canary route could not be available to the outside") + } + + var ( + buildPruningBaseDir = testdata.FixturePath("router") + testPodSvc = filepath.Join(buildPruningBaseDir, "web-server-signed-deploy.yaml") + srvrcInfo = "web-server-deploy" + secsvcName = "service-secure" + ) + + compat_otp.By("1.0: Create two server pods and the service") + ns := oc.Namespace() + createResourceFromWebServer(oc, ns, testPodSvc, srvrcInfo) + + compat_otp.By("2.0: Create a reencrypt route") + routehost := "reen35548" + ".apps." + getBaseDomain(oc) + createRoute(oc, ns, "reencrypt", "reen35548", secsvcName, []string{"--hostname=" + routehost}) + ensureRouteIsAdmittedByIngressController(oc, ns, "reen35548", "default") + + compat_otp.By("3.0: Curl the reencrypt route, and check set-cookie header, expect getting SameSite=None") + curlCmd := fmt.Sprintf(`curl https://%s -sSkI --connect-timeout 10`, routehost) + expectOutput := []string{"set-cookie:"} + result, _ := repeatCmdOnClient(oc, curlCmd, expectOutput, 60, 1) + o.Expect(result).To(o.ContainSubstring(`Secure; SameSite=None`)) + + compat_otp.By("4.0: Add Lax annotation to the reencrypt route, and then ensure the change in haproxy") + setAnnotation(oc, ns, "route/reen35548", "router.openshift.io/cookie-same-site=Lax") + routerPod := getOneRouterPodNameByIC(oc, "default") + ensureHaproxyBlockConfigContains(oc, routerPod, "be_secure:"+ns+":reen35548", []string{"SameSite=Lax"}) + + compat_otp.By("5.0: Curl the reencrypt route again, and check set-cookie header, expect getting SameSite=Lax") + result, _ = repeatCmdOnClient(oc, curlCmd, expectOutput, 60, 1) + o.Expect(result).To(o.ContainSubstring(`Secure; SameSite=Lax`)) + + compat_otp.By("6.0: Add Strict annotation to the reencrypt route, and then ensure the change in haproxy") + setAnnotation(oc, ns, "route/reen35548", "router.openshift.io/cookie-same-site=Strict") + ensureHaproxyBlockConfigContains(oc, routerPod, "be_secure:"+ns+":reen35548", []string{"SameSite=Strict"}) + + compat_otp.By("7.0: Curl the reencrypt route for the 3rd time, and check set-cookie header, expect getting SameSite=Strict") + result, _ = repeatCmdOnClient(oc, curlCmd, expectOutput, 60, 1) + o.Expect(result).To(o.ContainSubstring(`Secure; SameSite=Strict`)) + }) + + // author: mjoseph@redhat.com + g.It("Author:shudili-ROSA-OSD_CCS-ARO-Low-35549-router.openshift.io/cookie-same-site route annotation does not work with Passthrough routes", func() { + // if the ingress canary route isn't accessable from outside, skip it + if !isCanaryRouteAvailable(oc) { + g.Skip("Skip for the ingress canary route could not be available to the outside") + } + + var ( + buildPruningBaseDir = testdata.FixturePath("router") + testPodSvc = filepath.Join(buildPruningBaseDir, "web-server-signed-deploy.yaml") + srvrcInfo = "web-server-deploy" + secsvcName = "service-secure" + ) + + compat_otp.By("1.0: Create two server pods and the service") + ns := oc.Namespace() + createResourceFromWebServer(oc, ns, testPodSvc, srvrcInfo) + + compat_otp.By("2.0: Create a passthrough route") + routehost := "pass35549" + ".apps." + getBaseDomain(oc) + createRoute(oc, ns, "passthrough", "pass35549", secsvcName, []string{"--hostname=" + routehost}) + ensureRouteIsAdmittedByIngressController(oc, ns, "pass35549", "default") + + compat_otp.By("3.0: Curl the passthrough route, and check the response headers, expect NOT getting set-cookie header with SameSite=None") + curlCmd := fmt.Sprintf(`curl https://%s -sSkI --connect-timeout 10`, routehost) + expectOutput := []string{"HTTP.+200"} + result, _ := repeatCmdOnClient(oc, curlCmd, expectOutput, 60, 1) + o.Expect(result).NotTo(o.ContainSubstring(`Secure; SameSite=None`)) + + compat_otp.By("4.0: Add Lax annotation to the passthrough route, and then check there is NOT the cookie in haproxy") + setAnnotation(oc, ns, "route/pass35549", "router.openshift.io/cookie-same-site=Lax") + routerpod := getOneRouterPodNameByIC(oc, "default") + ensureHaproxyBlockConfigNotMatchRegexp(oc, routerpod, "backend be_tcp:"+ns+":pass35549", []string{`cookie: [0-9a-zA-Z]+`}) + + compat_otp.By("5.0: Curl the passthrough route again, and check the response headers, expect NOT getting set-cookie header with SameSite=Lax") + result, _ = repeatCmdOnClient(oc, curlCmd, expectOutput, 60, 1) + o.Expect(result).NotTo(o.ContainSubstring(`Secure; SameSite=Lax`)) + }) +}) diff --git a/tests-extension/test/e2e/dns-operator.go b/tests-extension/test/e2e/dns-operator.go new file mode 100644 index 000000000..1e2196742 --- /dev/null +++ b/tests-extension/test/e2e/dns-operator.go @@ -0,0 +1,292 @@ +package router + +import ( + "fmt" + "strings" + + g "github.com/onsi/ginkgo/v2" + o "github.com/onsi/gomega" + compat_otp "github.com/openshift/origin/test/extended/util/compat_otp" +) + +var _ = g.Describe("[sig-network-edge] Network_Edge Component_DNS", func() { + defer g.GinkgoRecover() + var oc = compat_otp.NewCLI("dns-operator", compat_otp.KubeConfigPath()) + + // incorporate OCP-26151 and OCP-23278 into one + // Test case creater: hongli@redhat.com - OCP-26151-Integrate DNS operator metrics with Prometheus + // Test case creater: hongli@redhat.com - OCP-23278-Integrate coredns metrics with monitoring component + g.It("Author:mjoseph-Critical-26151-Integrate DNS operator metrics with Prometheus", func() { + var ( + ons = "openshift-dns-operator" + dns = "openshift-dns" + label = `"openshift.io/cluster-monitoring":"true"` + prometheus = "prometheus-k8s" + ) + // OCP-26151 + compat_otp.By("1. Check the `cluster-monitoring` label exist in the dns operator namespace") + oplabels := getByJsonPath(oc, ons, "ns/openshift-dns-operator", "{.metadata.labels}") + o.Expect(oplabels).To(o.ContainSubstring(label)) + + compat_otp.By("2. Check whether servicemonitor exist in the dns operator namespace") + smname := getByJsonPath(oc, ons, "servicemonitor/dns-operator", "{.metadata.name}") + o.Expect(smname).To(o.ContainSubstring("dns-operator")) + + compat_otp.By("3. Check whether rolebinding exist in the dns operator namespace") + poname := getByJsonPath(oc, ons, "rolebinding/prometheus-k8s", "{.metadata.name}") + o.Expect(poname).To(o.ContainSubstring(prometheus)) + + // OCP-23278 + // Bug: 1688969 + compat_otp.By("4. Check the `cluster-monitoring` label exist in the dns namespace") + polabels := getByJsonPath(oc, dns, "ns/openshift-dns", "{.metadata.labels}") + o.Expect(polabels).To(o.ContainSubstring(label)) + + compat_otp.By("5. Check whether servicemonitor exist in the dns namespace") + smname1 := getByJsonPath(oc, dns, "servicemonitor/dns-default", "{.metadata.name}") + o.Expect(smname1).To(o.ContainSubstring("dns-default")) + + compat_otp.By("6. Check whether rolebinding exist in the dns namespace") + pdname := getByJsonPath(oc, dns, "rolebinding/prometheus-k8s", "{.metadata.name}") + o.Expect(pdname).To(o.ContainSubstring(prometheus)) + }) + + // Test case creater: hongli@redhat.com + // No dns operator namespace on HyperShift guest cluster so this case is not available + g.It("Author:mjoseph-NonHyperShiftHOST-High-37912-DNS operator should show clear error message when DNS service IP already allocated [Disruptive]", func() { + // Bug: 1813062 + // Store the clusterip from the cluster + clusterIp := getByJsonPath(oc, "openshift-dns", "service/dns-default", "{.spec.clusterIP}") + + compat_otp.By("Step1: Scale the CVO and DNS operator pod to zero and delete the default DNS service") + dnsOperatorPodName := getPodListByLabel(oc, "openshift-dns-operator", "name=dns-operator")[0] + defer func() { + compat_otp.By("Recover the CVO and DNS") + oc.AsAdmin().WithoutNamespace().Run("scale").Args("deployment/cluster-version-operator", "--replicas=1", "-n", "openshift-cluster-version").Output() + oc.AsAdmin().WithoutNamespace().Run("scale").Args("deployment/dns-operator", "--replicas=1", "-n", "openshift-dns-operator").Output() + // if the dns-default svc didn't came up in given time, dns operator restoration will help + deleteDnsOperatorToRestore(oc) + }() + _, err := oc.AsAdmin().WithoutNamespace().Run("scale").Args("deployment/cluster-version-operator", "--replicas=0", "-n", "openshift-cluster-version").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + _, err = oc.AsAdmin().WithoutNamespace().Run("scale").Args("deployment/dns-operator", "--replicas=0", "-n", "openshift-dns-operator").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + errPodDis := waitForResourceToDisappear(oc, "openshift-dns-operator", "pod/"+dnsOperatorPodName) + compat_otp.AssertWaitPollNoErr(errPodDis, fmt.Sprintf("resource %v does not disapper", "pod/"+dnsOperatorPodName)) + err1 := oc.AsAdmin().WithoutNamespace().Run("delete").Args("svc", "dns-default", "-n", "openshift-dns").Execute() + o.Expect(err1).NotTo(o.HaveOccurred()) + err = waitForResourceToDisappear(oc, "openshift-dns", "service/dns-default") + compat_otp.AssertWaitPollNoErr(err, "the service/dns-default does not disapper within allowed time") + + compat_otp.By("Step2: Create a test server with the Cluster IP and scale up the dns operator") + defer oc.AsAdmin().WithoutNamespace().Run("delete").Args("svc", "svc-37912", "-n", "openshift-dns").Execute() + err2 := oc.AsAdmin().WithoutNamespace().Run("create").Args( + "svc", "clusterip", "svc-37912", "--tcp=53:53", "--clusterip="+clusterIp, "-n", "openshift-dns").Execute() + o.Expect(err2).NotTo(o.HaveOccurred()) + oc.AsAdmin().WithoutNamespace().Run("scale").Args("deployment/dns-operator", "--replicas=1", "-n", "openshift-dns-operator").Output() + // wait for the dns operator pod to come up + ensurePodWithLabelReady(oc, "openshift-dns-operator", "name=dns-operator") + + compat_otp.By("Step3: Confirm the new dns service came with the given address") + newClusterIp := getByJsonPath(oc, "openshift-dns", "service/svc-37912", "{.spec.clusterIP}") + o.Expect(newClusterIp).To(o.BeEquivalentTo(clusterIp)) + + compat_otp.By("Step4: Confirm the error message from the DNS operator status") + outputOpcfg, errOpcfg := oc.AsAdmin().WithoutNamespace().Run("get").Args( + "dns.operator", "default", `-o=jsonpath={.status.conditions[?(@.type=="Degraded")].message}}`).Output() + o.Expect(errOpcfg).NotTo(o.HaveOccurred()) + o.Expect(outputOpcfg).To(o.ContainSubstring("No IP address is assigned to the DNS service")) + + compat_otp.By("Step5: Check the degraded status of dns operator among the cluster operators") + jsonPath := `{.status.conditions[?(@.type=="Available")].status}{.status.conditions[?(@.type=="Progressing")].status}{.status.conditions[?(@.type=="Degraded")].status}` + waitForOutputEquals(oc, "default", "co/dns", jsonPath, "FalseTrueTrue") + }) + + // author: mjoseph@redhat.com + g.It("Author:mjoseph-Critical-41049-DNS controlls pod placement by node selector [Disruptive]", func() { + podList := getAllDNSPodsNames(oc) + if len(podList) == 1 { + g.Skip("Skipping on SNO cluster (just has one dns pod)") + } + + var ( + ns = "openshift-dns" + label = "ne-dns-testing=true" + ) + + compat_otp.By("check the default dns nodeSelector is present") + nodePlacement := getByJsonPath(oc, ns, "ds/dns-default", "{.spec.template.spec.nodeSelector}") + o.Expect(nodePlacement).To(o.BeEquivalentTo(`{"kubernetes.io/os":"linux"}`)) + + // Since func forceOnlyOneDnsPodExist() has implemented DNS nodeSelector + // just ensure the pod is running on the node with label "ne-dns-testing=true" + compat_otp.By("check the nodeSelector can be updated") + defer deleteDnsOperatorToRestore(oc) + oneDnsPod := forceOnlyOneDnsPodExist(oc) + nodePlacement = getByJsonPath(oc, "openshift-dns", "ds/dns-default", "{.spec.template.spec.nodeSelector}") + o.Expect(nodePlacement).To(o.ContainSubstring(`{"ne-dns-testing":"true"}`)) + + compat_otp.By("check the dns pod is running on the expected node") + nodeByPod := getByJsonPath(oc, ns, "pod/"+oneDnsPod, "{.spec.nodeName}") + nodeByLabel := getByLabelAndJsonPath(oc, "default", "node", label, "{.items[*].metadata.name}") + o.Expect(nodeByPod).To(o.BeEquivalentTo(nodeByLabel)) + }) + + // author: mjoseph@redhat.com + g.It("Author:mjoseph-Critical-41050-DNS controll pod placement by tolerations [Disruptive]", func() { + // the case needs at least one worker node since dns pods will be removed from master + // so skip on SNO and Compact cluster that no dedicated worker node + output, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("nodes", "-l node-role.kubernetes.io/worker=,node-role.kubernetes.io/master!=", `-ojsonpath={.items[*].status.conditions[?(@.type=="Ready")].status}`).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + if strings.Count(output, "True") < 1 { + g.Skip("Skipping as there is no dedicated worker nodes") + } + + var ( + ns = "openshift-dns" + dnsCustomToleration = `[{"op":"replace", "path":"/spec/nodePlacement", "value":{"tolerations":[{"effect":"NoExecute","key":"my-dns-test","operator":"Equal","value":"abc"}]}}]` + ) + + compat_otp.By("check dns pod placement to confirm it is running on default tolerations") + tolerationCfg := getByJsonPath(oc, ns, "ds/dns-default", "{.spec.template.spec.tolerations}") + o.Expect(tolerationCfg).To(o.ContainSubstring(`{"key":"node-role.kubernetes.io/master","operator":"Exists"}`)) + + compat_otp.By("Patch dns operator config with custom tolerations of dns pod, not to tolerate master node taints") + dnsPodsList := getAllDNSPodsNames(oc) + jsonPath := `{.status.conditions[?(@.type=="Available")].status}{.status.conditions[?(@.type=="Progressing")].status}{.status.conditions[?(@.type=="Degraded")].status}` + defer deleteDnsOperatorToRestore(oc) + patchGlobalResourceAsAdmin(oc, "dns.operator.openshift.io/default", dnsCustomToleration) + waitForRangeOfPodsToDisappear(oc, ns, dnsPodsList) + waitForOutputEquals(oc, "default", "co/dns", jsonPath, "TrueFalseFalse") + // Get new created DNS pods and ensure they are not running on master + dnsPodsList = getAllDNSPodsNames(oc) + for _, podName := range dnsPodsList { + nodeName := getNodeNameByPod(oc, ns, podName) + nodeLabels := getByJsonPath(oc, "default", "node/"+nodeName, "{.metadata.labels}") + o.Expect(nodeLabels).NotTo(o.ContainSubstring("node-role.kubernetes.io/master")) + } + + compat_otp.By("check dns pod placement to check the custom tolerations") + tolerationCfg = getByJsonPath(oc, ns, "ds/dns-default", "{.spec.template.spec.tolerations}") + o.Expect(tolerationCfg).To(o.ContainSubstring(`{"effect":"NoExecute","key":"my-dns-test","operator":"Equal","value":"abc"}`)) + + compat_otp.By("check dns.operator status to see any error messages") + status := getByJsonPath(oc, "default", "dns.operator/default", "{.status}") + o.Expect(status).NotTo(o.ContainSubstring("error")) + }) + + // author: hongli@redhat.com + g.It("Author:hongli-High-46183-DNS operator supports Random, RoundRobin and Sequential policy for servers.forwardPlugin [Disruptive]", func() { + resourceName := "dns.operator.openshift.io/default" + jsonPatch := "[{\"op\":\"add\", \"path\":\"/spec/servers\", \"value\":[{\"forwardPlugin\":{\"policy\":\"Random\",\"upstreams\":[\"8.8.8.8\"]},\"name\":\"test\",\"zones\":[\"mytest.ocp\"]}]}]" + + compat_otp.By("Prepare the dns testing node and pod") + defer deleteDnsOperatorToRestore(oc) + oneDnsPod := forceOnlyOneDnsPodExist(oc) + + compat_otp.By("patch the dns.operator/default and add custom zones config, check Corefile and ensure the policy is Random") + patchGlobalResourceAsAdmin(oc, resourceName, jsonPatch) + policy := pollReadDnsCorefile(oc, oneDnsPod, "8.8.8.8", "-A2", "policy random") + o.Expect(policy).To(o.ContainSubstring(`policy random`)) + + compat_otp.By("updateh the custom zones policy to RoundRobin, check Corefile and ensure it is updated ") + patchGlobalResourceAsAdmin(oc, resourceName, "[{\"op\":\"replace\", \"path\":\"/spec/servers/0/forwardPlugin/policy\", \"value\":\"RoundRobin\"}]") + policy = pollReadDnsCorefile(oc, oneDnsPod, "8.8.8.8", "-A2", "policy round_robin") + o.Expect(policy).To(o.ContainSubstring(`policy round_robin`)) + + compat_otp.By("updateh the custom zones policy to Sequential, check Corefile and ensure it is updated") + patchGlobalResourceAsAdmin(oc, resourceName, "[{\"op\":\"replace\", \"path\":\"/spec/servers/0/forwardPlugin/policy\", \"value\":\"Sequential\"}]") + policy = pollReadDnsCorefile(oc, oneDnsPod, "8.8.8.8", "-A2", "policy sequential") + o.Expect(policy).To(o.ContainSubstring(`policy sequential`)) + }) + + // author: shudili@redhat.com + // no dns operator namespace on HyperShift guest cluster so this case is not available + g.It("Author:shudili-NonHyperShiftHOST-Medium-46873-Configure operatorLogLevel under the default dns operator and check the logs flag [Disruptive]", func() { + var ( + resourceName = "dns.operator.openshift.io/default" + cfgOploglevelDebug = "[{\"op\":\"replace\", \"path\":\"/spec/operatorLogLevel\", \"value\":\"Debug\"}]" + cfgOploglevelTrace = "[{\"op\":\"replace\", \"path\":\"/spec/operatorLogLevel\", \"value\":\"Trace\"}]" + cfgOploglevelNormal = "[{\"op\":\"replace\", \"path\":\"/spec/operatorLogLevel\", \"value\":\"Normal\"}]" + ) + defer deleteDnsOperatorToRestore(oc) + + compat_otp.By("Check default log level of dns operator") + outputOpcfg, errOpcfg := oc.AsAdmin().WithoutNamespace().Run("get").Args("dns.operator", "default", "-o=jsonpath={.spec.operatorLogLevel}").Output() + o.Expect(errOpcfg).NotTo(o.HaveOccurred()) + o.Expect(outputOpcfg).To(o.ContainSubstring("Normal")) + + //Remove the dns operator pod and wait for the new pod is created, which is useful to check the dns operator log + compat_otp.By("Remove dns operator pod") + dnsOperatorPodName := getPodListByLabel(oc, "openshift-dns-operator", "name=dns-operator")[0] + _, errDelpod := oc.AsAdmin().WithoutNamespace().Run("delete").Args("pod", dnsOperatorPodName, "-n", "openshift-dns-operator").Output() + o.Expect(errDelpod).NotTo(o.HaveOccurred()) + errPodDis := waitForResourceToDisappear(oc, "openshift-dns-operator", "pod/"+dnsOperatorPodName) + compat_otp.AssertWaitPollNoErr(errPodDis, fmt.Sprintf("the dns-operator pod isn't terminated")) + ensurePodWithLabelReady(oc, "openshift-dns-operator", "name=dns-operator") + + compat_otp.By("Patch dns operator with operator logLevel Debug") + patchGlobalResourceAsAdmin(oc, resourceName, cfgOploglevelDebug) + compat_otp.By("Check logLevel debug in dns operator") + outputOpcfg, errOpcfg = oc.AsAdmin().WithoutNamespace().Run("get").Args("dns.operator", "default", "-o=jsonpath={.spec.operatorLogLevel}").Output() + o.Expect(errOpcfg).NotTo(o.HaveOccurred()) + o.Expect(outputOpcfg).To(o.ContainSubstring("Debug")) + + compat_otp.By("Patch dns operator with operator logLevel trace") + patchGlobalResourceAsAdmin(oc, resourceName, cfgOploglevelTrace) + compat_otp.By("Check logLevel trace in dns operator") + outputOpcfg, errOpcfg = oc.AsAdmin().WithoutNamespace().Run("get").Args("dns.operator", "default", "-o=jsonpath={.spec.operatorLogLevel}").Output() + o.Expect(errOpcfg).NotTo(o.HaveOccurred()) + o.Expect(outputOpcfg).To(o.ContainSubstring("Trace")) + + compat_otp.By("Patch dns operator with operator logLevel normal") + patchGlobalResourceAsAdmin(oc, resourceName, cfgOploglevelNormal) + compat_otp.By("Check logLevel normal in dns operator") + outputOpcfg, errOpcfg = oc.AsAdmin().WithoutNamespace().Run("get").Args("dns.operator", "default", "-o=jsonpath={.spec.operatorLogLevel}").Output() + o.Expect(errOpcfg).NotTo(o.HaveOccurred()) + o.Expect(outputOpcfg).To(o.ContainSubstring("Normal")) + + compat_otp.By("Check logs of dns operator") + outputLogs, errLog := oc.AsAdmin().Run("logs").Args("deployment/dns-operator", "-n", "openshift-dns-operator", "-c", "dns-operator").Output() + o.Expect(errLog).NotTo(o.HaveOccurred()) + o.Expect(outputLogs).To(o.ContainSubstring("level=info")) + }) + + // Bug: OCPBUGS-6829 + g.It("Author:mjoseph-High-63512-Enbaling force_tcp for protocolStrategy field to allow DNS queries to send on TCP to upstream server [Disruptive]", func() { + var ( + resourceName = "dns.operator.openshift.io/default" + upstreamResolverPatch = "[{\"op\":\"add\", \"path\":\"/spec/upstreamResolvers/protocolStrategy\", \"value\":\"TCP\"}]" + upstreamResolverPatchRemove = "[{\"op\":\"replace\", \"path\":\"/spec/upstreamResolvers/protocolStrategy\", \"value\":\"\"}]" + dnsForwardPluginPatch = "[{\"op\":\"replace\", \"path\":\"/spec/servers\", \"value\":[{\"forwardPlugin\":{\"policy\":\"Sequential\",\"protocolStrategy\": \"TCP\",\"upstreams\":[\"8.8.8.8\"]},\"name\":\"test\",\"zones\":[\"mytest.ocp\"]}]}]" + ) + + compat_otp.By("1. Check the default dns operator config for “protocol strategy” is none") + output, err := oc.AsAdmin().Run("get").Args("cm/dns-default", "-n", "openshift-dns", "-o=jsonpath={.data.Corefile}").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(strings.Contains(output, "force_tcp")).NotTo(o.BeTrue()) + + compat_otp.By("2. Prepare the dns testing node and pod") + defer deleteDnsOperatorToRestore(oc) + oneDnsPod := forceOnlyOneDnsPodExist(oc) + + compat_otp.By("3. Patch dns operator with 'TCP' as protocol strategy for upstreamresolver") + patchGlobalResourceAsAdmin(oc, resourceName, upstreamResolverPatch) + + compat_otp.By("4. Check the upstreamresolver for “protocol strategy” is TCP in Corefile of coredns") + tcp := pollReadDnsCorefile(oc, oneDnsPod, "forward", "-A2", "force_tcp") + o.Expect(tcp).To(o.ContainSubstring("force_tcp")) + //remove the patch from upstreamresolver + patchGlobalResourceAsAdmin(oc, resourceName, upstreamResolverPatchRemove) + output, err = oc.AsAdmin().Run("get").Args("dns.operator.openshift.io/default", "-o=jsonpath={.spec.upstreamResolvers.protocolStrategy}").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(output).To(o.BeEmpty()) + + compat_otp.By("5. Patch dns operator with 'TCP' as protocol strategy for forwardPlugin") + patchGlobalResourceAsAdmin(oc, resourceName, dnsForwardPluginPatch) + + compat_otp.By("6. Check the protocol strategy value as 'TCP' in Corefile of coredns under forwardPlugin") + tcp1 := pollReadDnsCorefile(oc, oneDnsPod, "test", "-A5", "force_tcp") + o.Expect(tcp1).To(o.ContainSubstring("force_tcp")) + }) +}) diff --git a/tests-extension/test/e2e/dns.go b/tests-extension/test/e2e/dns.go new file mode 100644 index 000000000..004f1fd0a --- /dev/null +++ b/tests-extension/test/e2e/dns.go @@ -0,0 +1,1212 @@ +package router + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "slices" + "strings" + "time" + + g "github.com/onsi/ginkgo/v2" + o "github.com/onsi/gomega" + compat_otp "github.com/openshift/origin/test/extended/util/compat_otp" + "github.com/openshift/router-tests-extension/test/testdata" + e2e "k8s.io/kubernetes/test/e2e/framework" + netutils "k8s.io/utils/net" +) + +var _ = g.Describe("[sig-network-edge] Network_Edge Component_DNS", func() { + defer g.GinkgoRecover() + var oc = compat_otp.NewCLI("coredns", compat_otp.KubeConfigPath()) + + // author: shudili@redhat.com + g.It("Author:shudili-High-39842-CoreDNS supports dual stack ClusterIP Services for OCP4.8 or higher", func() { + var ( + buildPruningBaseDir = testdata.FixturePath("router") + testPodSvc = filepath.Join(buildPruningBaseDir, "web-server-v4v6rc.yaml") + unsecsvcName = "service-unsecurev4v6" + secsvcName = "service-securev4v6" + ) + + compat_otp.By("check the IP stack tpye, skip for non-dualstack platform") + ipStackType := checkIPStackType(oc) + e2e.Logf("the cluster IP stack type is: %v", ipStackType) + if ipStackType != "dualstack" { + g.Skip("Skip for non-dualstack platform") + } + + compat_otp.By("Create a backend pod and its services resources") + ns := oc.Namespace() + createResourceFromFile(oc, ns, testPodSvc) + ensurePodWithLabelReady(oc, ns, "name=web-server-v4v6rc") + srvPod := getPodListByLabel(oc, ns, "name=web-server-v4v6rc")[0] + + compat_otp.By("check the services v4v6 addresses") + IPAddresses := getByJsonPath(oc, ns, "service/"+unsecsvcName, "{.spec.clusterIPs}") + o.Expect(IPAddresses).To(o.MatchRegexp(`[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}`)) + o.Expect(strings.Count(IPAddresses, ":") >= 2).To(o.BeTrue()) + + IPAddresses = getByJsonPath(oc, ns, "service/"+secsvcName, "{.spec.clusterIPs}") + o.Expect(IPAddresses).To(o.MatchRegexp(`[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}`)) + o.Expect(strings.Count(IPAddresses, ":") >= 2).To(o.BeTrue()) + + compat_otp.By("check the services names can be resolved to their v4v6 addresses") + IPAddress1 := getByJsonPath(oc, ns, "service/"+unsecsvcName, "{.spec.clusterIPs[0]}") + IPAddress2 := getByJsonPath(oc, ns, "service/"+unsecsvcName, "{.spec.clusterIPs[1]}") + cmdOnPod := []string{"-n", ns, srvPod, "--", "getent", "ahosts", unsecsvcName} + repeatCmdOnClient(oc, cmdOnPod, IPAddress1, 30, 1) + repeatCmdOnClient(oc, cmdOnPod, IPAddress2, 30, 1) + + IPAddress1 = getByJsonPath(oc, ns, "service/"+secsvcName, "{.spec.clusterIPs[0]}") + IPAddress2 = getByJsonPath(oc, ns, "service/"+secsvcName, "{.spec.clusterIPs[1]}") + cmdOnPod = []string{"-n", ns, srvPod, "--", "getent", "ahosts", secsvcName} + repeatCmdOnClient(oc, cmdOnPod, IPAddress1, 30, 1) + repeatCmdOnClient(oc, cmdOnPod, IPAddress2, 30, 1) + }) + + // incorporate OCP-56047 and OCP-40718 into one + // Test case creater: shudili@redhat.com - OCP-56047-Set CoreDNS cache entries for forwarded zones + // Test case creater: jechen@redhat.com - OCP-40718-CoreDNS cache should use 900s for positive responses and 30s for negative responses + g.It("Author:shudili-Critical-40718-CoreDNS cache should use 900s for positive responses and 30s for negative responses [Disruptive]", func() { + compat_otp.By("Prepare the dns testing node and pod") + defer deleteDnsOperatorToRestore(oc) + oneDnsPod := forceOnlyOneDnsPodExist(oc) + + // OCP-40718 + compat_otp.By("1. Check the cache entries of the default corefiles in CoreDNS") + zoneInCoreFile1 := pollReadDnsCorefile(oc, oneDnsPod, ".:5353", "-A20", "cache 900") + o.Expect(zoneInCoreFile1).Should(o.And( + o.ContainSubstring("cache 900"), + o.ContainSubstring("denial 9984 30"))) + + // OCP-56047 + // bug: 2006803 + compat_otp.By("2. Patch the dns.operator/default and add a custom forward zone config") + resourceName := "dns.operator.openshift.io/default" + jsonPatch := "[{\"op\":\"add\", \"path\":\"/spec/servers\", \"value\":[{\"forwardPlugin\":{\"policy\":\"Random\",\"upstreams\":[\"8.8.8.8\"]},\"name\":\"test\",\"zones\":[\"mytest.ocp\"]}]}]" + patchGlobalResourceAsAdmin(oc, resourceName, jsonPatch) + + compat_otp.By("3. Check the cache entries of the custom forward zone in CoreDNS") + zoneInCoreFile := pollReadDnsCorefile(oc, oneDnsPod, "mytest.ocp", "-A15", "cache 900") + o.Expect(zoneInCoreFile).Should(o.And( + o.ContainSubstring("cache 900"), + o.ContainSubstring("denial 9984 30"))) + }) + + // Bug: 1916907 + g.It("Author:mjoseph-High-40867-Deleting the internal registry should not corrupt /etc/hosts [Disruptive]", func() { + compat_otp.By("Step1: Get the Cluster IP of image-registry") + // Skip the test case if openshift-image-registry namespace is not found + clusterIP, err := oc.AsAdmin().WithoutNamespace().Run("get").Args( + "service", "image-registry", "-n", "openshift-image-registry", "-o=jsonpath={.spec.clusterIP}").Output() + if err != nil || strings.Contains(clusterIP, `namespaces \"openshift-image-registry\" not found`) { + g.Skip("Skip for non-supported platform") + } + o.Expect(err).NotTo(o.HaveOccurred()) + + compat_otp.By("Step2: SSH to the node and confirm the /etc/hosts have the same clusterIP") + allNodeList, _ := compat_otp.GetAllNodes(oc) + // get a random node + node := getRandomElementFromList(allNodeList) + hostOutput, err := compat_otp.DebugNodeWithChroot(oc, node, "cat", "/etc/hosts") + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(hostOutput).To(o.And( + o.ContainSubstring("127.0.0.1 localhost localhost.localdomain localhost4 localhost4.localdomain4"), + o.ContainSubstring("::1 localhost localhost.localdomain localhost6 localhost6.localdomain6"), + o.ContainSubstring(clusterIP+" image-registry.openshift-image-registry.svc image-registry.openshift-image-registry.svc.cluster.local"))) + o.Expect(hostOutput).NotTo(o.And(o.ContainSubstring("error"), o.ContainSubstring("failed"), o.ContainSubstring("timed out"))) + + // Set status variables + expectedStatus := map[string]string{"Available": "True", "Progressing": "False", "Degraded": "False"} + + compat_otp.By("Step3: Delete the image-registry svc and check whether it receives a new Cluster IP") + err1 := oc.AsAdmin().WithoutNamespace().Run("delete").Args("svc", "image-registry", "-n", "openshift-image-registry").Execute() + o.Expect(err1).NotTo(o.HaveOccurred()) + err = waitCoBecomes(oc, "image-registry", 240, expectedStatus) + o.Expect(err).NotTo(o.HaveOccurred()) + + compat_otp.By("Step4: Get the new Cluster IP of image-registry") + newClusterIP, err2 := oc.AsAdmin().WithoutNamespace().Run("get").Args( + "service", "image-registry", "-n", "openshift-image-registry", "-o=jsonpath={.spec.clusterIP}").Output() + o.Expect(err2).NotTo(o.HaveOccurred()) + o.Expect(newClusterIP).NotTo(o.ContainSubstring(clusterIP)) + e2e.Logf("The new cluster IP is %v", newClusterIP) + + compat_otp.By("Step5: SSH to the node and confirm the /etc/hosts details, after deletion") + cmdList := []string{"cat", "/etc/hosts"} + expectedString := fmt.Sprintf(`%s image-registry.openshift-image-registry.svc image-registry.openshift-image-registry.svc.cluster.local # openshift-generated-node-resolver`, newClusterIP) + waitForDebugNodeOutputContains(oc, "default", node, cmdList, expectedString, 90*time.Second) + }) + + // incorporate OCP-40717 into existing OCP-46867 + // Test case creater: jechen@redhat.com - OCP-40717-Hostname lookup does not delay when master node dow + // Test case creater: shudili@redhat.com - OCP-46867-Configure upstream resolvers for CoreDNS flag + g.It("Author:shudili-Critical-46867-Configure upstream resolvers for CoreDNS flag [Disruptive]", func() { + var ( + resourceName = "dns.operator.openshift.io/default" + cfgMulIPv4Upstreams = "[{\"op\":\"replace\", \"path\":\"/spec/upstreamResolvers/upstreams\", \"value\":[" + + "{\"address\":\"10.100.1.11\",\"port\":53,\"type\":\"Network\"}, " + + "{\"address\":\"10.100.1.12\",\"port\":53,\"type\":\"Network\"}, " + + "{\"address\":\"10.100.1.13\",\"port\":5353,\"type\":\"Network\"}]}]" + expMulIPv4Upstreams = "forward . 10.100.1.11:53 10.100.1.12:53 10.100.1.13:5353" + cfgOneIPv4Upstreams = "[{\"op\":\"replace\", \"path\":\"/spec/upstreamResolvers/upstreams\", \"value\":[" + + "{\"address\":\"20.100.1.11\",\"port\":53,\"type\":\"Network\"}]}]" + expOneIPv4Upstreams = "forward . 20.100.1.11:53" + cfgMax15Upstreams = "[{\"op\":\"replace\", \"path\":\"/spec/upstreamResolvers/upstreams\", \"value\":[" + + "{\"address\":\"30.100.1.11\",\"port\":53,\"type\":\"Network\"}, " + + "{\"address\":\"30.100.1.12\",\"port\":53,\"type\":\"Network\"}, " + + "{\"address\":\"30.100.1.13\",\"port\":53,\"type\":\"Network\"}, " + + "{\"address\":\"30.100.1.14\",\"port\":53,\"type\":\"Network\"}, " + + "{\"address\":\"30.100.1.15\",\"port\":53,\"type\":\"Network\"}, " + + "{\"address\":\"30.100.1.16\",\"port\":53,\"type\":\"Network\"}, " + + "{\"address\":\"30.100.1.17\",\"port\":53,\"type\":\"Network\"}, " + + "{\"address\":\"30.100.1.18\",\"port\":53,\"type\":\"Network\"}, " + + "{\"address\":\"30.100.1.19\",\"port\":53,\"type\":\"Network\"}, " + + "{\"address\":\"30.100.1.20\",\"port\":53,\"type\":\"Network\"}, " + + "{\"address\":\"30.100.1.21\",\"port\":53,\"type\":\"Network\"}, " + + "{\"address\":\"30.100.1.22\",\"port\":53,\"type\":\"Network\"}, " + + "{\"address\":\"30.100.1.23\",\"port\":53,\"type\":\"Network\"}, " + + "{\"address\":\"30.100.1.24\",\"port\":53,\"type\":\"Network\"}, " + + "{\"address\":\"30.100.1.25\",\"port\":53,\"type\":\"Network\"}]}]" + expMax15Upstreams = "forward . 30.100.1.11:53 30.100.1.12:53 30.100.1.13:53 30.100.1.14:53 30.100.1.15:53 " + + "30.100.1.16:53 30.100.1.17:53 30.100.1.18:53 30.100.1.19:53 30.100.1.20:53 " + + "30.100.1.21:53 30.100.1.22:53 30.100.1.23:53 30.100.1.24:53 30.100.1.25:53" + cfgMulIPv6Upstreams = "[{\"op\":\"replace\", \"path\":\"/spec/upstreamResolvers/upstreams\", \"value\":[" + + "{\"address\":\"1001::aaaa\",\"port\":5353,\"type\":\"Network\"}, " + + "{\"address\":\"1001::BBBB\",\"port\":53,\"type\":\"Network\"}, " + + "{\"address\":\"1001::cccc\",\"port\":53,\"type\":\"Network\"}]}]" + expMulIPv6Upstreams = "forward . [1001::AAAA]:5353 [1001::BBBB]:53 [1001::CCCC]:53" + ) + compat_otp.By("Prepare the dns testing node and pod") + defer deleteDnsOperatorToRestore(oc) + oneDnsPod := forceOnlyOneDnsPodExist(oc) + + // OCP-40717 + compat_otp.By("Check the readiness probe period and timeout parameters are both set to 3 seconds") + output, err := oc.AsAdmin().Run("get").Args("pod/"+oneDnsPod, "-n", "openshift-dns", "-o=jsonpath={.spec.containers[0].readinessProbe.periodSeconds}").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(output).To(o.ContainSubstring(`3`)) + output1, err1 := oc.AsAdmin().Run("get").Args("pod/"+oneDnsPod, "-n", "openshift-dns", "-o=jsonpath={.spec.containers[0].readinessProbe.timeoutSeconds}").Output() + o.Expect(err1).NotTo(o.HaveOccurred()) + o.Expect(output1).To(o.ContainSubstring(`3`)) + + // OCP-46867 + compat_otp.By("Check default values of forward upstream resolvers for CoreDNS") + upstreams := pollReadDnsCorefile(oc, oneDnsPod, "forward", "-A2", "resolv.conf") + o.Expect(upstreams).To(o.ContainSubstring("forward . /etc/resolv.conf")) + + compat_otp.By("Patch dns operator with multiple ipv4 upstreams") + patchGlobalResourceAsAdmin(oc, resourceName, cfgMulIPv4Upstreams) + + compat_otp.By("Check multiple ipv4 forward upstream resolvers in CoreDNS") + upstreams = pollReadDnsCorefile(oc, oneDnsPod, "forward", "-A2", expMulIPv4Upstreams) + o.Expect(upstreams).To(o.ContainSubstring(expMulIPv4Upstreams)) + + compat_otp.By("Patch dns operator with a single ipv4 upstream, and then check the single ipv4 forward upstream resolver for CoreDNS") + patchGlobalResourceAsAdmin(oc, resourceName, cfgOneIPv4Upstreams) + upstreams = pollReadDnsCorefile(oc, oneDnsPod, "forward", "-A2", expOneIPv4Upstreams) + o.Expect(upstreams).To(o.ContainSubstring(expOneIPv4Upstreams)) + + compat_otp.By("Patch dns operator with max 15 ipv4 upstreams, and then the max 15 ipv4 forward upstream resolvers for CoreDNS") + patchGlobalResourceAsAdmin(oc, resourceName, cfgMax15Upstreams) + upstreams = pollReadDnsCorefile(oc, oneDnsPod, "forward", "-A2", expMax15Upstreams) + o.Expect(upstreams).To(o.ContainSubstring(expMax15Upstreams)) + + compat_otp.By("Patch dns operator with multiple ipv6 upstreams, and then check the multiple ipv6 forward upstream resolvers for CoreDNS") + patchGlobalResourceAsAdmin(oc, resourceName, cfgMulIPv6Upstreams) + upstreams = pollReadDnsCorefile(oc, oneDnsPod, "forward", "-A2", "1001") + o.Expect(upstreams).To(o.ContainSubstring(expMulIPv6Upstreams)) + }) + + // author: shudili@redhat.com + g.It("Author:shudili-Critical-46868-Configure forward policy for CoreDNS flag [Disruptive]", func() { + var ( + resourceName = "dns.operator.openshift.io/default" + cfgMulIPv4Upstreams = "[{\"op\":\"replace\", \"path\":\"/spec/upstreamResolvers/upstreams\", \"value\":[" + + "{\"address\":\"10.100.1.11\",\"port\":53,\"type\":\"Network\"}, " + + "{\"address\":\"10.100.1.12\",\"port\":53,\"type\":\"Network\"}, " + + "{\"address\":\"10.100.1.13\",\"port\":5353,\"type\":\"Network\"}]}]" + cfgPolicyRandom = "[{\"op\":\"replace\", \"path\":\"/spec/upstreamResolvers/policy\", \"value\":\"Random\"}]" + cfgPolicyRr = "[{\"op\":\"replace\", \"path\":\"/spec/upstreamResolvers/policy\", \"value\":\"RoundRobin\"}]" + cfgPolicySeq = "[{\"op\":\"replace\", \"path\":\"/spec/upstreamResolvers/policy\", \"value\":\"Sequential\"}]" + ) + compat_otp.By("Prepare the dns testing node and pod") + defer deleteDnsOperatorToRestore(oc) + oneDnsPod := forceOnlyOneDnsPodExist(oc) + + compat_otp.By("Check default values of forward policy for CoreDNS") + policy := pollReadDnsCorefile(oc, oneDnsPod, "forward", "-A2", "policy sequential") + o.Expect(policy).To(o.ContainSubstring("policy sequential")) + + compat_otp.By("Patch dns operator with multiple ipv4 upstreams, and check multiple ipv4 forward upstreams in CoreDNS") + patchGlobalResourceAsAdmin(oc, resourceName, cfgMulIPv4Upstreams) + upstreams := pollReadDnsCorefile(oc, oneDnsPod, "forward", "-A2", "10.100.1.11") + o.Expect(upstreams).To(o.ContainSubstring("forward . 10.100.1.11:53 10.100.1.12:53 10.100.1.13:5353")) + + compat_otp.By("Check default forward policy in CoreDNS after multiple ipv4 forward upstreams are configured") + o.Expect(upstreams).To(o.ContainSubstring("policy sequential")) + + compat_otp.By("Patch dns operator with policy random for upstream resolvers, and then check forward policy random in Corefile of coredns") + patchGlobalResourceAsAdmin(oc, resourceName, cfgPolicyRandom) + policy = pollReadDnsCorefile(oc, oneDnsPod, "forward", "-A2", "policy random") + o.Expect(policy).To(o.ContainSubstring("policy random")) + + compat_otp.By("Patch dns operator with policy roundrobin for upstream resolvers, and then check forward policy roundrobin in Corefile of coredns") + patchGlobalResourceAsAdmin(oc, resourceName, cfgPolicyRr) + policy = pollReadDnsCorefile(oc, oneDnsPod, "forward", "-A2", "policy round_robin") + o.Expect(policy).To(o.ContainSubstring("policy round_robin")) + + compat_otp.By("Patch dns operator with policy sequential for upstream resolvers, and then check forward policy sequential in Corefile of coredns") + patchGlobalResourceAsAdmin(oc, resourceName, cfgPolicySeq) + policy = pollReadDnsCorefile(oc, oneDnsPod, "forward", "-A2", "policy sequential") + o.Expect(policy).To(o.ContainSubstring("policy sequential")) + }) + + g.It("Author:shudili-Medium-46869-Negative test of configuring upstream resolvers and policy flag [Disruptive]", func() { + var ( + resourceName = "dns.operator.openshift.io/default" + cfgAddOneUpstreams = "[{\"op\":\"add\", \"path\":\"/spec/upstreamResolvers/upstreams\", \"value\":[" + + "{\"address\":\"30.100.1.11\",\"port\":53,\"type\":\"Network\"}, " + + "{\"address\":\"30.100.1.12\",\"port\":53,\"type\":\"Network\"}, " + + "{\"address\":\"30.100.1.13\",\"port\":53,\"type\":\"Network\"}, " + + "{\"address\":\"30.100.1.14\",\"port\":53,\"type\":\"Network\"}, " + + "{\"address\":\"30.100.1.15\",\"port\":53,\"type\":\"Network\"}, " + + "{\"address\":\"30.100.1.16\",\"port\":53,\"type\":\"Network\"}, " + + "{\"address\":\"30.100.1.17\",\"port\":53,\"type\":\"Network\"}, " + + "{\"address\":\"30.100.1.18\",\"port\":53,\"type\":\"Network\"}, " + + "{\"address\":\"30.100.1.19\",\"port\":53,\"type\":\"Network\"}, " + + "{\"address\":\"30.100.1.20\",\"port\":53,\"type\":\"Network\"}, " + + "{\"address\":\"30.100.1.21\",\"port\":53,\"type\":\"Network\"}, " + + "{\"address\":\"30.100.1.22\",\"port\":53,\"type\":\"Network\"}, " + + "{\"address\":\"30.100.1.23\",\"port\":53,\"type\":\"Network\"}, " + + "{\"address\":\"30.100.1.24\",\"port\":53,\"type\":\"Network\"}, " + + "{\"address\":\"30.100.1.25\",\"port\":53,\"type\":\"Network\"}, " + + "{\"address\":\"30.100.1.26\",\"port\":53,\"type\":\"Network\"}]}]" + invalidCfgStringUpstreams = "[{\"op\":\"replace\", \"path\":\"/spec/upstreamResolvers/upstreams\", \"value\":[" + + "{\"address\":\"str_test\",\"port\":53,\"type\":\"Network\"}]}]" + invalidCfgNumberUpstreams = "[{\"op\":\"replace\", \"path\":\"/spec/upstreamResolvers/upstreams\", \"value\":[" + + "{\"address\":\"100\",\"port\":53,\"type\":\"Network\"}]}]" + invalidCfgSringPolicy = "[{\"op\":\"replace\", \"path\":\"/spec/upstreamResolvers/policy\", \"value\":\"string_test\"}]" + invalidCfgNumberPolicy = "[{\"op\":\"replace\", \"path\":\"/spec/upstreamResolvers/policy\", \"value\":\"2\"}]" + invalidCfgRandomPolicy = "[{\"op\":\"replace\", \"path\":\"/spec/upstreamResolvers/policy\", \"value\":\"random\"}]" + ) + compat_otp.By("Prepare the dns testing node and pod") + defer deleteDnsOperatorToRestore(oc) + forceOnlyOneDnsPodExist(oc) + + compat_otp.By("Try to add one more upstream resolver, totally 16 upstream resolvers by patching dns operator") + output, _ := oc.AsAdmin().WithoutNamespace().Run("patch").Args(resourceName, "--patch="+cfgAddOneUpstreams, "--type=json").Output() + o.Expect(output).To(o.ContainSubstring("have at most 15 items")) + + compat_otp.By("Try to add a upstream resolver with a string as an address") + output, _ = oc.AsAdmin().WithoutNamespace().Run("patch").Args(resourceName, "--patch="+invalidCfgStringUpstreams, "--type=json").Output() + o.Expect(output).To(o.ContainSubstring("Invalid value: \"str_test\"")) + + compat_otp.By("Try to add a upstream resolver with a number as an address") + output, _ = oc.AsAdmin().WithoutNamespace().Run("patch").Args(resourceName, "--patch="+invalidCfgNumberUpstreams, "--type=json").Output() + o.Expect(output).To(o.ContainSubstring("Invalid value: \"100\"")) + + compat_otp.By("Try to configure the polciy with a string") + output, _ = oc.AsAdmin().WithoutNamespace().Run("patch").Args(resourceName, "--patch="+invalidCfgSringPolicy, "--type=json").Output() + o.Expect(output).To(o.ContainSubstring("Unsupported value: \"string_test\"")) + + compat_otp.By("Try to configure the polciy with a number") + output, _ = oc.AsAdmin().WithoutNamespace().Run("patch").Args(resourceName, "--patch="+invalidCfgNumberPolicy, "--type=json").Output() + o.Expect(output).To(o.ContainSubstring("Unsupported value: \"2\"")) + + compat_otp.By("Try to configure the polciy with a similar string like random") + output, _ = oc.AsAdmin().WithoutNamespace().Run("patch").Args(resourceName, "--patch="+invalidCfgRandomPolicy, "--type=json").Output() + o.Expect(output).To(o.ContainSubstring("Unsupported value: \"random\"")) + }) + + // author: shudili@redhat.com + g.It("Author:shudili-Critical-46872-Configure logLevel for CoreDNS under DNS operator flag [Disruptive]", func() { + var ( + resourceName = "dns.operator.openshift.io/default" + cfgLogLevelDebug = "[{\"op\":\"replace\", \"path\":\"/spec/logLevel\", \"value\":\"Debug\"}]" + cfgLogLevelTrace = "[{\"op\":\"replace\", \"path\":\"/spec/logLevel\", \"value\":\"Trace\"}]" + ) + compat_otp.By("Prepare the dns testing node and pod") + defer deleteDnsOperatorToRestore(oc) + oneDnsPod := forceOnlyOneDnsPodExist(oc) + + compat_otp.By("Check default log level of CoreDNS") + logOutput := pollReadDnsCorefile(oc, oneDnsPod, "log", "-A2", "class error") + o.Expect(logOutput).To(o.ContainSubstring("class error")) + + compat_otp.By("Patch dns operator with logLevel Debug for CoreDNS, and then check log class for logLevel Debug in both CM and the Corefile of coredns") + patchGlobalResourceAsAdmin(oc, resourceName, cfgLogLevelDebug) + logOutput = pollReadDnsCorefile(oc, oneDnsPod, "log", "-A2", "class denial error") + o.Expect(logOutput).To(o.ContainSubstring("class denial error")) + + compat_otp.By("Patch dns operator with logLevel Trace for CoreDNS, and then check log class for logLevel Trace in Corefile of coredns") + patchGlobalResourceAsAdmin(oc, resourceName, cfgLogLevelTrace) + logOutput = pollReadDnsCorefile(oc, oneDnsPod, "log", "-A2", "class all") + o.Expect(logOutput).To(o.ContainSubstring("class all")) + }) + + g.It("Author:shudili-Medium-46874-negative test for configuring logLevel and operatorLogLevel flag [Disruptive]", func() { + var ( + resourceName = "dns.operator.openshift.io/default" + invalidCfgStringLogLevel = "[{\"op\":\"replace\", \"path\":\"/spec/logLevel\", \"value\":\"string_test\"}]" + invalidCfgNumberLogLevel = "[{\"op\":\"replace\", \"path\":\"/spec/logLevel\", \"value\":\"2\"}]" + invalidCfgTraceLogLevel = "[{\"op\":\"replace\", \"path\":\"/spec/logLevel\", \"value\":\"trace\"}]" + invalidCfgStringOPLogLevel = "[{\"op\":\"replace\", \"path\":\"/spec/operatorLogLevel\", \"value\":\"string_test\"}]" + invalidCfgNumberOPLogLevel = "[{\"op\":\"replace\", \"path\":\"/spec/operatorLogLevel\", \"value\":\"2\"}]" + invalidCfgTraceOPLogLevel = "[{\"op\":\"replace\", \"path\":\"/spec/operatorLogLevel\", \"value\":\"trace\"}]" + ) + compat_otp.By("Prepare the dns testing node and pod") + defer deleteDnsOperatorToRestore(oc) + forceOnlyOneDnsPodExist(oc) + + compat_otp.By("Try to configure log level with a string") + output, _ := oc.AsAdmin().WithoutNamespace().Run("patch").Args(resourceName, "--patch="+invalidCfgStringLogLevel, "--type=json").Output() + o.Expect(output).To(o.ContainSubstring("Unsupported value: \"string_test\"")) + + compat_otp.By("Try to configure log level with a number") + output, _ = oc.AsAdmin().WithoutNamespace().Run("patch").Args(resourceName, "--patch="+invalidCfgNumberLogLevel, "--type=json").Output() + o.Expect(output).To(o.ContainSubstring("Unsupported value: \"2\"")) + + compat_otp.By("Try to configure log level with a similar string like trace") + output, _ = oc.AsAdmin().WithoutNamespace().Run("patch").Args(resourceName, "--patch="+invalidCfgTraceLogLevel, "--type=json").Output() + o.Expect(output).To(o.ContainSubstring("Unsupported value: \"trace\"")) + + compat_otp.By("Try to configure dns operator log level with a string") + output, _ = oc.AsAdmin().WithoutNamespace().Run("patch").Args(resourceName, "--patch="+invalidCfgStringOPLogLevel, "--type=json").Output() + o.Expect(output).To(o.ContainSubstring("Unsupported value: \"string_test\"")) + + compat_otp.By("Try to configure dns operator log level with a number") + output, _ = oc.AsAdmin().WithoutNamespace().Run("patch").Args(resourceName, "--patch="+invalidCfgNumberOPLogLevel, "--type=json").Output() + o.Expect(output).To(o.ContainSubstring("Unsupported value: \"2\"")) + + compat_otp.By("Try to configure dns operator log level with a similar string like trace") + output, _ = oc.AsAdmin().WithoutNamespace().Run("patch").Args(resourceName, "--patch="+invalidCfgTraceOPLogLevel, "--type=json").Output() + o.Expect(output).To(o.ContainSubstring("Unsupported value: \"trace\"")) + }) + + g.It("Author:shudili-Low-46875-Different LogLevel logging function of CoreDNS flag [Disruptive]", func() { + var ( + buildPruningBaseDir = testdata.FixturePath("router") + clientPod = filepath.Join(buildPruningBaseDir, "test-client-pod.yaml") + clientPodName = "hello-pod" + clientPodLabel = "app=hello-pod" + coreDNSSrvPod = filepath.Join(buildPruningBaseDir, "coreDNS-pod.yaml") + srvPodName = "test-coredns" + srvPodLabel = "name=test-coredns" + failedDNSReq = "failed.not-myocp-test.com" + nxDNSReq = "notexist.myocp-test.com" + normalDNSReq = "www.myocp-test.com" + resourceName = "dns.operator.openshift.io/default" + cfgDebug = "[{\"op\":\"replace\", \"path\":\"/spec/logLevel\", \"value\":\"Debug\"}]" + cfgTrace = "[{\"op\":\"replace\", \"path\":\"/spec/logLevel\", \"value\":\"Trace\"}]" + ) + compat_otp.By("Prepare the dns testing node and pod") + defer deleteDnsOperatorToRestore(oc) + oneDnsPod := forceOnlyOneDnsPodExist(oc) + podList := []string{oneDnsPod} + + compat_otp.By("Create a dns server pod") + ns := oc.Namespace() + defer compat_otp.RecoverNamespaceRestricted(oc, ns) + compat_otp.SetNamespacePrivileged(oc, ns) + replaceCoreDnsImage(oc, coreDNSSrvPod) + err := oc.AsAdmin().Run("create").Args("-f", coreDNSSrvPod, "-n", ns).Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + ensurePodWithLabelReady(oc, ns, srvPodLabel) + + compat_otp.By("get the user's dns server pod's IP") + srvPodIP := getPodv4Address(oc, srvPodName, ns) + + compat_otp.By("patch upstream dns resolver with the user's dns server, and then wait the corefile is updated") + dnsUpstreamResolver := "[{\"op\":\"replace\", \"path\":\"/spec/upstreamResolvers/upstreams\", \"value\":[{\"address\":\"" + srvPodIP + "\",\"port\":53,\"type\":\"Network\"}]}]" + patchGlobalResourceAsAdmin(oc, resourceName, dnsUpstreamResolver) + // Converting the IPV6 address to upper case for searching in the coreDNS file + if strings.Count(srvPodIP, ":") >= 2 { + srvPodIP = fmt.Sprintf("%s", strings.ToUpper(srvPodIP)) + srvPodIP = "[" + srvPodIP + "]" + } + pollReadDnsCorefile(oc, oneDnsPod, "forward", "-A2", srvPodIP) + + compat_otp.By("create a client pod") + createResourceFromFile(oc, ns, clientPod) + ensurePodWithLabelReady(oc, ns, clientPodLabel) + + compat_otp.By("Let client send out SERVFAIL nslookup to the dns server, and check the desired SERVFAIL logs from a coredns pod") + output := nslookupsAndWaitForDNSlog(oc, clientPodName, failedDNSReq, podList, failedDNSReq+".") + o.Expect(output).To(o.ContainSubstring(failedDNSReq)) + + compat_otp.By("Patch dns operator with logLevel Debug for CoreDNS, and wait the Corefile is updated") + patchGlobalResourceAsAdmin(oc, resourceName, cfgDebug) + pollReadDnsCorefile(oc, oneDnsPod, "log", "-A2", "class denial error") + + compat_otp.By("Let client send out NXDOMAIN nslookup to the dns server, and check the desired NXDOMAIN logs from a coredns pod") + output = nslookupsAndWaitForDNSlog(oc, clientPodName, nxDNSReq, podList, "-type=mx", nxDNSReq+".") + o.Expect(output).To(o.ContainSubstring(nxDNSReq)) + + compat_otp.By("Patch dns operator with logLevel Trace for CoreDNS, and wait the Corefile is updated") + patchGlobalResourceAsAdmin(oc, resourceName, cfgTrace) + pollReadDnsCorefile(oc, oneDnsPod, "log", "-A2", "class all") + + compat_otp.By("Let client send out normal nslookup which will get correct response, and check the desired TRACE logs from a coredns pod") + output = nslookupsAndWaitForDNSlog(oc, clientPodName, normalDNSReq, podList, normalDNSReq+".") + o.Expect(output).To(o.ContainSubstring(normalDNSReq)) + }) + + g.It("Author:mjoseph-NonHyperShiftHOST-Critical-51536-Support CoreDNS forwarding DNS requests over TLS using ForwardPlugin [Disruptive]", func() { + var ( + buildPruningBaseDir = testdata.FixturePath("router") + cmFile = filepath.Join(buildPruningBaseDir, "ca-bundle.pem") + coreDNSSrvPod = filepath.Join(buildPruningBaseDir, "coreDNS-pod.yaml") + srvPodName = "test-coredns" + srvPodLabel = "name=test-coredns" + resourceName = "dns.operator.openshift.io/default" + ) + + compat_otp.By("1.Prepare the dns testing node and pod") + defer deleteDnsOperatorToRestore(oc) + oneDnsPod := forceOnlyOneDnsPodExist(oc) + + compat_otp.By("2.Create a dns server pod") + ns := oc.Namespace() + defer compat_otp.RecoverNamespaceRestricted(oc, ns) + compat_otp.SetNamespacePrivileged(oc, ns) + replaceCoreDnsImage(oc, coreDNSSrvPod) + err := oc.AsAdmin().Run("create").Args("-f", coreDNSSrvPod, "-n", ns).Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + ensurePodWithLabelReady(oc, ns, srvPodLabel) + + compat_otp.By("3.Get the user's dns server pod's IP") + srvPodIP := getPodv4Address(oc, srvPodName, ns) + + compat_otp.By("4.Create configmap client-ca-xxxxx in namespace openshift-config") + defer deleteConfigMap(oc, "openshift-config", "ca-51536-bundle") + createConfigMapFromFile(oc, "openshift-config", "ca-51536-bundle", cmFile) + + compat_otp.By("5.Patch the dns.operator/default with transport option as TLS for forwardplugin") + dnsForwardPlugin := "[{\"op\":\"replace\", \"path\":\"/spec\", \"value\":{\"servers\":[{\"forwardPlugin\":{\"policy\":\"Sequential\",\"transportConfig\": {\"tls\":{\"caBundle\": {\"name\": \"ca-51536-bundle\"}, \"serverName\": \"dns.ocp51536.ocp\"}, \"transport\": \"TLS\"}, \"upstreams\":[\"" + srvPodIP + "\"]}, \"name\": \"test\", \"zones\":[\"ocp51536.ocp\"]}]}}]" + patchGlobalResourceAsAdmin(oc, resourceName, dnsForwardPlugin) + + compat_otp.By("6.Check and confirm the upstream resolver's IP(srvPodIP) and custom CAbundle name appearing in the dns pod") + forward := pollReadDnsCorefile(oc, oneDnsPod, srvPodIP, "-b6", "ocp51536") + o.Expect(forward).To(o.ContainSubstring("ocp51536.ocp:5353")) + o.Expect(forward).To(o.ContainSubstring("forward . tls://" + srvPodIP)) + o.Expect(forward).To(o.ContainSubstring("tls_servername dns.ocp51536.ocp")) + o.Expect(forward).To(o.ContainSubstring("tls /etc/pki/dns.ocp51536.ocp-ca-ca-51536-bundle")) + + compat_otp.By("7.Check no error logs from dns operator pod") + dnsOperatorPodName := getPodListByLabel(oc, "openshift-dns-operator", "name=dns-operator") + podLogs, errLogs := compat_otp.GetSpecificPodLogs(oc, "openshift-dns-operator", "dns-operator", dnsOperatorPodName[0], `ocp51536.ocp:5353 -A3`) + o.Expect(errLogs).NotTo(o.HaveOccurred(), "Error in getting logs from the pod") + o.Expect(podLogs).To(o.ContainSubstring(`msg="reconciling request: /default"`)) + }) + + g.It("Author:mjoseph-NonHyperShiftHOST-Low-51857-Support CoreDNS forwarding DNS requests over TLS - non existing CA bundle [Disruptive]", func() { + var ( + buildPruningBaseDir = testdata.FixturePath("router") + coreDNSSrvPod = filepath.Join(buildPruningBaseDir, "coreDNS-pod.yaml") + srvPodName = "test-coredns" + srvPodLabel = "name=test-coredns" + resourceName = "dns.operator.openshift.io/default" + ) + + compat_otp.By("1.Prepare the dns testing node and pod") + defer deleteDnsOperatorToRestore(oc) + oneDnsPod := forceOnlyOneDnsPodExist(oc) + + compat_otp.By("2.Create a dns server pod") + ns := oc.Namespace() + defer compat_otp.RecoverNamespaceRestricted(oc, ns) + compat_otp.SetNamespacePrivileged(oc, ns) + replaceCoreDnsImage(oc, coreDNSSrvPod) + err := oc.AsAdmin().Run("create").Args("-f", coreDNSSrvPod, "-n", ns).Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + ensurePodWithLabelReady(oc, ns, srvPodLabel) + + compat_otp.By("3.Get the user's dns server pod's IP") + srvPodIP := getPodv4Address(oc, srvPodName, ns) + + compat_otp.By("4.Patch the dns.operator/default with non existing CA bundle for forwardplugin") + dnsForwardPlugin := "[{\"op\":\"replace\", \"path\":\"/spec\", \"value\":{\"servers\":[{\"forwardPlugin\":{\"policy\":\"Sequential\",\"transportConfig\": {\"tls\":{\"caBundle\": {\"name\": \"ca-51857-bundle\"}, \"serverName\": \"dns.ocp51857.ocp\"}, \"transport\": \"TLS\"}, \"upstreams\":[\"" + srvPodIP + "\"]}, \"name\": \"test\", \"zones\":[\"ocp51857.ocp\"]}]}}]" + patchGlobalResourceAsAdmin(oc, resourceName, dnsForwardPlugin) + + compat_otp.By("5.Check and confirm the upstream resolver's IP(srvPodIP) appearing without the custom CAbundle name") + forward := pollReadDnsCorefile(oc, oneDnsPod, srvPodIP, "-b6", "ocp51857") + o.Expect(forward).To(o.ContainSubstring("ocp51857.ocp:5353")) + o.Expect(forward).To(o.ContainSubstring("forward . tls://" + srvPodIP)) + o.Expect(forward).To(o.ContainSubstring("tls_servername dns.ocp51857.ocp")) + o.Expect(forward).To(o.ContainSubstring("tls")) + o.Expect(forward).NotTo(o.ContainSubstring("/etc/pki/dns.ocp51857.ocp-ca-ca-51857-bundle")) + + compat_otp.By("6.Check and confirm the non configured CABundle warning message from dns operator pod") + dnsOperatorPodName := getPodListByLabel(oc, "openshift-dns-operator", "name=dns-operator") + podLogs1, errLogs := compat_otp.GetSpecificPodLogs(oc, "openshift-dns-operator", "dns-operator", dnsOperatorPodName[0], `ocp51857.ocp:5353 -A3`) + o.Expect(errLogs).NotTo(o.HaveOccurred(), "Error in getting logs from the pod") + o.Expect(podLogs1).To(o.ContainSubstring(`level=warning msg="source ca bundle configmap ca-51857-bundle does not exist"`)) + o.Expect(podLogs1).To(o.ContainSubstring(`level=warning msg="failed to get destination ca bundle configmap ca-ca-51857-bundle: configmaps \"ca-ca-51857-bundle\" not found"`)) + }) + + g.It("Author:mjoseph-NonHyperShiftHOST-Critical-51946-Support CoreDNS forwarding DNS requests over TLS using UpstreamResolvers [Disruptive]", func() { + var ( + buildPruningBaseDir = testdata.FixturePath("router") + coreDNSSrvPod = filepath.Join(buildPruningBaseDir, "coreDNS-pod.yaml") + srvPodName = "test-coredns" + srvPodLabel = "name=test-coredns" + resourceName = "dns.operator.openshift.io/default" + dirname = "/tmp/OCP-51946-ca/" + caKey = dirname + "ca.key" + caCert = dirname + "ca-bundle.crt" + caSubj = "/CN=NE-Test-Root-CA" + dnsPodLabel = "dns.operator.openshift.io/daemonset-dns=default" + ) + + compat_otp.By("1.Prepare the dns testing node and pod") + defer deleteDnsOperatorToRestore(oc) + oneDnsPod := forceOnlyOneDnsPodExist(oc) + + compat_otp.By("2.Create a dns server pod") + ns := oc.Namespace() + defer compat_otp.RecoverNamespaceRestricted(oc, ns) + compat_otp.SetNamespacePrivileged(oc, ns) + replaceCoreDnsImage(oc, coreDNSSrvPod) + err := oc.AsAdmin().Run("create").Args("-f", coreDNSSrvPod, "-n", ns).Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + ensurePodWithLabelReady(oc, ns, srvPodLabel) + srvPodIP := getPodv4Address(oc, srvPodName, ns) + + compat_otp.By("3.Generate a new self-signed CA") + defer os.RemoveAll(dirname) + err = os.MkdirAll(dirname, 0755) + o.Expect(err).NotTo(o.HaveOccurred()) + e2e.Logf("Generate the CA private key") + opensslCmd := fmt.Sprintf(`openssl genrsa -out %s 4096`, caKey) + _, err = exec.Command("bash", "-c", opensslCmd).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + + e2e.Logf("Create the CA certificate") + opensslCmd = fmt.Sprintf(`openssl req -x509 -new -nodes -key %s -sha256 -days 1 -out %s -subj %s`, caKey, caCert, caSubj) + _, err = exec.Command("bash", "-c", opensslCmd).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + + compat_otp.By("4.Create configmap ca-xxxxx-bundle in namespace openshift-config") + defer deleteConfigMap(oc, "openshift-config", "ca-51946-bundle") + createConfigMapFromFile(oc, "openshift-config", "ca-51946-bundle", caCert) + + compat_otp.By("5.Patch the dns.operator/default with transport option as TLS for upstreamresolver") + dnsUpstreamResolver := "[{\"op\":\"replace\", \"path\":\"/spec/upstreamResolvers\", \"value\":{\"transportConfig\": {\"tls\":{\"caBundle\": {\"name\": \"ca-51946-bundle\"}, \"serverName\": \"dns.ocp51946.ocp\"}, \"transport\": \"TLS\"}, \"upstreams\":[{\"address\":\"" + srvPodIP + "\", \"port\": 853, \"type\":\"Network\"}]}}]" + patchGlobalResourceAsAdmin(oc, resourceName, dnsUpstreamResolver) + + compat_otp.By("6.Check and confirm the upstream resolver's IP(srvPodIP) and custom CAbundle name appearing in the dns pod") + // Converting the IPV6 address to upper case for searching in the coreDNS file + if strings.Count(srvPodIP, ":") >= 2 { + srvPodIP = fmt.Sprintf("%s", strings.ToUpper(srvPodIP)) + srvPodIP = "[" + srvPodIP + "]" + } + // since new configmap is mounted so dns pod is restarted + waitErr := waitForResourceToDisappear(oc, "openshift-dns", "pod/"+oneDnsPod) + compat_otp.AssertWaitPollNoErr(waitErr, fmt.Sprintf("max time reached but pod %s is not terminated", oneDnsPod)) + ensurePodWithLabelReady(oc, "openshift-dns", dnsPodLabel) + newDnsPod := getDNSPodName(oc) + upstreams := readDNSCorefile(oc, newDnsPod, srvPodIP, "-A4") + o.Expect(upstreams).To(o.ContainSubstring("forward . tls://" + srvPodIP + ":853")) + o.Expect(upstreams).To(o.ContainSubstring("tls_servername dns.ocp51946.ocp")) + o.Expect(upstreams).To(o.ContainSubstring("tls /etc/pki/dns.ocp51946.ocp-ca-ca-51946-bundle")) + + compat_otp.By("7.Check no error logs from dns operator pod") + dnsOperatorPodName := getPodListByLabel(oc, "openshift-dns-operator", "name=dns-operator") + podLogs, errLogs := compat_otp.GetSpecificPodLogs(oc, "openshift-dns-operator", "dns-operator", dnsOperatorPodName[0], srvPodIP+` -A3`) + o.Expect(errLogs).NotTo(o.HaveOccurred(), "Error in getting logs from the pod") + o.Expect(podLogs).To(o.ContainSubstring(`msg="reconciling request: /default"`)) + }) + + g.It("Author:mjoseph-NonHyperShiftHOST-High-52077-CoreDNS forwarding DNS requests over TLS with CLEAR TEXT [Disruptive]", func() { + var ( + buildPruningBaseDir = testdata.FixturePath("router") + coreDNSSrvPod = filepath.Join(buildPruningBaseDir, "coreDNS-pod.yaml") + srvPodName = "test-coredns" + srvPodLabel = "name=test-coredns" + resourceName = "dns.operator.openshift.io/default" + ) + + compat_otp.By("1.Prepare the dns testing node and pod") + defer deleteDnsOperatorToRestore(oc) + oneDnsPod := forceOnlyOneDnsPodExist(oc) + + compat_otp.By("2.Create a dns server pod") + ns := oc.Namespace() + defer compat_otp.RecoverNamespaceRestricted(oc, ns) + compat_otp.SetNamespacePrivileged(oc, ns) + replaceCoreDnsImage(oc, coreDNSSrvPod) + err := oc.AsAdmin().Run("create").Args("-f", coreDNSSrvPod, "-n", ns).Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + ensurePodWithLabelReady(oc, ns, srvPodLabel) + + compat_otp.By("3.Get the user's dns server pod's IP") + srvPodIP := getPodv4Address(oc, srvPodName, ns) + + compat_otp.By("4.Patch the dns.operator/default with transport option as Cleartext for forwardplugin") + dnsForwardPlugin := "[{\"op\":\"add\", \"path\":\"/spec/servers\", \"value\":[{\"forwardPlugin\":{\"policy\":\"Sequential\",\"transportConfig\": {\"transport\": \"Cleartext\"}, \"upstreams\":[\"" + srvPodIP + "\"]}, \"name\": \"test\", \"zones\":[\"ocp52077.ocp\"]}]}]" + patchGlobalResourceAsAdmin(oc, resourceName, dnsForwardPlugin) + + compat_otp.By("5.Check and confirm the upstream resolver's IP(srvPodIP) appearing in the dns pod") + forward := pollReadDnsCorefile(oc, oneDnsPod, srvPodIP, "-b6", "ocp52077") + o.Expect(forward).To(o.ContainSubstring("ocp52077.ocp:5353")) + o.Expect(forward).To(o.ContainSubstring("forward . " + srvPodIP)) + + compat_otp.By("6.Check no error logs from dns operator pod") + dnsOperatorPodName := getPodListByLabel(oc, "openshift-dns-operator", "name=dns-operator") + podLogs1, errLogs1 := compat_otp.GetSpecificPodLogs(oc, "openshift-dns-operator", "dns-operator", dnsOperatorPodName[0], `ocp52077.ocp:5353 -A3`) + o.Expect(errLogs1).NotTo(o.HaveOccurred(), "Error in getting logs from the pod") + o.Expect(podLogs1).To(o.ContainSubstring(`msg="reconciling request: /default"`)) + // Patching to remove the forwardplugin configurations. + dnsDefault := "[{\"op\":\"remove\", \"path\":\"/spec/servers\"}]" + patchGlobalResourceAsAdmin(oc, resourceName, dnsDefault) + + compat_otp.By("7.Patch dns.operator/default with transport option as Cleartext for upstreamresolver") + dnsUpstreamResolver := "[{\"op\":\"replace\", \"path\":\"/spec/upstreamResolvers\", \"value\":{\"transportConfig\":{\"transport\":\"Cleartext\"}, \"upstreams\":[{\"address\":\"" + srvPodIP + "\", \"type\":\"Network\"}]}}]" + patchGlobalResourceAsAdmin(oc, resourceName, dnsUpstreamResolver) + + compat_otp.By("8.Check and confirm the upstream resolver's IP(srvPodIP) appearing in the dns pod") + // Converting the IPV6 address to upper case for searching in the coreDNS file + if strings.Count(srvPodIP, ":") >= 2 { + srvPodIP = fmt.Sprintf("%s", strings.ToUpper(srvPodIP)) + srvPodIP = "[" + srvPodIP + "]" + } + upstreams := pollReadDnsCorefile(oc, oneDnsPod, srvPodIP, "-A2", "forward") + o.Expect(upstreams).To(o.ContainSubstring("forward . " + srvPodIP + ":53")) + + compat_otp.By("9.Check no error logs from dns operator pod") + podLogs, errLogs := compat_otp.GetSpecificPodLogs(oc, "openshift-dns-operator", "dns-operator", dnsOperatorPodName[0], srvPodIP+`:53 -A3`) + o.Expect(errLogs).NotTo(o.HaveOccurred(), "Error in getting logs from the pod") + o.Expect(podLogs).To(o.ContainSubstring(`msg="reconciling request: /default"`)) + }) + + g.It("Author:mjoseph-NonHyperShiftHOST-High-52497-Support CoreDNS forwarding DNS requests over TLS - using system CA [Disruptive]", func() { + var ( + buildPruningBaseDir = testdata.FixturePath("router") + coreDNSSrvPod = filepath.Join(buildPruningBaseDir, "coreDNS-pod.yaml") + srvPodName = "test-coredns" + srvPodLabel = "name=test-coredns" + resourceName = "dns.operator.openshift.io/default" + ) + + compat_otp.By("1.Prepare the dns testing node and pod") + defer deleteDnsOperatorToRestore(oc) + oneDnsPod := forceOnlyOneDnsPodExist(oc) + + compat_otp.By("2.Create a dns server pod") + ns := oc.Namespace() + defer compat_otp.RecoverNamespaceRestricted(oc, ns) + compat_otp.SetNamespacePrivileged(oc, ns) + replaceCoreDnsImage(oc, coreDNSSrvPod) + err := oc.AsAdmin().Run("create").Args("-f", coreDNSSrvPod, "-n", ns).Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + ensurePodWithLabelReady(oc, ns, srvPodLabel) + + compat_otp.By("3.Get the user's dns server pod's IP") + srvPodIP := getPodv4Address(oc, srvPodName, ns) + + compat_otp.By("4.Patch the dns.operator/default with transport option as tls for forwardplugin") + dnsForwardPlugin := "[{\"op\":\"add\", \"path\":\"/spec/servers\", \"value\":[{\"forwardPlugin\":{\"policy\":\"Sequential\",\"transportConfig\": {\"tls\":{\"serverName\": \"dns.ocp52497.ocp\"}, \"transport\": \"TLS\"}, \"upstreams\":[\"" + srvPodIP + "\"]}, \"name\": \"test\", \"zones\":[\"ocp52497.ocp\"]}]}]" + patchGlobalResourceAsAdmin(oc, resourceName, dnsForwardPlugin) + + compat_otp.By("5.Check and confirm the upstream resolver's IP(srvPodIP) appearing in the dns pod") + forward := pollReadDnsCorefile(oc, oneDnsPod, srvPodIP, "-b6", "ocp52497") + o.Expect(forward).To(o.ContainSubstring("ocp52497.ocp:5353")) + o.Expect(forward).To(o.ContainSubstring("forward . tls://" + srvPodIP)) + o.Expect(forward).To(o.ContainSubstring("tls_servername dns.ocp52497.ocp")) + o.Expect(forward).To(o.ContainSubstring("tls")) + + compat_otp.By("6.Check no error logs from dns operator pod") + dnsOperatorPodName := getPodListByLabel(oc, "openshift-dns-operator", "name=dns-operator") + podLogs1, errLogs1 := compat_otp.GetSpecificPodLogs(oc, "openshift-dns-operator", "dns-operator", dnsOperatorPodName[0], `ocp52497.ocp:5353 -A3`) + o.Expect(errLogs1).NotTo(o.HaveOccurred(), "Error in getting logs from the pod") + o.Expect(podLogs1).To(o.ContainSubstring(`msg="reconciling request: /default"`)) + // Patching to remove the forwardplugin configurations. + dnsDefault := "[{\"op\":\"remove\", \"path\":\"/spec/servers\"}]" + patchGlobalResourceAsAdmin(oc, resourceName, dnsDefault) + + compat_otp.By("7.Patch dns.operator/default with transport option as tls for upstreamresolver") + dnsUpstreamResolver := "[{\"op\":\"replace\", \"path\":\"/spec/upstreamResolvers\", \"value\":{\"transportConfig\": {\"tls\":{\"serverName\": \"dns.ocp52497.ocp\"}, \"transport\": \"TLS\"}, \"upstreams\":[{\"address\":\"" + srvPodIP + "\", \"port\": 853, \"type\":\"Network\"}]}}]" + patchGlobalResourceAsAdmin(oc, resourceName, dnsUpstreamResolver) + + compat_otp.By("8.Check and confirm the upstream resolver's IP(srvPodIP) appearing in the dns pod") + // Converting the IPV6 address to upper case for searching in the coreDNS file + if strings.Count(srvPodIP, ":") >= 2 { + srvPodIP = fmt.Sprintf("%s", strings.ToUpper(srvPodIP)) + srvPodIP = "[" + srvPodIP + "]" + } + upstreams := pollReadDnsCorefile(oc, oneDnsPod, srvPodIP, "-A3", "forward") + o.Expect(upstreams).To(o.ContainSubstring("forward . tls://" + srvPodIP + ":853")) + o.Expect(upstreams).To(o.ContainSubstring("tls_servername dns.ocp52497.ocp")) + o.Expect(upstreams).To(o.ContainSubstring("tls")) + + compat_otp.By("9.Check no error logs from dns operator pod") + podLogs, errLogs := compat_otp.GetSpecificPodLogs(oc, "openshift-dns-operator", "dns-operator", dnsOperatorPodName[0], srvPodIP+` -A3`) + o.Expect(errLogs).NotTo(o.HaveOccurred(), "Error in getting logs from the pod") + o.Expect(podLogs).To(o.ContainSubstring(`msg="reconciling request: /default"`)) + }) + + g.It("Author:mjoseph-Critical-54042-Configuring CoreDNS caching and TTL parameters [Disruptive]", func() { + var ( + resourceName = "dns.operator.openshift.io/default" + cacheValue = "[{\"op\":\"replace\", \"path\":\"/spec/cache\", \"value\":{\"negativeTTL\":\"1800s\", \"positiveTTL\":\"604801s\"}}]" + cacheSmallValue = "[{\"op\":\"replace\", \"path\":\"/spec/cache\", \"value\":{\"negativeTTL\":\"1s\", \"positiveTTL\":\"1s\"}}]" + cacheDecimalValue = "[{\"op\":\"replace\", \"path\":\"/spec/cache\", \"value\":{\"negativeTTL\":\"1.9s\", \"positiveTTL\":\"1.6m\"}}]" + cacheWrongValue = "[{\"op\":\"replace\", \"path\":\"/spec/cache\", \"value\":{\"negativeTTL\":\"-9s\", \"positiveTTL\":\"1.6\"}}]" + ) + + compat_otp.By("1. Prepare the dns testing node and pod") + defer deleteDnsOperatorToRestore(oc) + oneDnsPod := forceOnlyOneDnsPodExist(oc) + + compat_otp.By("2. Patch the dns.operator/default with postive and negative cache values") + patchGlobalResourceAsAdmin(oc, resourceName, cacheValue) + + compat_otp.By("3. Check the cache value in Corefile of coredn") + cache := pollReadDnsCorefile(oc, oneDnsPod, "cache 604801", "-A2", "denial") + o.Expect(cache).To(o.ContainSubstring("denial 9984 1800")) + + compat_otp.By("4. Patch the dns.operator/default with smallest cache values and verify the same") + patchGlobalResourceAsAdmin(oc, resourceName, cacheSmallValue) + cache1 := pollReadDnsCorefile(oc, oneDnsPod, "cache 1", "-A2", "denial") + o.Expect(cache1).To(o.ContainSubstring("denial 9984 1")) + + compat_otp.By("5. Patch the dns.operator/default with decimal cache values and verify the same") + patchGlobalResourceAsAdmin(oc, resourceName, cacheDecimalValue) + cache2 := pollReadDnsCorefile(oc, oneDnsPod, "cache 96", "-A2", "denial") + o.Expect(cache2).To(o.ContainSubstring("denial 9984 2")) + + compat_otp.By("6. Patch the dns.operator/default with unrelasitc cache values and check the error messages") + output, _ := oc.AsAdmin().WithoutNamespace().Run("patch").Args(resourceName, "--patch="+cacheWrongValue, "--type=json").Output() + o.Expect(output).To(o.ContainSubstring("spec.cache.positiveTTL: Invalid value: \"1.6\"")) + o.Expect(output).To(o.ContainSubstring("spec.cache.negativeTTL: Invalid value: \"-9s\"")) + }) + + // Bug: 1949361, 1884053, 1756344 + g.It("Author:mjoseph-NonHyperShiftHOST-High-55821-Check CoreDNS default bufsize, readinessProbe path and policy", func() { + var ( + buildPruningBaseDir = testdata.FixturePath("router") + clientPod = filepath.Join(buildPruningBaseDir, "test-client-pod.yaml") + clientPodLabel = "app=hello-pod" + clientPodName = "hello-pod" + ptrValue = "10.0.30.172.in-addr.arpa" + ) + ns := oc.Namespace() + + compat_otp.By("Check updated value in dns operator file") + output, err := oc.AsAdmin().Run("get").Args("cm/dns-default", "-n", "openshift-dns", "-o=jsonpath={.data.Corefile}").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(output).To(o.ContainSubstring("bufsize 1232")) + + compat_otp.By("Check the cache value in Corefile of coredns under all dns-default-xxx pods") + podList := getAllDNSPodsNames(oc) + keepSearchInAllDNSPods(oc, podList, "bufsize 1232") + + compat_otp.By("Create a client pod") + createResourceFromFile(oc, ns, clientPod) + ensurePodWithLabelReady(oc, ns, clientPodLabel) + + compat_otp.By("Client send out a dig for google.com to check response") + digOutput, err2 := oc.Run("exec").Args(clientPodName, "--", "dig", "google.com").Output() + o.Expect(err2).NotTo(o.HaveOccurred()) + o.Expect(digOutput).To(o.ContainSubstring("udp: 1232")) + + compat_otp.By("Client send out a dig for NXDOMAIN to check response") + digOutput1, err3 := oc.Run("exec").Args(clientPodName, "--", "dig", "nxdomain.google.com").Output() + o.Expect(err3).NotTo(o.HaveOccurred()) + o.Expect(digOutput1).To(o.ContainSubstring("udp: 1232")) + + compat_otp.By("Check the different DNS records") + ingressContPod := getPodListByLabel(oc, "openshift-ingress-operator", "name=ingress-operator") + // To identify which address type the cluster IP belongs + clusterIP := getSvcClusterIPByName(oc, "openshift-dns", "dns-default") + if netutils.IsIPv6String(clusterIP) { + ptrValue = convertV6AddressToPTR(clusterIP) + } + + // To find the PTR record + digOutput3, err3 := oc.AsAdmin().Run("exec").Args("-n", "openshift-ingress-operator", ingressContPod[0], + "--", "dig", "+short", ptrValue, "PTR").Output() + o.Expect(err3).NotTo(o.HaveOccurred()) + o.Expect(digOutput3).To(o.ContainSubstring("dns-default.openshift-dns.svc.cluster.local.")) + + // To find the SRV record + digOutput4, err4 := oc.AsAdmin().Run("exec").Args("-n", "openshift-ingress-operator", ingressContPod[0], "--", "dig", + "+short", "_8443-tcp._tcp.ingress-canary.openshift-ingress-canary.svc.cluster.local", "SRV").Output() + o.Expect(err4).NotTo(o.HaveOccurred()) + o.Expect(digOutput4).To(o.ContainSubstring("ingress-canary.openshift-ingress-canary.svc.cluster.local.")) + + // bug:- 1884053 + compat_otp.By("Check Readiness probe configured to use the '/ready' path") + dnsPodName2 := getRandomElementFromList(podList) + output2, err4 := oc.AsAdmin().Run("get").Args("pod/"+dnsPodName2, "-n", "openshift-dns", "-o=jsonpath={.spec.containers[0].readinessProbe.httpGet}").Output() + o.Expect(err4).NotTo(o.HaveOccurred()) + o.Expect(output2).To(o.ContainSubstring(`"path":"/ready"`)) + + // bug:- 1756344 + compat_otp.By("Check the policy is sequential in Corefile of coredns under all dns-default-xxx pods") + keepSearchInAllDNSPods(oc, podList, "policy sequential") + }) + + // Bug: 2061244 + // no master nodes on HyperShift guest cluster so this case is not available + g.It("Author:mjoseph-NonHyperShiftHOST-High-56325-DNS pod should not work on nodes with taint configured [Disruptive]", func() { + + compat_otp.By("Check whether the dns pods eviction annotation is set or not") + podList := getAllDNSPodsNames(oc) + dnsPodName := getRandomElementFromList(podList) + findAnnotation := getAnnotation(oc, "openshift-dns", "po", dnsPodName) + o.Expect(findAnnotation).To(o.ContainSubstring(`cluster-autoscaler.kubernetes.io/enable-ds-eviction":"true`)) + + // get the worker and master node name + masterNodes := getByLabelAndJsonPath(oc, "default", "node", "node-role.kubernetes.io/master", "{.items[*].metadata.name}") + workerNodes := getByLabelAndJsonPath(oc, "default", "node", "node-role.kubernetes.io/worker", "{.items[*].metadata.name}") + masterNodeName := getRandomElementFromList(strings.Split(masterNodes, " ")) + workerNodeName := getRandomElementFromList(strings.Split(workerNodes, " ")) + + compat_otp.By("Apply NoSchedule taint to worker node and confirm the dns pod is not scheduled") + defer deleteTaint(oc, "node", workerNodeName, "dedicated-") + addTaint(oc, "node", workerNodeName, "dedicated=Kafka:NoSchedule") + // Confirming one node is not schedulable with dns pod + podOut, err := oc.AsAdmin().WithoutNamespace().Run("describe").Args("-n", "openshift-dns", "ds", "dns-default").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + if !strings.Contains(podOut, "Number of Nodes Misscheduled: 1") { + e2e.Logf("Number of Nodes Misscheduled: 1 is not expected") + } + + compat_otp.By("Apply NoSchedule taint to master node and confirm the dns pod is not scheduled on it") + defer deleteTaint(oc, "node", masterNodeName, "dns-taint-") + addTaint(oc, "node", masterNodeName, "dns-taint=test:NoSchedule") + // Confirming two nodes are not schedulable with dns pod + podOut2, err := oc.AsAdmin().WithoutNamespace().Run("describe").Args("-n", "openshift-dns", "ds", "dns-default").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + if !strings.Contains(podOut2, "Number of Nodes Misscheduled: 2") { + e2e.Logf("Number of Nodes Misscheduled: 2 is not expected") + } + }) + + // Bug: 1916907 + // Bug: OCPBUGS-35063 + g.It("Author:mjoseph-NonHyperShiftHOST-Longduration-NonPreRelease-High-56539-Disabling the internal registry should not corrupt /etc/hosts [Disruptive]", func() { + compat_otp.By("Step1: Get the Cluster IP of image-registry") + // Skip the test case if openshift-image-registry namespace is not found + clusterIP, err := oc.AsAdmin().WithoutNamespace().Run("get").Args( + "service", "image-registry", "-n", "openshift-image-registry", "-o=jsonpath={.spec.clusterIP}").Output() + if err != nil || strings.Contains(clusterIP, `namespaces \"openshift-image-registry\" not found`) { + g.Skip("Skip for non-supported platform") + } + o.Expect(err).NotTo(o.HaveOccurred()) + + compat_otp.By("Step2: SSH to the node and confirm the /etc/hosts have the same clusterIP") + allNodeList, _ := compat_otp.GetAllNodes(oc) + // get a random node + node := getRandomElementFromList(allNodeList) + hostOutput, err := compat_otp.DebugNodeWithChroot(oc, node, "cat", "/etc/hosts") + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(hostOutput).To(o.And( + o.ContainSubstring("127.0.0.1 localhost localhost.localdomain localhost4 localhost4.localdomain4"), + o.ContainSubstring("::1 localhost localhost.localdomain localhost6 localhost6.localdomain6"), + o.ContainSubstring(clusterIP))) + o.Expect(hostOutput).NotTo(o.And(o.ContainSubstring("error"), o.ContainSubstring("failed"), o.ContainSubstring("timed out"))) + + // Set status variables + expectedStatus := map[string]string{"Available": "True", "Progressing": "False", "Degraded": "False"} + + compat_otp.By("Step3: Disable the internal registry and check /host details") + defer func() { + compat_otp.By("Recover image registry change") + err4 := oc.AsAdmin().Run("patch").Args("configs.imageregistry/cluster", "-p", "{\"spec\":{\"managementState\":\"Managed\"}}", "--type=merge").Execute() + o.Expect(err4).NotTo(o.HaveOccurred()) + err = waitCoBecomes(oc, "image-registry", 240, expectedStatus) + o.Expect(err).NotTo(o.HaveOccurred()) + err = waitCoBecomes(oc, "openshift-apiserver", 480, expectedStatus) + o.Expect(err).NotTo(o.HaveOccurred()) + err = waitCoBecomes(oc, "kube-apiserver", 800, expectedStatus) + o.Expect(err).NotTo(o.HaveOccurred()) + }() + // Set image registry to 'Removed' + _, err = oc.WithoutNamespace().AsAdmin().Run("patch").Args("configs.imageregistry/cluster", "-p", `{"spec":{"managementState":"Removed"}}`, "--type=merge").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + + compat_otp.By("Step4: SSH to the node and confirm the /etc/hosts details, after disabling") + hostOutput2, err5 := compat_otp.DebugNodeWithChroot(oc, node, "cat", "/etc/hosts") + o.Expect(err5).NotTo(o.HaveOccurred()) + o.Expect(hostOutput2).To(o.And( + o.ContainSubstring("127.0.0.1 localhost localhost.localdomain localhost4 localhost4.localdomain4"), + o.ContainSubstring("::1 localhost localhost.localdomain localhost6 localhost6.localdomain6"))) + o.Expect(hostOutput2).NotTo(o.And(o.ContainSubstring("error"), o.ContainSubstring("failed"), o.ContainSubstring("timed out"))) + }) + + g.It("Author:mjoseph-ROSA-OSD_CCS-ARO-Critical-56884-Confirm the coreDNS version and Kubernetes version of the oc client", func() { + var kubernetesVersion = "v1.34" + var coreDNS = "CoreDNS-1.13.1" + + compat_otp.By("1.Check the Kubernetes version") + ocClientOutput, err := oc.AsAdmin().WithoutNamespace().Run("version").Args("--client=false").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(ocClientOutput).To(o.ContainSubstring(kubernetesVersion)) + + compat_otp.By("2.Check all default dns pods for coredns version") + cmd := fmt.Sprintf("coredns --version") + podList := getAllDNSPodsNames(oc) + dnsPod := getRandomElementFromList(podList) + output, err := oc.AsAdmin().Run("exec").Args("-n", "openshift-dns", dnsPod, "-c", "dns", "--", "bash", "-c", cmd).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(output).To(o.ContainSubstring(coreDNS)) + }) + + g.It("Author:mjoseph-Critical-60350-Check the max number of domains in the search path list of any pod", func() { + var ( + buildPruningBaseDir = testdata.FixturePath("router") + clientPod = filepath.Join(buildPruningBaseDir, "testpod-60350.yaml") + clientPodLabel = "app=testpod-60350" + clientPodName = "testpod-60350" + ) + ns := oc.Namespace() + + compat_otp.By("Create a pod with 32 DNS search list") + createResourceFromFile(oc, ns, clientPod) + ensurePodWithLabelReady(oc, ns, clientPodLabel) + + compat_otp.By("Check the pod event logs and confirm there is no Search Line limits") + checkPodEvent := describePodResource(oc, clientPodName, ns) + o.Expect(checkPodEvent).NotTo(o.ContainSubstring("Warning DNSConfigForming")) + + compat_otp.By("Check the resulting pod have all those search entries in its /etc/resolf.conf") + execOutput, err := oc.Run("exec").Args(clientPodName, "--", "sh", "-c", "cat /etc/resolv.conf").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(execOutput).To(o.ContainSubstring("8th.com 9th.com 10th.com 11th.com 12th.com 13th.com 14th.com 15th.com 16th.com 17th.com 18th.com 19th.com 20th.com 21th.com 22th.com 23th.com 24th.com 25th.com 26th.com 27th.com 28th.com 29th.com 30th.com 31th.com 32th.com")) + }) + + g.It("Author:mjoseph-Critical-60492-Check the max number of characters in the search path of any pod", func() { + var ( + buildPruningBaseDir = testdata.FixturePath("router") + clientPod = filepath.Join(buildPruningBaseDir, "testpod-60492.yaml") + clientPodLabel = "app=testpod-60492" + clientPodName = "testpod-60492" + ) + ns := oc.Namespace() + + compat_otp.By("Create a pod with a single search path with 253 characters") + createResourceFromFile(oc, ns, clientPod) + ensurePodWithLabelReady(oc, ns, clientPodLabel) + + compat_otp.By("Check the pod event logs and confirm there is no Search Line limits") + checkPodEvent := describePodResource(oc, clientPodName, ns) + o.Expect(checkPodEvent).NotTo(o.ContainSubstring("Warning DNSConfigForming")) + + compat_otp.By("Check the resulting pod have all those search entries in its /etc/resolf.conf") + execOutput, err := oc.Run("exec").Args(clientPodName, "--", "sh", "-c", "cat /etc/resolv.conf").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(execOutput).To(o.ContainSubstring("t47x6d4lzz1zxm1bakrmiceb0tljzl9n8r19kqu9s3731ectkllp9mezn7cldozt25nlenyh5jus5b9rr687u2icimakjpyf4rsux3c66giulc0d2ipsa6bpa6dykgd0mc25r1m89hvzjcix73sdwfbu5q67t0c131i1fqne0o7we20ve2emh1046h9m854wfxo0spb2gv5d65v9x2ibuiti7rhr2y8u72hil5cutp63sbhi832kf3v4vuxa0")) + }) + + // Bug: 2095941, OCPBUGS-5943 + g.It("Author:mjoseph-ROSA-OSD_CCS-ARO-High-63553-Annotation 'TopologyAwareHints' presents should not cause any pathological events", func() { + // OCPBUGS-5943 + compat_otp.By("Check dns daemon set for minReadySeconds to 9, maxSurge to 10% and maxUnavailable to 0") + jsonPath := `{.spec.minReadySeconds}-{.spec.updateStrategy.rollingUpdate.maxSurge}-{.spec.updateStrategy.rollingUpdate.maxUnavailable}` + spec := getByJsonPath(oc, "openshift-dns", "daemonset/dns-default", jsonPath) + o.Expect(spec).To(o.ContainSubstring("9-10%-0")) + + // Checking whether there are windows nodes + windowNodeList, err := compat_otp.GetAllNodesbyOSType(oc, "windows") + o.Expect(err).NotTo(o.HaveOccurred()) + + if len(windowNodeList) > 1 { + g.Skip("This case will not work on clusters having windows nodes") + } + + compat_otp.By("Check whether the topology-aware-hints annotation is auto set or not") + // Get all dns pods then check the resident nodes labels one by one + // search unique `topology.kubernetes.io/zone` info on worker nodes + zoneList := []string{} + for _, dnsPod := range getAllDNSPodsNames(oc) { + node := getByJsonPath(oc, "openshift-dns", "pod/"+dnsPod, "{.spec.nodeName}") + labels := getByJsonPath(oc, "default", "node/"+node, "{.metadata.labels}") + // excluding the master nodes + if strings.Contains(labels, "node-role.kubernetes.io/master") || strings.Contains(labels, "node-role.kubernetes.io/control-plane") { + continue + } + zoneInfo := getByJsonPath(oc, "default", "node/"+node, `{.metadata.labels.topology\.kubernetes\.io/zone}`) + // set zone as invalid if no zone label or its value is "" + if zoneInfo == "" { + zoneList = append(zoneList, "Invalid") + break + } + if !slices.Contains(zoneList, zoneInfo) { + e2e.Logf("new zone is found: %v", zoneInfo) + zoneList = append(zoneList, zoneInfo) + } + } + e2e.Logf("all found zones are: %v", zoneList) + + // Topology-aware hints annotation present only if all nodes having the topology.kubernetes.io/zone label and from at least two zones + findAnnotation := getAnnotation(oc, "openshift-dns", "svc", "dns-default") + if slices.Contains(zoneList, "Invalid") || len(zoneList) < 2 { + o.Expect(findAnnotation).NotTo(o.ContainSubstring(`"service.kubernetes.io/topology-aware-hints":"auto"`)) + } else { + o.Expect(findAnnotation).To(o.ContainSubstring(`"service.kubernetes.io/topology-aware-hints":"auto"`)) + } + }) + + g.It("Author:mjoseph-NonHyperShiftHOST-ConnectedOnly-Critical-73379-DNSNameResolver CR get updated with IP addresses and TTL of the DNS name [Serial]", func() { + // skip the test if featureSet is not there + if !compat_otp.IsTechPreviewNoUpgrade(oc) { + g.Skip("featureSet: TechPreviewNoUpgrade is required for this test, skipping") + } + var ( + buildPruningBaseDir = testdata.FixturePath("router") + clientPod = filepath.Join(buildPruningBaseDir, "test-client-pod.yaml") + clientPodLabel = "app=hello-pod" + clientPodName = "hello-pod" + egressFirewall = filepath.Join(buildPruningBaseDir, "egressfirewall-wildcard.yaml") + ) + + compat_otp.By("1. Create egressfirewall file") + ns := oc.Namespace() + operateResourceFromFile(oc, "create", ns, egressFirewall) + waitEgressFirewallApplied(oc, "default", ns) + + compat_otp.By("2. Create a client pod") + createResourceFromFile(oc, ns, clientPod) + ensurePodWithLabelReady(oc, ns, clientPodLabel) + + compat_otp.By("3. Verify the record created with the dns name in the DNSNameResolver CR") + wildcardDnsName := getByJsonPath(oc, "openshift-ovn-kubernetes", "dnsnameresolver", "{.items..spec.name}") + o.Expect(wildcardDnsName).To(o.ContainSubstring("*.google.com.")) + + compat_otp.By("4. Verify the allowed rules which matches the wildcard take effect.") + // as per the egress firewall, only domains having "*.google.com" will only allowed + checkDomainReachability(oc, clientPodName, ns, "www.google.com", true) + checkDomainReachability(oc, clientPodName, ns, "www.redhat.com", false) + checkDomainReachability(oc, clientPodName, ns, "calendar.google.com", true) + + compat_otp.By("5. Confirm the wildcard entry is resolved to dnsName with IP address and TTL value") + // resolved DNS names + dnsName := getByJsonPath(oc, "openshift-ovn-kubernetes", "dnsnameresolver", "{.items..status.resolvedNames..dnsName}") + o.Expect(dnsName).To(o.ContainSubstring("www.google.com. calendar.google.com.")) + // resolved TTL values + ttlValues := getByJsonPath(oc, "openshift-ovn-kubernetes", "dnsnameresolver", "{.items..status.resolvedNames..resolvedAddresses..ttlSeconds}") + o.Expect(ttlValues).To(o.MatchRegexp(`[0-9]{1,3}`)) + // resolved IP address + ipAddress := getByJsonPath(oc, "openshift-ovn-kubernetes", "dnsnameresolver", "{.items..status.resolvedNames..resolvedAddresses..ip}") + o.Expect(ipAddress).To(o.MatchRegexp(`[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}`)) + o.Expect(strings.Count(ipAddress, ":") >= 2).To(o.BeTrue()) + }) + + // Bug: OCPBUGS-33750 + g.It("Author:mjoseph-NonHyperShiftHOST-ConnectedOnly-High-75426-DNSNameResolver CR should resolve multiple DNS names [Serial]", func() { + // skip the test if featureSet is not there + if !compat_otp.IsTechPreviewNoUpgrade(oc) { + g.Skip("featureSet: TechPreviewNoUpgrade is required for this test, skipping") + } + var ( + buildPruningBaseDir = testdata.FixturePath("router") + clientPod = filepath.Join(buildPruningBaseDir, "test-client-pod.yaml") + clientPodLabel = "app=hello-pod" + clientPodName = "hello-pod" + egressFirewall = filepath.Join(buildPruningBaseDir, "egressfirewall-wildcard.yaml") + egressFirewall2 = filepath.Join(buildPruningBaseDir, "egressfirewall-multiDomain.yaml") + ) + + compat_otp.By("1. Create four egressfirewall rules and client pods in different namepaces, then wait until there are available") + var project []string + for i := range 4 { + project = append(project, oc.Namespace()) + compat_otp.SetNamespacePrivileged(oc, project[i]) + operateResourceFromFile(oc, "create", project[i], clientPod) + operateResourceFromFile(oc, "create", project[i], egressFirewall) + ensurePodWithLabelReady(oc, project[i], clientPodLabel) + waitEgressFirewallApplied(oc, "default", project[i]) + oc.SetupProject() + } + + compat_otp.By("2. Check whether the default dnsnameresolver CR got created and its resolved dns name") + wildcardDnsName := getByJsonPath(oc, "openshift-ovn-kubernetes", "dnsnameresolver", "{.items..spec.name}") + o.Expect(wildcardDnsName).To(o.ContainSubstring("*.google.com.")) + randomNS := getRandomElementFromList(project) + checkDomainReachability(oc, clientPodName, randomNS, "www.google.com", true) + + compat_otp.By("3. Edit some egressfirewalls") + updateValueTest1 := "[{\"op\":\"replace\",\"path\":\"/spec/egress/0/to/dnsName\", \"value\":\"www.yahoo.com\"}]" + updateValueTest2 := "[{\"op\":\"add\",\"path\":\"/spec/egress/1\", \"value\":{\"type\":\"Deny\",\"to\":{\"dnsName\":\"www.redhat.com\"}}}]" + updateValueTest3 := "[{\"op\":\"add\",\"path\":\"/spec/egress/0\", \"value\":{\"type\":\"Deny\",\"to\":{\"dnsName\":\"calendar.google.com\"}}}]" + updateValueTest4 := "[{\"op\":\"add\",\"path\":\"/spec/egress/1\", \"value\":{\"type\":\"Deny\",\"to\":{\"dnsName\":\"calendar.google.com\"}}}]" + patchResourceAsAdminAnyType(oc, project[0], "egressfirewall.k8s.ovn.org/default", updateValueTest1, "json") + patchResourceAsAdminAnyType(oc, project[1], "egressfirewall.k8s.ovn.org/default", updateValueTest2, "json") + patchResourceAsAdminAnyType(oc, project[2], "egressfirewall.k8s.ovn.org/default", updateValueTest3, "json") + patchResourceAsAdminAnyType(oc, project[3], "egressfirewall.k8s.ovn.org/default", updateValueTest4, "json") + waitEgressFirewallApplied(oc, "default", project[0]) + waitEgressFirewallApplied(oc, "default", project[1]) + waitEgressFirewallApplied(oc, "default", project[2]) + waitEgressFirewallApplied(oc, "default", project[3]) + + compat_otp.By("4. Check the changes made to dnsnameresolver CR and its resolved dns name in different namespace") + wildcardDnsName = getByJsonPath(oc, "openshift-ovn-kubernetes", "dnsnameresolver", "{.items..spec.name}") + o.Expect(wildcardDnsName).To(o.And(o.ContainSubstring( + "calendar.google.com."), o.ContainSubstring( + "*.google.com."), o.ContainSubstring( + "www.redhat.com."), o.ContainSubstring( + "www.yahoo.com."))) + checkDomainReachability(oc, clientPodName, project[0], "www.yahoo.com", true) + checkDomainReachability(oc, clientPodName, project[0], "www.google.com", false) + checkDomainReachability(oc, clientPodName, project[1], "www.google.com", true) + checkDomainReachability(oc, clientPodName, project[1], "www.redhat.com", false) + checkDomainReachability(oc, clientPodName, project[2], "calendar.google.com", false) + checkDomainReachability(oc, clientPodName, project[2], "www.google.com", true) + checkDomainReachability(oc, clientPodName, project[3], "calendar.google.com", true) + + compat_otp.By("5. Delete an egressfirewall and confirm the same") + err1 := oc.AsAdmin().WithoutNamespace().Run("delete").Args("egressfirewall", "default", "-n", project[0]).Execute() + o.Expect(err1).NotTo(o.HaveOccurred()) + // the firewall was previous blocking the dns resolution of 'google.com' in the namespace and now not + checkDomainReachability(oc, clientPodName, project[0], "www.google.com", true) + wildcardDnsName = getByJsonPath(oc, "openshift-ovn-kubernetes", "dnsnameresolver", "{.items..spec.name}") + o.Expect(wildcardDnsName).NotTo(o.ContainSubstring("www.yahoo.com.")) + + compat_otp.By("6. Recreate an egressfirewall and confirm the same") + // Updating in the yaml file with dnsName '*.google.com' as 'amazon.com' + sedCmd := fmt.Sprintf(`sed -i'' -e 's|"\*.google.com\"|www.amazon.com|g' %s`, egressFirewall) + _, sedErr := exec.Command("bash", "-c", sedCmd).Output() + o.Expect(sedErr).NotTo(o.HaveOccurred()) + operateResourceFromFile(oc, "create", project[0], egressFirewall) + waitEgressFirewallApplied(oc, "default", project[0]) + checkDomainReachability(oc, clientPodName, project[0], "www.amazon.com", true) + wildcardDnsName = getByJsonPath(oc, "openshift-ovn-kubernetes", "dnsnameresolver", "{.items..spec.name}") + o.Expect(wildcardDnsName).To(o.ContainSubstring("www.amazon.com.")) + + compat_otp.By("7. Create another egressfirewall and its client pod in a different namespace") + project5 := oc.Namespace() + compat_otp.SetNamespacePrivileged(oc, project5) + operateResourceFromFile(oc, "create", project5, egressFirewall2) + waitEgressFirewallApplied(oc, "default", project5) + operateResourceFromFile(oc, "create", project5, clientPod) + ensurePodWithLabelReady(oc, project5, clientPodLabel) + + compat_otp.By("8. Verify the three dnsnameresolver records created in DNSNameResolver CR") + wildcardDnsNames := getByJsonPath(oc, "openshift-ovn-kubernetes", "dnsnameresolver", "{.items..spec.name}") + o.Expect(wildcardDnsNames).To(o.And(o.ContainSubstring("*.google.com."), o.ContainSubstring( + "www.facebook.com."), o.ContainSubstring("registry-1.docker.io."))) + + compat_otp.By("9. Verify the dns records are resolved based on allowed rules only") + checkDomainReachability(oc, clientPodName, project5, "www.facebook.com:80", true) + checkDomainReachability(oc, clientPodName, project5, "registry-1.docker.io", true) + // as per the egress firewall, domain name having "www.facebook.com" with port 80 will only resolved + checkDomainReachability(oc, clientPodName, project5, "www.facebook.com:443", false) + + compat_otp.By("10. Confirm the dns records are resolved with IP address and TTL value") + // resolved DNS names + dnsName := getByJsonPath(oc, "openshift-ovn-kubernetes", "dnsnameresolver", "{.items..status.resolvedNames..dnsName}") + o.Expect(dnsName).To(o.And(o.ContainSubstring("www.facebook.com."), o.ContainSubstring("registry-1.docker.io."))) + // resolved TTL values + ttlValues := getByJsonPath(oc, "openshift-ovn-kubernetes", "dnsnameresolver", "{.items..status.resolvedNames..resolvedAddresses..ttlSeconds}") + o.Expect(ttlValues).To(o.MatchRegexp(`[0-9]{1,3}`)) + // resolved IP address + ipAddress := getByJsonPath(oc, "openshift-ovn-kubernetes", "dnsnameresolver", "{.items..status.resolvedNames..resolvedAddresses..ip}") + o.Expect(ipAddress).To(o.MatchRegexp(`[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}`)) + o.Expect(strings.Count(ipAddress, ":") >= 2).To(o.BeTrue()) + }) +}) diff --git a/tests-extension/test/e2e/externaldns.go b/tests-extension/test/e2e/externaldns.go new file mode 100644 index 000000000..a3791f215 --- /dev/null +++ b/tests-extension/test/e2e/externaldns.go @@ -0,0 +1,351 @@ +package router + +import ( + "fmt" + "os/exec" + "path/filepath" + "strings" + "time" + + "github.com/openshift/origin/test/extended/util/compat_otp/architecture" + + g "github.com/onsi/ginkgo/v2" + o "github.com/onsi/gomega" + + compat_otp "github.com/openshift/origin/test/extended/util/compat_otp" + "github.com/openshift/router-tests-extension/test/testdata" +) + +var _ = g.Describe("[sig-network-edge] Network_Edge Component_ExtDNS", func() { + defer g.GinkgoRecover() + + var ( + oc = compat_otp.NewCLI("externaldns", compat_otp.KubeConfigPath()) + operatorNamespace = "external-dns-operator" + operatorLabel = "name=external-dns-operator" + operandLabelKey = "app.kubernetes.io/instance=" + addLabel = "external-dns.mydomain.org/publish=yes" + delLabel = "external-dns.mydomain.org/publish-" + recordsReadyLog = "All records are already up to date" + ) + + g.BeforeEach(func() { + // skip ARM64 arch + architecture.SkipNonAmd64SingleArch(oc) + // skip if OLM capability is disabled + compat_otp.SkipNoOLMCore(oc) + + compat_otp.By("Deploy External DNS konflux FBC") + createExternalDNSCatalogSource(oc) + }) + + // author: hongli@redhat.com + g.It("Author:hongli-ConnectedOnly-ROSA-OSD_CCS-LEVEL0-High-48138-Support External DNS on AWS platform", func() { + var ( + buildPruningBaseDir = testdata.FixturePath("router/extdns") + sampleAWS = filepath.Join(buildPruningBaseDir, "sample-aws-rt.yaml") + crName = "sample-aws-rt" + operandLabel = operandLabelKey + crName + routeNamespace = "openshift-ingress-canary" + routeName = "canary" + ) + + compat_otp.By("Ensure the case is runnable on the cluster") + compat_otp.SkipIfPlatformTypeNot(oc, "AWS") + if compat_otp.IsSTSCluster(oc) { + g.Skip("Skip on STS cluster") + } + baseDomain, _ := oc.AsAdmin().WithoutNamespace().Run("get").Args("dns.config", "cluster", "-o=jsonpath={.spec.baseDomain}").Output() + // this case cannot be executed on a shared vpc cluster + privateZoneIAMRole, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("dns.config", "cluster", "-o=jsonpath={.spec.platform.aws}").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + if strings.Contains(privateZoneIAMRole, "privateZoneIAMRole") { + g.Skip("Skipping since this case will not run on a shared vpc cluster") + } + createExternalDNSOperator(oc) + + compat_otp.By("Create CR ExternalDNS sample-aws-rt and ensure operand pod is ready") + ensurePodWithLabelReady(oc, operatorNamespace, operatorLabel) + defer oc.AsAdmin().WithoutNamespace().Run("delete").Args("externaldns", crName).Output() + // To avoid connection refused flake error, as the controller CR creation needs extra prepare time after the operator pod is ready + time.Sleep(3 * time.Second) + sedCmd := fmt.Sprintf(`sed -i'' -e 's/basedomain/%s/g' %s`, baseDomain, sampleAWS) + _, err = exec.Command("bash", "-c", sedCmd).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + _, err = oc.AsAdmin().WithoutNamespace().Run("create").Args("-f", sampleAWS).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + ensurePodWithLabelReady(oc, operatorNamespace, operandLabel) + ensureLogsContainString(oc, operatorNamespace, operandLabel, recordsReadyLog) + + compat_otp.By("Add label to canary route, ensure ExternalDNS added the record") + defer oc.AsAdmin().WithoutNamespace().Run("label").Args("-n", routeNamespace, "route", routeName, delLabel, "--overwrite").Output() + _, err = oc.AsAdmin().WithoutNamespace().Run("label").Args("-n", routeNamespace, "route", routeName, addLabel).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + ensureLogsContainString(oc, operatorNamespace, operandLabel, "Desired change: CREATE external-dns-canary-openshift-ingress-canary") + + compat_otp.By("Remove label from the canary route, ensure ExternalDNS deleted the record") + _, err = oc.AsAdmin().WithoutNamespace().Run("label").Args("-n", routeNamespace, "route", routeName, delLabel, "--overwrite").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + ensureLogsContainString(oc, operatorNamespace, operandLabel, "Desired change: DELETE external-dns-canary-openshift-ingress-canary") + }) + + // author: hongli@redhat.com + g.It("ConnectedOnly-ARO-Author:hongli-High-48139-Support External DNS on Azure DNS provider", func() { + var ( + buildPruningBaseDir = testdata.FixturePath("router/extdns") + sampleAzure = filepath.Join(buildPruningBaseDir, "sample-azure-rt.yaml") + crName = "sample-azure-rt" + operandLabel = operandLabelKey + crName + routeNamespace = "openshift-ingress-canary" + routeName = "canary" + ) + + compat_otp.By("Ensure the case is runnable on the cluster") + compat_otp.SkipIfPlatformTypeNot(oc, "Azure") + if compat_otp.IsSTSCluster(oc) { + g.Skip("Skip on STS cluster") + } + cloudName, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("infrastructure", "cluster", "-o=jsonpath={.status.platformStatus.azure.cloudName}").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + if strings.ToLower(cloudName) == "azureusgovernmentcloud" { + g.Skip("Skip on MAG (Microsoft Azure Gov) cloud") + } + zoneID, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("dns.config", "cluster", "-o=jsonpath={.spec.privateZone.id}").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + if !strings.Contains(zoneID, "openshift") { + g.Skip("Skip since no valid DNS privateZone is configured in this cluster") + } + createExternalDNSOperator(oc) + + compat_otp.By("Create CR ExternalDNS sample-azure-svc with invalid zone ID") + ensurePodWithLabelReady(oc, operatorNamespace, operatorLabel) + defer oc.AsAdmin().WithoutNamespace().Run("delete").Args("externaldns", crName).Output() + time.Sleep(3 * time.Second) + _, err = oc.AsAdmin().WithoutNamespace().Run("create").Args("-f", sampleAzure).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + ensurePodWithLabelReady(oc, operatorNamespace, operandLabel) + ensureLogsContainString(oc, operatorNamespace, operandLabel, "Found 0 Azure DNS zone") + operandPod := getPodListByLabel(oc, operatorNamespace, operandLabel) + + compat_otp.By("Patch externaldns with valid privateZone ID and wait until new operand pod ready") + patchStr := "[{\"op\":\"replace\",\"path\":\"/spec/zones/0\",\"value\":" + zoneID + "}]" + patchGlobalResourceAsAdmin(oc, "externaldnses.externaldns.olm.openshift.io/"+crName, patchStr) + err = waitForResourceToDisappear(oc, operatorNamespace, "pod/"+operandPod[0]) + compat_otp.AssertWaitPollNoErr(err, fmt.Sprintf("resource %v does not disapper", "pod/"+operandPod[0])) + ensurePodWithLabelReady(oc, operatorNamespace, operandLabel) + ensureLogsContainString(oc, operatorNamespace, operandLabel, "Found 1 Azure Private DNS zone") + + compat_otp.By("Add label to canary route, ensure ExternalDNS added the record") + defer oc.AsAdmin().WithoutNamespace().Run("label").Args("-n", routeNamespace, "route", routeName, delLabel, "--overwrite").Output() + _, err = oc.AsAdmin().WithoutNamespace().Run("label").Args("-n", routeNamespace, "route", routeName, addLabel).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + ensureLogsContainString(oc, operatorNamespace, operandLabel, "Updating TXT record named 'external-dns-canary") + + compat_otp.By("Remove label from the canary route, ensure ExternalDNS deleted the record") + _, err = oc.AsAdmin().WithoutNamespace().Run("label").Args("-n", routeNamespace, "route", routeName, delLabel, "--overwrite").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + ensureLogsContainString(oc, operatorNamespace, operandLabel, "Deleting TXT record named 'external-dns-canary") + }) + + // author: hongli@redhat.com + g.It("Author:hongli-ConnectedOnly-High-48140-Support External DNS on GCP DNS provider", func() { + var ( + buildPruningBaseDir = testdata.FixturePath("router/extdns") + sampleGCP = filepath.Join(buildPruningBaseDir, "sample-gcp-svc.yaml") + crName = "sample-gcp-svc" + operandLabel = operandLabelKey + crName + serviceNamespace = "openshift-ingress-canary" + serviceName = "ingress-canary" + ) + + compat_otp.By("Ensure the case is runnable on the cluster") + compat_otp.SkipIfPlatformTypeNot(oc, "GCP") + if compat_otp.IsSTSCluster(oc) { + g.Skip("Skip on STS cluster") + } + zoneID, _ := oc.AsAdmin().WithoutNamespace().Run("get").Args("dns.config", "cluster", "-o=jsonpath={.spec.privateZone.id}").Output() + if !strings.Contains(zoneID, "private") { + g.Skip("Skip since no valid DNS privateZone is configured in this cluster") + } + // Extract zone name from the full path format (4.20+) or use as-is (4.19 and earlier) + zoneName := extractGCPZoneName(zoneID) + createExternalDNSOperator(oc) + baseDomain := getBaseDomain(oc) + + compat_otp.By("Create CR ExternalDNS sample-gcp-svc and ensure operand pod is ready") + ensurePodWithLabelReady(oc, operatorNamespace, operatorLabel) + defer oc.AsAdmin().WithoutNamespace().Run("delete").Args("externaldns", crName).Output() + time.Sleep(3 * time.Second) + _, err := oc.AsAdmin().WithoutNamespace().Run("create").Args("-f", sampleGCP).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + ensurePodWithLabelReady(oc, operatorNamespace, operandLabel) + ensureLogsContainString(oc, operatorNamespace, operandLabel, "No zones in the project") + operandPod := getPodListByLabel(oc, operatorNamespace, operandLabel) + + compat_otp.By("Patch externaldns with valid privateZone ID and wait until new operand pod ready") + patchStr := "[{\"op\":\"replace\",\"path\":\"/spec/source/fqdnTemplate/0\",\"value\":'{{.Name}}." + baseDomain + "'},{\"op\":\"replace\",\"path\":\"/spec/zones/0\",\"value\":" + zoneName + "}]" + patchGlobalResourceAsAdmin(oc, "externaldnses.externaldns.olm.openshift.io/"+crName, patchStr) + waitErr := waitForResourceToDisappear(oc, operatorNamespace, "pod/"+operandPod[0]) + compat_otp.AssertWaitPollNoErr(waitErr, fmt.Sprintf("resource %v does not disapper", "pod/"+operandPod[0])) + ensurePodWithLabelReady(oc, operatorNamespace, operandLabel) + ensureLogsContainString(oc, operatorNamespace, operandLabel, recordsReadyLog) + + compat_otp.By("Add label to canary service, ensure ExternalDNS added the record") + defer oc.AsAdmin().WithoutNamespace().Run("label").Args("-n", serviceNamespace, "service", serviceName, delLabel, "--overwrite").Output() + _, err = oc.AsAdmin().WithoutNamespace().Run("label").Args("-n", serviceNamespace, "service", serviceName, addLabel).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + ensureLogsContainString(oc, operatorNamespace, operandLabel, "Add records: external-dns-ingress-canary") + + compat_otp.By("Remove label from the canary service, ensure ExternalDNS deleted the record") + _, err = oc.AsAdmin().WithoutNamespace().Run("label").Args("-n", serviceNamespace, "service", serviceName, delLabel, "--overwrite").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + ensureLogsContainString(oc, operatorNamespace, operandLabel, "Del records: external-dns-ingress-canary") + }) + + // author: mjoseph@redhat.com + g.It("Author:mjoseph-ConnectedOnly-ROSA-OSD_CCS-Critical-68826-External DNS support for preexisting Route53 for Shared VPC clusters", func() { + var ( + buildPruningBaseDir = testdata.FixturePath("router/extdns") + sampleAWSVPC = filepath.Join(buildPruningBaseDir, "sample-aws-sharedvpc-rt.yaml") + crName = "sample-aws-sharedvpc-rt" + operandLabel = operandLabelKey + crName + routeNamespace = "openshift-ingress-canary" + routeName = "canary" + ) + + compat_otp.By("Ensure the case is runnable on the cluster") + compat_otp.SkipIfPlatformTypeNot(oc, "AWS") + if compat_otp.IsSTSCluster(oc) { + g.Skip("Skip on STS cluster") + } + baseDomain, _ := oc.AsAdmin().WithoutNamespace().Run("get").Args("dns.config", "cluster", "-o=jsonpath={.spec.baseDomain}").Output() + + // privateZoneIAMRole needs to be present for shared vpc cluster + privateZoneIAMRole, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("dns.config", "cluster", "-o=jsonpath={.spec.platform.aws.privateZoneIAMRole}").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + if !strings.Contains(privateZoneIAMRole, "arn:aws:iam::") { + g.Skip("Skip since this is not a shared vpc cluster") + } + + compat_otp.By("1. Check the STS Role in the cluster") + output, ouputErr := oc.AsAdmin().WithoutNamespace().Run("get").Args("CredentialsRequest/openshift-ingress", "-n", "openshift-cloud-credential-operator", "-o=jsonpath={.spec.providerSpec.statementEntries[0].action}").Output() + o.Expect(ouputErr).NotTo(o.HaveOccurred()) + o.Expect(output).To(o.ContainSubstring("sts:AssumeRole")) + // Getting the private zone id from dns config") + privateZoneId := getByJsonPath(oc, "openshift-dns", "dns.config/cluster", "{.spec.privateZone.id}") + + compat_otp.By("2. Create External DNS Operator in the cluster") + createExternalDNSOperator(oc) + ensurePodWithLabelReady(oc, operatorNamespace, operatorLabel) + + compat_otp.By("3. Create CR ExternalDNS sample-aws-sharedvpc-rt and ensure operand pod is ready") + defer oc.AsAdmin().WithoutNamespace().Run("delete").Args("externaldns", crName).Output() + // To avoid connection refused flake error, as the controller CR creation needs extra prepare time after the operator pod is ready + time.Sleep(3 * time.Second) + // Updating the yaml file with basedomin and ARN value + sedCmd := fmt.Sprintf(`sed -i'' -e 's@basedomain@%s@g;s@privatezoneiamrole@%v@g' %s`, baseDomain, privateZoneIAMRole, sampleAWSVPC) + _, err = exec.Command("bash", "-c", sedCmd).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + _, err = oc.AsAdmin().WithoutNamespace().Run("create").Args("-f", sampleAWSVPC).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + ensurePodWithLabelReady(oc, operatorNamespace, operandLabel) + ensureLogsContainString(oc, operatorNamespace, operandLabel, recordsReadyLog) + + compat_otp.By("4. Add label to canary route, ensure ExternalDNS added the record") + defer oc.AsAdmin().WithoutNamespace().Run("label").Args("-n", routeNamespace, "route", routeName, delLabel, "--overwrite").Output() + _, err = oc.AsAdmin().WithoutNamespace().Run("label").Args("-n", routeNamespace, "route", routeName, addLabel).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + ensureLogsContainString(oc, operatorNamespace, operandLabel, "Desired change: CREATE canary-openshift-ingress-canary.apps."+baseDomain+" CNAME [Id: /hostedzone/"+privateZoneId+"]") + + compat_otp.By("5. Remove label from the canary route, ensure ExternalDNS deleted the record") + _, err = oc.AsAdmin().WithoutNamespace().Run("label").Args("-n", routeNamespace, "route", routeName, delLabel, "--overwrite").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + ensureLogsContainString(oc, operatorNamespace, operandLabel, "Desired change: DELETE canary-openshift-ingress-canary.apps."+baseDomain+" CNAME [Id: /hostedzone/"+privateZoneId+"]") + }) +}) + +var _ = g.Describe("[sig-network-edge] Network_Edge Component_ExtDNS on STS", func() { + defer g.GinkgoRecover() + + var ( + oc = compat_otp.NewCLI("externaldnssts", compat_otp.KubeConfigPath()) + operatorNamespace = "external-dns-operator" + operatorLabel = "name=external-dns-operator" + operandLabelKey = "app.kubernetes.io/instance=" + addLabel = "external-dns.mydomain.org/publish=yes" + delLabel = "external-dns.mydomain.org/publish-" + recordsReadyLog = "All records are already up to date" + ) + g.BeforeEach(func() { + // skip ARM64 arch + architecture.SkipNonAmd64SingleArch(oc) + // skip if OLM capability is disabled + compat_otp.SkipNoOLMCore(oc) + + compat_otp.By("Deploy External DNS konflux FBC") + createExternalDNSCatalogSource(oc) + }) + + // this case runs only on AWS STS and hypershift cluster + g.It("Author:mjoseph-ConnectedOnly-ROSA-OSD_CCS-High-74949-ExternalDNS operand support on STS cluster", func() { + var ( + buildPruningBaseDir = testdata.FixturePath("router/extdns") + sampleAWSSTS = filepath.Join(buildPruningBaseDir, "sample-aws-sts-rt.yaml") + crName = "sample-aws-sts-rt" + operandLabel = operandLabelKey + crName + routeNamespace = "openshift-ingress-canary" + routeName = "canary" + ) + + compat_otp.By("1. Ensure the case is runnable on the cluster") + compat_otp.SkipIfPlatformTypeNot(oc, "AWS") + // Skip in Gov cluster + region, err := compat_otp.GetAWSClusterRegion(oc) + o.Expect(err).NotTo(o.HaveOccurred()) + if strings.Contains(region, "us-gov") { + g.Skip("Skipping for the aws cluster in us-gov region") + } + // Skip in non STS cluster + if !compat_otp.IsSTSCluster(oc) { + g.Skip("Skip for non-STS cluster") + } + // Skip in Shared VPC cluster + privateZoneIAMRole, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("dns.config", "cluster", "-o=jsonpath={.spec.platform.aws}").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + if strings.Contains(privateZoneIAMRole, "privateZoneIAMRole") { + g.Skip("Skipping since this case will not run on a shared vpc cluster") + } + + compat_otp.By("2. Create the External DNS Operator") + createExternalDNSOperator(oc) + ensurePodWithLabelReady(oc, operatorNamespace, operatorLabel) + + compat_otp.By("3. Prepare the STS related credentials like role, policy and secret for the cluster") + defer clearUpExDnsStsCluster(oc, "74949") + prepareStsCredForCluster(oc, "74949") + + compat_otp.By("4. Create STS ExternalDNS CR `sample-aws-sts-rt` and ensure operand pod is ready") + baseDomain := getBaseDomain(oc) + // get the Hosted zone ID from AWS route53 + privateZoneID := getPrivateZoneID(oc, baseDomain) + updateFilebySedCmd(sampleAWSSTS, "basedomain", baseDomain) + updateFilebySedCmd(sampleAWSSTS, "privatezone", privateZoneID) + + defer oc.AsAdmin().WithoutNamespace().Run("delete").Args("externaldns", crName).Output() + _, err = oc.AsAdmin().WithoutNamespace().Run("create").Args("-f", sampleAWSSTS).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + ensurePodWithLabelReady(oc, operatorNamespace, operandLabel) + ensureLogsContainString(oc, operatorNamespace, operandLabel, recordsReadyLog) + + compat_otp.By("5. Add label to canary route, ensure ExternalDNS added the record") + defer oc.AsAdmin().WithoutNamespace().Run("label").Args("-n", routeNamespace, "route", routeName, delLabel, "--overwrite").Output() + _, err = oc.AsAdmin().WithoutNamespace().Run("label").Args("-n", routeNamespace, "route", routeName, addLabel).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + ensureLogsContainString(oc, operatorNamespace, operandLabel, "Desired change: CREATE external-dns-canary-openshift-ingress-canary") + + compat_otp.By("6. Remove label from the canary route, ensure ExternalDNS deleted the record") + _, err = oc.AsAdmin().WithoutNamespace().Run("label").Args("-n", routeNamespace, "route", routeName, delLabel, "--overwrite").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + ensureLogsContainString(oc, operatorNamespace, operandLabel, "Desired change: DELETE external-dns-canary-openshift-ingress-canary") + }) +}) diff --git a/tests-extension/test/e2e/externaldns_util.go b/tests-extension/test/e2e/externaldns_util.go new file mode 100644 index 000000000..de3e8b926 --- /dev/null +++ b/tests-extension/test/e2e/externaldns_util.go @@ -0,0 +1,114 @@ +package router + +import ( + "context" + "fmt" + "os" + "path/filepath" + "strings" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/iam" + "github.com/aws/aws-sdk-go/service/route53" + o "github.com/onsi/gomega" + exutil "github.com/openshift/origin/test/extended/util" + compat_otp "github.com/openshift/origin/test/extended/util/compat_otp" + "github.com/openshift/router-tests-extension/test/testdata" + e2e "k8s.io/kubernetes/test/e2e/framework" +) + +// Create External DNS Controller (operand) Role and inline policy +func createExDnsRolePolicy(iamClient *iam.Client, infraID string, oidcArnPrefix string, oidcName string) string { + buildPruningBaseDir := testdata.FixturePath("router/extdns") + exDnsPermissionPolicyFile := filepath.Join(buildPruningBaseDir, "sts-exdns-perms-policy.json") + exDnsRoleName := infraID + "-exdns-role" + exDnsPolicyName := infraID + "-exdns-perms-policy" + + exdDnsTrustPolicy := `{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Principal": { + "Federated": "%s" + }, + "Action": "sts:AssumeRoleWithWebIdentity", + "Condition": { + "StringEquals": { + "%s:sub": "system:serviceaccount:external-dns-operator:external-dns-sample-aws-sts-rt" + } + } + } + ] +}` + oidcArn := oidcArnPrefix + oidcName + exdDnsTrustPolicy = fmt.Sprintf(exdDnsTrustPolicy, oidcArn, oidcName) + // create role + exDnsRoleArn := iamCreateRole(iamClient, exdDnsTrustPolicy, exDnsRoleName) + + exDnsPermissionPolicy, err := os.ReadFile(exDnsPermissionPolicyFile) + o.Expect(err).NotTo(o.HaveOccurred()) + // create policy + iamPutRolePolicy(iamClient, string(exDnsPermissionPolicy), exDnsPolicyName, exDnsRoleName) + return exDnsRoleArn +} + +// Remove External DNS Operand role and policy on the STS cluster +func deleteExDnsRolePolicy(iamClient *iam.Client, infraID, prefix string) { + exDnsRoleName := infraID + "-" + prefix + "-exdns-role" + exDnsPolicyName := infraID + "-" + prefix + "-exdns-perms-policy" + iamDeleteRolePolicy(iamClient, exDnsPolicyName, exDnsRoleName) + iamDeleteRole(iamClient, exDnsRoleName) +} + +// Prepare roles, policies and secrets for STS cluster +func prepareStsCredForCluster(oc *exutil.CLI, prefix string) { + infraID, _ := compat_otp.GetInfraID(oc) + oidcName := getOidc(oc) + iamClient := newIamClient() + stsClient := newStsClient() + account := getAwsAccount(stsClient) + oidcArnPrefix := "arn:aws:iam::" + account + ":oidc-provider/" + + // create role and policy + exDnsRoleArn := createExDnsRolePolicy(iamClient, infraID+"-"+prefix, oidcArnPrefix, oidcName) + // create a secret with the external dns ARN role + createSecretUsingRoleARN(oc, "external-dns-operator", exDnsRoleArn) +} + +// Clear up all roles, policies and secrets of the STS cluster +func clearUpExDnsStsCluster(oc *exutil.CLI, prefix string) { + infraID, _ := compat_otp.GetInfraID(oc) + iamClient := newIamClient() + deleteExDnsRolePolicy(iamClient, infraID, prefix) + + // deleting secret + oc.AsAdmin().WithoutNamespace().Run("delete").Args("secret", "-n", "external-dns-operator", "aws-sts-creds").Output() +} + +// Create the STS secret with the external dns ARN role +func createSecretUsingRoleARN(oc *exutil.CLI, ns, exDnsRoleArn string) { + buildPruningBaseDir := testdata.FixturePath("router/extdns") + awsStsCredSecret := filepath.Join(buildPruningBaseDir, "aws-sts-creds-secret.yaml") + updateFilebySedCmd(awsStsCredSecret, "external-dns-role-arn", exDnsRoleArn) + + _, err := oc.AsAdmin().WithoutNamespace().Run("create").Args("-f", awsStsCredSecret).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + // verify the secret creation + output := getByJsonPath(oc, ns, "secret", "{.items[*].metadata.name}") + o.Expect(output).Should(o.ContainSubstring("aws-sts-creds")) +} + +// Collect the Zone details from the AWS route53 and return the Hosted zone ID +func getPrivateZoneID(oc *exutil.CLI, domainName string) string { + route53Client := compat_otp.NewRoute53Client() + var hostedZoneDetails *route53.ListHostedZonesByNameOutput + + // collect the hostZone Details from the AWS route53 using the domain name + hostedZoneDetails, err := route53Client.ListHostedZonesByNameWithContext( + context.Background(), &route53.ListHostedZonesByNameInput{ + DNSName: aws.String(domainName)}) + o.Expect(err).NotTo(o.HaveOccurred()) + e2e.Logf("The zone Id of the host domain '%s' is '%s'", *hostedZoneDetails.HostedZones[0].Name, *hostedZoneDetails.HostedZones[0].Id) + return strings.TrimPrefix(*hostedZoneDetails.HostedZones[0].Id, "/hostedzone/") +} diff --git a/tests-extension/test/e2e/gatewayapi.go b/tests-extension/test/e2e/gatewayapi.go new file mode 100644 index 000000000..36f2a99b7 --- /dev/null +++ b/tests-extension/test/e2e/gatewayapi.go @@ -0,0 +1,273 @@ +package router + +import ( + "fmt" + "path/filepath" + "strings" + "time" + + g "github.com/onsi/ginkgo/v2" + o "github.com/onsi/gomega" + compat_otp "github.com/openshift/origin/test/extended/util/compat_otp" + "github.com/openshift/router-tests-extension/test/testdata" + wait "k8s.io/apimachinery/pkg/util/wait" + e2e "k8s.io/kubernetes/test/e2e/framework" +) + +var _ = g.Describe("[sig-network-edge] Network_Edge Component_Router", func() { + defer g.GinkgoRecover() + + var oc = compat_otp.NewCLI("gatewayapi", compat_otp.KubeConfigPath()) + + g.BeforeEach(func() { + platforms := map[string]bool{ + "aws": true, + "azure": true, + "gcp": true, + "ibmcloud": true, + "powervs": true, + } + if !platforms[compat_otp.CheckPlatform(oc)] { + g.Skip("Skip for non-cloud platforms") + } + + if isInternalLBScopeInDefaultIngresscontroller(oc) { + g.Skip("Skip for private cluster since GatewayClass is not accepted in cluster") + } + }) + + // author: iamin@redhat.com + g.It("Author:iamin-High-81976-Ensure that RBAC works correctly for gatewayAPI resources for unprivileged users", func() { + var ( + buildPruningBaseDir = testdata.FixturePath("router") + gwcFile = filepath.Join(buildPruningBaseDir, "gatewayclass.yaml") + gwcName = "openshift-default" + gwFile = filepath.Join(buildPruningBaseDir, "gateway.yaml") + testPodSvc = filepath.Join(buildPruningBaseDir, "web-server-deploy.yaml") + httpRouteFile = filepath.Join(buildPruningBaseDir, "httproute.yaml") + + gateway1 = gatewayDescription{ + name: "gateway", + namespace: "openshift-ingress", + hostname: "", + template: gwFile, + } + + httpRoute = httpRouteDescription{ + name: "route81976", + namespace: "", + gwName: "", + hostname: "", + template: httpRouteFile, + } + ) + + compat_otp.By("1.0: Create a gatewayClass object") + output, err := oc.AsAdmin().WithoutNamespace().Run("create").Args("-f", gwcFile).Output() + if err != nil && !strings.Contains(output, "AlreadyExists") { + e2e.Failf("Failed to create gatewayclass: %v", err) + } + + waitErr := wait.PollImmediate(2*time.Second, 300*time.Second, func() (bool, error) { + output, _ := oc.AsAdmin().WithoutNamespace().Run("get").Args("gatewayclass", gwcName, "-ojsonpath={.status.conditions}").Output() + if strings.Contains(output, "True") { + e2e.Logf("the gatewayclass is accepted") + return true, nil + } + e2e.Logf("Waiting for the GatewayClass to be accepted") + return false, nil + }) + o.Expect(waitErr).NotTo(o.HaveOccurred(), "The GatewayClass was never accepted") + + compat_otp.By("2.0: Create a gateway object as an admin") + baseDomain := getBaseDomain(oc) + gateway1.hostname = "*.gwapi." + baseDomain + defer gateway1.delete(oc) + gateway1.create(oc) + + waitErr = wait.PollImmediate(2*time.Second, 120*time.Second, func() (bool, error) { + output, _ := oc.AsAdmin().WithoutNamespace().Run("get").Args("gateway", gateway1.name, "-n", gateway1.namespace).Output() + if strings.Contains(output, "True") { + e2e.Logf("the Gateway is programmed") + return true, nil + } + e2e.Logf("Waiting for the Gateway to be accepted") + return false, nil + }) + o.Expect(waitErr).NotTo(o.HaveOccurred(), "The Gateway was never accepted") + + compat_otp.By("3.0: Attempt to Create a gateway object as a test user in 'openshift-ingress' namespace") + output, err = createUserResourceToNsFromTemplate(oc, "openshift-ingress", "--ignore-unknown-parameters=true", "-f", gwFile) + o.Expect(err).To(o.HaveOccurred()) + o.Expect(output).To(o.ContainSubstring(`cannot create resource "gateways" in API group "gateway.networking.k8s.io" in the namespace "openshift-ingress`)) + + compat_otp.By("4.0: Attempt to Create a gateway object as a test user in namespace with admin access") + ns := oc.Namespace() + output, err = createUserResourceToNsFromTemplate(oc, ns, "--ignore-unknown-parameters=true", "-f", gwFile, "-p", "NAMESPACE="+ns) + o.Expect(err).To(o.HaveOccurred()) + o.Expect(output).To(o.ContainSubstring(`cannot create resource "gateways" in API group "gateway.networking.k8s.io" in the namespace ` + `"` + ns + `"`)) + + compat_otp.By("5.0: Create a web-server application") + createResourceFromFile(oc, ns, testPodSvc) + ensurePodWithLabelReady(oc, ns, "name=web-server-deploy") + + compat_otp.By("6.0: Create a HTTPRoute as a test user") + httpRoute.namespace = ns + httpRoute.gwName = gateway1.name + httpRoute.hostname = "ocp81976.gwapi." + baseDomain + httpRoute.userCreate(oc) + + waitForOutputEquals(oc, ns, "httproute/"+httpRoute.name, `{.status.parents[].conditions[?(@.type=="Accepted")].status}`, "False") + output = getByJsonPath(oc, ns, "httproute/"+httpRoute.name, `{.status.parents[].conditions[?(@.type=="Accepted")]}`) + o.Expect(output).To(o.And(o.ContainSubstring(`namespace \"`+ns+`\" is not allowed by the parent`), o.ContainSubstring(`NotAllowedByListeners`))) + + compat_otp.By("7.0: Label the namespace with the Gateway Selector") + err = oc.AsAdmin().WithoutNamespace().Run("label").Args("ns", ns, "app=gwapi").Execute() + o.Expect(err).NotTo(o.HaveOccurred(), "Adding label to the namespace failed") + + compat_otp.By("8.0: Re-check the HTTPRoute") + waitForOutputEquals(oc, ns, "httproute/"+httpRoute.name, `{.status.parents[].conditions[?(@.type=="Accepted")].status}`, "True") + + compat_otp.By("Clean up cluster resources") + err = oc.AsAdmin().WithoutNamespace().Run("delete").Args("-f", gwcFile).Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + }) + + // author: hongli@redhat.com + // https://issues.redhat.com/browse/OCPBUGS-58358 + g.It("Author:hongli-ROSA-OSD_CCS-ARO-NonPreRelease-PreChkUpgrade-High-83185-Ensure GatewayAPI works well after upgrade", func() { + var ( + buildPruningBaseDir = testdata.FixturePath("router") + gwcFile = filepath.Join(buildPruningBaseDir, "gatewayclass.yaml") + gwcName = "openshift-default" + gwFile = filepath.Join(buildPruningBaseDir, "gateway.yaml") + testPodSvc = filepath.Join(buildPruningBaseDir, "web-server-deploy.yaml") + httpRouteFile = filepath.Join(buildPruningBaseDir, "httproute.yaml") + ns = "e2e-test-gatewayapi-ocp83185" + + gateway = gatewayDescription{ + name: "gateway", + namespace: "openshift-ingress", + hostname: "", + template: gwFile, + } + + httpRoute = httpRouteDescription{ + name: "route83185", + namespace: "", + gwName: "", + hostname: "", + template: httpRouteFile, + } + ) + + compat_otp.By("1.0: Create a gatewayClass object and wait until it is Accepted") + output, err := oc.AsAdmin().WithoutNamespace().Run("create").Args("-f", gwcFile).Output() + if err != nil && !strings.Contains(output, "AlreadyExists") { + e2e.Failf("Failed to create gatewayclass: %v", err) + } + waitForOutputEquals(oc, "default", "gatewayclass/"+gwcName, `{.status.conditions[?(@.type=="Accepted")].status}`, "True") + + compat_otp.By("2.0: Create a gateway object and wait until it is Programmed") + baseDomain := getBaseDomain(oc) + gateway.hostname = "*.gwapi." + baseDomain + gateway.create(oc) + waitForOutputEquals(oc, "openshift-ingress", "gateway/gateway", `{.status.conditions[?(@.type=="Programmed")].status}`, "True") + + // Use admin user to create the pod/svc/httproute since they should be kept until post upgrade + compat_otp.By("3.0: Create a project and apply required label, then create web-server application") + oc.CreateSpecifiedNamespaceAsAdmin(ns) + err = oc.AsAdmin().WithoutNamespace().Run("label").Args("ns", ns, "app=gwapi").Execute() + o.Expect(err).NotTo(o.HaveOccurred(), "Adding label to the namespace failed") + operateResourceFromFile(oc, "create", ns, testPodSvc) + ensurePodWithLabelReady(oc, ns, "name=web-server-deploy") + + compat_otp.By("4.0: Create a HTTPRoute and wait until it is accepted") + httpRoute.namespace = ns + httpRoute.gwName = gateway.name + httpRoute.hostname = "ocp83185.gwapi." + baseDomain + jsonCfg := parseToJSON(oc, []string{"--ignore-unknown-parameters=true", "-f", httpRoute.template, "-p", "NAME=" + httpRoute.name, "NAMESPACE=" + httpRoute.namespace, "GWNAME=" + httpRoute.gwName, "HOSTNAME=" + httpRoute.hostname}) + err = oc.AsAdmin().WithoutNamespace().Run("create").Args("-n", ns, "-f", jsonCfg).Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + waitForOutputEquals(oc, ns, "httproute/route83185", `{.status.parents[].conditions[?(@.type=="Accepted")].status}`, "True") + }) + + // author: hongli@redhat.com + // https://issues.redhat.com/browse/OCPBUGS-58358 + g.It("Author:hongli-ROSA-OSD_CCS-ARO-NonPreRelease-PstChkUpgrade-High-83185-Ensure GatewayAPI works well after upgrade", func() { + var ( + ns = "e2e-test-gatewayapi-ocp83185" + ) + + output, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("ns", ns).Output() + if err != nil || strings.Contains(output, "NotFound") { + g.Skip("Skipping since can not find the namespace that should be created in Pre-Upgrade test") + } + + compat_otp.By("1.0: Check the GatewayClass status in post upgrade") + status := getByJsonPath(oc, "default", "gatewayclass/openshift-default", `{.status.conditions[?(@.type=="Accepted")].status}`) + o.Expect(status).To(o.ContainSubstring("True")) + + compat_otp.By("2.0: Check istio status in post upgrade") + status = getByJsonPath(oc, "default", "istio/openshift-gateway", `{.status.state}`) + o.Expect(status).To(o.ContainSubstring("Healthy")) + + compat_otp.By("3.0: Check the Gateway status in post upgrade") + status = getByJsonPath(oc, "openshift-ingress", "gateway/gateway", `{.status.conditions[?(@.type=="Programmed")].status}`) + o.Expect(status).To(o.ContainSubstring("True")) + + compat_otp.By("4.0: Check the HTTPRoute in post upgrade") + status = getByJsonPath(oc, ns, "httproute/route83185", `{.status.parents[].conditions[?(@.type=="Accepted")].status}`) + o.Expect(status).To(o.ContainSubstring("True")) + }) + + // author: iamin@redhat.com + g.It("Author:iamin-ROSA-OSD_CCS-ARO-High-86110-View metrics showing who is using Gateway API", func() { + var ( + buildPruningBaseDir = testdata.FixturePath("router") + gwcFile = filepath.Join(buildPruningBaseDir, "gatewayclass.yaml") + gwcName = "openshift-default" + gwFile = filepath.Join(buildPruningBaseDir, "gateway.yaml") + + gateway1 = gatewayDescription{ + name: "gateway", + namespace: "openshift-ingress", + hostname: "", + template: gwFile, + } + ) + + compat_otp.By("1.0: Create a gatewayClass object") + output, err := oc.AsAdmin().WithoutNamespace().Run("create").Args("-f", gwcFile).Output() + if err != nil && !strings.Contains(output, "AlreadyExists") { + e2e.Failf("Failed to create gatewayclass: %v", err) + } + + waitForOutputContains(oc, "default", "gatewayclass/"+gwcName, "{.status.conditions}", "True", 300*time.Second) + + compat_otp.By("2.0: Create a gateway object under the openshift gatewayClass") + baseDomain := getBaseDomain(oc) + gateway1.hostname = "*.gwapi." + baseDomain + defer gateway1.delete(oc) + gateway1.create(oc) + + waitForOutputContains(oc, gateway1.namespace, "gateway/"+gateway1.name, "{.status.conditions}", "True", 120*time.Second) + + compat_otp.By("3.0: Check prometheus metrics for the created gateways") + token, err := getSAToken(oc, "prometheus-k8s", "openshift-monitoring") + o.Expect(err).NotTo(o.HaveOccurred()) + + url := "https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/" + query := "query?query=openshift:gateway_api_usage:count" + + curlCmd := []string{"-n", "openshift-monitoring", "-c", "prometheus", "prometheus-k8s-0", "-i", "--", "curl", "-k", "-s", "-H", fmt.Sprintf("Authorization: Bearer %v", token), fmt.Sprintf("%s%s", url, query), "--connect-timeout", "10"} + repeatCmdOnClient(oc, curlCmd, `"gateway_class_type":"openshift"`, 300, 1) + + compat_otp.By("4.0: Confirm the metrics values are correct") + repeatCmdOnClient(oc, curlCmd, `"1"`, 120, 1) + + compat_otp.By("Clean up cluster resources") + err = oc.AsAdmin().WithoutNamespace().Run("delete").Args("-f", gwcFile).Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + }) +}) diff --git a/tests-extension/test/e2e/haproxy-router.go b/tests-extension/test/e2e/haproxy-router.go new file mode 100644 index 000000000..25ae49596 --- /dev/null +++ b/tests-extension/test/e2e/haproxy-router.go @@ -0,0 +1,2893 @@ +package router + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "strconv" + "strings" + "time" + + g "github.com/onsi/ginkgo/v2" + o "github.com/onsi/gomega" + "github.com/tidwall/gjson" + e2e "k8s.io/kubernetes/test/e2e/framework" + + compat_otp "github.com/openshift/origin/test/extended/util/compat_otp" + "github.com/openshift/router-tests-extension/test/testdata" +) + +var _ = g.Describe("[sig-network-edge] Network_Edge Component_Router", func() { + defer g.GinkgoRecover() + + var oc = compat_otp.NewCLI("router-env", compat_otp.KubeConfigPath()) + + // incorporate OCP-15044, OCP-15049, OCP-15050 and OCP-15051 into one + // Test case creater: hongli@redhat.com - OCP-15044 The backend health check interval of unsecure route can be set by annotation + // Test case creater: hongli@redhat.com - OCP-15049 The backend health check interval of edge route can be set by annotation + // Test case creater: hongli@redhat.com - OCP-15050 The backend health check interval of passthrough route can be set by annotation + // Test case creater: hongli@redhat.com - OCP-15051 The backend health check interval of reencrypt route can be set by annotation + g.It("Author:mjoseph-ROSA-OSD_CCS-ARO-Critical-15044-The backend health check interval of unsecure/edge/passthrough/reencrypt route can be set by annotation", func() { + var ( + buildPruningBaseDir = testdata.FixturePath("router") + testPodSvc = filepath.Join(buildPruningBaseDir, "web-server-signed-deploy.yaml") + unSecSvcName = "service-unsecure" + secSvcName = "service-secure" + ) + + compat_otp.By("1. Create a server pod and its service") + ns := oc.Namespace() + defaultContPod := getOneRouterPodNameByIC(oc, "default") + // need two replicas of the server to test this scenario + updateFilebySedCmd(testPodSvc, "replicas: 1", "replicas: 2") + createResourceFromWebServer(oc, ns, testPodSvc, "web-server-deploy") + + compat_otp.By("2.0: Create an unsecure, edge, reencrypt and passthrough route") + createRoute(oc, ns, "http", "route-http", unSecSvcName, []string{}) + ensureRouteIsAdmittedByIngressController(oc, ns, "route-http", "default") + createRoute(oc, ns, "edge", "route-edge", unSecSvcName, []string{}) + ensureRouteIsAdmittedByIngressController(oc, ns, "route-edge", "default") + createRoute(oc, ns, "passthrough", "route-pass", secSvcName, []string{}) + ensureRouteIsAdmittedByIngressController(oc, ns, "route-pass", "default") + createRoute(oc, ns, "reencrypt", "route-reen", secSvcName, []string{}) + ensureRouteIsAdmittedByIngressController(oc, ns, "route-reen", "default") + + compat_otp.By("3.0: Annotate unsecure, edge, reencrypt and passthrough route") + setAnnotation(oc, ns, "route/route-http", `router.openshift.io/haproxy.health.check.interval=200ms`) + setAnnotation(oc, ns, "route/route-edge", `router.openshift.io/haproxy.health.check.interval=300ms`) + setAnnotation(oc, ns, "route/route-pass", `router.openshift.io/haproxy.health.check.interval=400ms`) + setAnnotation(oc, ns, "route/route-reen", `router.openshift.io/haproxy.health.check.interval=500ms`) + + compat_otp.By("4. Check the router pod and ensure the routes are loaded in haproxy.config of default controller") + ensureHaproxyBlockConfigContains(oc, defaultContPod, ns+":"+"route-http", []string{"check inter 200ms"}) + ensureHaproxyBlockConfigContains(oc, defaultContPod, ns+":"+"route-edge", []string{"check inter 300ms"}) + ensureHaproxyBlockConfigContains(oc, defaultContPod, ns+":"+"route-pass", []string{"check inter 400ms"}) + ensureHaproxyBlockConfigContains(oc, defaultContPod, ns+":"+"route-reen", []string{"check inter 500ms"}) + }) + + // author: hongli@redhat.com + g.It("Author:shudili-ROSA-OSD_CCS-ARO-High-16870-No health check when there is only one endpoint for a route", func() { + // skip the test if featureSet is set there + if compat_otp.IsTechPreviewNoUpgrade(oc) { + g.Skip("Skip for not supporting DynamicConfigurationManager") + } + + var ( + buildPruningBaseDir = testdata.FixturePath("router") + testPodSvc = filepath.Join(buildPruningBaseDir, "web-server-deploy.yaml") + deploymentName = "web-server-deploy" + unSecSvcName = "service-unsecure" + ) + + compat_otp.By("1.0: Create a single pod and the service") + ns := oc.Namespace() + createResourceFromFile(oc, ns, testPodSvc) + ensurePodWithLabelReady(oc, ns, "name=web-server-deploy") + output, err := oc.Run("get").Args("service").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(output).To(o.ContainSubstring(unSecSvcName)) + + compat_otp.By("2.0: Create an unsecure route") + createRoute(oc, ns, "http", unSecSvcName, unSecSvcName, []string{}) + ensureRouteIsAdmittedByIngressController(oc, ns, unSecSvcName, "default") + + compat_otp.By("3.0: Check the haproxy.config that the health check should not exist in the backend server slots") + jsonPath := fmt.Sprintf(`{.items[?(@.metadata.generateName=="%s-")].metadata.name}`, unSecSvcName) + epSliceName := getByJsonPath(oc, ns, "EndpointSlice", jsonPath) + jsonPath = "{.endpoints[0].addresses[0]}" + epIP := getByJsonPath(oc, ns, "EndpointSlice/"+epSliceName, jsonPath) + routerpod := getOneRouterPodNameByIC(oc, "default") + backendConfig := ensureHaproxyBlockConfigContains(oc, routerpod, "be_http:"+ns+":"+unSecSvcName, []string{epIP}) + o.Expect(backendConfig).NotTo(o.ContainSubstring("check inter")) + + compat_otp.By("4.0: Scale up the deployment with replicas 2, then check the haproxy.config that the health check should exist in the backend server slots") + scaleDeploy(oc, ns, deploymentName, 2) + backendConfig = ensureHaproxyBlockConfigMatchRegexp(oc, routerpod, "be_http:"+ns+":"+unSecSvcName, []string{epIP + ".+check inter"}) + o.Expect(strings.Count(backendConfig, "check inter") == 2).To(o.BeTrue()) + }) + + // Test case creater: zzhao@redhat.com + g.It("Author:mjoseph-ROSA-OSD_CCS-ARO-High-16872-Health check when there are multi service and each service has one backend", func() { + var ( + buildPruningBaseDir = testdata.FixturePath("router") + webServerTemplate = filepath.Join(buildPruningBaseDir, "template-web-server-deploy.yaml") + webServerDeploy1 = webServerDeployDescription{ + deployName: "web-server-deploy1", + svcSecureName: "service-secure1", + svcUnsecureName: "service-unsecure1", + template: webServerTemplate, + namespace: "", + } + + webServerDeploy2 = webServerDeployDescription{ + deployName: "web-server-deploy2", + svcSecureName: "service-secure2", + svcUnsecureName: "service-unsecure2", + template: webServerTemplate, + namespace: "", + } + deploy1Label = "name=" + webServerDeploy1.deployName + deploy2Label = "name=" + webServerDeploy2.deployName + unsecureRouteName = "16872" + ) + + compat_otp.By("Deploy two sets of web-server and services") + ns := oc.Namespace() + webServerDeploy1.namespace = ns + webServerDeploy2.namespace = ns + webServerDeploy1.create(oc) + webServerDeploy2.create(oc) + ensurePodWithLabelReady(oc, ns, deploy1Label) + ensurePodWithLabelReady(oc, ns, deploy2Label) + routerPod := getOneRouterPodNameByIC(oc, "default") + + compat_otp.By("1. Create unsecure route and set route-backends with multi serivces") + createRoute(oc, ns, "http", unsecureRouteName, webServerDeploy1.svcUnsecureName, []string{}) + ensureRouteIsAdmittedByIngressController(oc, ns, unsecureRouteName, "default") + // Confirm there is no check inter in the haproxy.config before setting route-backends with multi services. + httpSearchOutput := ensureHaproxyBlockConfigContains(oc, routerPod, ns+":"+unsecureRouteName, []string{unsecureRouteName}) + o.Expect(httpSearchOutput).NotTo(o.ContainSubstring("check inter 500ms")) + // Note: the "balance roundrobin" is used for the route once set route-backends, no need to annotate the route" + err := oc.Run("set").Args("route-backends", unsecureRouteName, "service-unsecure1=20", "service-unsecure2=80").Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + + compat_otp.By("2. Check unsecure route health check interval in haproxy.config") + ensureHaproxyBlockConfigMatchRegexp(oc, routerPod, "be_http:"+ns+":"+unsecureRouteName, []string{"server pod:" + webServerDeploy1.deployName + ".+check inter 5000ms", "server pod:" + webServerDeploy2.deployName + ".+check inter 5000ms"}) + }) + + // author: shudili@redhat.com + // Includes OCP-21766: Integrate router metrics with the monitoring component + // OCP-10903: The router pod should have default resource limits + g.It("Author:shudili-ROSA-OSD_CCS-ARO-High-21766-misc tests for haproxy router", func() { + var ( + namespace = "openshift-ingress" + servicemonitorName = "router-default" + rolebindingName = "prometheus-k8s" + ) + compat_otp.By(fmt.Sprintf("Check whether servicemonitor %s exists or not", servicemonitorName)) + jsonPath := "{.items[*].metadata.name}" + servicemonitorList := getByJsonPath(oc, namespace, "servicemonitor", jsonPath) + o.Expect(servicemonitorList).To(o.ContainSubstring(servicemonitorName)) + + compat_otp.By(fmt.Sprintf("Check whether rolebinding prometheus-k8s exists or not", rolebindingName)) + rolebindingList := getByJsonPath(oc, namespace, "rolebinding", jsonPath) + o.Expect(rolebindingList).To(o.ContainSubstring(rolebindingName)) + + compat_otp.By(fmt.Sprintf("check the openshift.io/cluster-monitoring label of the namespace %s, which should be true", namespace)) + jsonPath = `{.metadata.labels.openshift\.io/cluster-monitoring}` + value := getByJsonPath(oc, "default", "namespace/"+namespace, jsonPath) + o.Expect(value).To(o.ContainSubstring("true")) + + // OCP-10903 + compat_otp.By("check the default resources limits of the router container in the router-default deployment") + jsonPath = `{.spec.template.spec.containers[?(@.name=="router")].resources.requests.cpu}{.spec.template.spec.containers[?(@.name=="router")].resources.requests.memory}` + resources := getByJsonPath(oc, "openshift-ingress", "deployments/router-default", jsonPath) + o.Expect(resources).To(o.ContainSubstring("100m256Mi")) + }) + + // author: shudili@redhat.com + g.It("Author:shudili-ROSA-OSD_CCS-ARO-High-27628-router support HTTP2 for passthrough route and reencrypt route with custom certs", func() { + var ( + buildPruningBaseDir = testdata.FixturePath("router") + testPodSvc = filepath.Join(buildPruningBaseDir, "web-server-signed-deploy.yaml") + srvdmInfo = "web-server-deploy" + svcName = "service-secure" + clientPod = filepath.Join(buildPruningBaseDir, "test-client-pod.yaml") + clientPodName = "hello-pod" + clientPodLabel = "app=hello-pod" + podDirname = "/data/OCP-27628-ca" + podCaCrt = podDirname + "/27628-ca.crt" + podUsrCrt = podDirname + "/27628-usr.crt" + podUsrKey = podDirname + "/27628-usr.key" + dirname = "/tmp/OCP-27628-ca" + caSubj = "/CN=NE-Test-Root-CA" + caCrt = dirname + "/27628-ca.crt" + caKey = dirname + "/27628-ca.key" + userSubj = "/CN=example-ne.com" + usrCrt = dirname + "/27628-usr.crt" + usrKey = dirname + "/27628-usr.key" + usrCsr = dirname + "/27628-usr.csr" + cmName = "ocp27628" + ) + + // enabled mTLS for http/2 traffic testing, if not, the frontend haproxy will use http/1.1 + baseTemp := filepath.Join(buildPruningBaseDir, "ingresscontroller-np.yaml") + extraParas := fmt.Sprintf(` + clientTLS: + clientCA: + name: %s + clientCertificatePolicy: Required +`, cmName) + customTemp := addExtraParametersToYamlFile(baseTemp, "spec:", extraParas) + defer os.Remove(customTemp) + + var ( + ingctrl = ingressControllerDescription{ + name: "ocp27628", + namespace: "openshift-ingress-operator", + domain: "", + template: customTemp, + } + ) + + compat_otp.By("1.0 Get the domain info for testing") + ingctrl.domain = ingctrl.name + "." + getBaseDomain(oc) + routehost := "reen27628" + "." + ingctrl.domain + + compat_otp.By("2.0: Start to use openssl to create ca certification&key and user certification&key") + defer os.RemoveAll(dirname) + err := os.MkdirAll(dirname, 0755) + o.Expect(err).NotTo(o.HaveOccurred()) + + compat_otp.By("2.1: Create a new self-signed CA including the ca certification and ca key") + opensslNewCa(caKey, caCrt, caSubj) + + compat_otp.By("2.2: Create a user CSR and the user key for the reen route") + opensslNewCsr(usrKey, usrCsr, userSubj) + + compat_otp.By("2.3: Sign the user CSR and generate the certificate for the reen route") + san := "subjectAltName = DNS:*." + ingctrl.domain + opensslSignCsr(san, usrCsr, caCrt, caKey, usrCrt) + + compat_otp.By("3.0: create a cm with date ca certification, then create the custom ingresscontroller") + defer deleteConfigMap(oc, "openshift-config", cmName) + createConfigMapFromFile(oc, "openshift-config", cmName, "ca-bundle.pem="+caCrt) + + defer ingctrl.delete(oc) + ingctrl.create(oc) + ensureCustomIngressControllerAvailable(oc, ingctrl.name) + + compat_otp.By("4.0: enable http2 on the custom ingresscontroller by the annotation if env ROUTER_DISABLE_HTTP2 is true") + jsonPath := "{.spec.template.spec.containers[0].env[?(@.name==\"ROUTER_DISABLE_HTTP2\")].value}" + envValue := getByJsonPath(oc, "openshift-ingress", "deployment/router-"+ingctrl.name, jsonPath) + if envValue == "true" { + setAnnotationAsAdmin(oc, ingctrl.namespace, "ingresscontroller/"+ingctrl.name, `ingress.operator.openshift.io/default-enable-http2=true`) + ensureRouterDeployGenerationIs(oc, ingctrl.name, "2") + } + + compat_otp.By("5.0 Create a deployment and a client pod") + ns := oc.Namespace() + createResourceFromFile(oc, ns, testPodSvc) + ensurePodWithLabelReady(oc, ns, "name="+srvdmInfo) + err = oc.AsAdmin().WithoutNamespace().Run("create").Args("-n", ns, "-f", clientPod).Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + ensurePodWithLabelReady(oc, ns, clientPodLabel) + err = oc.AsAdmin().WithoutNamespace().Run("cp").Args("-n", ns, dirname, ns+"/"+clientPodName+":"+podDirname).Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + + compat_otp.By("6.0 Create a reencrypt route and a passthrough route inside the namespace") + createRoute(oc, ns, "reencrypt", "route-reen", svcName, []string{"--hostname=" + routehost, "--ca-cert=" + caCrt, "--cert=" + usrCrt, "--key=" + usrKey}) + ensureRouteIsAdmittedByIngressController(oc, ns, "route-reen", "default") + routehost2 := "pass27628" + "." + ingctrl.domain + createRoute(oc, ns, "passthrough", "route-pass", svcName, []string{"--hostname=" + routehost2}) + ensureRouteIsAdmittedByIngressController(oc, ns, "route-pass", "default") + + compat_otp.By("7.0 Check the cert_config.map for the reencypt route with custom cert") + routerpod := getOneNewRouterPodFromRollingUpdate(oc, ingctrl.name) + output, err := oc.AsAdmin().WithoutNamespace().Run("exec").Args("-n", "openshift-ingress", routerpod, "--", "bash", "-c", "cat cert_config.map").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(output).To(o.ContainSubstring(`route-reen.pem [alpn h2,http/1.1] ` + routehost)) + + compat_otp.By("8.0 Curl the reencrypt route with specified protocol http2") + podIP := getPodv4Address(oc, routerpod, "openshift-ingress") + toDst := routehost + ":443:" + podIP + curlCmd := []string{"-n", ns, clientPodName, "--", "curl", "https://" + routehost, "-sI", "--cacert", podCaCrt, "--cert", podUsrCrt, "--key", podUsrKey, "--http2", "--resolve", toDst, "--connect-timeout", "10"} + repeatCmdOnClient(oc, curlCmd, "HTTP/2 200", 60, 1) + + compat_otp.By("9.0 Curl the pass route with specified protocol http2") + toDst = routehost2 + ":443:" + podIP + curlCmd = []string{"-n", ns, clientPodName, "--", "curl", "https://" + routehost2, "-skI", "--http2", "--resolve", toDst, "--connect-timeout", "10"} + repeatCmdOnClient(oc, curlCmd, "HTTP/2 200", 60, 1) + }) + + // author: hongli@redhat.com + g.It("Author:shudili-ROSA-OSD_CCS-ARO-High-37714-Ingresscontroller routes traffic only to ready pods/backends", func() { + // skip the test if featureSet is set there + if compat_otp.IsTechPreviewNoUpgrade(oc) { + g.Skip("Skip for DCM enabled, the haproxy has the dynamic server slot configration for the only one endpoint, not the static") + } + + // if the ingress canary route isn't accessable from outside, skip it + if !isCanaryRouteAvailable(oc) { + g.Skip("Skip for the ingress canary route could not be available to the outside") + } + + var ( + buildPruningBaseDir = testdata.FixturePath("router") + testPodSvc = filepath.Join(buildPruningBaseDir, "web-server-signed-deploy.yaml") + unSecSvcName = "service-unsecure" + ) + + compat_otp.By("1.0: updated the deployment file with readinessProbe") + extraParas := fmt.Sprintf(` + readinessProbe: + exec: + command: + - cat + - /data/ready + initialDelaySeconds: 5 + periodSeconds: 5 +`) + updatedDeployFile := addContenToFileWithMatchedOrder(testPodSvc, " - name: nginx", extraParas, 1) + defer os.Remove(updatedDeployFile) + + compat_otp.By("2.0 Create a deployment and a client pod") + ns := oc.Namespace() + createResourceFromFile(oc, ns, updatedDeployFile) + waitForOutputEquals(oc, ns, "deployment/web-server-deploy", "{.status.replicas}", "1", 180*time.Second) + serverPod := getPodListByLabel(oc, ns, "name=web-server-deploy")[0] + waitForOutputEquals(oc, ns, "pod/"+serverPod, "{.status.phase}", "Running") + waitForOutputEquals(oc, ns, "pod/"+serverPod, "{.status.conditions[?(@.type==\"Ready\")].status}", "False") + + compat_otp.By("3.0 Create a http route") + routehost := "unsecure37714" + ".apps." + getBaseDomain(oc) + createRoute(oc, ns, "http", unSecSvcName, unSecSvcName, []string{"--hostname=" + routehost}) + ensureRouteIsAdmittedByIngressController(oc, ns, unSecSvcName, "default") + + compat_otp.By("4.0 Check haproxy.config which should not have the server slot") + routerpod := getOneRouterPodNameByIC(oc, "default") + serverSlot := "pod:" + serverPod + ":" + unSecSvcName + ":http" + backendConfig := ensureHaproxyBlockConfigContains(oc, routerpod, "be_http:"+ns+":"+unSecSvcName, []string{unSecSvcName}) + o.Expect(backendConfig).NotTo(o.ContainSubstring(serverSlot)) + + compat_otp.By("5.0 Curl the http route, expect to get 503 for the server pod is not ready") + curlCmd := fmt.Sprintf(`curl http://%s -sI --connect-timeout 10`, routehost) + repeatCmdOnClient(oc, curlCmd, "503", 60, 1) + + compat_otp.By("6.0 Create the /data/ready under the server pod") + _, err := oc.AsAdmin().WithoutNamespace().Run("exec").Args("-n", ns, serverPod, "--", "touch", "/data/ready").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + waitForOutputEquals(oc, ns, "pod/"+serverPod, "{.status.conditions[?(@.type==\"Ready\")].status}", "True") + + compat_otp.By("7.0 Check haproxy.config which should have the server slot") + ensureHaproxyBlockConfigContains(oc, routerpod, "be_http:"+ns+":"+unSecSvcName, []string{serverSlot}) + + compat_otp.By("8.0 Curl the http route again, expect to get 200 ok for the server pod is ready") + repeatCmdOnClient(oc, curlCmd, "200 OK", 60, 1) + }) + + // author: aiyengar@redhat.com + g.It("Author:aiyengar-Critical-40675-Ingresscontroller with endpointPublishingStrategy of hostNetwork allows PROXY protocol for source forwarding [Flaky]", func() { + buildPruningBaseDir := testdata.FixturePath("router") + customTemp := filepath.Join(buildPruningBaseDir, "ingresscontroller-hn-PROXY.yaml") + var ( + ingctrl = ingressControllerDescription{ + name: "ocp40675", + namespace: "openshift-ingress-operator", + domain: "", + template: customTemp, + } + ) + + compat_otp.By("check whether there are more than two worker nodes present for testing hostnetwork") + workerNodeCount, _ := exactNodeDetails(oc) + if workerNodeCount <= 2 { + g.Skip("Skipping as we need more than two worker nodes") + } + + compat_otp.By("Create a hostNetwork ingresscontroller with PROXY protocol set") + baseDomain := getBaseDomain(oc) + ingctrl.domain = ingctrl.name + "." + baseDomain + defer ingctrl.delete(oc) + ingctrl.create(oc) + ensureRouterDeployGenerationIs(oc, ingctrl.name, "1") + + compat_otp.By("Check the router env to verify the PROXY variable is applied") + routername := getOneNewRouterPodFromRollingUpdate(oc, ingctrl.name) + pollReadPodData(oc, "openshift-ingress", routername, "/usr/bin/env", `ROUTER_USE_PROXY_PROTOCOL=true`) + }) + + // author: aiyengar@redhat.com + g.It("Author:aiyengar-Critical-40677-Ingresscontroller with endpointPublishingStrategy of nodePort allows PROXY protocol for source forwarding", func() { + buildPruningBaseDir := testdata.FixturePath("router") + customTemp := filepath.Join(buildPruningBaseDir, "ingresscontroller-np-PROXY.yaml") + var ( + ingctrl = ingressControllerDescription{ + name: "ocp40677", + namespace: "openshift-ingress-operator", + domain: "", + template: customTemp, + } + ) + + compat_otp.By("Create a NP ingresscontroller with PROXY protocol set") + baseDomain := getBaseDomain(oc) + ingctrl.domain = ingctrl.name + "." + baseDomain + defer ingctrl.delete(oc) + ingctrl.create(oc) + ensureRouterDeployGenerationIs(oc, ingctrl.name, "1") + + compat_otp.By("Check the router env to verify the PROXY variable is applied") + podname := getOneNewRouterPodFromRollingUpdate(oc, ingctrl.name) + dssearch := readRouterPodEnv(oc, podname, "ROUTER_USE_PROXY_PROTOCOL") + o.Expect(dssearch).To(o.ContainSubstring(`ROUTER_USE_PROXY_PROTOCOL=true`)) + }) + + // author: shudili@redhat.com + g.It("Author:shudili-Medium-40679-The endpointPublishingStrategy parameter allow TCP/PROXY/empty definition for HostNetwork or NodePort type strategies", func() { + var ( + buildPruningBaseDir = testdata.FixturePath("router") + customTemp = filepath.Join(buildPruningBaseDir, "ingresscontroller-hn-PROXY.yaml") + ingctrl = ingressControllerDescription{ + name: "ocp40679", + namespace: "openshift-ingress-operator", + domain: "", + template: customTemp, + } + ingctrlResource = "ingresscontroller/" + ingctrl.name + ) + + compat_otp.By("check whether there are more than two worker nodes present for testing hostnetwork") + workerNodeCount, _ := exactNodeDetails(oc) + if workerNodeCount <= 2 { + g.Skip("Skipping as we need more than two worker nodes") + } + + compat_otp.By("Create a hostNetwork ingresscontroller with protocol PROXY set by the template") + baseDomain := getBaseDomain(oc) + ingctrl.domain = ingctrl.name + "." + baseDomain + defer ingctrl.delete(oc) + ingctrl.create(oc) + ensureRouterDeployGenerationIs(oc, ingctrl.name, "1") + + compat_otp.By("Check the router env to verify the PROXY variable is applied") + routerpod := getOneNewRouterPodFromRollingUpdate(oc, ingctrl.name) + pollReadPodData(oc, "openshift-ingress", routerpod, "/usr/bin/env", `ROUTER_USE_PROXY_PROTOCOL=true`) + + compat_otp.By("Patch the hostNetwork ingresscontroller with protocol TCP") + patchPath := "{\"spec\":{\"endpointPublishingStrategy\":{\"hostNetwork\":{\"protocol\": \"TCP\"}}}}" + patchResourceAsAdmin(oc, ingctrl.namespace, ingctrlResource, patchPath) + ensureRouterDeployGenerationIs(oc, ingctrl.name, "2") + + compat_otp.By("Check the configuration and router env for protocol TCP") + routerpod = getOneNewRouterPodFromRollingUpdate(oc, ingctrl.name) + cmd := fmt.Sprintf("/usr/bin/env | grep %s", `ROUTER_USE_PROXY_PROTOCOL`) + jsonPath := "{.spec.endpointPublishingStrategy.hostNetwork.protocol}" + output := getByJsonPath(oc, ingctrl.namespace, ingctrlResource, jsonPath) + o.Expect(output).To(o.ContainSubstring("TCP")) + err := oc.AsAdmin().WithoutNamespace().Run("exec").Args("-n", ingctrl.namespace, routerpod, "--", "bash", "-c", cmd).Execute() + o.Expect(err).To(o.HaveOccurred()) + + compat_otp.By("Patch the hostNetwork ingresscontroller with protocol empty") + patchPath = `{"spec":{"endpointPublishingStrategy":{"hostNetwork":{"protocol": ""}}}}` + patchResourceAsAdmin(oc, ingctrl.namespace, ingctrlResource, patchPath) + + compat_otp.By("Check the configuration and router env for protocol empty") + output = getByJsonPath(oc, ingctrl.namespace, ingctrlResource, jsonPath) + o.Expect(output).To(o.BeEmpty()) + err = oc.AsAdmin().WithoutNamespace().Run("exec").Args("-n", ingctrl.namespace, routerpod, "--", "bash", "-c", cmd).Execute() + o.Expect(err).To(o.HaveOccurred()) + }) + + g.It("Author:aiyengar-ROSA-OSD_CCS-ARO-LEVEL0-High-41042-The Power-of-two balancing features defaults to random LB algorithm instead of leastconn for REEN/Edge/insecure routes", func() { + var ( + baseDomain = getBaseDomain(oc) + defaultPod = getOneRouterPodNameByIC(oc, "default") + unsecsvcName = "service-unsecure" + secsvcName = "service-secure" + ) + buildPruningBaseDir := testdata.FixturePath("router") + testPodSvc := filepath.Join(buildPruningBaseDir, "web-server-signed-deploy.yaml") + addSvc := filepath.Join(buildPruningBaseDir, "svc-additional-backend.yaml") + + compat_otp.By("Create pods and service resources") + ns := oc.Namespace() + createResourceFromFile(oc, ns, testPodSvc) + createResourceFromFile(oc, ns, addSvc) + ensurePodWithLabelReady(oc, ns, "name=web-server-deploy") + + compat_otp.By("Expose a edge/insecure/REEN/passthrough type routes via the services inside the namespace") + edgeRoute := "route-edge" + "-" + ns + "." + baseDomain + reenRoute := "route-reen" + "-" + ns + "." + baseDomain + createRoute(oc, ns, "edge", "route-edge", unsecsvcName, []string{"--hostname=" + edgeRoute}) + output, err := oc.Run("get").Args("route").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(output).To(o.ContainSubstring("route-edge")) + createRoute(oc, ns, "reencrypt", "route-reen", secsvcName, []string{"--hostname=" + reenRoute}) + output, err = oc.Run("get").Args("route").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(output).To(o.ContainSubstring("route-reen")) + createRoute(oc, ns, "http", unsecsvcName, unsecsvcName, []string{}) + output, err = oc.Run("get").Args("route").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(output).To(o.ContainSubstring(unsecsvcName)) + + compat_otp.By("Check the default loadbalance algorithm inside proxy pod") + edgeBackend := "be_edge_http:" + ns + ":route-edge" + reenBackend := "be_secure:" + ns + ":route-reen" + insecBackend := "be_http:" + ns + ":service-unsecure" + ensureHaproxyBlockConfigContains(oc, defaultPod, edgeBackend, []string{"balance random"}) + ensureHaproxyBlockConfigContains(oc, defaultPod, reenBackend, []string{"balance random"}) + ensureHaproxyBlockConfigContains(oc, defaultPod, insecBackend, []string{"balance random"}) + }) + + g.It("Author:aiyengar-ROSA-OSD_CCS-ARO-Critical-41186-The Power-of-two balancing features switches to roundrobin mode for REEN/Edge/insecure/passthrough routes with multiple backends configured with weights", func() { + var ( + baseDomain = getBaseDomain(oc) + defaultPod = getOneRouterPodNameByIC(oc, "default") + unsecsvcName = "service-unsecure" + secsvcName = "service-secure" + ) + buildPruningBaseDir := testdata.FixturePath("router") + testPodSvc := filepath.Join(buildPruningBaseDir, "web-server-signed-deploy.yaml") + addSvc := filepath.Join(buildPruningBaseDir, "svc-additional-backend.yaml") + + compat_otp.By("Create pods and service resources") + ns := oc.Namespace() + createResourceFromFile(oc, ns, testPodSvc) + createResourceFromFile(oc, ns, addSvc) + ensurePodWithLabelReady(oc, ns, "name=web-server-deploy") + + compat_otp.By("Expose a edge/insecure/REEN/passthrough type routes via the services inside the namespace") + edgeRoute := "route-edge" + "-" + ns + "." + baseDomain + reenRoute := "route-reen" + "-" + ns + "." + baseDomain + passthRoute := "route-passth" + "-" + ns + "." + baseDomain + createRoute(oc, ns, "edge", "route-edge", unsecsvcName, []string{"--hostname=" + edgeRoute}) + output, err := oc.Run("get").Args("route").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(output).To(o.ContainSubstring("route-edge")) + createRoute(oc, ns, "reencrypt", "route-reen", secsvcName, []string{"--hostname=" + reenRoute}) + output, err = oc.Run("get").Args("route").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(output).To(o.ContainSubstring("route-reen")) + createRoute(oc, ns, "passthrough", "route-passth", unsecsvcName, []string{"--hostname=" + passthRoute}) + output, err = oc.Run("get").Args("route").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(output).To(o.ContainSubstring("route-passth")) + createRoute(oc, ns, "http", unsecsvcName, unsecsvcName, []string{}) + output, err = oc.Run("get").Args("route").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(output).To(o.ContainSubstring(unsecsvcName)) + + compat_otp.By("Check the default loadbalance algorithm inside proxy pod") + edgeBackend := "be_edge_http:" + ns + ":route-edge" + reenBackend := "be_secure:" + ns + ":route-reen" + insecBackend := "be_http:" + ns + ":service-unsecure" + ensureHaproxyBlockConfigContains(oc, defaultPod, edgeBackend, []string{"balance random"}) + ensureHaproxyBlockConfigContains(oc, defaultPod, reenBackend, []string{"balance random"}) + ensureHaproxyBlockConfigContains(oc, defaultPod, insecBackend, []string{"balance random"}) + + compat_otp.By("Add service as weighted backend to the routes and check the balancing algorithm value") + passthBackend := "be_tcp:" + ns + ":route-passth" + _, edgerr := oc.Run("set").WithoutNamespace().Args("route-backends", "route-edge", "service-unsecure1=100", "service-unsecure2=150").Output() + o.Expect(edgerr).NotTo(o.HaveOccurred()) + _, reenerr := oc.Run("set").WithoutNamespace().Args("route-backends", "route-reen", "service-secure1=100", "service-secure2=150").Output() + o.Expect(reenerr).NotTo(o.HaveOccurred()) + _, passtherr := oc.Run("set").WithoutNamespace().Args("route-backends", "route-passth", "service-secure1=100", "service-secure2=150").Output() + o.Expect(passtherr).NotTo(o.HaveOccurred()) + _, insecerr := oc.Run("set").WithoutNamespace().Args("route-backends", "service-unsecure", "service-unsecure1=100", "service-unsecure2=150").Output() + o.Expect(insecerr).NotTo(o.HaveOccurred()) + ensureHaproxyBlockConfigContains(oc, defaultPod, edgeBackend, []string{"balance roundrobin"}) + ensureHaproxyBlockConfigContains(oc, defaultPod, reenBackend, []string{"balance roundrobin"}) + ensureHaproxyBlockConfigContains(oc, defaultPod, insecBackend, []string{"balance roundrobin"}) + ensureHaproxyBlockConfigContains(oc, defaultPod, passthBackend, []string{"balance roundrobin"}) + }) + + g.It("Author:aiyengar-High-41187-The Power of two balancing honours the per route balancing algorithm defined via haproxy.router.openshift.io/balance annotation", func() { + var ( + defaultPod = getOneRouterPodNameByIC(oc, "default") + unsecsvcName = "service-unsecure" + ) + buildPruningBaseDir := testdata.FixturePath("router") + testPodSvc := filepath.Join(buildPruningBaseDir, "web-server-signed-deploy.yaml") + + compat_otp.By("Create pods and service resources") + ns := oc.Namespace() + createResourceFromFile(oc, ns, testPodSvc) + ensurePodWithLabelReady(oc, ns, "name=web-server-deploy") + + compat_otp.By("Expose a route from the namespace and set route LB annotation") + createRoute(oc, ns, "http", unsecsvcName, unsecsvcName, []string{}) + output, err := oc.Run("get").Args("route").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(output).To(o.ContainSubstring(unsecsvcName)) + setAnnotation(oc, ns, "route/service-unsecure", "haproxy.router.openshift.io/balance=leastconn") + findAnnotation, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("route", unsecsvcName, "-n", ns, "-o=jsonpath={.metadata.annotations}").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + getAlgoValue := gjson.Get(string(findAnnotation), "haproxy\\.router\\.openshift\\.io/balance").String() + o.Expect(getAlgoValue).To(o.ContainSubstring("leastconn")) + + compat_otp.By("Check the default loadbalance algorithm inside proxy pod and check the default LB variable to confirm power-of-two is active") + insecBackend := "be_http:" + ns + ":service-unsecure" + rtrParamCheck := readPodEnv(oc, defaultPod, "openshift-ingress", "ROUTER_LOAD_BALANCE_ALGORITHM") + o.Expect(rtrParamCheck).To(o.ContainSubstring("random")) + ensureHaproxyBlockConfigContains(oc, defaultPod, insecBackend, []string{"balance leastconn"}) + }) + + g.It("Author:aiyengar-High-41206-Power-of-two feature allows unsupportedConfigOverrides ingress operator option to enable leastconn balancing algorithm", func() { + buildPruningBaseDir := testdata.FixturePath("router") + customTemp := filepath.Join(buildPruningBaseDir, "ingresscontroller-np.yaml") + var ( + ingctrl = ingressControllerDescription{ + name: "41206", + namespace: "openshift-ingress-operator", + domain: "", + template: customTemp, + } + ) + testPodSvc := filepath.Join(buildPruningBaseDir, "web-server-signed-deploy.yaml") + + compat_otp.By("Create a custom ingresscontroller, and get its router name") + baseDomain := getBaseDomain(oc) + ingctrl.domain = ingctrl.name + "." + baseDomain + defer ingctrl.delete(oc) + ingctrl.create(oc) + ensureRouterDeployGenerationIs(oc, ingctrl.name, "1") + + compat_otp.By("Patch ingresscontroller with unsupportedConfigOverrides option") + ingctrlResource := "ingresscontrollers/" + ingctrl.name + patchResourceAsAdmin(oc, ingctrl.namespace, ingctrlResource, "{\"spec\":{\"unsupportedConfigOverrides\":{\"loadBalancingAlgorithm\":\"leastconn\"}}}") + ensureRouterDeployGenerationIs(oc, ingctrl.name, "2") + newrouterpod := getOneNewRouterPodFromRollingUpdate(oc, ingctrl.name) + + compat_otp.By("verify ROUTER_LOAD_BALANCE_ALGORITHM variable of the deployed router pod") + checkenv := readRouterPodEnv(oc, newrouterpod, "ROUTER_LOAD_BALANCE_ALGORITHM") + o.Expect(checkenv).To(o.ContainSubstring(`ROUTER_LOAD_BALANCE_ALGORITHM=leastconn`)) + + compat_otp.By("deploy pod resource and expose a route via the ingresscontroller") + ns := oc.Namespace() + edgeRoute := "route-edge" + "-" + ns + "." + ingctrl.domain + createResourceFromFile(oc, ns, testPodSvc) + ensurePodWithLabelReady(oc, ns, "name=web-server-deploy") + createRoute(oc, ns, "edge", "route-edge", "service-unsecure", []string{"--hostname=" + edgeRoute}) + output, err := oc.Run("get").Args("route").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(output).To(o.ContainSubstring("route-edge")) + + compat_otp.By("Check the router config for the default LB algorithm set at the backend") + edgeBackend := "be_edge_http:" + ns + ":route-edge" + ensureHaproxyBlockConfigContains(oc, newrouterpod, edgeBackend, []string{"balance leastconn"}) + }) + + g.It("Author:mjoseph-High-41929-Haproxy router continues to function normally with the service selector of exposed route gets removed/deleted", func() { + var ( + buildPruningBaseDir = testdata.FixturePath("router") + customTemp = filepath.Join(buildPruningBaseDir, "ingresscontroller-np.yaml") + testPodSvc = filepath.Join(buildPruningBaseDir, "web-server-deploy.yaml") + ingctrl = ingressControllerDescription{ + name: "ocp41929", + namespace: "openshift-ingress-operator", + domain: "", + template: customTemp, + } + ) + + compat_otp.By("Create a custom ingresscontroller") + baseDomain := getBaseDomain(oc) + ingctrl.domain = ingctrl.name + "." + baseDomain + defer ingctrl.delete(oc) + ingctrl.create(oc) + ensureRouterDeployGenerationIs(oc, ingctrl.name, "1") + custContPod := getOneNewRouterPodFromRollingUpdate(oc, ingctrl.name) + + compat_otp.By("Deploy a backend pod and its service resources") + ns := oc.Namespace() + createResourceFromFile(oc, ns, testPodSvc) + ensurePodWithLabelReady(oc, ns, "name=web-server-deploy") + + compat_otp.By("Expose a route with the unsecure service inside the namespace") + routehost := "service-unsecure-" + ns + "." + ingctrl.domain + SrvErr := oc.Run("expose").Args("svc/service-unsecure", "--hostname="+routehost).Execute() + o.Expect(SrvErr).NotTo(o.HaveOccurred()) + routeOutput := getRoutes(oc, ns) + o.Expect(routeOutput).To(o.ContainSubstring("service-unsecure")) + + compat_otp.By("Cross check the selector value of the 'service-unsecure' service") + jpath := "{.spec.selector}" + output := getByJsonPath(oc, ns, "svc/service-unsecure", jpath) + o.Expect(output).To(o.ContainSubstring(`"name":"web-server-deploy"`)) + + compat_otp.By("Delete the service selector for the 'service-unsecure' service") + patchPath := `{"spec":{"selector":null}}` + patchResourceAsAdmin(oc, ns, "svc/service-unsecure", patchPath) + + compat_otp.By("Check the service config to confirm the value of the selector is empty") + output = getByJsonPath(oc, ns, "svc/service-unsecure", jpath) + o.Expect(output).To(o.BeEmpty()) + + compat_otp.By("Check the router pod logs and confirm there is no reload error message") + log, err := oc.AsAdmin().WithoutNamespace().Run("logs").Args("-n", "openshift-ingress", custContPod).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + if strings.Contains(log, "error reloading router") { + e2e.Failf("Router reloaded after removing service selector") + } + }) + + // author: shudili@redhat.com + g.It("Author:shudili-Medium-42878-Errorfile stanzas and dummy default html files have been added to the router", func() { + compat_otp.By("Get pod (router) in openshift-ingress namespace") + podname := getOneRouterPodNameByIC(oc, "default") + + compat_otp.By("Check if there are default 404 and 503 error pages on the router") + searchOutput := readRouterPodData(oc, podname, "ls -l", "error-page") + o.Expect(searchOutput).To(o.ContainSubstring(`error-page-404.http`)) + o.Expect(searchOutput).To(o.ContainSubstring(`error-page-503.http`)) + + compat_otp.By("Check if errorfile stanzas have been added into haproxy-config.template") + searchOutput = readRouterPodData(oc, podname, "cat haproxy-config.template", "errorfile") + o.Expect(searchOutput).To(o.ContainSubstring(`ROUTER_ERRORFILE_404`)) + o.Expect(searchOutput).To(o.ContainSubstring(`ROUTER_ERRORFILE_503`)) + }) + + // author: shudili@redhat.com + g.It("Author:shudili-Medium-42940-User can customize HAProxy 2.0 Error Page", func() { + var ( + buildPruningBaseDir = testdata.FixturePath("router") + customTemp = filepath.Join(buildPruningBaseDir, "ingresscontroller-np.yaml") + testPodSvc = filepath.Join(buildPruningBaseDir, "web-server-deploy.yaml") + srvrcInfo = "web-server-deploy" + srvName = "service-unsecure" + clientPod = filepath.Join(buildPruningBaseDir, "test-client-pod.yaml") + clientPodName = "hello-pod" + clientPodLabel = "app=hello-pod" + http404page = filepath.Join(buildPruningBaseDir, "error-page-404.http") + http503page = filepath.Join(buildPruningBaseDir, "error-page-503.http") + cmName = "my-custom-error-code-pages-42940" + patchHTTPErrorPage = "{\"spec\": {\"httpErrorCodePages\": {\"name\": \"" + cmName + "\"}}}" + ingctrl = ingressControllerDescription{ + name: "ocp42940", + namespace: "openshift-ingress-operator", + domain: "", + template: customTemp, + } + ingctrlResource = "ingresscontrollers/" + ingctrl.name + ) + + compat_otp.By("Create a ConfigMap with custom 404 and 503 error pages") + cmCrtErr := oc.AsAdmin().WithoutNamespace().Run("create").Args("configmap", cmName, "--from-file="+http404page, "--from-file="+http503page, "-n", "openshift-config").Execute() + o.Expect(cmCrtErr).NotTo(o.HaveOccurred()) + defer deleteConfigMap(oc, "openshift-config", cmName) + cmOutput, cmErr := oc.WithoutNamespace().AsAdmin().Run("get").Args("configmap", "-n", "openshift-config").Output() + o.Expect(cmErr).NotTo(o.HaveOccurred()) + o.Expect(cmOutput).To(o.ContainSubstring(cmName)) + cmOutput, cmErr = oc.WithoutNamespace().AsAdmin().Run("get").Args("configmap", cmName, "-o=jsonpath={.data}", "-n", "openshift-config").Output() + o.Expect(cmErr).NotTo(o.HaveOccurred()) + o.Expect(cmOutput).To(o.ContainSubstring("error-page-404.http")) + o.Expect(cmOutput).To(o.ContainSubstring("Custom error page:The requested document was not found")) + o.Expect(cmOutput).To(o.ContainSubstring("error-page-503.http")) + o.Expect(cmOutput).To(o.ContainSubstring("Custom error page:The requested application is not available")) + + compat_otp.By("Create one custom ingresscontroller") + baseDomain := getBaseDomain(oc) + ingctrl.domain = ingctrl.name + "." + baseDomain + defer ingctrl.delete(oc) + ingctrl.create(oc) + ensureRouterDeployGenerationIs(oc, ingctrl.name, "1") + + compat_otp.By("patch the custom ingresscontroller with the http error code pages") + patchResourceAsAdmin(oc, ingctrl.namespace, ingctrlResource, patchHTTPErrorPage) + ensureRouterDeployGenerationIs(oc, ingctrl.name, "2") + + compat_otp.By("get one custom ingress-controller router pod's IP") + podname := getOneNewRouterPodFromRollingUpdate(oc, ingctrl.name) + podIP := getPodv4Address(oc, podname, "openshift-ingress") + + compat_otp.By("Create a client pod, a backend pod and its service resources") + ns := oc.Namespace() + compat_otp.By("create a client pod") + createResourceFromFile(oc, ns, clientPod) + ensurePodWithLabelReady(oc, ns, clientPodLabel) + compat_otp.By("create an unsecure service and its backend pod") + createResourceFromFile(oc, ns, testPodSvc) + ensurePodWithLabelReady(oc, ns, "name="+srvrcInfo) + + compat_otp.By("Expose an route with the unsecure service inside the namespace") + routehost := srvName + "-" + ns + "." + ingctrl.domain + srvErr := oc.Run("expose").Args("service", srvName, "--hostname="+routehost).Execute() + o.Expect(srvErr).NotTo(o.HaveOccurred()) + waitForOutputEquals(oc, ns, "route", "{.items[0].metadata.name}", srvName) + + compat_otp.By("curl a normal route from the client pod") + routestring := srvName + "-" + ns + "." + ingctrl.name + "." + waitForCurl(oc, clientPodName, baseDomain, routestring, "200 OK", podIP) + + compat_otp.By("curl a non-existing route, expect to get custom http 404 Not Found error") + notExistRoute := "notexistroute" + "-" + ns + "." + ingctrl.domain + toDst := routehost + ":80:" + podIP + toDst2 := notExistRoute + ":80:" + podIP + output, errCurlRoute := oc.Run("exec").Args(clientPodName, "--", "curl", "-v", "http://"+notExistRoute, "--resolve", toDst2, "--connect-timeout", "10").Output() + o.Expect(errCurlRoute).NotTo(o.HaveOccurred()) + o.Expect(output).To(o.ContainSubstring("404 Not Found")) + o.Expect(output).To(o.ContainSubstring("Custom error page:The requested document was not found")) + + compat_otp.By("delete the backend pod and try to curl the route, expect to get custom http 503 Service Unavailable") + podname, err := oc.Run("get").Args("pods", "-l", "name="+srvrcInfo, "-o=jsonpath={.items[0].metadata.name}").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + err = oc.Run("delete").Args("deployment", srvrcInfo).Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + err = waitForResourceToDisappear(oc, ns, "pod/"+podname) + compat_otp.AssertWaitPollNoErr(err, fmt.Sprintf("resource %v does not disapper", "pod/"+podname)) + output, err = oc.Run("exec").Args(clientPodName, "--", "curl", "-v", "http://"+routehost, "--resolve", toDst, "--connect-timeout", "10").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(output).To(o.ContainSubstring("503 Service Unavailable")) + o.Expect(output).To(o.ContainSubstring("Custom error page:The requested application is not available")) + }) + + // author: jechen@redhat.com + g.It("Author:jechen-High-43115-Configmap mounted on router volume after ingresscontroller has spec field HttpErrorCodePage populated with configmap name", func() { + buildPruningBaseDir := testdata.FixturePath("router") + customTemp := filepath.Join(buildPruningBaseDir, "ingresscontroller-np.yaml") + var ( + ingctrl = ingressControllerDescription{ + name: "ocp43115", + namespace: "openshift-ingress-operator", + domain: "", + template: customTemp, + } + ) + + compat_otp.By("1. create a custom ingresscontroller, and get its router name") + baseDomain := getBaseDomain(oc) + ingctrl.domain = ingctrl.name + "." + baseDomain + defer ingctrl.delete(oc) + ingctrl.create(oc) + ensureRouterDeployGenerationIs(oc, ingctrl.name, "1") + + compat_otp.By("2. Configure a customized error page configmap from files in openshift-config namespace") + configmapName := "custom-43115-error-code-pages" + cmFile1 := filepath.Join(buildPruningBaseDir, "error-page-503.http") + cmFile2 := filepath.Join(buildPruningBaseDir, "error-page-404.http") + _, error := oc.AsAdmin().WithoutNamespace().Run("create").Args("configmap", configmapName, "--from-file="+cmFile1, "--from-file="+cmFile2, "-n", "openshift-config").Output() + o.Expect(error).NotTo(o.HaveOccurred()) + defer oc.AsAdmin().WithoutNamespace().Run("delete").Args("configmap", configmapName, "-n", "openshift-config").Output() + + compat_otp.By("3. Check if configmap is successfully configured in openshift-config namesapce") + err := checkConfigMap(oc, "openshift-config", configmapName) + compat_otp.AssertWaitPollNoErr(err, fmt.Sprintf("cm %v not found", configmapName)) + + compat_otp.By("4. Patch the configmap created above to the custom ingresscontroller in openshift-ingress namespace") + ingctrlResource := "ingresscontrollers/" + ingctrl.name + patchResourceAsAdmin(oc, ingctrl.namespace, ingctrlResource, "{\"spec\":{\"httpErrorCodePages\":{\"name\":\"custom-43115-error-code-pages\"}}}") + + compat_otp.By("5. Check if configmap is successfully patched into openshift-ingress namesapce, configmap with name ingctrl.name-errorpages should be created") + expectedCmName := ingctrl.name + `-errorpages` + err = checkConfigMap(oc, "openshift-ingress", expectedCmName) + compat_otp.AssertWaitPollNoErr(err, fmt.Sprintf("cm %v not found", expectedCmName)) + + compat_otp.By("6. Obtain new router pod created, and check if error_code_pages directory is created on it") + ensureRouterDeployGenerationIs(oc, ingctrl.name, "2") + newrouterpod := getOneNewRouterPodFromRollingUpdate(oc, ingctrl.name) + + compat_otp.By("Check /var/lib/haproxy/conf directory to see if error_code_pages subdirectory is created on the router") + searchOutput := readRouterPodData(oc, newrouterpod, "ls -al /var/lib/haproxy/conf", "error_code_pages") + o.Expect(searchOutput).To(o.ContainSubstring(`error_code_pages`)) + + compat_otp.By("7. Check if custom error code pages have been mounted") + searchOutput = readRouterPodData(oc, newrouterpod, "ls -al /var/lib/haproxy/conf/error_code_pages", "error") + o.Expect(searchOutput).To(o.ContainSubstring(`error-page-503.http -> ..data/error-page-503.http`)) + o.Expect(searchOutput).To(o.ContainSubstring(`error-page-404.http -> ..data/error-page-404.http`)) + + searchOutput = readRouterPodData(oc, newrouterpod, "cat /var/lib/haproxy/conf/error_code_pages/error-page-503.http", "Unavailable") + o.Expect(searchOutput).To(o.ContainSubstring(`HTTP/1.0 503 Service Unavailable`)) + o.Expect(searchOutput).To(o.ContainSubstring(`Custom:Application Unavailable`)) + + searchOutput = readRouterPodData(oc, newrouterpod, "cat /var/lib/haproxy/conf/error_code_pages/error-page-404.http", "Not Found") + o.Expect(searchOutput).To(o.ContainSubstring(`HTTP/1.0 404 Not Found`)) + o.Expect(searchOutput).To(o.ContainSubstring(`Custom:Not Found`)) + + }) + + // author: shudili@redhat.com + g.It("Author:shudili-Medium-43292-User can delete configmap and update configmap with new custom error page", func() { + var ( + buildPruningBaseDir = testdata.FixturePath("router") + customTemp = filepath.Join(buildPruningBaseDir, "ingresscontroller-np.yaml") + testPodSvc = filepath.Join(buildPruningBaseDir, "web-server-deploy.yaml") + srvrcInfo = "web-server-deploy" + srvName = "service-unsecure" + clientPod = filepath.Join(buildPruningBaseDir, "test-client-pod.yaml") + clientPodName = "hello-pod" + clientPodLabel = "app=hello-pod" + http404page = filepath.Join(buildPruningBaseDir, "error-page-404.http") + http503page = filepath.Join(buildPruningBaseDir, "error-page-503.http") + http404page2 = filepath.Join(buildPruningBaseDir, "error-page2-404.http") + http503page2 = filepath.Join(buildPruningBaseDir, "error-page2-503.http") + cmName = "my-custom-error-code-pages-43292" + patchHTTPErrorPage = "{\"spec\": {\"httpErrorCodePages\": {\"name\": \"" + cmName + "\"}}}" + rmHTTPErrorPage = "{\"spec\": {\"httpErrorCodePages\": {\"name\": \"\"}}}" + ingctrl = ingressControllerDescription{ + name: "ocp43292", + namespace: "openshift-ingress-operator", + domain: "", + template: customTemp, + } + ingctrlResource = "ingresscontrollers/" + ingctrl.name + ) + + compat_otp.By("Create a ConfigMap with custom 404 and 503 error pages") + defer deleteConfigMap(oc, "openshift-config", cmName) + cmCrtErr := oc.AsAdmin().WithoutNamespace().Run("create").Args("configmap", cmName, "--from-file="+http404page, "--from-file="+http503page, "-n", "openshift-config").Execute() + o.Expect(cmCrtErr).NotTo(o.HaveOccurred()) + cmOutput, cmErr := oc.WithoutNamespace().AsAdmin().Run("get").Args("configmap", cmName, "-o=jsonpath={.data}", "-n", "openshift-config").Output() + o.Expect(cmErr).NotTo(o.HaveOccurred()) + o.Expect(cmOutput).Should(o.And( + o.ContainSubstring("error-page-404.http"), + o.ContainSubstring("Custom error page:The requested document was not found"), + o.ContainSubstring("error-page-503.http"), + o.ContainSubstring("Custom error page:The requested application is not available"))) + + compat_otp.By("Create one custom ingresscontroller") + baseDomain := getBaseDomain(oc) + ingctrl.domain = ingctrl.name + "." + baseDomain + defer ingctrl.delete(oc) + ingctrl.create(oc) + ensureRouterDeployGenerationIs(oc, ingctrl.name, "1") + + compat_otp.By("patch the custom ingresscontroller with the http error code pages") + patchResourceAsAdmin(oc, ingctrl.namespace, ingctrlResource, patchHTTPErrorPage) + ensureRouterDeployGenerationIs(oc, ingctrl.name, "2") + + compat_otp.By("get one custom ingress-controller router pod's IP") + podname := getOneNewRouterPodFromRollingUpdate(oc, ingctrl.name) + podIP := getPodv4Address(oc, podname, "openshift-ingress") + + compat_otp.By("Create a client pod, a backend pod and its service resources") + ns := oc.Namespace() + compat_otp.By("create a client pod") + createResourceFromFile(oc, ns, clientPod) + ensurePodWithLabelReady(oc, ns, clientPodLabel) + compat_otp.By("create an unsecure service and its backend pod") + createResourceFromFile(oc, ns, testPodSvc) + ensurePodWithLabelReady(oc, ns, "name="+srvrcInfo) + + compat_otp.By("Expose an route with the unsecure service inside the namespace") + routehost := srvName + "-" + ns + "." + ingctrl.domain + toDst := routehost + ":80:" + podIP + output, SrvErr := oc.Run("expose").Args("service", srvName, "--hostname="+routehost).Output() + o.Expect(SrvErr).NotTo(o.HaveOccurred()) + o.Expect(output).To(o.ContainSubstring(srvName)) + cmdOnPod := []string{"-n", ns, clientPodName, "--", "curl", "-I", "http://" + routehost, "--resolve", toDst, "--connect-timeout", "10"} + repeatCmdOnClient(oc, cmdOnPod, "200", 60, 1) + + compat_otp.By("curl a non-existing route, expect to get custom http 404 Not Found error") + notExistRoute := "notexistroute" + "-" + ns + "." + ingctrl.domain + toDst = notExistRoute + ":80:" + podIP + output, err := oc.Run("exec").Args("-n", ns, clientPodName, "--", "curl", "-v", "http://"+notExistRoute, "--resolve", toDst, "--connect-timeout", "10").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(output).Should(o.And( + o.ContainSubstring("404 Not Found"), + o.ContainSubstring("Custom error page:The requested document was not found"))) + + compat_otp.By("remove the custom error page from the ingress-controller") + patchResourceAsAdmin(oc, ingctrl.namespace, ingctrlResource, rmHTTPErrorPage) + ensureRouterDeployGenerationIs(oc, ingctrl.name, "3") + getOneNewRouterPodFromRollingUpdate(oc, ingctrl.name) + + compat_otp.By("delete the configmap") + cmDltErr := oc.AsAdmin().WithoutNamespace().Run("delete").Args("configmap", cmName, "-n", "openshift-config").Execute() + o.Expect(cmDltErr).NotTo(o.HaveOccurred()) + + compat_otp.By("Create the ConfigMap with another 404 and 503 error pages") + cmCrtErr = oc.AsAdmin().WithoutNamespace().Run("create").Args("configmap", cmName, "--from-file="+http404page2, "--from-file="+http503page2, "-n", "openshift-config").Execute() + o.Expect(cmCrtErr).NotTo(o.HaveOccurred()) + cmOutput, cmErr = oc.WithoutNamespace().AsAdmin().Run("get").Args("configmap", cmName, "-o=jsonpath={.data}", "-n", "openshift-config").Output() + o.Expect(cmErr).NotTo(o.HaveOccurred()) + o.Expect(cmOutput).Should(o.And( + o.ContainSubstring("error-page2-404.http"), + o.ContainSubstring("Custom error page:THE REQUESTED DOCUMENT WAS NOT FOUND YET!"), + o.ContainSubstring("error-page2-503.http"), + o.ContainSubstring("Custom error page:THE REQUESTED APPLICATION IS NOT AVAILABLE YET!"))) + + // the following test step will be added after bug 1990020 is fixed(https://bugzilla.redhat.com/show_bug.cgi?id=1990020) + // compat_otp.By("curl the non-existing route, expect to get the new custom http 404 Not Found error") + // output, err = oc.Run("exec").Args(clientPodName, "--", "curl", "-v", "http://"+notExistRoute, "--resolve", toDst).Output() + // o.Expect(err).NotTo(o.HaveOccurred()) + // o.Expect(output).Should(o.And( + // o.ContainSubstring("404 Not Found"), + // o.ContainSubstring("Custom error page:Custom error page:THE REQUESTED DOCUMENT WAS NOT FOUND YET!"))) + + }) + + // author: aiyengar@redhat.com + g.It("Author:aiyengar-Critical-43414-The logEmptyRequests ingresscontroller parameter set to Ignore add the dontlognull option in the haproxy configuration", func() { + buildPruningBaseDir := testdata.FixturePath("router") + customTemp := filepath.Join(buildPruningBaseDir, "ingresscontroller-np.yaml") + var ( + ingctrl = ingressControllerDescription{ + name: "43414", + namespace: "openshift-ingress-operator", + domain: "", + template: customTemp, + } + ) + + compat_otp.By("Create a custom ingresscontroller, and get its router name") + baseDomain := getBaseDomain(oc) + ingctrl.domain = ingctrl.name + "." + baseDomain + defer ingctrl.delete(oc) + ingctrl.create(oc) + ensureRouterDeployGenerationIs(oc, ingctrl.name, "1") + + compat_otp.By("Patch ingresscontroller with logEmptyRequests set to Ignore option") + ingctrlResource := "ingresscontrollers/" + ingctrl.name + patchResourceAsAdmin(oc, ingctrl.namespace, ingctrlResource, "{\"spec\":{\"logging\":{\"access\":{\"destination\":{\"type\":\"Container\"},\"logEmptyRequests\":\"Ignore\"}}}}") + ensureRouterDeployGenerationIs(oc, ingctrl.name, "2") + newrouterpod := getOneNewRouterPodFromRollingUpdate(oc, ingctrl.name) + + compat_otp.By("verify the Dontlog variable inside the router pod") + checkenv := readRouterPodEnv(oc, newrouterpod, "ROUTER_DONT_LOG_NULL") + o.Expect(checkenv).To(o.ContainSubstring(`ROUTER_DONT_LOG_NULL=true`)) + + compat_otp.By("Verify the parameter set in the haproxy configuration of the router pod") + checkoutput, err := oc.AsAdmin().WithoutNamespace().Run("exec").Args("-n", "openshift-ingress", newrouterpod, "--", "bash", "-c", `cat haproxy.config | grep -w "dontlognull"`).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(checkoutput).To(o.ContainSubstring(`option dontlognull`)) + + }) + + // author: aiyengar@redhat.com + g.It("Author:aiyengar-Critical-43416-httpEmptyRequestsPolicy ingresscontroller parameter set to ignore adds the http-ignore-probes option in the haproxy configuration", func() { + buildPruningBaseDir := testdata.FixturePath("router") + customTemp := filepath.Join(buildPruningBaseDir, "ingresscontroller-np.yaml") + var ( + ingctrl = ingressControllerDescription{ + name: "43416", + namespace: "openshift-ingress-operator", + domain: "", + template: customTemp, + } + ) + + compat_otp.By("Create a custom ingresscontroller, and get its router name") + baseDomain := getBaseDomain(oc) + ingctrl.domain = ingctrl.name + "." + baseDomain + defer ingctrl.delete(oc) + ingctrl.create(oc) + ensureRouterDeployGenerationIs(oc, ingctrl.name, "1") + + compat_otp.By("Patch ingresscontroller with logEmptyRequests set to Ignore option") + ingctrlResource := "ingresscontrollers/" + ingctrl.name + patchResourceAsAdmin(oc, ingctrl.namespace, ingctrlResource, "{\"spec\":{\"httpEmptyRequestsPolicy\":\"Ignore\"}}") + ensureRouterDeployGenerationIs(oc, ingctrl.name, "2") + newrouterpod := getOneNewRouterPodFromRollingUpdate(oc, ingctrl.name) + compat_otp.By("verify the Dontlog variable inside the router pod") + checkenv := readRouterPodEnv(oc, newrouterpod, "ROUTER_HTTP_IGNORE_PROBES") + o.Expect(checkenv).To(o.ContainSubstring(`ROUTER_HTTP_IGNORE_PROBES=true`)) + + compat_otp.By("Verify the parameter set in the haproxy configuration of the router pod") + checkoutput, err := oc.AsAdmin().WithoutNamespace().Run("exec").Args("-n", "openshift-ingress", newrouterpod, "--", "bash", "-c", `cat haproxy.config | grep -w "http-ignore-probes"`).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(checkoutput).To(o.ContainSubstring(`option http-ignore-probes`)) + }) + + // author: shudili@redhat.com + g.It("Author:shudili-High-43454-The logEmptyRequests option only gets applied when the access logging is configured for the ingresscontroller", func() { + var ( + buildPruningBaseDir = testdata.FixturePath("router") + customTemp = filepath.Join(buildPruningBaseDir, "ingresscontroller-np.yaml") + ingctrl = ingressControllerDescription{ + name: "43454", + namespace: "openshift-ingress-operator", + domain: "", + template: customTemp, + } + ingctrlResource = "ingresscontrollers/" + ingctrl.name + ) + + compat_otp.By("create a custom ingresscontroller") + baseDomain := getBaseDomain(oc) + ingctrl.domain = ingctrl.name + "." + baseDomain + defer ingctrl.delete(oc) + ingctrl.create(oc) + ensureRouterDeployGenerationIs(oc, ingctrl.name, "1") + + compat_otp.By("check the default .spec.logging") + jpath := "{.spec.logging}" + logging := getByJsonPath(oc, ingctrl.namespace, ingctrlResource, jpath) + o.Expect(logging).To(o.ContainSubstring("")) + + compat_otp.By("patch the custom ingresscontroller with .spec.logging.access.destination.container") + patchPath := `{"spec":{"logging":{"access":{"destination":{"type":"Container"}}}}}` + patchResourceAsAdmin(oc, ingctrl.namespace, ingctrlResource, patchPath) + ensureRouterDeployGenerationIs(oc, ingctrl.name, "2") + + compat_otp.By("check the .spec.logging") + logging = getByJsonPath(oc, ingctrl.namespace, ingctrlResource, jpath) + expLogStr := "\"logEmptyRequests\":\"Log\"" + o.Expect(logging).To(o.ContainSubstring(expLogStr)) + }) + + // author: mjoseph@redhat.com + g.It("Author:mjoseph-High-46571-Setting ROUTER_ENABLE_COMPRESSION and ROUTER_COMPRESSION_MIME in HAProxy", func() { + buildPruningBaseDir := testdata.FixturePath("router") + customTemp := filepath.Join(buildPruningBaseDir, "ingresscontroller-np.yaml") + var ( + ingctrl = ingressControllerDescription{ + name: "46571", + namespace: "openshift-ingress-operator", + domain: "", + template: customTemp, + } + ) + + compat_otp.By("Create a custom ingresscontroller, and get its router name") + baseDomain := getBaseDomain(oc) + ingctrl.domain = ingctrl.name + "." + baseDomain + defer ingctrl.delete(oc) + ingctrl.create(oc) + ensureRouterDeployGenerationIs(oc, ingctrl.name, "1") + + compat_otp.By("Patch ingresscontroller with httpCompression option") + ingctrlResource := "ingresscontrollers/" + ingctrl.name + patchResourceAsAdmin(oc, ingctrl.namespace, ingctrlResource, "{\"spec\":{\"httpCompression\":{\"mimeTypes\":[\"text/html\",\"text/css; charset=utf-8\",\"application/json\"]}}}") + ensureRouterDeployGenerationIs(oc, ingctrl.name, "2") + newrouterpod := getOneNewRouterPodFromRollingUpdate(oc, ingctrl.name) + + compat_otp.By("check the env variable of the router pod") + checkenv1 := readRouterPodEnv(oc, newrouterpod, "ROUTER_ENABLE_COMPRESSION") + o.Expect(checkenv1).To(o.ContainSubstring(`ROUTER_ENABLE_COMPRESSION=true`)) + checkenv2 := readRouterPodEnv(oc, newrouterpod, "ROUTER_COMPRESSION_MIME") + o.Expect(checkenv2).To(o.ContainSubstring(`ROUTER_COMPRESSION_MIME=text/html "text/css; charset=utf-8" application/json`)) + + compat_otp.By("check the haproxy config on the router pod for compression algorithm") + algo := readRouterPodData(oc, newrouterpod, "cat haproxy.config", "compression") + o.Expect(algo).To(o.ContainSubstring(`compression algo gzip`)) + o.Expect(algo).To(o.ContainSubstring(`compression type text/html "text/css; charset=utf-8" application/json`)) + }) + + // author: mjoseph@redhat.com + g.It("Author:mjoseph-Low-46898-Setting wrong data in ROUTER_ENABLE_COMPRESSION and ROUTER_COMPRESSION_MIME in HAProxy", func() { + buildPruningBaseDir := testdata.FixturePath("router") + customTemp := filepath.Join(buildPruningBaseDir, "ingresscontroller-np.yaml") + var ( + ingctrl = ingressControllerDescription{ + name: "46898", + namespace: "openshift-ingress-operator", + domain: "", + template: customTemp, + } + ) + + compat_otp.By("Create a custom ingresscontroller, and get its router name") + baseDomain := getBaseDomain(oc) + ingctrl.domain = ingctrl.name + "." + baseDomain + defer ingctrl.delete(oc) + ingctrl.create(oc) + ensureRouterDeployGenerationIs(oc, ingctrl.name, "1") + routerpod := getOneNewRouterPodFromRollingUpdate(oc, ingctrl.name) + + compat_otp.By("Patch ingresscontroller with wrong httpCompression data and check whether it is configurable") + output, _ := oc.AsAdmin().WithoutNamespace().Run("patch").Args("ingresscontroller/46898", "-p", "{\"spec\":{\"httpCompression\":{\"mimeTypes\":[\"text/\",\"text/css; charset=utf-8\",\"//\"]}}}", "--type=merge", "-n", ingctrl.namespace).Output() + o.Expect(output).To(o.ContainSubstring("Invalid value: \"text/\": spec.httpCompression.mimeTypes[0] in body should match")) + o.Expect(output).To(o.ContainSubstring("application|audio|image|message|multipart|text|video")) + + compat_otp.By("check the env variable of the router pod") + output1, _ := oc.AsAdmin().WithoutNamespace().Run("exec").Args("-n", "openshift-ingress", routerpod, "--", "bash", "-c", "/usr/bin/env | grep ROUTER_ENABLE_COMPRESSION").Output() + o.Expect(output1).NotTo(o.ContainSubstring(`ROUTER_ENABLE_COMPRESSION=true`)) + + compat_otp.By("check the haproxy config on the router pod for compression algorithm") + output2, _ := oc.AsAdmin().WithoutNamespace().Run("exec").Args("-n", "openshift-ingress", routerpod, "--", "bash", "-c", "cat haproxy.config | grep compression").Output() + o.Expect(output2).NotTo(o.ContainSubstring(`compression algo gzip`)) + }) + + // author: hongli@redhat.com + g.It("Author:hongli-LEVEL0-Critical-47344-check haproxy router v4v6 mode", func() { + compat_otp.By("Get ROUTER_IP_V4_V6_MODE env, if NotFound then v4 is using by default") + defaultRouterPod := getOneRouterPodNameByIC(oc, "default") + checkEnv := readRouterPodEnv(oc, defaultRouterPod, "ROUTER_IP_V4_V6_MODE") + ipStackType := checkIPStackType(oc) + e2e.Logf("the cluster IP stack type is: %v", ipStackType) + if ipStackType == "ipv6single" { + o.Expect(checkEnv).To(o.ContainSubstring("=v6")) + } else if ipStackType == "dualstack" { + o.Expect(checkEnv).To(o.ContainSubstring("=v4v6")) + } else { + o.Expect(checkEnv).To(o.ContainSubstring("NotFound")) + } + }) + + // author: shudili@redhat.com + g.It("Author:shudili-High-49131-check haproxy's version and router base image", func() { + var expVersion = "haproxy28-2.8.10-1.rhaos4.21.el9" + compat_otp.By("Try to get HAProxy's version in a default router pod") + haproxyVer := getHAProxyRPMVersion(oc) + compat_otp.By("show haproxy version(" + haproxyVer + "), and check if it is updated successfully") + o.Expect(haproxyVer).To(o.ContainSubstring(expVersion)) + // in 4.16, OCP-73373 - Bump openshift-router image to RHEL9" + routerpod := getOneRouterPodNameByIC(oc, "default") + output, err := oc.AsAdmin().WithoutNamespace().Run("exec").Args("-n", "openshift-ingress", routerpod, "--", "bash", "-c", "cat /etc/redhat-release").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(output).To(o.ContainSubstring("Red Hat Enterprise Linux release 9")) + // added OCP-75905([OCPBUGS-33900] [OCPBUGS-32369] HAProxy shouldn't consume high cpu usage) in 4.14+ + output2, err2 := oc.AsAdmin().WithoutNamespace().Run("exec").Args("-n", "openshift-ingress", routerpod, "--", "bash", "-c", "rpm -qa haproxy28 --changelog | grep -A2 OCPBUGS-32369").Output() + o.Expect(err2).NotTo(o.HaveOccurred()) + o.Expect(output2).Should(o.And( + o.ContainSubstring(`Resolve https://issues.redhat.com/browse/OCPBUGS-32369`), + o.ContainSubstring(`Carry fix for https://github.com/haproxy/haproxy/issues/2537`), + o.ContainSubstring(`Fix for issue 2537 picked from https://git.haproxy.org/?p=haproxy.git;a=commit;h=4a9e3e102e192b9efd17e3241a6cc659afb7e7dc`))) + }) + + // author: shudili@redhat.com + g.It("Author:shudili-High-50074-Allow Ingress to be modified on the settings of livenessProbe and readinessProbe", func() { + buildPruningBaseDir := testdata.FixturePath("router") + customTemp := filepath.Join(buildPruningBaseDir, "ingresscontroller-np.yaml") + timeout5 := "{\"spec\":{\"template\":{\"spec\":{\"containers\":[{\"name\":\"router\",\"livenessProbe\":{\"timeoutSeconds\":5},\"readinessProbe\":{\"timeoutSeconds\":5}}]}}}}" + timeoutmax := "{\"spec\":{\"template\":{\"spec\":{\"containers\":[{\"name\":\"router\",\"livenessProbe\":{\"timeoutSeconds\":2147483647},\"readinessProbe\":{\"timeoutSeconds\":2147483647}}]}}}}" + var ( + ingctrl = ingressControllerDescription{ + name: "ocp50074", + namespace: "openshift-ingress-operator", + domain: "", + template: customTemp, + } + ) + + compat_otp.By("Create one custom ingresscontroller") + baseDomain := getBaseDomain(oc) + ingctrl.domain = ingctrl.name + "." + baseDomain + defer ingctrl.delete(oc) + ingctrl.create(oc) + ensureRouterDeployGenerationIs(oc, ingctrl.name, "1") + getOneNewRouterPodFromRollingUpdate(oc, ingctrl.name) + + compat_otp.By("check the default liveness probe and readiness probe parameters in the json outut of the router deployment") + routerDeploymentName := "router-" + ingctrl.name + output, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("deployment", routerDeploymentName, "-o=jsonpath={..livenessProbe}", "-n", "openshift-ingress").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(output).To(o.ContainSubstring("\"timeoutSeconds\":1")) + output, err = oc.AsAdmin().WithoutNamespace().Run("get").Args("deployment", routerDeploymentName, "-o=jsonpath={..readinessProbe}", "-n", "openshift-ingress").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(output).To(o.ContainSubstring("\"timeoutSeconds\":1")) + + compat_otp.By("patch livenessProbe and readinessProbe with 5s to the router deployment") + _, err = oc.AsAdmin().WithoutNamespace().Run("patch").Args("deployment", routerDeploymentName, "--type=strategic", "--patch="+timeout5, "-n", "openshift-ingress").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + ensureRouterDeployGenerationIs(oc, ingctrl.name, "2") + + compat_otp.By("check liveness probe and readiness probe 5s in the json output of the router deployment") + output, err = oc.AsAdmin().WithoutNamespace().Run("get").Args("deployment", routerDeploymentName, "-o=jsonpath={..livenessProbe}", "-n", "openshift-ingress").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(output).To(o.ContainSubstring("\"timeoutSeconds\":5")) + output, err = oc.AsAdmin().WithoutNamespace().Run("get").Args("deployment", routerDeploymentName, "-o=jsonpath={..readinessProbe}", "-n", "openshift-ingress").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(output).To(o.ContainSubstring("\"timeoutSeconds\":5")) + + compat_otp.By("patch livenessProbe and readinessProbe with max 2147483647s to the router deployment") + _, err = oc.AsAdmin().WithoutNamespace().Run("patch").Args("deployment", routerDeploymentName, "--type=strategic", "--patch="+timeoutmax, "-n", "openshift-ingress").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + ensureRouterDeployGenerationIs(oc, ingctrl.name, "3") + + compat_otp.By("check liveness probe and readiness probe max 2147483647s in the json output of the router deployment") + output, err = oc.AsAdmin().WithoutNamespace().Run("get").Args("deployment", routerDeploymentName, "-o=jsonpath={..livenessProbe}", "-n", "openshift-ingress").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(output).To(o.ContainSubstring("\"timeoutSeconds\":2147483647")) + output, err = oc.AsAdmin().WithoutNamespace().Run("get").Args("deployment", routerDeploymentName, "-o=jsonpath={..readinessProbe}", "-n", "openshift-ingress").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(output).To(o.ContainSubstring("\"timeoutSeconds\":2147483647")) + + compat_otp.By("check liveness probe and readiness probe max 2147483647s in the json output of the router pod") + podname := getOneNewRouterPodFromRollingUpdate(oc, ingctrl.name) + output, err = oc.AsAdmin().WithoutNamespace().Run("get").Args("pod", podname, "-o=jsonpath={..livenessProbe}", "-n", "openshift-ingress").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(output).To(o.ContainSubstring("\"timeoutSeconds\":2147483647")) + output, err = oc.AsAdmin().WithoutNamespace().Run("get").Args("pod", podname, "-o=jsonpath={..readinessProbe}", "-n", "openshift-ingress").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(output).To(o.ContainSubstring("\"timeoutSeconds\":2147483647")) + }) + + // author: shudili@redhat.com + g.It("Author:shudili-Low-50075-Negative test of allow Ingress to be modified on the settings of livenessProbe and readinessProbe", func() { + buildPruningBaseDir := testdata.FixturePath("router") + customTemp := filepath.Join(buildPruningBaseDir, "ingresscontroller-np.yaml") + timeoutMinus := "{\"spec\":{\"template\":{\"spec\":{\"containers\":[{\"name\":\"router\",\"livenessProbe\":{\"timeoutSeconds\":-1},\"readinessProbe\":{\"timeoutSeconds\":-1}}]}}}}" + timeoutString := "{\"spec\":{\"template\":{\"spec\":{\"containers\":[{\"name\":\"router\",\"livenessProbe\":{\"timeoutSeconds\":\"abc\"},\"readinessProbe\":{\"timeoutSeconds\":\"abc\"}}]}}}}" + var ( + ingctrl = ingressControllerDescription{ + name: "ocp50075", + namespace: "openshift-ingress-operator", + domain: "", + template: customTemp, + } + ) + + compat_otp.By("Create one custom ingresscontroller") + baseDomain := getBaseDomain(oc) + ingctrl.domain = ingctrl.name + "." + baseDomain + defer ingctrl.delete(oc) + ingctrl.create(oc) + ensureRouterDeployGenerationIs(oc, ingctrl.name, "1") + + compat_otp.By("try to patch livenessProbe and readinessProbe with a minus number -1 to the router deployment") + routerDeploymentName := "router-" + ingctrl.name + output, _ := oc.AsAdmin().WithoutNamespace().Run("patch").Args("deployment", routerDeploymentName, "--type=strategic", "--patch="+timeoutMinus, "-n", "openshift-ingress").Output() + o.Expect(output).To(o.ContainSubstring("spec.template.spec.containers[0].livenessProbe.timeoutSeconds: Invalid value: -1: must be greater than or equal to 0")) + o.Expect(output).To(o.ContainSubstring("spec.template.spec.containers[0].readinessProbe.timeoutSeconds: Invalid value: -1: must be greater than or equal to 0")) + + compat_otp.By("try to patch livenessProbe and readinessProbe with string type of value to the router deployment") + output, _ = oc.AsAdmin().WithoutNamespace().Run("patch").Args("deployment", routerDeploymentName, "--type=strategic", "--patch="+timeoutString, "-n", "openshift-ingress").Output() + o.Expect(output).To(o.ContainSubstring("The request is invalid: patch: Invalid value: \"map[spec:map[template:map[spec:map[containers:[map[livenessProbe:map[timeoutSeconds:abc] name:router readinessProbe:map[timeoutSeconds:abc]]]]]]]\": unrecognized type: int32")) + }) + + g.It("Author:shudili-High-50405-Multiple routers with hostnetwork endpoint strategy can be deployed on same worker node with different http/https/stat port numbers", func() { + var ( + buildPruningBaseDir = testdata.FixturePath("router") + customTemp = filepath.Join(buildPruningBaseDir, "ingresscontroller-hostnetwork-only.yaml") + ingctrlhp1 = ingctrlHostPortDescription{ + name: "ocp50405one", + namespace: "openshift-ingress-operator", + domain: "", + httpport: 10080, + httpsport: 10443, + statsport: 10936, + template: customTemp, + } + + ingctrlhp2 = ingctrlHostPortDescription{ + name: "ocp50405two", + namespace: "openshift-ingress-operator", + domain: "", + httpport: 11080, + httpsport: 11443, + statsport: 11936, + template: customTemp, + } + ingctrlResource1 = "ingresscontrollers/" + ingctrlhp1.name + ingctrlResource2 = "ingresscontrollers/" + ingctrlhp2.name + ns = "openshift-ingress" + ) + + compat_otp.By("Pre-flight check for the platform type and number of worker nodes in the environment") + platformtype := compat_otp.CheckPlatform(oc) + platforms := map[string]bool{ + // ‘None’ also for Baremetal + "none": true, + "baremetal": true, + "vsphere": true, + "openstack": true, + "nutanix": true, + } + if !platforms[platformtype] { + g.Skip("Skip for non-supported platform") + } + workerNodeCount, _ := exactNodeDetails(oc) + if workerNodeCount < 1 { + g.Skip("Skipping as we at least need one worker node") + } + + compat_otp.By("Collect nodename of one of the default haproxy pods") + defRouterPod := getOneRouterPodNameByIC(oc, "default") + defNodeName := getNodeNameByPod(oc, ns, defRouterPod) + + compat_otp.By("Create two custom ingresscontrollers") + baseDomain := getBaseDomain(oc) + ingctrlhp1.domain = ingctrlhp1.name + "." + baseDomain + ingctrlhp2.domain = ingctrlhp2.name + "." + baseDomain + + defer ingctrlhp1.delete(oc) + ingctrlhp1.create(oc) + ensureRouterDeployGenerationIs(oc, ingctrlhp1.name, "1") + + defer ingctrlhp2.delete(oc) + ingctrlhp2.create(oc) + ensureRouterDeployGenerationIs(oc, ingctrlhp2.name, "1") + + compat_otp.By("Patch the two custom ingress-controllers with nodePlacement") + patchSelectNode := "{\"spec\":{\"nodePlacement\":{\"nodeSelector\":{\"matchLabels\":{\"kubernetes.io/hostname\": \"" + defNodeName + "\"}}}}}" + err := oc.AsAdmin().WithoutNamespace().Run("patch").Args(ingctrlResource1, "-p", patchSelectNode, "--type=merge", "-n", ingctrlhp1.namespace).Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + err = oc.AsAdmin().WithoutNamespace().Run("patch").Args(ingctrlResource2, "-p", patchSelectNode, "--type=merge", "-n", ingctrlhp2.namespace).Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + ensureRouterDeployGenerationIs(oc, ingctrlhp1.name, "2") + ensureRouterDeployGenerationIs(oc, ingctrlhp2.name, "2") + + compat_otp.By("Check the node names on which the route pods of the custom ingress-controllers reside on") + routerPod1 := getOneNewRouterPodFromRollingUpdate(oc, ingctrlhp1.name) + routerPod2 := getOneNewRouterPodFromRollingUpdate(oc, ingctrlhp2.name) + routerNodeName1 := getNodeNameByPod(oc, ns, routerPod1) + routerNodeName2 := getNodeNameByPod(oc, ns, routerPod2) + o.Expect(defNodeName).Should(o.And( + o.ContainSubstring(routerNodeName1), + o.ContainSubstring(routerNodeName2))) + + compat_otp.By("Verify the http/https/statsport of the custom proxy pod") + checkPodEnv := describePodResource(oc, routerPod1, "openshift-ingress") + o.Expect(checkPodEnv).Should(o.And( + o.ContainSubstring("ROUTER_SERVICE_HTTPS_PORT: 10443"), + o.ContainSubstring("ROUTER_SERVICE_HTTP_PORT: 10080"), + o.ContainSubstring("STATS_PORT: 10936"))) + + }) + + // author: shudili@redhat.com + g.It("Author:shudili-Low-50406-The http/https/stat port field in the ingresscontroller does not accept negative values during configuration", func() { + var ( + buildPruningBaseDir = testdata.FixturePath("router") + customTemp = filepath.Join(buildPruningBaseDir, "ingresscontroller-hostnetwork-only.yaml") + ingctrlhp = ingctrlHostPortDescription{ + name: "ocp50406", + namespace: "openshift-ingress-operator", + domain: "", + httpport: 10080, + httpsport: 10443, + statsport: 10936, + template: customTemp, + } + + ingctrlResource = "ingresscontrollers/" + ingctrlhp.name + ) + + compat_otp.By("Pre-flight check for the platform type and number of worker nodes in the environment") + platformtype := compat_otp.CheckPlatform(oc) + platforms := map[string]bool{ + // ‘None’ also for Baremetal + "none": true, + "baremetal": true, + "vsphere": true, + "openstack": true, + "nutanix": true, + } + if !platforms[platformtype] { + g.Skip("Skip for non-supported platform") + } + workerNodeCount, _ := exactNodeDetails(oc) + if workerNodeCount < 1 { + g.Skip("Skipping as we atleast need one worker node") + } + + compat_otp.By("Create a custom ingresscontrollers") + baseDomain := getBaseDomain(oc) + ingctrlhp.domain = ingctrlhp.name + "." + baseDomain + defer ingctrlhp.delete(oc) + ingctrlhp.create(oc) + ensureRouterDeployGenerationIs(oc, ingctrlhp.name, "1") + + compat_otp.By("Patch the the custom ingress-controllers with invalid hostNetwork configutations") + jsonPath := "{\"spec\":{\"endpointPublishingStrategy\":{\"hostNetwork\":{\"httpPort\": -10090}}}}" + output, err := oc.AsAdmin().WithoutNamespace().Run("patch").Args(ingctrlResource, "-p", jsonPath, "--type=merge", "-n", ingctrlhp.namespace).Output() + o.Expect(err).To(o.HaveOccurred()) + o.Expect(output).To(o.ContainSubstring("Invalid value: -10090")) + + jsonPath = "{\"spec\":{\"endpointPublishingStrategy\":{\"hostNetwork\":{\"httpPort\": -11443}}}}" + output, err = oc.AsAdmin().WithoutNamespace().Run("patch").Args(ingctrlResource, "-p", jsonPath, "--type=merge", "-n", ingctrlhp.namespace).Output() + o.Expect(err).To(o.HaveOccurred()) + o.Expect(output).To(o.ContainSubstring("Invalid value: -11443")) + + jsonPath = "{\"spec\":{\"endpointPublishingStrategy\":{\"hostNetwork\":{\"httpPort\": -12936}}}}" + output, err = oc.AsAdmin().WithoutNamespace().Run("patch").Args(ingctrlResource, "-p", jsonPath, "--type=merge", "-n", ingctrlhp.namespace).Output() + o.Expect(err).To(o.HaveOccurred()) + o.Expect(output).To(o.ContainSubstring("Invalid value: -12936")) + }) + + // author: shudili@redhat.com + g.It("Author:shudili-High-50819-Routers with hostnetwork endpoint strategy with same http/https/stat port numbers cannot be deployed on the same worker node", func() { + var ( + buildPruningBaseDir = testdata.FixturePath("router") + customTemp = filepath.Join(buildPruningBaseDir, "ingresscontroller-hostnetwork-only.yaml") + ingctrlhp1 = ingctrlHostPortDescription{ + name: "ocp50819one", + namespace: "openshift-ingress-operator", + domain: "", + httpport: 10080, + httpsport: 10443, + statsport: 10936, + template: customTemp, + } + + ingctrlhp2 = ingctrlHostPortDescription{ + name: "ocp50819two", + namespace: "openshift-ingress-operator", + domain: "", + httpport: 10080, + httpsport: 10433, + statsport: 10936, + template: customTemp, + } + ) + + compat_otp.By("Pre-flight check for the platform type and number of worker nodes in the environment") + platformtype := compat_otp.CheckPlatform(oc) + platforms := map[string]bool{ + // ‘None’ also for Baremetal + "none": true, + "baremetal": true, + "vsphere": true, + "openstack": true, + "nutanix": true, + } + if !platforms[platformtype] { + g.Skip("Skip for non-supported platform") + } + workerNodeCount, _ := exactNodeDetails(oc) + if workerNodeCount < 1 { + g.Skip("Skipping as we atleast need one worker node") + } + + compat_otp.By("Create one custom ingresscontroller") + baseDomain := getBaseDomain(oc) + ingctrlhp1.domain = ingctrlhp1.name + "." + baseDomain + ingctrlhp2.domain = ingctrlhp2.name + "." + baseDomain + + defer ingctrlhp1.delete(oc) + ingctrlhp1.create(oc) + ensureRouterDeployGenerationIs(oc, ingctrlhp1.name, "1") + + compat_otp.By("Patch the first custom IC with max replicas, so each node has a custom router pod ") + jpath := "{.status.readyReplicas}" + if workerNodeCount > 1 { + ingctrl1Resource := "ingresscontrollers/" + ingctrlhp1.name + patchResourceAsAdmin(oc, ingctrlhp1.namespace, ingctrl1Resource, "{\"spec\":{\"replicas\":"+strconv.Itoa(workerNodeCount)+"}}") + ensureRouterDeployGenerationIs(oc, ingctrlhp1.name, "2") + waitForOutputEquals(oc, "openshift-ingress", "deployment/router-"+ingctrlhp1.name, jpath, strconv.Itoa(workerNodeCount)) + } + + compat_otp.By("Try to create another custom IC with the same http/https/stat port numbers as the first custom IC") + defer ingctrlhp2.delete(oc) + ingctrlhp2.create(oc) + err := waitForPodWithLabelAppear(oc, "openshift-ingress", "ingresscontroller.operator.openshift.io/deployment-ingresscontroller=ocp50819two") + compat_otp.AssertWaitPollNoErr(err, "router pod of the second custom IC does not appear within allowed time!") + customICRouterPod := getPodListByLabel(oc, "openshift-ingress", "ingresscontroller.operator.openshift.io/deployment-ingresscontroller=ocp50819two") + checkPodMsg := getByJsonPath(oc, "openshift-ingress", "pod/"+customICRouterPod[0], "{.status..message}") + o.Expect(checkPodMsg).To(o.ContainSubstring("node(s) didn't have free ports for the requested pod ports")) + }) + + g.It("Author:aiyengar-ROSA-OSD_CCS-ARO-High-52738-The Power-of-two balancing features switches to source algorithm for passthrough routes", func() { + var ( + baseDomain = getBaseDomain(oc) + defaultPod = getOneRouterPodNameByIC(oc, "default") + ) + buildPruningBaseDir := testdata.FixturePath("router") + testPodSvc := filepath.Join(buildPruningBaseDir, "web-server-signed-deploy.yaml") + + compat_otp.By("Create pods and service resources") + ns := oc.Namespace() + createResourceFromFile(oc, ns, testPodSvc) + ensurePodWithLabelReady(oc, ns, "name=web-server-deploy") + + compat_otp.By("Expose a passthrough type routes via the services inside the namespace") + passthRoute := "route-passth" + "-" + ns + "." + baseDomain + createRoute(oc, ns, "passthrough", "route-passth", "service-secure", []string{"--hostname=" + passthRoute}) + output, err := oc.Run("get").Args("route").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(output).To(o.ContainSubstring("route-passth")) + + compat_otp.By("Check the default loadbalance algorithm inside proxy pod and check the default LB variable to confirm power-of-two is active") + rtrParamCheck := readPodEnv(oc, defaultPod, "openshift-ingress", "ROUTER_LOAD_BALANCE_ALGORITHM") + o.Expect(rtrParamCheck).To(o.ContainSubstring("random")) + passthBackend := "be_tcp:" + ns + ":route-passth" + ensureHaproxyBlockConfigContains(oc, defaultPod, passthBackend, []string{"balance source"}) + }) + + // author: shudili@redhat.com + g.It("Author:shudili-Medium-53048-Ingresscontroller with private endpoint publishing strategy supports PROXY protocol", func() { + var ( + buildPruningBaseDir = testdata.FixturePath("router") + customTemp = filepath.Join(buildPruningBaseDir, "ingresscontroller-private.yaml") + ingctrl = ingressControllerDescription{ + name: "53048", + namespace: "openshift-ingress-operator", + domain: "", + template: customTemp, + } + ingctrlResource = "ingresscontrollers/" + ingctrl.name + ) + + compat_otp.By("create a custom ingresscontroller") + baseDomain := getBaseDomain(oc) + ingctrl.domain = ingctrl.name + "." + baseDomain + defer ingctrl.delete(oc) + ingctrl.create(oc) + ensureRouterDeployGenerationIs(oc, ingctrl.name, "1") + + compat_otp.By("check the default value of .status.endpointPublishingStrategy.private.protocol, which should be TCP") + jpath := "{.status.endpointPublishingStrategy.private.protocol}" + protocol := getByJsonPath(oc, ingctrl.namespace, ingctrlResource, jpath) + o.Expect(protocol).To(o.ContainSubstring("TCP")) + + compat_otp.By("patch the custom ingresscontroller with protocol proxy") + patchPath := "{\"spec\":{\"endpointPublishingStrategy\":{\"private\":{\"protocol\":\"PROXY\"}}}}" + patchResourceAsAdmin(oc, ingctrl.namespace, ingctrlResource, patchPath) + ensureRouterDeployGenerationIs(oc, ingctrl.name, "2") + + compat_otp.By("check the changed value of .endpointPublishingStrategy.private.protocol, which should be PROXY") + jpath = "{.spec.endpointPublishingStrategy.private.protocol}{.status.endpointPublishingStrategy.private.protocol}" + protocol = getByJsonPath(oc, ingctrl.namespace, ingctrlResource, jpath) + o.Expect(protocol).To(o.ContainSubstring("PROXYPROXY")) + + compat_otp.By("check the custom ingresscontroller's status, which should indicate that PROXY protocol is enabled") + jsonPath := "{.status.endpointPublishingStrategy}" + status := getByJsonPath(oc, ingctrl.namespace, ingctrlResource, jsonPath) + o.Expect(status).To(o.ContainSubstring(`{"private":{"protocol":"PROXY"},"type":"Private"}`)) + + compat_otp.By("check the private deployment, which should have PROXY protocol enabled") + jsonPath = `{.spec.template.spec.containers[0].env[?(@.name=="ROUTER_USE_PROXY_PROTOCOL")]}` + proxyProtocol := getByJsonPath(oc, "openshift-ingress", "deployments/router-"+ingctrl.name, jsonPath) + o.Expect(proxyProtocol).To(o.ContainSubstring(`{"name":"ROUTER_USE_PROXY_PROTOCOL","value":"true"}`)) + + compat_otp.By("check the ROUTER_USE_PROXY_PROTOCOL env, which should be true") + routerpod := getOneNewRouterPodFromRollingUpdate(oc, ingctrl.name) + proxyEnv := readRouterPodEnv(oc, routerpod, "ROUTER_USE_PROXY_PROTOCOL") + o.Expect(proxyEnv).To(o.ContainSubstring("ROUTER_USE_PROXY_PROTOCOL=true")) + + compat_otp.By("check the accept-proxy in haproxy.config of a router pod") + bindCfg, err := oc.AsAdmin().WithoutNamespace().Run("exec").Args("-n", "openshift-ingress", routerpod, "--", "bash", "-c", "cat haproxy.config | grep \"bind :\"").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + IPStackType := checkIPStackType(oc) + if IPStackType == "ipv4single" { + o.Expect(strings.Count(bindCfg, "bind :")).To(o.Equal(2)) + ensureHaproxyBlockConfigContains(oc, routerpod, "bind :80 accept-proxy", []string{"bind :80 accept-proxy"}) + ensureHaproxyBlockConfigContains(oc, routerpod, "bind :443 accept-proxy", []string{"bind :443 accept-proxy"}) + } else if IPStackType == "ipv6single" { + o.Expect(strings.Count(bindCfg, "bind :")).To(o.Equal(2)) + ensureHaproxyBlockConfigContains(oc, routerpod, "bind :::80 v6only accept-proxy", []string{"bind :::80 v6only accept-proxy"}) + ensureHaproxyBlockConfigContains(oc, routerpod, "bind :::443 v6only accept-proxy", []string{"bind :::443 v6only accept-proxy"}) + } else if IPStackType == "dualstack" { + o.Expect(strings.Count(bindCfg, "bind :")).To(o.Equal(4)) + ensureHaproxyBlockConfigContains(oc, routerpod, "bind :80 accept-proxy", []string{"bind :80 accept-proxy"}) + ensureHaproxyBlockConfigContains(oc, routerpod, "bind :443 accept-proxy", []string{"bind :443 accept-proxy"}) + ensureHaproxyBlockConfigContains(oc, routerpod, "bind :::80 v6only accept-proxy", []string{"bind :::80 v6only accept-proxy"}) + ensureHaproxyBlockConfigContains(oc, routerpod, "bind :::443 v6only accept-proxy", []string{"bind :::443 v6only accept-proxy"}) + } + }) + + // Bug: 2044682 + g.It("Author:shudili-ROSA-OSD_CCS-ARO-High-54998-Set Cookie2 by an application in a route should not kill all router pods", func() { + var ( + buildPruningBaseDir = testdata.FixturePath("router") + customTemp = filepath.Join(buildPruningBaseDir, "ingresscontroller-np.yaml") + testPodSvc = filepath.Join(buildPruningBaseDir, "web-server-deploy.yaml") + srvrcInfo = "web-server-deploy" + srvName = "service-unsecure" + clientPod = filepath.Join(buildPruningBaseDir, "test-client-pod.yaml") + clientPodName = "hello-pod" + clientPodLabel = "app=hello-pod" + ingctrl = ingressControllerDescription{ + name: "54998", + namespace: "openshift-ingress-operator", + domain: "", + template: customTemp, + } + ) + + compat_otp.By("create a custom ingresscontroller") + baseDomain := getBaseDomain(oc) + ingctrl.domain = ingctrl.name + "." + baseDomain + defer ingctrl.delete(oc) + ingctrl.create(oc) + ensureRouterDeployGenerationIs(oc, ingctrl.name, "1") + + compat_otp.By("get one custom ingress-controller router pod's IP") + podname := getOneNewRouterPodFromRollingUpdate(oc, ingctrl.name) + podIP := getPodv4Address(oc, podname, "openshift-ingress") + + compat_otp.By("create an unsecure service and its backend pod") + ns := oc.Namespace() + sedCmd := fmt.Sprintf(`sed -i'' -e 's/8080/10081/g' %s`, testPodSvc) + _, err := exec.Command("bash", "-c", sedCmd).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + createResourceFromFile(oc, ns, testPodSvc) + ensurePodWithLabelReady(oc, ns, "name="+srvrcInfo) + + compat_otp.By("start the service on the backend server port 10081 by socat command") + jsonPath := "{.items[0].metadata.name}" + srvPodName := getByJsonPath(oc, ns, "pods", jsonPath) + cidr, errCidr := oc.AsAdmin().WithoutNamespace().Run("get").Args("network.config", "cluster", "-o=jsonpath={.spec.clusterNetwork[].cidr}").Output() + o.Expect(errCidr).NotTo(o.HaveOccurred()) + // set ipv4 socat or ipv6 socat command on the server + resWithSetCookie2 := `nohup socat -T 1 -6 -d -d tcp-l:10081,reuseaddr,fork,crlf system:'echo -e "\"HTTP/1.0 200 OK\nDocumentType: text/html\nHeader: Set-Cookie2 X=Y;\n\nthis is a test\""'` + if strings.Contains(cidr, ".") { + resWithSetCookie2 = `nohup socat -T 1 -d -d tcp-l:10081,reuseaddr,fork,crlf system:'echo -e "\"HTTP/1.0 200 OK\nDocumentType: text/html\nHeader: Set-Cookie2 X=Y;\n\nthis is a test\""'` + } + cmd1, _, _, errSetCookie2 := oc.AsAdmin().Run("exec").Args("-n", ns, srvPodName, "--", "bash", "-c", resWithSetCookie2).Background() + defer cmd1.Process.Kill() + o.Expect(errSetCookie2).NotTo(o.HaveOccurred()) + + compat_otp.By("expose a route with the unsecure service inside the namespace") + routehost := srvName + "-" + ns + "." + ingctrl.domain + output, SrvErr := oc.Run("expose").Args("service", srvName, "--hostname="+routehost).Output() + o.Expect(SrvErr).NotTo(o.HaveOccurred()) + o.Expect(output).To(o.ContainSubstring(srvName)) + + compat_otp.By("create a client pod to send traffic") + createResourceFromFile(oc, ns, clientPod) + ensurePodWithLabelReady(oc, ns, clientPodLabel) + + compat_otp.By("curl the route from the client pod") + toDst := routehost + ":80:" + podIP + cmdOnPod := []string{"-n", ns, clientPodName, "--", "curl", "-I", "http://" + routehost, "--resolve", toDst, "--connect-timeout", "10"} + result, _ := repeatCmdOnClient(oc, cmdOnPod, "Set-Cookie2 X=Y", 60, 1) + o.Expect(result).To(o.ContainSubstring("Set-Cookie2 X=Y")) + }) + + // Bug: 1967228 + g.It("Author:shudili-High-55825-the 503 Error page should not contain license for a vulnerable release of Bootstrap", func() { + var ( + buildPruningBaseDir = testdata.FixturePath("router") + customTemp = filepath.Join(buildPruningBaseDir, "ingresscontroller-np.yaml") + clientPod = filepath.Join(buildPruningBaseDir, "test-client-pod.yaml") + clientPodName = "hello-pod" + clientPodLabel = "app=hello-pod" + ingctrl = ingressControllerDescription{ + name: "55825", + namespace: "openshift-ingress-operator", + domain: "", + template: customTemp, + } + ) + + compat_otp.By("create a custom ingresscontroller") + baseDomain := getBaseDomain(oc) + ingctrl.domain = ingctrl.name + "." + baseDomain + defer ingctrl.delete(oc) + ingctrl.create(oc) + ensureRouterDeployGenerationIs(oc, ingctrl.name, "1") + + compat_otp.By("Create a client pod used to send traffic") + ns := oc.Namespace() + compat_otp.By("create a client pod") + createResourceFromFile(oc, ns, clientPod) + ensurePodWithLabelReady(oc, ns, clientPodLabel) + + compat_otp.By("curl a non-existing route, and then check that Bootstrap portion of the license is removed") + podname := getOneNewRouterPodFromRollingUpdate(oc, ingctrl.name) + podIP := getPodv4Address(oc, podname, "openshift-ingress") + notExistRoute := "notexistroute" + "-" + ns + "." + ingctrl.domain + toDst := notExistRoute + ":80:" + podIP + output, err2 := oc.Run("exec").Args(clientPodName, "--", "curl", "-Iv", "http://"+notExistRoute, "--resolve", toDst, "--connect-timeout", "10").Output() + o.Expect(err2).NotTo(o.HaveOccurred()) + o.Expect(output).To(o.ContainSubstring("503")) + o.Expect(output).ShouldNot(o.And( + o.ContainSubstring("Bootstrap"), + o.ContainSubstring("Copyright 2011-2015 Twitter"), + o.ContainSubstring("Licensed under MIT"), + o.ContainSubstring("normalize.css v3.0.3"))) + + }) + + // author: mjoseph@redhat.com + g.It("Author:mjoseph-ROSA-OSD_CCS-ARO-High-56898-Accessing the route should wake up the idled resources", func() { + // This test case works only on OVN or SDN network type + networkType := compat_otp.CheckNetworkType(oc) + if !(strings.Contains(networkType, "openshiftsdn") || strings.Contains(networkType, "ovn")) { + g.Skip(fmt.Sprintf("Skipping because idling is not supported on the '%s' network type", networkType)) + } + var ( + buildPruningBaseDir = testdata.FixturePath("router") + customTemp = filepath.Join(buildPruningBaseDir, "ingresscontroller-np.yaml") + testPodSvc = filepath.Join(buildPruningBaseDir, "web-server-deploy.yaml") + clientPod = filepath.Join(buildPruningBaseDir, "test-client-pod.yaml") + clientPodName = "hello-pod" + clientPodLabel = "app=hello-pod" + ingctrl = ingressControllerDescription{ + name: "ocp56898", + namespace: "openshift-ingress-operator", + domain: "", + template: customTemp, + } + ) + + compat_otp.By("Create a custom ingresscontroller") + baseDomain := getBaseDomain(oc) + ingctrl.domain = ingctrl.name + "." + baseDomain + defer ingctrl.delete(oc) + ingctrl.create(oc) + ensureRouterDeployGenerationIs(oc, ingctrl.name, "1") + custContPod := getOneNewRouterPodFromRollingUpdate(oc, ingctrl.name) + custContIP := getPodv4Address(oc, custContPod, "openshift-ingress") + + compat_otp.By("Deploy a backend pod and its service resources") + ns := oc.Namespace() + createResourceFromFile(oc, ns, testPodSvc) + ensurePodWithLabelReady(oc, ns, "name=web-server-deploy") + + compat_otp.By("Create a client pod") + createResourceFromFile(oc, ns, clientPod) + ensurePodWithLabelReady(oc, ns, clientPodLabel) + + compat_otp.By("Expose a route with the unsecure service inside the namespace") + routehost := "service-unsecure-" + ns + "." + ingctrl.domain + SrvErr := oc.Run("expose").Args("svc/service-unsecure", "--hostname="+routehost).Execute() + o.Expect(SrvErr).NotTo(o.HaveOccurred()) + routeOutput := getRoutes(oc, ns) + o.Expect(routeOutput).To(o.ContainSubstring("service-unsecure")) + + compat_otp.By("Check the router pod and ensure the routes are loaded in haproxy.config") + haproxyOutput := pollReadPodData(oc, "openshift-ingress", custContPod, "cat haproxy.config", "service-unsecure") + o.Expect(haproxyOutput).To(o.ContainSubstring("backend be_http:" + ns + ":service-unsecure")) + + compat_otp.By("Check the reachability of the insecure route") + waitForCurl(oc, clientPodName, baseDomain, "service-unsecure-"+ns+"."+"ocp56898.", "HTTP/1.1 200 OK", custContIP) + + compat_otp.By("Idle the insecure service") + idleOutput, err := oc.AsAdmin().WithoutNamespace().Run("idle").Args("service-unsecure", "-n", ns).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(idleOutput).To(o.ContainSubstring("The service \"" + ns + "/service-unsecure\" has been marked as idled")) + + compat_otp.By("Verify the Idle annotation") + findAnnotation := getAnnotation(oc, ns, "svc", "service-unsecure") + o.Expect(findAnnotation).To(o.ContainSubstring("idling.alpha.openshift.io/idled-at")) + o.Expect(findAnnotation).To(o.ContainSubstring(`idling.alpha.openshift.io/unidle-targets":"[{\"kind\":\"Deployment\",\"name\":\"web-server-deploy\",\"group\":\"apps\",\"replicas\":1}]`)) + + compat_otp.By("Wake the Idle resource by accessing its route") + waitForCurl(oc, clientPodName, baseDomain, "service-unsecure-"+ns+"."+"ocp56898.", "HTTP/1.1 200 OK", custContIP) + + compat_otp.By("Confirm the Idle annotation got removed") + findAnnotation = getAnnotation(oc, ns, "svc", "service-unsecure") + o.Expect(findAnnotation).NotTo(o.ContainSubstring("idling.alpha.openshift.io/idled-at")) + }) + + // bug: 1826225 + g.It("Author:shudili-High-57001-edge terminated h2 (gRPC) connections need a haproxy template change to work correctly", func() { + var ( + buildPruningBaseDir = testdata.FixturePath("router") + srvPodSvc = filepath.Join(buildPruningBaseDir, "bug1826225-proh2-deploy.yaml") + srvrcInfo = "web-server-deploy" + svcName = "service-h2c-57001" + routeName = "myedge1" + ) + + compat_otp.By("Create a backend pod and its service resources") + ns := oc.Namespace() + compat_otp.By("create a h2c service and its backend pod") + createResourceFromFile(oc, ns, srvPodSvc) + ensurePodWithLabelReady(oc, ns, "name="+srvrcInfo) + + compat_otp.By("Create an edge route with the h2c service inside the namespace") + output, routeErr := oc.AsAdmin().WithoutNamespace().Run("create").Args("route", "edge", routeName, "--service="+svcName, "-n", ns).Output() + o.Expect(routeErr).NotTo(o.HaveOccurred()) + o.Expect(output).To(o.ContainSubstring(routeName)) + + compat_otp.By("Check the Haproxy backend configuration and make sure proto h2 is added for the route") + podname := getOneRouterPodNameByIC(oc, "default") + backendConfig := pollReadPodData(oc, "openshift-ingress", podname, "cat haproxy.config", svcName) + o.Expect(backendConfig).To(o.ContainSubstring("proto h2")) + }) + + // bugzilla: 1976894 + g.It("Author:mjoseph-Medium-57404-Idling StatefulSet is not supported", func() { + var ( + buildPruningBaseDir = testdata.FixturePath("router") + customTemp = filepath.Join(buildPruningBaseDir, "ingresscontroller-np.yaml") + testPodSvc = filepath.Join(buildPruningBaseDir, "ocp57404-stateful-set.yaml") + clientPod = filepath.Join(buildPruningBaseDir, "test-client-pod.yaml") + ingctrl = ingressControllerDescription{ + name: "ocp57404", + namespace: "openshift-ingress-operator", + domain: "", + template: customTemp, + } + ) + + compat_otp.By("Create a custom ingresscontroller") + baseDomain := getBaseDomain(oc) + ingctrl.domain = ingctrl.name + "." + baseDomain + defer ingctrl.delete(oc) + ingctrl.create(oc) + ensureRouterDeployGenerationIs(oc, ingctrl.name, "1") + custContPod := getOneNewRouterPodFromRollingUpdate(oc, ingctrl.name) + custContIP := getPodv4Address(oc, custContPod, "openshift-ingress") + + compat_otp.By("Deploy the statefulset and its service resources") + ns := oc.Namespace() + createResourceFromFile(oc, ns, testPodSvc) + ensurePodWithLabelReady(oc, ns, "app=echoenv-sts") + + compat_otp.By("Create a client pod") + createResourceFromFile(oc, ns, clientPod) + ensurePodWithLabelReady(oc, ns, "app=hello-pod") + + compat_otp.By("Expose a route with the unsecure service inside the namespace") + routehost := "echoenv-statefulset-service-" + ns + "." + ingctrl.domain + SrvErr := oc.Run("expose").Args("svc/echoenv-statefulset-service", "--hostname="+routehost).Execute() + o.Expect(SrvErr).NotTo(o.HaveOccurred()) + routeOutput := getRoutes(oc, ns) + o.Expect(routeOutput).To(o.ContainSubstring("echoenv-statefulset-service")) + + compat_otp.By("Check the reachability of the insecure route") + waitForCurl(oc, "hello-pod", baseDomain, "echoenv-statefulset-service-"+ns+"."+"ocp57404.", "HTTP/1.1 200 OK", custContIP) + + compat_otp.By("Trying to idle the statefulset-service") + idleOutput, _ := oc.AsAdmin().WithoutNamespace().Run("idle").Args("echoenv-statefulset-service", "-n", ns).Output() + o.Expect(idleOutput).To(o.ContainSubstring("idling StatefulSet is not supported yet")) + + compat_otp.By("Verify the Idle annotation is not present") + findAnnotation := getAnnotation(oc, ns, "svc", "echoenv-statefulset-service") + o.Expect(findAnnotation).NotTo(o.ContainSubstring("idling.alpha.openshift.io/idled-at")) + + compat_otp.By("Recheck the reachability of the insecure route") + waitForCurl(oc, "hello-pod", baseDomain, "echoenv-statefulset-service-"+ns+"."+"ocp57404.", "HTTP/1.1 200 OK", custContIP) + }) + + // bugzilla: 1941592 1859134 + g.It("Author:mjoseph-Medium-57406-HAProxyDown message only for pods and No reaper messages for zombie processes", func() { + compat_otp.By("Verify there will be precise message pointing to the router nod, when HAProxy is down") + output := getByJsonPath(oc, "openshift-ingress-operator", "PrometheusRule", `{.items[0].spec.groups[0].rules[?(@.alert=="HAProxyDown")].annotations.message}`) + o.Expect(output).To(o.ContainSubstring(`HAProxy metrics are reporting that HAProxy is down on pod {{ $labels.namespace }} / {{ $labels.pod }}`)) + + compat_otp.By("Check the router pod logs and confirm there is no periodic reper error message for zombie process") + podname := getOneRouterPodNameByIC(oc, "default") + log, err := oc.AsAdmin().WithoutNamespace().Run("logs").Args("-n", "openshift-ingress", podname).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + if strings.Contains(log, "waitid: no child processes") { + e2e.Failf("waitid: no child processes generated") + } + }) + + // author: mjoseph@redhat.com + g.It("Author:mjoseph-Critical-59951-IngressController option to use PROXY protocol with IBM Cloud load-balancers - TCP, PROXY and omitted", func() { + // This test case is only for IBM cluster + compat_otp.SkipIfPlatformTypeNot(oc, "IBMCloud") + var ( + buildPruningBaseDir = testdata.FixturePath("router") + customTemp = filepath.Join(buildPruningBaseDir, "ingresscontroller-IBMproxy.yaml") + ingctrl = ingressControllerDescription{ + name: "ocp59951", + namespace: "openshift-ingress-operator", + domain: "", + template: customTemp, + } + ingctrlResource = "ingresscontrollers/" + ingctrl.name + ) + + compat_otp.By("create a custom ingresscontroller") + baseDomain := getBaseDomain(oc) + ingctrl.domain = ingctrl.name + "." + baseDomain + defer ingctrl.delete(oc) + ingctrl.create(oc) + ensureRouterDeployGenerationIs(oc, ingctrl.name, "1") + + compat_otp.By("check the value of .status.endpointPublishingStrategy.loadBalancer.providerParameters.ibm.protocol, which should be PROXY") + jpath := "{.status.endpointPublishingStrategy.loadBalancer.providerParameters.ibm.protocol}" + protocol := getByJsonPath(oc, ingctrl.namespace, ingctrlResource, jpath) + o.Expect(protocol).To(o.ContainSubstring("PROXY")) + + compat_otp.By("check the ROUTER_USE_PROXY_PROTOCOL env, which should be true") + routerpod := getOneNewRouterPodFromRollingUpdate(oc, ingctrl.name) + pollReadPodData(oc, "openshift-ingress", routerpod, "/usr/bin/env", `ROUTER_USE_PROXY_PROTOCOL=true`) + + compat_otp.By("Ensure the proxy-protocol annotation is added to the LB service") + findAnnotation, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("svc", "router-ocp59951", "-n", "openshift-ingress", "-o=jsonpath={.metadata.annotations}").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(findAnnotation).To(o.ContainSubstring(`"service.kubernetes.io/ibm-load-balancer-cloud-provider-enable-features":"proxy-protocol"`)) + + compat_otp.By("patch the custom ingresscontroller with protocol option TCP") + patchPath := "{\"spec\":{\"endpointPublishingStrategy\":{\"loadBalancer\":{\"providerParameters\":{\"ibm\":{\"protocol\":\"TCP\"}}}}}}" + patchResourceAsAdmin(oc, ingctrl.namespace, ingctrlResource, patchPath) + + ensureRouterDeployGenerationIs(oc, ingctrl.name, "2") + compat_otp.By("check the value of .status.endpointPublishingStrategy.loadBalancer.providerParameters.ibm.protocol, which should be TCP") + jpath = "{.status.endpointPublishingStrategy.loadBalancer.providerParameters.ibm.protocol}" + protocol = getByJsonPath(oc, ingctrl.namespace, ingctrlResource, jpath) + o.Expect(protocol).To(o.ContainSubstring("TCP")) + + compat_otp.By("check the ROUTER_USE_PROXY_PROTOCOL env, which should not present") + routerpod = getOneNewRouterPodFromRollingUpdate(oc, ingctrl.name) + proxyEnv, _ := oc.AsAdmin().WithoutNamespace().Run("exec").Args("-n", "openshift-ingress", routerpod, "--", "bash", "-c", "/usr/bin/env | grep ROUTER_USE_PROXY_PROTOCOL").Output() + o.Expect(proxyEnv).NotTo(o.ContainSubstring("ROUTER_USE_PROXY_PROTOCOL")) + + compat_otp.By("patch the custom ingresscontroller with protocol option omitted") + patchPath = `{"spec":{"endpointPublishingStrategy":{"loadBalancer":{"providerParameters":{"ibm":{"protocol":""}}}}}}` + patchResourceAsAdmin(oc, ingctrl.namespace, ingctrlResource, patchPath) + + compat_otp.By(`check the value of .status.endpointPublishingStrategy.loadBalancer.providerParameters.ibm.protocol, which should be ""`) + jpath = "{.status.endpointPublishingStrategy.loadBalancer.providerParameters.ibm}" + protocol = getByJsonPath(oc, ingctrl.namespace, ingctrlResource, jpath) + o.Expect(protocol).To(o.ContainSubstring(`{}`)) + }) + + // OCPBUGS-4573 + g.It("Author:shudili-ROSA-OSD_CCS-ARO-High-62926-Ingress controller stats port is not set according to endpointPublishingStrategy", func() { + var ( + buildPruningBaseDir = testdata.FixturePath("router") + customTemp = filepath.Join(buildPruningBaseDir, "ingresscontroller-hostnetwork-only.yaml") + ingctrlhp = ingctrlHostPortDescription{ + name: "ocp62926", + namespace: "openshift-ingress-operator", + domain: "", + httpport: 16080, + httpsport: 16443, + statsport: 16936, + template: customTemp, + } + + ingctrlResource = "ingresscontrollers/" + ingctrlhp.name + ) + + compat_otp.By("Pre-flight check for the platform type and number of worker nodes in the environment") + platformtype := compat_otp.CheckPlatform(oc) + platforms := map[string]bool{ + // ‘None’ also for Baremetal + "none": true, + "baremetal": true, + "vsphere": true, + "openstack": true, + "nutanix": true, + } + if !platforms[platformtype] { + g.Skip("Skip for non-supported platform") + } + workerNodeCount, _ := exactNodeDetails(oc) + if workerNodeCount < 1 { + g.Skip("Skipping as we atleast need one worker node") + } + + compat_otp.By("Create a custom ingress-controller") + baseDomain := getBaseDomain(oc) + ingctrlhp.domain = ingctrlhp.name + "." + baseDomain + defer ingctrlhp.delete(oc) + ingctrlhp.create(oc) + ensureRouterDeployGenerationIs(oc, ingctrlhp.name, "1") + + compat_otp.By("Patch the the custom ingress-controller with httpPort 17080, httpsPort 17443 and statsPort 17936") + jsonPath := "{\"spec\":{\"endpointPublishingStrategy\":{\"hostNetwork\":{\"httpPort\":17080, \"httpsPort\":17443, \"statsPort\":17936}}}}" + patchResourceAsAdmin(oc, ingctrlhp.namespace, ingctrlResource, jsonPath) + ensureRouterDeployGenerationIs(oc, ingctrlhp.name, "2") + + compat_otp.By("Check STATS_PORT env under a custom router pod, which should be 17936") + routerpod := getOneNewRouterPodFromRollingUpdate(oc, ingctrlhp.name) + jsonPath = `{.spec.containers[].env[?(@.name=="STATS_PORT")].value}` + output := getByJsonPath(oc, "openshift-ingress", "pod/"+routerpod, jsonPath) + o.Expect(output).To(o.ContainSubstring("17936")) + + compat_otp.By("Check http/https/metrics ports under a custom router pod, which should be 17080/17443/17936") + jsonPath = `{.spec.containers[].ports[?(@.name=="http")].hostPort}-{.spec.containers[].ports[?(@.name=="https")].hostPort}-{.spec.containers[].ports[?(@.name=="metrics")].hostPort}` + output = getByJsonPath(oc, "openshift-ingress", "pod/"+routerpod, jsonPath) + o.Expect(output).To(o.ContainSubstring("17080-17443-17936")) + + compat_otp.By("Check the custom router-internal service, make sure the targetPort of the metrics port is changed to metrics instead of port number 1936") + jsonPath = `{.spec.ports[?(@.name=="metrics")].targetPort}` + output = getByJsonPath(oc, "openshift-ingress", "service/router-internal-"+ingctrlhp.name, jsonPath) + o.Expect(output).To(o.ContainSubstring("metrics")) + + compat_otp.By("Check http/https/metrics ports under the router endpoints, which should be 17080/17443/17936") + jsonPath = `{.subsets[].ports[?(@.name=="http")].port}-{.subsets[].ports[?(@.name=="https")].port}-{.subsets[].ports[?(@.name=="metrics")].port}` + output = getByJsonPath(oc, "openshift-ingress", "endpoints/router-internal-"+ingctrlhp.name, jsonPath) + o.Expect(output).To(o.ContainSubstring("17080-17443-17936")) + }) + + // Test case creater: shudili@redhat.com + g.It("Author:mjoseph-DEPRECATED-ROSA-OSD_CCS-ARO-High-77862-Check whether required ENV varibales are configured after enabling Dynamic Configuration Manager", func() { + // skip the test if featureSet is not there + if !compat_otp.IsTechPreviewNoUpgrade(oc) { + g.Skip("featureSet: TechPreviewNoUpgrade is required for this test, skipping") + } + + buildPruningBaseDir := testdata.FixturePath("router") + customTemp := filepath.Join(buildPruningBaseDir, "ingresscontroller-np.yaml") + testPodSvc := filepath.Join(buildPruningBaseDir, "web-server-deploy.yaml") + srvrcInfo := "web-server-deploy" + srvName := "service-unsecure" + var ( + ingctrl = ingressControllerDescription{ + name: "ocp77862", + namespace: "openshift-ingress-operator", + domain: "", + template: customTemp, + } + ) + + compat_otp.By("1. Create a custom ingresscontroller, and get its router name") + baseDomain := getBaseDomain(oc) + ns := oc.Namespace() + ingctrl.domain = ingctrl.name + "." + baseDomain + defer ingctrl.delete(oc) + ingctrl.create(oc) + ensureRouterDeployGenerationIs(oc, ingctrl.name, "1") + podname := getOneNewRouterPodFromRollingUpdate(oc, ingctrl.name) + defaultPodname := getOneRouterPodNameByIC(oc, "default") + + compat_otp.By("2. Create an unsecure service and its backend pod") + createResourceFromFile(oc, ns, testPodSvc) + ensurePodWithLabelReady(oc, ns, "name="+srvrcInfo) + + compat_otp.By("3. Expose a route with the unsecure service inside the namespace") + createRoute(oc, ns, "http", srvName, srvName, []string{}) + waitForOutputEquals(oc, ns, "route", "{.items[0].metadata.name}", srvName) + + compat_otp.By("4. Check the env variable of the default router pod") + checkenv := readRouterPodEnv(oc, defaultPodname, "ROUTER") + o.Expect(checkenv).To(o.ContainSubstring(`ROUTER_BLUEPRINT_ROUTE_POOL_SIZE=0`)) + o.Expect(checkenv).To(o.ContainSubstring(`ROUTER_MAX_DYNAMIC_SERVERS=1`)) + o.Expect(checkenv).To(o.ContainSubstring(`ROUTER_HAPROXY_CONFIG_MANAGER=true`)) + + compat_otp.By("5. Check the env variable of the custom router pod") + checkenv2 := readRouterPodEnv(oc, podname, "ROUTER") + o.Expect(checkenv2).To(o.ContainSubstring(`ROUTER_BLUEPRINT_ROUTE_POOL_SIZE=0`)) + o.Expect(checkenv2).To(o.ContainSubstring(`ROUTER_MAX_DYNAMIC_SERVERS=1`)) + o.Expect(checkenv2).To(o.ContainSubstring(`ROUTER_HAPROXY_CONFIG_MANAGER=true`)) + + compat_otp.By("6. Check the haproxy config on the router pod for dynamic pod config") + insecBackend := "be_http:" + ns + ":service-unsecure" + ensureHaproxyBlockConfigContains(oc, podname, insecBackend, []string{"dynamic-cookie-key", "server-template _dynamic-pod- 1-1 172.4.0.4:8765 check disabled"}) + }) + + // author: shudili@redhat.com + g.It("Author:shudili-DEPRECATED-ROSA-OSD_CCS-ARO-High-77892-Dynamic Configuration Manager for plain HTTP route [Serial]", func() { + // skip the test if featureSet is not there + if !compat_otp.IsTechPreviewNoUpgrade(oc) { + g.Skip("featureSet: TechPreviewNoUpgrade is required for this test, skipping") + } + + var ( + buildPruningBaseDir = testdata.FixturePath("router") + baseTemp = filepath.Join(buildPruningBaseDir, "ingresscontroller-np.yaml") + testPodSvc = filepath.Join(buildPruningBaseDir, "web-server-signed-deploy.yaml") + srvdmInfo = "web-server-deploy" + svcName = "service-unsecure" + clientPod = filepath.Join(buildPruningBaseDir, "test-client-pod.yaml") + clientPodName = "hello-pod" + clientPodLabel = "app=hello-pod" + desiredReplicas = 8 + ingctrl = ingressControllerDescription{ + name: "ocp77892", + namespace: "openshift-ingress-operator", + domain: "", + template: baseTemp, + } + ) + + compat_otp.By("1.0: Create a custom ingresscontroller") + ingctrl.domain = ingctrl.name + "." + getBaseDomain(oc) + routehost := "unsecure77892" + "." + ingctrl.domain + defer func() { + // added debug info, in case the original router pod was terminated + routerpod2 := getOneNewRouterPodFromRollingUpdate(oc, ingctrl.name) + e2e.Logf("Before end of testing, the routerpod is: %s", routerpod2) + ingctrl.delete(oc) + }() + ingctrl.create(oc) + ensureCustomIngressControllerAvailable(oc, ingctrl.name) + + compat_otp.By("2.0 Create a deployment, a HTTP route and a client pod") + ns := oc.Namespace() + createResourceFromFile(oc, ns, testPodSvc) + ensurePodWithLabelReady(oc, ns, "name="+srvdmInfo) + createRoute(oc, ns, "http", "unsecure77892", svcName, []string{"--hostname=" + routehost}) + ensureRouteIsAdmittedByIngressController(oc, ns, "unsecure77892", "default") + createResourceFromFile(oc, ns, clientPod) + ensurePodWithLabelReady(oc, ns, clientPodLabel) + + compat_otp.By("3.0 Curl the HTTP route") + routerpod := getOneRouterPodNameByIC(oc, ingctrl.name) + e2e.Logf("init routerpod is: %s", routerpod) + podIP := getPodv4Address(oc, routerpod, "openshift-ingress") + toDst := routehost + ":80:" + podIP + curlCmd := []string{"-n", ns, clientPodName, "--", "curl", "http://" + routehost, "-s", "--resolve", toDst, "--connect-timeout", "10"} + repeatCmdOnClient(oc, curlCmd, "Hello-OpenShift", 60, 1) + + compat_otp.By("4.0 Check the route's backend configuration including server pod, dynamic pool, dynamic cookie") + backend := "be_http:" + ns + ":unsecure77892" + checkDcmBackendCfg(oc, routerpod, backend) + + compat_otp.By("5.0 Use debug command to check the dynamic server's state") + // used the socat command under the router pod to get all the route's endpoints status + socatCmd := fmt.Sprintf(`echo "show servers state %s" | socat stdio /var/lib/haproxy/run/haproxy.sock | sed 1d | grep -v '^#' | cut -d ' ' -f2-6 | sed -e 's/0$/STOP/' -e 's/1$/STARTING/' -e 's/2$/UP/' -e 's/3$/SOFTSTOP/'`, backend) + initSrvStates := checkDcmUpEndpoints(oc, routerpod, socatCmd, 1) + currentSrvStates := "" + + compat_otp.By("6.0 get the initial router reloaded log") + log, err := oc.AsAdmin().WithoutNamespace().Run("logs").Args("-n", "openshift-ingress", routerpod).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + initReloadedNum := strings.Count(log, `"msg"="router reloaded" "logger"="template" "output"=`) + + compat_otp.By("7.0 keep scaling up the deployment with replicas 1") + for i := 1; i < desiredReplicas; i++ { + compat_otp.By("7." + strconv.Itoa(i) + ".1: scale up the deployment with replicas " + strconv.Itoa(i+1)) + scaleDeploy(oc, ns, srvdmInfo, i+1) + waitForOutputEquals(oc, ns, "deployment/"+srvdmInfo, "{.status.availableReplicas}", strconv.Itoa(i+1)) + + compat_otp.By("7." + strconv.Itoa(i) + ".2: Check the route's backend configuration including server pod, dynamic pool, dynamic cookie") + checkDcmBackendCfg(oc, routerpod, backend) + + compat_otp.By("7." + strconv.Itoa(i) + ".3: Use debug command to check the dynamic server's state") + currentSrvStates = checkDcmUpEndpoints(oc, routerpod, socatCmd, i+1) + + compat_otp.By("7." + strconv.Itoa(i) + ".4: check whether got the router reloaded log or not") + initReloadedNum = checkRouterReloadedLogs(oc, routerpod, initReloadedNum, initSrvStates, currentSrvStates) + initSrvStates = currentSrvStates + } + + compat_otp.By("8.0 keep scaling down the deployment with replicas 1") + for i := desiredReplicas - 1; i >= 0; i-- { + compat_otp.By("8." + strconv.Itoa(desiredReplicas-i) + ".1: scale up the deployment with replicas " + strconv.Itoa(i)) + scaleDeploy(oc, ns, srvdmInfo, i) + + compat_otp.By("8." + strconv.Itoa(desiredReplicas-i) + ".2: Check the route's backend configuration including server pod, dynamic pool, dynamic cookie") + checkDcmBackendCfg(oc, routerpod, backend) + + compat_otp.By("8." + strconv.Itoa(desiredReplicas-i) + ".3: Use debug command to check the dynamic server's state") + currentSrvStates = checkDcmUpEndpoints(oc, routerpod, socatCmd, i) + + compat_otp.By("8." + strconv.Itoa(desiredReplicas-i) + ".4: check whether got the router reloaded log or not") + initReloadedNum = checkRouterReloadedLogs(oc, routerpod, initReloadedNum, initSrvStates, currentSrvStates) + initSrvStates = currentSrvStates + } + + compat_otp.By("9.0 keep scaling up the deployment with replicas 2") + maxReplicas := 0 + for i := 2; i <= desiredReplicas-2; i = i + 2 { + compat_otp.By("9." + strconv.Itoa((i+2)/2) + ".1: scale up the deployment with replicas " + strconv.Itoa(i)) + scaleDeploy(oc, ns, srvdmInfo, i) + + compat_otp.By("9." + strconv.Itoa(i/2) + ".2: Check the route's backend configuration including server pod, dynamic pool, dynamic cookie") + checkDcmBackendCfg(oc, routerpod, backend) + + compat_otp.By("9." + strconv.Itoa(i/2) + ".3: Use debug command to check the dynamic server's state") + currentSrvStates = checkDcmUpEndpoints(oc, routerpod, socatCmd, i) + + compat_otp.By("9." + strconv.Itoa(i/2) + ".4: check whether got the router reloaded log or not") + initReloadedNum = checkRouterReloadedLogs(oc, routerpod, initReloadedNum, initSrvStates, currentSrvStates) + initSrvStates = currentSrvStates + maxReplicas = i + } + + compat_otp.By("10.0 keep scaling down the deployment with replicas 2") + for i := maxReplicas - 2; i >= 0; i = i - 2 { + compat_otp.By("10." + strconv.Itoa((maxReplicas-i)/2) + ".1: scale up the deployment with replicas " + strconv.Itoa(i)) + scaleDeploy(oc, ns, srvdmInfo, i) + + compat_otp.By("10." + strconv.Itoa((maxReplicas-i)/2) + ".2: Check the route's backend configuration including server pod, dynamic pool, dynamic cookie") + checkDcmBackendCfg(oc, routerpod, backend) + + compat_otp.By("10." + strconv.Itoa((maxReplicas-i)/2) + ".3: Use debug command to check the dynamic server's state") + currentSrvStates = checkDcmUpEndpoints(oc, routerpod, socatCmd, i) + + compat_otp.By("10." + strconv.Itoa((maxReplicas-i)/2) + ".4: get the router reloaded log") + initReloadedNum = checkRouterReloadedLogs(oc, routerpod, initReloadedNum, initSrvStates, currentSrvStates) + initSrvStates = currentSrvStates + } + }) + + // author: shudili@redhat.com + g.It("Author:shudili-DEPRECATED-ROSA-OSD_CCS-ARO-High-77906-Enable ALPN for reencrypt routes when DCM is enabled", func() { + // skip the test if featureSet is not there + if !compat_otp.IsTechPreviewNoUpgrade(oc) { + g.Skip("featureSet: TechPreviewNoUpgrade is required for this test, skipping") + } + + var ( + buildPruningBaseDir = testdata.FixturePath("router") + testPodSvc = filepath.Join(buildPruningBaseDir, "web-server-signed-deploy.yaml") + srvdmInfo = "web-server-deploy" + svcName = "service-secure" + clientPod = filepath.Join(buildPruningBaseDir, "test-client-pod.yaml") + clientPodName = "hello-pod" + clientPodLabel = "app=hello-pod" + podDirname = "/data/OCP-77906-ca" + podCaCrt = podDirname + "/77906-ca.crt" + podUsrCrt = podDirname + "/77906-usr.crt" + podUsrKey = podDirname + "/77906-usr.key" + dirname = "/tmp/OCP-77906-ca" + caSubj = "/CN=NE-Test-Root-CA" + caCrt = dirname + "/77906-ca.crt" + caKey = dirname + "/77906-ca.key" + userSubj = "/CN=example-ne.com" + usrCrt = dirname + "/77906-usr.crt" + usrKey = dirname + "/77906-usr.key" + usrCsr = dirname + "/77906-usr.csr" + cmName = "ocp77906" + ) + + // enabled mTLS for http/2 traffic testing, if not, the frontend haproxy will use http/1.1 + baseTemp := filepath.Join(buildPruningBaseDir, "ingresscontroller-np.yaml") + extraParas := fmt.Sprintf(` + clientTLS: + clientCA: + name: %s + clientCertificatePolicy: Required +`, cmName) + customTemp := addExtraParametersToYamlFile(baseTemp, "spec:", extraParas) + defer os.Remove(customTemp) + + var ( + ingctrl = ingressControllerDescription{ + name: "ocp77906", + namespace: "openshift-ingress-operator", + domain: "", + template: customTemp, + } + ) + + compat_otp.By("1.0 Get the domain info for testing") + ingctrl.domain = ingctrl.name + "." + getBaseDomain(oc) + routehost := "reen77906" + "." + ingctrl.domain + + compat_otp.By("2.0: Start to use openssl to create ca certification&key and user certification&key") + defer os.RemoveAll(dirname) + err := os.MkdirAll(dirname, 0755) + o.Expect(err).NotTo(o.HaveOccurred()) + + compat_otp.By("2.1: Create a new self-signed CA including the ca certification and ca key") + opensslNewCa(caKey, caCrt, caSubj) + + compat_otp.By("2.2: Create a user CSR and the user key") + opensslNewCsr(usrKey, usrCsr, userSubj) + + compat_otp.By("2.3: Sign the user CSR and generate the certificate") + san := "subjectAltName = DNS:*." + ingctrl.domain + opensslSignCsr(san, usrCsr, caCrt, caKey, usrCrt) + + compat_otp.By("3.0: create a cm with date ca certification, then create the custom ingresscontroller") + defer deleteConfigMap(oc, "openshift-config", cmName) + createConfigMapFromFile(oc, "openshift-config", cmName, "ca-bundle.pem="+caCrt) + + defer ingctrl.delete(oc) + ingctrl.create(oc) + ensureCustomIngressControllerAvailable(oc, ingctrl.name) + + compat_otp.By("4.0: enable http2 on the custom ingresscontroller by the annotation if env ROUTER_DISABLE_HTTP2 is true") + jsonPath := "{.spec.template.spec.containers[0].env[?(@.name==\"ROUTER_DISABLE_HTTP2\")].value}" + envValue := getByJsonPath(oc, "openshift-ingress", "deployment/router-"+ingctrl.name, jsonPath) + if envValue == "true" { + setAnnotationAsAdmin(oc, ingctrl.namespace, "ingresscontroller/"+ingctrl.name, `ingress.operator.openshift.io/default-enable-http2=true`) + ensureRouterDeployGenerationIs(oc, ingctrl.name, "2") + } + + compat_otp.By("5.0 Create a deployment and a client pod") + ns := oc.Namespace() + createResourceFromFile(oc, ns, testPodSvc) + ensurePodWithLabelReady(oc, ns, "name="+srvdmInfo) + err = oc.AsAdmin().WithoutNamespace().Run("create").Args("-n", ns, "-f", clientPod).Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + ensurePodWithLabelReady(oc, ns, clientPodLabel) + err = oc.AsAdmin().WithoutNamespace().Run("cp").Args("-n", ns, dirname, ns+"/"+clientPodName+":"+podDirname).Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + + compat_otp.By("6.0 Create a reencrypt route inside the namespace") + createRoute(oc, ns, "reencrypt", "route-reen", svcName, []string{"--hostname=" + routehost, "--ca-cert=" + caCrt, "--cert=" + usrCrt, "--key=" + usrKey}) + ensureRouteIsAdmittedByIngressController(oc, ns, "route-reen", "default") + + compat_otp.By("7.0 Check the haproxy.config, make sure alpn is enabled for the reencrypt route's backend endpoint") + backend := "be_secure:" + ns + ":route-reen" + routerpod := getOneNewRouterPodFromRollingUpdate(oc, ingctrl.name) + ensureHaproxyBlockConfigContains(oc, routerpod, backend, []string{"ssl alpn h2,http/1.1"}) + + compat_otp.By("8.0 Curl the reencrypt route with specified protocol http2") + podIP := getPodv4Address(oc, routerpod, "openshift-ingress") + toDst := routehost + ":443:" + podIP + curlCmd := []string{"-n", ns, clientPodName, "--", "curl", "https://" + routehost, "-sI", "--cacert", podCaCrt, "--cert", podUsrCrt, "--key", podUsrKey, "--http2", "--resolve", toDst, "--connect-timeout", "10"} + repeatCmdOnClient(oc, curlCmd, "HTTP/2 200", 60, 1) + + compat_otp.By("9.0 Curl the reencrypt route with specified protocol http1.1") + curlCmd = []string{"-n", ns, clientPodName, "--", "curl", "https://" + routehost, "-sI", "--cacert", podCaCrt, "--cert", podUsrCrt, "--key", podUsrKey, "--http1.1", "--resolve", toDst, "--connect-timeout", "10"} + repeatCmdOnClient(oc, curlCmd, "HTTP/1.1 200", 60, 1) + }) + + // author: shudili@redhat.com + g.It("Author:shudili-DEPRECATED-ROSA-OSD_CCS-ARO-High-77973-Dynamic Configuration Manager for edge route [Serial]", func() { + // skip the test if featureSet is not there + if !compat_otp.IsTechPreviewNoUpgrade(oc) { + g.Skip("featureSet: TechPreviewNoUpgrade is required for this test, skipping") + } + + var ( + buildPruningBaseDir = testdata.FixturePath("router") + baseTemp = filepath.Join(buildPruningBaseDir, "ingresscontroller-np.yaml") + testPodSvc = filepath.Join(buildPruningBaseDir, "web-server-signed-deploy.yaml") + srvdmInfo = "web-server-deploy" + svcName = "service-unsecure" + clientPod = filepath.Join(buildPruningBaseDir, "test-client-pod.yaml") + clientPodName = "hello-pod" + clientPodLabel = "app=hello-pod" + desiredReplicas = 8 + ingctrl = ingressControllerDescription{ + name: "ocp77973", + namespace: "openshift-ingress-operator", + domain: "", + template: baseTemp, + } + ) + + compat_otp.By("1.0: Create a custom ingresscontroller") + ingctrl.domain = ingctrl.name + "." + getBaseDomain(oc) + routehost := "edge77973" + "." + ingctrl.domain + defer func() { + // added debug info, in case the original router pod was terminated + routerpod2 := getOneNewRouterPodFromRollingUpdate(oc, ingctrl.name) + e2e.Logf("Before end of testing, the routerpod is: %s", routerpod2) + ingctrl.delete(oc) + }() + ingctrl.create(oc) + ensureCustomIngressControllerAvailable(oc, ingctrl.name) + + compat_otp.By("2.0 Create a deployment, an edge route and a client pod") + ns := oc.Namespace() + createResourceFromFile(oc, ns, testPodSvc) + ensurePodWithLabelReady(oc, ns, "name="+srvdmInfo) + createRoute(oc, ns, "edge", "edge77973", svcName, []string{"--hostname=" + routehost}) + ensureRouteIsAdmittedByIngressController(oc, ns, "edge77973", "default") + createResourceFromFile(oc, ns, clientPod) + ensurePodWithLabelReady(oc, ns, clientPodLabel) + + compat_otp.By("3.0 Curl the edge route") + routerpod := getOneRouterPodNameByIC(oc, ingctrl.name) + e2e.Logf("init routerpod is: %s", routerpod) + podIP := getPodv4Address(oc, routerpod, "openshift-ingress") + toDst := routehost + ":443:" + podIP + curlCmd := []string{"-n", ns, clientPodName, "--", "curl", "https://" + routehost, "-ks", "--resolve", toDst, "--connect-timeout", "10"} + repeatCmdOnClient(oc, curlCmd, "Hello-OpenShift", 60, 1) + + compat_otp.By("4.0 Check the route's backend configuration including server pod, dynamic pool and dynamic cookie") + backend := "be_edge_http:" + ns + ":edge77973" + checkDcmBackendCfg(oc, routerpod, backend) + + compat_otp.By("5.0 Use debug command to check the dynamic server's state") + // used the socat command under the router pod to get all the route's endpoints status + socatCmd := fmt.Sprintf(`echo "show servers state %s" | socat stdio /var/lib/haproxy/run/haproxy.sock | sed 1d | grep -v '^#' | cut -d ' ' -f2-6 | sed -e 's/0$/STOP/' -e 's/1$/STARTING/' -e 's/2$/UP/' -e 's/3$/SOFTSTOP/'`, backend) + initSrvStates := checkDcmUpEndpoints(oc, routerpod, socatCmd, 1) + currentSrvStates := "" + + compat_otp.By("6.0 get the initial router reloaded log") + log, err := oc.AsAdmin().WithoutNamespace().Run("logs").Args("-n", "openshift-ingress", routerpod).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + initReloadedNum := strings.Count(log, `"msg"="router reloaded" "logger"="template" "output"=`) + + compat_otp.By("7.0 keep scaling up the deployment with replicas 1") + for i := 1; i < desiredReplicas; i++ { + compat_otp.By("7." + strconv.Itoa(i) + ".1: scale up the deployment with replicas " + strconv.Itoa(i+1)) + scaleDeploy(oc, ns, srvdmInfo, i+1) + waitForOutputEquals(oc, ns, "deployment/"+srvdmInfo, "{.status.availableReplicas}", strconv.Itoa(i+1)) + + compat_otp.By("7." + strconv.Itoa(i) + ".2: Check the route's backend configuration including server pod, dynamic pool, dynamic cookie") + checkDcmBackendCfg(oc, routerpod, backend) + + compat_otp.By("7." + strconv.Itoa(i) + ".3: Use debug command to check the dynamic server's state") + currentSrvStates = checkDcmUpEndpoints(oc, routerpod, socatCmd, i+1) + + compat_otp.By("7." + strconv.Itoa(i) + ".4: check whether got the router reloaded log or not") + initReloadedNum = checkRouterReloadedLogs(oc, routerpod, initReloadedNum, initSrvStates, currentSrvStates) + initSrvStates = currentSrvStates + } + + compat_otp.By("8.0 keep scaling down the deployment with replicas 1") + for i := desiredReplicas - 1; i >= 0; i-- { + compat_otp.By("8." + strconv.Itoa(desiredReplicas-i) + ".1: scale up the deployment with replicas " + strconv.Itoa(i)) + scaleDeploy(oc, ns, srvdmInfo, i) + + compat_otp.By("8." + strconv.Itoa(desiredReplicas-i) + ".2: Check the route's backend configuration including server pod, dynamic pool, dynamic cookie") + checkDcmBackendCfg(oc, routerpod, backend) + + compat_otp.By("8." + strconv.Itoa(desiredReplicas-i) + ".3: Use debug command to check the dynamic server's state") + currentSrvStates = checkDcmUpEndpoints(oc, routerpod, socatCmd, i) + + compat_otp.By("8." + strconv.Itoa(desiredReplicas-i) + ".4: check whether got the router reloaded log or not") + initReloadedNum = checkRouterReloadedLogs(oc, routerpod, initReloadedNum, initSrvStates, currentSrvStates) + initSrvStates = currentSrvStates + } + + compat_otp.By("9.0 keep scaling up the deployment with replicas 2") + maxReplicas := 0 + for i := 2; i <= desiredReplicas-2; i = i + 2 { + compat_otp.By("9." + strconv.Itoa((i+2)/2) + ".1: scale up the deployment with replicas " + strconv.Itoa(i)) + scaleDeploy(oc, ns, srvdmInfo, i) + + compat_otp.By("9." + strconv.Itoa(i/2) + ".2: Check the route's backend configuration including server pod, dynamic pool, dynamic cookie") + checkDcmBackendCfg(oc, routerpod, backend) + + compat_otp.By("9." + strconv.Itoa(i/2) + ".3: Use debug command to check the dynamic server's state") + currentSrvStates = checkDcmUpEndpoints(oc, routerpod, socatCmd, i) + + compat_otp.By("9." + strconv.Itoa(i/2) + ".4: check whether got the router reloaded log or not") + initReloadedNum = checkRouterReloadedLogs(oc, routerpod, initReloadedNum, initSrvStates, currentSrvStates) + initSrvStates = currentSrvStates + maxReplicas = i + } + + compat_otp.By("10.0 keep scaling down the deployment with replicas 2") + for i := maxReplicas - 2; i >= 0; i = i - 2 { + compat_otp.By("10." + strconv.Itoa((maxReplicas-i)/2) + ".1: scale up the deployment with replicas " + strconv.Itoa(i)) + scaleDeploy(oc, ns, srvdmInfo, i) + + compat_otp.By("10." + strconv.Itoa((maxReplicas-i)/2) + ".2: Check the route's backend configuration including server pod, dynamic pool, dynamic cookie") + checkDcmBackendCfg(oc, routerpod, backend) + + compat_otp.By("10." + strconv.Itoa((maxReplicas-i)/2) + ".3: Use debug command to check the dynamic server's state") + currentSrvStates = checkDcmUpEndpoints(oc, routerpod, socatCmd, i) + + compat_otp.By("10." + strconv.Itoa((maxReplicas-i)/2) + ".4: get the router reloaded log") + initReloadedNum = checkRouterReloadedLogs(oc, routerpod, initReloadedNum, initSrvStates, currentSrvStates) + initSrvStates = currentSrvStates + } + }) + + // author: shudili@redhat.com + g.It("Author:shudili-DEPRECATED-ROSA-OSD_CCS-ARO-High-77974-Dynamic Configuration Manager for passthrough route [Serial]", func() { + // skip the test if featureSet is not there + if !compat_otp.IsTechPreviewNoUpgrade(oc) { + g.Skip("featureSet: TechPreviewNoUpgrade is required for this test, skipping") + } + + var ( + buildPruningBaseDir = testdata.FixturePath("router") + baseTemp = filepath.Join(buildPruningBaseDir, "ingresscontroller-np.yaml") + testPodSvc = filepath.Join(buildPruningBaseDir, "web-server-signed-deploy.yaml") + srvdmInfo = "web-server-deploy" + svcName = "service-secure" + clientPod = filepath.Join(buildPruningBaseDir, "test-client-pod.yaml") + clientPodName = "hello-pod" + clientPodLabel = "app=hello-pod" + desiredReplicas = 8 + ingctrl = ingressControllerDescription{ + name: "ocp77974", + namespace: "openshift-ingress-operator", + domain: "", + template: baseTemp, + } + ) + + compat_otp.By("1.0: Create a custom ingresscontroller") + ingctrl.domain = ingctrl.name + "." + getBaseDomain(oc) + routehost := "passth77974" + "." + ingctrl.domain + defer func() { + // added debug info, in case the original router pod was terminated + routerpod2 := getOneNewRouterPodFromRollingUpdate(oc, ingctrl.name) + e2e.Logf("Before end of testing, the routerpod is: %s", routerpod2) + ingctrl.delete(oc) + }() + ingctrl.create(oc) + ensureCustomIngressControllerAvailable(oc, ingctrl.name) + + compat_otp.By("2.0 Create a deployment, a passthrough route and a client pod") + ns := oc.Namespace() + createResourceFromFile(oc, ns, testPodSvc) + ensurePodWithLabelReady(oc, ns, "name="+srvdmInfo) + createRoute(oc, ns, "passthrough", "passth77974", svcName, []string{"--hostname=" + routehost}) + ensureRouteIsAdmittedByIngressController(oc, ns, "passth77974", "default") + createResourceFromFile(oc, ns, clientPod) + ensurePodWithLabelReady(oc, ns, clientPodLabel) + + compat_otp.By("3.0 Curl the passthrough route") + routerpod := getOneRouterPodNameByIC(oc, ingctrl.name) + e2e.Logf("init routerpod is: %s", routerpod) + podIP := getPodv4Address(oc, routerpod, "openshift-ingress") + toDst := routehost + ":443:" + podIP + curlCmd := []string{"-n", ns, clientPodName, "--", "curl", "https://" + routehost, "-ks", "--resolve", toDst, "--connect-timeout", "10"} + repeatCmdOnClient(oc, curlCmd, "Hello-OpenShift", 60, 1) + + compat_otp.By("4.0 Check the route's backend configuration including server pod and dynamic pool") + backend := "be_tcp:" + ns + ":passth77974" + checkDcmBackendCfg(oc, routerpod, backend) + + compat_otp.By("5.0 Use debug command to check the dynamic server's state") + // used the socat command under the router pod to get all the route's endpoints status + socatCmd := fmt.Sprintf(`echo "show servers state %s" | socat stdio /var/lib/haproxy/run/haproxy.sock | sed 1d | grep -v '^#' | cut -d ' ' -f2-6 | sed -e 's/0$/STOP/' -e 's/1$/STARTING/' -e 's/2$/UP/' -e 's/3$/SOFTSTOP/'`, backend) + initSrvStates := checkDcmUpEndpoints(oc, routerpod, socatCmd, 1) + currentSrvStates := "" + + compat_otp.By("6.0 get the initial router reloaded log") + log, err := oc.AsAdmin().WithoutNamespace().Run("logs").Args("-n", "openshift-ingress", routerpod).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + initReloadedNum := strings.Count(log, `"msg"="router reloaded" "logger"="template" "output"=`) + + compat_otp.By("7.0 keep scaling up the deployment with replicas 1") + for i := 1; i < desiredReplicas; i++ { + compat_otp.By("7." + strconv.Itoa(i) + ".1: scale up the deployment with replicas " + strconv.Itoa(i+1)) + scaleDeploy(oc, ns, srvdmInfo, i+1) + waitForOutputEquals(oc, ns, "deployment/"+srvdmInfo, "{.status.availableReplicas}", strconv.Itoa(i+1)) + + compat_otp.By("7." + strconv.Itoa(i) + ".2: Check the route's backend configuration including server pod, dynamic pool, dynamic cookie") + checkDcmBackendCfg(oc, routerpod, backend) + + compat_otp.By("7." + strconv.Itoa(i) + ".3: Use debug command to check the dynamic server's state") + currentSrvStates = checkDcmUpEndpoints(oc, routerpod, socatCmd, i+1) + + compat_otp.By("7." + strconv.Itoa(i) + ".4: check whether got the router reloaded log or not") + initReloadedNum = checkRouterReloadedLogs(oc, routerpod, initReloadedNum, initSrvStates, currentSrvStates) + initSrvStates = currentSrvStates + } + + compat_otp.By("8.0 keep scaling down the deployment with replicas 1") + for i := desiredReplicas - 1; i >= 0; i-- { + compat_otp.By("8." + strconv.Itoa(desiredReplicas-i) + ".1: scale up the deployment with replicas " + strconv.Itoa(i)) + scaleDeploy(oc, ns, srvdmInfo, i) + + compat_otp.By("8." + strconv.Itoa(desiredReplicas-i) + ".2: Check the route's backend configuration including server pod, dynamic pool, dynamic cookie") + checkDcmBackendCfg(oc, routerpod, backend) + + compat_otp.By("8." + strconv.Itoa(desiredReplicas-i) + ".3: Use debug command to check the dynamic server's state") + currentSrvStates = checkDcmUpEndpoints(oc, routerpod, socatCmd, i) + + compat_otp.By("8." + strconv.Itoa(desiredReplicas-i) + ".4: check whether got the router reloaded log or not") + initReloadedNum = checkRouterReloadedLogs(oc, routerpod, initReloadedNum, initSrvStates, currentSrvStates) + initSrvStates = currentSrvStates + } + + compat_otp.By("9.0 keep scaling up the deployment with replicas 2") + maxReplicas := 0 + for i := 2; i <= desiredReplicas-2; i = i + 2 { + compat_otp.By("9." + strconv.Itoa((i+2)/2) + ".1: scale up the deployment with replicas " + strconv.Itoa(i)) + scaleDeploy(oc, ns, srvdmInfo, i) + + compat_otp.By("9." + strconv.Itoa(i/2) + ".2: Check the route's backend configuration including server pod, dynamic pool, dynamic cookie") + checkDcmBackendCfg(oc, routerpod, backend) + + compat_otp.By("9." + strconv.Itoa(i/2) + ".3: Use debug command to check the dynamic server's state") + currentSrvStates = checkDcmUpEndpoints(oc, routerpod, socatCmd, i) + + compat_otp.By("9." + strconv.Itoa(i/2) + ".4: check whether got the router reloaded log or not") + initReloadedNum = checkRouterReloadedLogs(oc, routerpod, initReloadedNum, initSrvStates, currentSrvStates) + initSrvStates = currentSrvStates + maxReplicas = i + } + + compat_otp.By("10.0 keep scaling down the deployment with replicas 2") + for i := maxReplicas - 2; i >= 0; i = i - 2 { + compat_otp.By("10." + strconv.Itoa((maxReplicas-i)/2) + ".1: scale up the deployment with replicas " + strconv.Itoa(i)) + scaleDeploy(oc, ns, srvdmInfo, i) + + compat_otp.By("10." + strconv.Itoa((maxReplicas-i)/2) + ".2: Check the route's backend configuration including server pod, dynamic pool, dynamic cookie") + checkDcmBackendCfg(oc, routerpod, backend) + + compat_otp.By("10." + strconv.Itoa((maxReplicas-i)/2) + ".3: Use debug command to check the dynamic server's state") + currentSrvStates = checkDcmUpEndpoints(oc, routerpod, socatCmd, i) + + compat_otp.By("10." + strconv.Itoa((maxReplicas-i)/2) + ".4: get the router reloaded log") + initReloadedNum = checkRouterReloadedLogs(oc, routerpod, initReloadedNum, initSrvStates, currentSrvStates) + initSrvStates = currentSrvStates + } + }) + + // author: shudili@redhat.com + g.It("Author:shudili-DEPRECATED-ROSA-OSD_CCS-ARO-High-77975-Dynamic Configuration Manager for reencrypt route [Serial]", func() { + // skip the test if featureSet is not there + if !compat_otp.IsTechPreviewNoUpgrade(oc) { + g.Skip("featureSet: TechPreviewNoUpgrade is required for this test, skipping") + } + + var ( + buildPruningBaseDir = testdata.FixturePath("router") + baseTemp = filepath.Join(buildPruningBaseDir, "ingresscontroller-np.yaml") + testPodSvc = filepath.Join(buildPruningBaseDir, "web-server-signed-deploy.yaml") + srvdmInfo = "web-server-deploy" + svcName = "service-secure" + clientPod = filepath.Join(buildPruningBaseDir, "test-client-pod.yaml") + clientPodName = "hello-pod" + clientPodLabel = "app=hello-pod" + desiredReplicas = 8 + ingctrl = ingressControllerDescription{ + name: "ocp77975", + namespace: "openshift-ingress-operator", + domain: "", + template: baseTemp, + } + ) + + compat_otp.By("1.0: Create a custom ingresscontroller") + ingctrl.domain = ingctrl.name + "." + getBaseDomain(oc) + routehost := "reen77975" + "." + ingctrl.domain + defer func() { + // added debug info, in case the original router pod was terminated + routerpod2 := getOneNewRouterPodFromRollingUpdate(oc, ingctrl.name) + e2e.Logf("Before end of testing, the routerpod is: %s", routerpod2) + ingctrl.delete(oc) + }() + ingctrl.create(oc) + ensureCustomIngressControllerAvailable(oc, ingctrl.name) + + compat_otp.By("2.0 Create a deployment, a reencrypt route and a client pod") + ns := oc.Namespace() + createResourceFromFile(oc, ns, testPodSvc) + ensurePodWithLabelReady(oc, ns, "name="+srvdmInfo) + createRoute(oc, ns, "reencrypt", "reen77975", svcName, []string{"--hostname=" + routehost}) + ensureRouteIsAdmittedByIngressController(oc, ns, "reen77975", "default") + createResourceFromFile(oc, ns, clientPod) + ensurePodWithLabelReady(oc, ns, clientPodLabel) + + compat_otp.By("3.0 Curl the reencrypt route") + routerpod := getOneRouterPodNameByIC(oc, ingctrl.name) + e2e.Logf("init routerpod is: %s", routerpod) + podIP := getPodv4Address(oc, routerpod, "openshift-ingress") + toDst := routehost + ":443:" + podIP + curlCmd := []string{"-n", ns, clientPodName, "--", "curl", "https://" + routehost, "-ks", "--resolve", toDst, "--connect-timeout", "10"} + repeatCmdOnClient(oc, curlCmd, "Hello-OpenShift", 60, 1) + + compat_otp.By("4.0 Check the route's backend configuration including server pod, dynamic pool, dynamic cookie") + backend := "be_secure:" + ns + ":reen77975" + checkDcmBackendCfg(oc, routerpod, backend) + + compat_otp.By("5.0 Use debug command to check the dynamic server's state") + // used the socat command under the router pod to get all the route's endpoints status + socatCmd := fmt.Sprintf(`echo "show servers state %s" | socat stdio /var/lib/haproxy/run/haproxy.sock | sed 1d | grep -v '^#' | cut -d ' ' -f2-6 | sed -e 's/0$/STOP/' -e 's/1$/STARTING/' -e 's/2$/UP/' -e 's/3$/SOFTSTOP/'`, backend) + initSrvStates := checkDcmUpEndpoints(oc, routerpod, socatCmd, 1) + currentSrvStates := "" + + compat_otp.By("6.0 get the initial router reloaded log") + log, err := oc.AsAdmin().WithoutNamespace().Run("logs").Args("-n", "openshift-ingress", routerpod).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + initReloadedNum := strings.Count(log, `"msg"="router reloaded" "logger"="template" "output"=`) + + compat_otp.By("7.0 keep scaling up the deployment with replicas 1") + for i := 1; i < desiredReplicas; i++ { + compat_otp.By("7." + strconv.Itoa(i) + ".1: scale up the deployment with replicas " + strconv.Itoa(i+1)) + scaleDeploy(oc, ns, srvdmInfo, i+1) + waitForOutputEquals(oc, ns, "deployment/"+srvdmInfo, "{.status.availableReplicas}", strconv.Itoa(i+1)) + + compat_otp.By("7." + strconv.Itoa(i) + ".2: Check the route's backend configuration including server pod, dynamic pool, dynamic cookie") + checkDcmBackendCfg(oc, routerpod, backend) + + compat_otp.By("7." + strconv.Itoa(i) + ".3: Use debug command to check the dynamic server's state") + currentSrvStates = checkDcmUpEndpoints(oc, routerpod, socatCmd, i+1) + + compat_otp.By("7." + strconv.Itoa(i) + ".4: check whether got the router reloaded log or not") + initReloadedNum = checkRouterReloadedLogs(oc, routerpod, initReloadedNum, initSrvStates, currentSrvStates) + initSrvStates = currentSrvStates + } + + compat_otp.By("8.0 keep scaling down the deployment with replicas 1") + for i := desiredReplicas - 1; i >= 0; i-- { + compat_otp.By("8." + strconv.Itoa(desiredReplicas-i) + ".1: scale up the deployment with replicas " + strconv.Itoa(i)) + scaleDeploy(oc, ns, srvdmInfo, i) + + compat_otp.By("8." + strconv.Itoa(desiredReplicas-i) + ".2: Check the route's backend configuration including server pod, dynamic pool and dynamic cookie") + checkDcmBackendCfg(oc, routerpod, backend) + + compat_otp.By("8." + strconv.Itoa(desiredReplicas-i) + ".3: Use debug command to check the dynamic server's state") + currentSrvStates = checkDcmUpEndpoints(oc, routerpod, socatCmd, i) + + compat_otp.By("8." + strconv.Itoa(desiredReplicas-i) + ".4: check whether got the router reloaded log or not") + initReloadedNum = checkRouterReloadedLogs(oc, routerpod, initReloadedNum, initSrvStates, currentSrvStates) + initSrvStates = currentSrvStates + } + + compat_otp.By("9.0 keep scaling up the deployment with replicas 2") + maxReplicas := 0 + for i := 2; i <= desiredReplicas-2; i = i + 2 { + compat_otp.By("9." + strconv.Itoa((i+2)/2) + ".1: scale up the deployment with replicas " + strconv.Itoa(i)) + scaleDeploy(oc, ns, srvdmInfo, i) + + compat_otp.By("9." + strconv.Itoa(i/2) + ".2: Check the route's backend configuration including server pod, dynamic pool, dynamic cookie") + checkDcmBackendCfg(oc, routerpod, backend) + + compat_otp.By("9." + strconv.Itoa(i/2) + ".3: Use debug command to check the dynamic server's state") + currentSrvStates = checkDcmUpEndpoints(oc, routerpod, socatCmd, i) + + compat_otp.By("9." + strconv.Itoa(i/2) + ".4: check whether got the router reloaded log or not") + initReloadedNum = checkRouterReloadedLogs(oc, routerpod, initReloadedNum, initSrvStates, currentSrvStates) + initSrvStates = currentSrvStates + maxReplicas = i + } + + compat_otp.By("10.0 keep scaling down the deployment with replicas 2") + for i := maxReplicas - 2; i >= 0; i = i - 2 { + compat_otp.By("10." + strconv.Itoa((maxReplicas-i)/2) + ".1: scale up the deployment with replicas " + strconv.Itoa(i)) + scaleDeploy(oc, ns, srvdmInfo, i) + + compat_otp.By("10." + strconv.Itoa((maxReplicas-i)/2) + ".2: Check the route's backend configuration including server pod, dynamic pool, dynamic cookie") + checkDcmBackendCfg(oc, routerpod, backend) + + compat_otp.By("10." + strconv.Itoa((maxReplicas-i)/2) + ".3: Use debug command to check the dynamic server's state") + currentSrvStates = checkDcmUpEndpoints(oc, routerpod, socatCmd, i) + + compat_otp.By("10." + strconv.Itoa((maxReplicas-i)/2) + ".4: get the router reloaded log") + initReloadedNum = checkRouterReloadedLogs(oc, routerpod, initReloadedNum, initSrvStates, currentSrvStates) + initSrvStates = currentSrvStates + } + }) + + // author: shudili@redhat.com + g.It("Author:shudili-DEPRECATED-ROSA-OSD_CCS-ARO-Medium-78239-traffic test for dynamic servers", func() { + // skip the test if featureSet is not there + if !compat_otp.IsTechPreviewNoUpgrade(oc) { + g.Skip("featureSet: TechPreviewNoUpgrade is required for this test, skipping") + } + + var ( + buildPruningBaseDir = testdata.FixturePath("router") + baseTemp = filepath.Join(buildPruningBaseDir, "ingresscontroller-np.yaml") + testPodSvc = filepath.Join(buildPruningBaseDir, "web-server-signed-deploy.yaml") + srvdmInfo = "web-server-deploy" + unsecSvcName = "service-unsecure" + secSvcName = "service-secure" + clientPod = filepath.Join(buildPruningBaseDir, "test-client-pod.yaml") + clientPodName = "hello-pod" + clientPodLabel = "app=hello-pod" + desiredReplicas = 8 + ingctrl = ingressControllerDescription{ + name: "ocp78239", + namespace: "openshift-ingress-operator", + domain: "", + template: baseTemp, + } + ) + + compat_otp.By("1.0: Create a custom ingresscontroller") + ingctrl.domain = ingctrl.name + "." + getBaseDomain(oc) + httpRoutehost := "unsecure78239" + "." + ingctrl.domain + reenRoutehost := "reen78239" + "." + ingctrl.domain + defer ingctrl.delete(oc) + ingctrl.create(oc) + ensureCustomIngressControllerAvailable(oc, ingctrl.name) + + compat_otp.By("2.0 Create a deployment and a client pod") + ns := oc.Namespace() + createResourceFromFile(oc, ns, testPodSvc) + ensurePodWithLabelReady(oc, ns, "name="+srvdmInfo) + createResourceFromFile(oc, ns, clientPod) + ensurePodWithLabelReady(oc, ns, clientPodLabel) + + compat_otp.By("3.0 Create the HTTP route and the reencrypt route") + createRoute(oc, ns, "http", "unsecure78239", unsecSvcName, []string{"--hostname=" + httpRoutehost}) + createRoute(oc, ns, "reencrypt", "reen78239", secSvcName, []string{"--hostname=" + reenRoutehost}) + ensureRouteIsAdmittedByIngressController(oc, ns, "unsecure78239", "default") + ensureRouteIsAdmittedByIngressController(oc, ns, "reen78239", "default") + + compat_otp.By("4.0 set lb roundrobin to the routes") + setAnnotation(oc, ns, "route/unsecure78239", "haproxy.router.openshift.io/balance=roundrobin") + setAnnotation(oc, ns, "route/reen78239", "haproxy.router.openshift.io/balance=roundrobin") + + compat_otp.By("5.0 Scale up the deployment with the desired replicas strconv.Itoa(desiredReplicas)") + podList := scaleDeploy(oc, ns, srvdmInfo, desiredReplicas) + waitForOutputEquals(oc, ns, "deployment/"+srvdmInfo, "{.status.availableReplicas}", strconv.Itoa(desiredReplicas)) + + compat_otp.By("6.0 Keep curing the http route, make sure all backend endpoints are hit") + routerpod := getOneRouterPodNameByIC(oc, ingctrl.name) + podIP := getPodv4Address(oc, routerpod, "openshift-ingress") + toDst := httpRoutehost + ":80:" + podIP + curlCmd := []string{"-n", ns, clientPodName, "--", "curl", "http://" + httpRoutehost, "-s", "--resolve", toDst, "--connect-timeout", "10"} + checkDcmServersAccessible(oc, curlCmd, podList, 180, desiredReplicas) + + compat_otp.By("7.0 Keep curing the reencrypt route, make sure all backend endpoints are hit") + toDst = reenRoutehost + ":443:" + podIP + curlCmd = []string{"-n", ns, clientPodName, "--", "curl", "https://" + reenRoutehost, "-ks", "--resolve", toDst, "--connect-timeout", "10"} + checkDcmServersAccessible(oc, curlCmd, podList, 180, desiredReplicas) + }) + + // author: hongli@redhat.com + // https://issues.redhat.com/browse/OCPBUGS-43745 and https://issues.redhat.com/browse/OCPBUGS-43811 + g.It("Author:hongli-ROSA-OSD_CCS-ARO-Critical-79514-haproxy option idle-close-on-response is configurable", func() { + buildPruningBaseDir := testdata.FixturePath("router") + customTemp := filepath.Join(buildPruningBaseDir, "ingresscontroller-np.yaml") + var ( + ingctrl = ingressControllerDescription{ + name: "ocp79514", + namespace: "openshift-ingress-operator", + domain: "", + template: customTemp, + } + ingctrlResource = "ingresscontrollers/" + ingctrl.name + ) + + compat_otp.By("Create a custom ingresscontroller") + baseDomain := getBaseDomain(oc) + ingctrl.domain = ingctrl.name + "." + baseDomain + defer ingctrl.delete(oc) + ingctrl.create(oc) + ensureCustomIngressControllerAvailable(oc, ingctrl.name) + routerPod := getOneRouterPodNameByIC(oc, ingctrl.name) + + compat_otp.By("Verify default spec.idleConnectionTerminationPolicy is Immediate in 4.19+") + output := getByJsonPath(oc, ingctrl.namespace, ingctrlResource, `{.spec.idleConnectionTerminationPolicy}`) + o.Expect(output).To(o.ContainSubstring("Immediate")) + + compat_otp.By("Verify no variable ROUTER_IDLE_CLOSE_ON_RESPONSE in deployed router pod") + checkEnv := readRouterPodEnv(oc, routerPod, "ROUTER_IDLE_CLOSE_ON_RESPONSE") + o.Expect(checkEnv).To(o.ContainSubstring("NotFound")) + + compat_otp.By("Verify no option idle-close-on-response in haproxy.config") + _, err := oc.AsAdmin().WithoutNamespace().Run("exec").Args("-n", "openshift-ingress", routerPod, "--", "grep", "idle-close-on-response", "haproxy.config").Output() + o.Expect(err).To(o.HaveOccurred()) + + compat_otp.By("Patch custom ingresscontroller spec.idleConnectionTerminationPolicy with Deferred") + patchResourceAsAdmin(oc, ingctrl.namespace, ingctrlResource, `{"spec":{"idleConnectionTerminationPolicy":"Deferred"}}`) + ensureRouterDeployGenerationIs(oc, ingctrl.name, "2") + routerPod = getOneNewRouterPodFromRollingUpdate(oc, ingctrl.name) + + compat_otp.By("Verify variable ROUTER_IDLE_CLOSE_ON_RESPONSE of the deployed router pod") + checkEnv = readRouterPodEnv(oc, routerPod, "ROUTER_IDLE_CLOSE_ON_RESPONSE") + o.Expect(checkEnv).To(o.ContainSubstring(`ROUTER_IDLE_CLOSE_ON_RESPONSE=true`)) + + compat_otp.By("Check the router haproxy.config for option idle-close-on-response") + output = readRouterPodData(oc, routerPod, "cat haproxy.config", "idle-close-on-response") + o.Expect(strings.Count(output, "option idle-close-on-response")).To(o.Equal(3)) + }) +}) diff --git a/tests-extension/test/e2e/headers.go b/tests-extension/test/e2e/headers.go new file mode 100644 index 000000000..80e5ccfea --- /dev/null +++ b/tests-extension/test/e2e/headers.go @@ -0,0 +1,1974 @@ +package router + +import ( + "encoding/base64" + "fmt" + "os" + "os/exec" + "path/filepath" + "regexp" + "strconv" + "strings" + + g "github.com/onsi/ginkgo/v2" + o "github.com/onsi/gomega" + e2e "k8s.io/kubernetes/test/e2e/framework" + + compat_otp "github.com/openshift/origin/test/extended/util/compat_otp" + "github.com/openshift/router-tests-extension/test/testdata" +) + +var _ = g.Describe("[sig-network-edge] Network_Edge Component_Router", func() { + defer g.GinkgoRecover() + + var oc = compat_otp.NewCLI("router-headers", compat_otp.KubeConfigPath()) + + // author: shudili@redhat.com + // incorporate OCP-34157 and OCP-34163 into one + // OCP-34157 [HAProxy-frontend-capture] capture and log specific http Request header via "httpCaptureHeaders" option + // OCP-34163 [HAProxy-frontend-capture] capture and log specific http Response headers via "httpCaptureHeaders" option + g.It("Author:shudili-ROSA-OSD_CCS-ARO-ConnectedOnly-Critical-34157-NetworkEdge capture and log specific http Request header via httpCaptureHeaders option", func() { + buildPruningBaseDir := testdata.FixturePath("router") + testPodSvc := filepath.Join(buildPruningBaseDir, "httpbin-deploy.yaml") + unsecsvcName := "httpbin-svc-insecure" + clientPod := filepath.Join(buildPruningBaseDir, "test-client-pod.yaml") + clientPodName := "hello-pod" + clientPodLabel := "app=hello-pod" + srv := "gunicorn" + + baseTemp := filepath.Join(buildPruningBaseDir, "ingresscontroller-np.yaml") + extraParas := fmt.Sprintf(` + logging: + access: + destination: + type: Container + httpCaptureHeaders: + request: + - name: Host + maxLength: 100 + response: + - name: Server + maxLength: 100 +`) + + customTemp := addExtraParametersToYamlFile(baseTemp, "spec:", extraParas) + defer os.Remove(customTemp) + + ingctrl := ingressControllerDescription{ + name: "ocp34157", + namespace: "openshift-ingress-operator", + domain: "", + template: customTemp, + } + + compat_otp.By("1.0 Create a custom ingresscontroller") + baseDomain := getBaseDomain(oc) + ingctrl.domain = ingctrl.name + "." + baseDomain + defer ingctrl.delete(oc) + ingctrl.create(oc) + ensureCustomIngressControllerAvailable(oc, ingctrl.name) + + compat_otp.By("2.0 Create a client pod, a backend pod and its service resources") + ns := oc.Namespace() + createResourceFromFile(oc, ns, clientPod) + ensurePodWithLabelReady(oc, ns, clientPodLabel) + createResourceFromFile(oc, ns, testPodSvc) + ensurePodWithLabelReady(oc, ns, "name=httpbin-pod") + + compat_otp.By("3.0 Create a http route, and then curl the route") + routehost := unsecsvcName + "34157" + ".apps." + getBaseDomain(oc) + routerpod := getOneRouterPodNameByIC(oc, ingctrl.name) + podIP := getPodv4Address(oc, routerpod, "openshift-ingress") + toDst := routehost + ":80:" + podIP + curlCmd := []string{"-n", ns, clientPodName, "--", "curl", "http://" + routehost + "/headers", "-I", "--resolve", toDst, "--connect-timeout", "10"} + createRoute(oc, ns, "http", unsecsvcName, unsecsvcName, []string{"--hostname=" + routehost}) + ensureRouteIsAdmittedByIngressController(oc, ns, unsecsvcName, "default") + repeatCmdOnClient(oc, curlCmd, "200", 60, 1) + + // check for OCP-34157 + compat_otp.By("4.0: check the log which should contain the host") + waitRouterLogsAppear(oc, routerpod, routehost) + + // check for OCP-34163 + compat_otp.By("5.0: check the log which should contain the backend server info") + waitRouterLogsAppear(oc, routerpod, srv) + }) + + // includes OCP-34231 and OCP-34247 + // author: shudili@redhat.com + g.It("Author:shudili-ROSA-OSD_CCS-ARO-Critical-34231-Configure Ingresscontroller to preserve existing header with forwardedHeaderPolicy set to Append", func() { + buildPruningBaseDir := testdata.FixturePath("router") + baseTemp := filepath.Join(buildPruningBaseDir, "ingresscontroller-np.yaml") + extraParas := fmt.Sprintf(` + httpHeaders: + forwardedHeaderPolicy: Append +`) + customTemp := addExtraParametersToYamlFile(baseTemp, "spec:", extraParas) + defer os.Remove(customTemp) + + var ( + testPodSvc = filepath.Join(buildPruningBaseDir, "httpbin-deploy.yaml") + unsecSvcName = "httpbin-svc-insecure" + clientPod = filepath.Join(buildPruningBaseDir, "test-client-pod.yaml") + clientPodName = "hello-pod" + clientPodLabel = "app=hello-pod" + exampleXForwardedHost = "www.example-ne.com" + ingctrl = ingressControllerDescription{ + name: "ocp34231", + namespace: "openshift-ingress-operator", + domain: "", + template: customTemp, + } + ) + + compat_otp.By("1.0 Create an custom IC with forwardedHeaderPolicy Append") + baseDomain := getBaseDomain(oc) + ingctrl.domain = ingctrl.name + "." + baseDomain + defer ingctrl.delete(oc) + ingctrl.create(oc) + ensureRouterDeployGenerationIs(oc, ingctrl.name, "1") + + compat_otp.By("2.0: Create a client pod, a deployment and the services in a namespace") + ns := oc.Namespace() + createResourceFromFile(oc, ns, clientPod) + ensurePodWithLabelReady(oc, ns, clientPodLabel) + createResourceFromFile(oc, ns, testPodSvc) + ensurePodWithLabelReady(oc, ns, "name=httpbin-pod") + + compat_otp.By("3.0: Create a http route for testing OCP-34231") + routehost := "unsecure34231" + "." + ingctrl.domain + createRoute(oc, ns, "http", "route-http", unsecSvcName, []string{"--hostname=" + routehost}) + ensureRouteIsAdmittedByIngressController(oc, ns, "route-http", ingctrl.name) + + compat_otp.By("4.0: Check the forwardedHeaderPolicy env, which should be append") + routerpod := getOneNewRouterPodFromRollingUpdate(oc, ingctrl.name) + pollReadPodData(oc, "openshift-ingress", routerpod, "/usr/bin/env", `ROUTER_SET_FORWARDED_HEADERS=append`) + + compat_otp.By("5.0: Curl the http route with a specified X-Forwarded-Host, then check the X-Forwarded-Host header value, which should contain both the exampleXForwardedHost and routehost") + podIP := getPodv4Address(oc, routerpod, "openshift-ingress") + toDst := routehost + ":80:" + podIP + curlCmd := []string{"-n", ns, clientPodName, "--", "curl", "http://" + routehost + "/headers", "-sv", "-H", "X-Forwarded-Host: " + exampleXForwardedHost, "--resolve", toDst, "--connect-timeout", "10"} + output, _ := repeatCmdOnClient(oc, curlCmd, "200 OK", 60, 1) + o.Expect(output).To(o.ContainSubstring(fmt.Sprintf(`"X-Forwarded-Host": "%s,%s"`, exampleXForwardedHost, routehost))) + + // OCP-34247(Different Routes can have different policy with haproxy.router.openshift.io/set-forwarded-headers annotations) + compat_otp.By("6.0: Create two http routes for testing OCP-34247") + routehostOCP34247a := "unsecure34247a" + "." + ingctrl.domain + routehostOCP34247b := "unsecure34247b" + "." + ingctrl.domain + createRoute(oc, ns, "http", "route-http34247a", unsecSvcName, []string{"--hostname=" + routehostOCP34247a}) + createRoute(oc, ns, "http", "route-http34247b", unsecSvcName, []string{"--hostname=" + routehostOCP34247b}) + ensureRouteIsAdmittedByIngressController(oc, ns, "route-http34247a", ingctrl.name) + ensureRouteIsAdmittedByIngressController(oc, ns, "route-http34247b", ingctrl.name) + + compat_otp.By("7.0 Add the set-forwarded-headers annotations to the two routes, one with replace, the other with never, then check it in haproxy") + setAnnotation(oc, ns, "route/route-http34247a", "haproxy.router.openshift.io/set-forwarded-headers=replace") + setAnnotation(oc, ns, "route/route-http34247b", "haproxy.router.openshift.io/set-forwarded-headers=never") + backend1Start := "backend be_http:" + ns + ":route-http34247a" + backend2Start := "backend be_http:" + ns + ":route-http34247b" + ensureHaproxyBlockConfigContains(oc, routerpod, backend1Start, []string{"http-request set-header X-Forwarded-Host %[req.hdr(host)]"}) + backend2Cfg := getBlockConfig(oc, routerpod, backend2Start) + o.Expect(backend2Cfg).NotTo(o.ContainSubstring(`http-request set-header X-Forwarded-Host`)) + + compat_otp.By("8.0: Curl the replace annotation http route with a specified X-Forwarded-Host, then check the header value which should be replaced by routehostOCP34247a") + toDst = routehostOCP34247a + ":80:" + podIP + curlCmd = []string{"-n", ns, clientPodName, "--", "curl", "http://" + routehostOCP34247a + "/headers", "-sv", "-H", "X-Forwarded-Host: " + exampleXForwardedHost, "--resolve", toDst, "--connect-timeout", "10"} + output, _ = repeatCmdOnClient(oc, curlCmd, "200 OK", 60, 1) + o.Expect(output).To(o.ContainSubstring(fmt.Sprintf(`"X-Forwarded-Host": "%s"`, routehostOCP34247a))) + + compat_otp.By("9.0: Curl the never annotation http route, then check the http headers which should not contain the X-Forwarded-Host header") + toDst = routehostOCP34247b + ":80:" + podIP + curlCmd = []string{"-n", ns, clientPodName, "--", "curl", "http://" + routehostOCP34247b + "/headers", "-sv", "--resolve", toDst, "--connect-timeout", "10"} + output, _ = repeatCmdOnClient(oc, curlCmd, "200 OK", 60, 1) + o.Expect(output).NotTo(o.ContainSubstring(`X-Forwarded-Host`)) + }) + + // includes OCP-34233 and OCP-34246 + // author: shudili@redhat.com + g.It("Author:shudili-ROSA-OSD_CCS-ARO-Critical-34233-Configure Ingresscontroller to replace any existing Forwarded header with forwardedHeaderPolicy set to Replace", func() { + buildPruningBaseDir := testdata.FixturePath("router") + baseTemp := filepath.Join(buildPruningBaseDir, "ingresscontroller-np.yaml") + extraParas := fmt.Sprintf(` + httpHeaders: + forwardedHeaderPolicy: Replace +`) + customTemp := addExtraParametersToYamlFile(baseTemp, "spec:", extraParas) + defer os.Remove(customTemp) + + var ( + testPodSvc = filepath.Join(buildPruningBaseDir, "httpbin-deploy.yaml") + unsecSvcName = "httpbin-svc-insecure" + clientPod = filepath.Join(buildPruningBaseDir, "test-client-pod.yaml") + clientPodName = "hello-pod" + clientPodLabel = "app=hello-pod" + exampleXForwardedHost = "www.example-ne.com" + ingctrl = ingressControllerDescription{ + name: "ocp34233", + namespace: "openshift-ingress-operator", + domain: "", + template: customTemp, + } + ) + + compat_otp.By("1.0 Create an custom IC with forwardedHeaderPolicy Replace") + baseDomain := getBaseDomain(oc) + ingctrl.domain = ingctrl.name + "." + baseDomain + defer ingctrl.delete(oc) + ingctrl.create(oc) + ensureRouterDeployGenerationIs(oc, ingctrl.name, "1") + + compat_otp.By("2.0: Create a client pod, a deployment and the services in a namespace") + ns := oc.Namespace() + createResourceFromFile(oc, ns, clientPod) + ensurePodWithLabelReady(oc, ns, clientPodLabel) + createResourceFromFile(oc, ns, testPodSvc) + ensurePodWithLabelReady(oc, ns, "name=httpbin-pod") + + compat_otp.By("3.0: Create a http route for testing OCP-34233") + routehost := "unsecure34233" + "." + ingctrl.domain + createRoute(oc, ns, "http", "route-http", unsecSvcName, []string{"--hostname=" + routehost}) + ensureRouteIsAdmittedByIngressController(oc, ns, "route-http", ingctrl.name) + + compat_otp.By("4.0: Check the forwardedHeaderPolicy env, which should be replace") + routerpod := getOneNewRouterPodFromRollingUpdate(oc, ingctrl.name) + pollReadPodData(oc, "openshift-ingress", routerpod, "/usr/bin/env", `ROUTER_SET_FORWARDED_HEADERS=replace`) + + compat_otp.By("5.0: Curl the http route, then check the X-Forwarded-Host header value: exampleXForwardedHost should be replaced by routehost") + podIP := getPodv4Address(oc, routerpod, "openshift-ingress") + toDst := routehost + ":80:" + podIP + curlCmd := []string{"-n", ns, clientPodName, "--", "curl", "http://" + routehost + "/headers", "-sv", "-H", "X-Forwarded-Host: " + exampleXForwardedHost, "--resolve", toDst, "--connect-timeout", "10"} + output, _ := repeatCmdOnClient(oc, curlCmd, "200 OK", 60, 1) + o.Expect(output).To(o.ContainSubstring(fmt.Sprintf(`"X-Forwarded-Host": "%s"`, routehost))) + + // OCP-34246(Configure a different header policy for the route with haproxy.router.openshift.io/set-forwarded-headers annotations) + compat_otp.By("6.0: Create a http route for the testing OCP-34246") + routehost = "unsecure34246" + "." + ingctrl.domain + createRoute(oc, ns, "http", "route-http34246", unsecSvcName, []string{"--hostname=" + routehost}) + ensureRouteIsAdmittedByIngressController(oc, ns, "route-http34246", ingctrl.name) + + compat_otp.By("7.0 Add set-forwarded-headers=if-none annotation to the route, then check it in haproxy") + setAnnotation(oc, ns, "route/route-http34246", "haproxy.router.openshift.io/set-forwarded-headers=if-none") + backendStart := "be_http:" + ns + ":route-http34246" + ensureHaproxyBlockConfigContains(oc, routerpod, backendStart, []string{"option forwardfor if-none"}) + + // For if-none, if the http request already has the X-Forwarded-Host header, cluster won't append or replace with its info + compat_otp.By("8.0: Curl the http route with a specified X-Forwarded-Host, then check the X-Forwarded-Host header value, which should not be replaced by routehost or not be appended by routehost") + toDst = routehost + ":80:" + podIP + curlCmd = []string{"-n", ns, clientPodName, "--", "curl", "http://" + routehost + "/headers", "-sv", "-H", "X-Forwarded-Host: " + exampleXForwardedHost, "--resolve", toDst, "--connect-timeout", "10"} + output, _ = repeatCmdOnClient(oc, curlCmd, "200 OK", 60, 1) + o.Expect(output).To(o.ContainSubstring(fmt.Sprintf(`"X-Forwarded-Host": "%s"`, exampleXForwardedHost))) + + // For if-none, if the http request has not the X-Forwarded-Proto-Version header, cluster will create it + compat_otp.By("9.0: Curl the http route with a specified X-Forwarded-Proto-Version, then check the header value is added") + curlCmd = []string{"-n", ns, clientPodName, "--", "curl", "http://" + routehost + "/headers", "-sv", "-H", "X-Forwarded-Proto-Version: http2", "--resolve", toDst, "--connect-timeout", "10"} + output, _ = repeatCmdOnClient(oc, curlCmd, "200 OK", 60, 1) + o.Expect(output).To(o.ContainSubstring(fmt.Sprintf(`"X-Forwarded-Proto-Version": "%s"`, "http2"))) + }) + + // includes OCP-34234, OCP-34235 and OCP-34236 + // author: shudili@redhat.com + g.It("Author:shudili-ROSA-OSD_CCS-ARO-High-34234-Configure Ingresscontroller to set the headers if they are not already set with forwardedHeaderPolicy set to Ifnone", func() { + // OCP-34236(forwardedHeaderPolicy option defaults to Append if none is defined in the ingresscontroller configuration) + var ( + buildPruningBaseDir = testdata.FixturePath("router") + baseTemp = filepath.Join(buildPruningBaseDir, "ingresscontroller-np.yaml") + testPodSvc = filepath.Join(buildPruningBaseDir, "httpbin-deploy.yaml") + unsecSvcName = "httpbin-svc-insecure" + clientPod = filepath.Join(buildPruningBaseDir, "test-client-pod.yaml") + clientPodName = "hello-pod" + clientPodLabel = "app=hello-pod" + exampleXForwardedHost = "www.example-ne.com" + ingctrl = ingressControllerDescription{ + name: "ocp34236", + namespace: "openshift-ingress-operator", + domain: "", + template: baseTemp, + } + ingctrlResource = "ingresscontroller/" + ingctrl.name + ) + + compat_otp.By("1.0 Create an custom IC with default forwardedHeaderPolicy Append") + baseDomain := getBaseDomain(oc) + ingctrl.domain = ingctrl.name + "." + baseDomain + defer ingctrl.delete(oc) + ingctrl.create(oc) + ensureRouterDeployGenerationIs(oc, ingctrl.name, "1") + + compat_otp.By("2.0: Create a client pod, a deployment and the services in a namespace") + ns := oc.Namespace() + createResourceFromFile(oc, ns, clientPod) + ensurePodWithLabelReady(oc, ns, clientPodLabel) + createResourceFromFile(oc, ns, testPodSvc) + ensurePodWithLabelReady(oc, ns, "name=httpbin-pod") + + compat_otp.By("3.0: Create a http route for the testing") + routehost := "unsecure34236" + "." + ingctrl.domain + createRoute(oc, ns, "http", "route-http", unsecSvcName, []string{"--hostname=" + routehost}) + ensureRouteIsAdmittedByIngressController(oc, ns, "route-http", ingctrl.name) + + compat_otp.By("4.0: Check httpCaptureHeaders configuration in haproxy") + routerpod := getOneNewRouterPodFromRollingUpdate(oc, ingctrl.name) + pollReadPodData(oc, "openshift-ingress", routerpod, "/usr/bin/env", `ROUTER_SET_FORWARDED_HEADERS=append`) + + compat_otp.By("5.0: Curl the http route with specified X-Forwarded-Host, then check the X-Forwarded-Host header value, which should contain both the exampleXForwardedHost and routehost") + podIP := getPodv4Address(oc, routerpod, "openshift-ingress") + toDst := routehost + ":80:" + podIP + curlCmd := []string{"-n", ns, clientPodName, "--", "curl", "http://" + routehost + "/headers", "-sv", "-H", "X-Forwarded-Host: " + exampleXForwardedHost, "--resolve", toDst, "--connect-timeout", "10"} + output, _ := repeatCmdOnClient(oc, curlCmd, "200 OK", 60, 1) + o.Expect(output).To(o.ContainSubstring(fmt.Sprintf(`"X-Forwarded-Host": "%s,%s"`, exampleXForwardedHost, routehost))) + + // OCP-34234(Configure Ingresscontroller to set the headers if they are not already set with forwardedHeaderPolicy set to Ifnone) + compat_otp.By("6.0 Patch the httpHeaders with forwardedHeaderPolicy IfNone") + patchPath := `{"spec":{"httpHeaders":{"forwardedHeaderPolicy":"IfNone"}}}` + patchResourceAsAdmin(oc, ingctrl.namespace, ingctrlResource, patchPath) + ensureRouterDeployGenerationIs(oc, ingctrl.name, "2") + + compat_otp.By("7.0: Check httpCaptureHeaders configuration in haproxy") + routerpod = getOneNewRouterPodFromRollingUpdate(oc, ingctrl.name) + pollReadPodData(oc, "openshift-ingress", routerpod, "/usr/bin/env", `ROUTER_SET_FORWARDED_HEADERS=if-none`) + + // For if-none, if the http request already has the X-Forwarded-Host header, cluster won't append or replace with its info + compat_otp.By("8.0: Curl the http route with a specified X-Forwarded-Host, then check the X-Forwarded-Host header value, which should not be replaced by routehost or not be appended by routehost") + podIP = getPodv4Address(oc, routerpod, "openshift-ingress") + toDst = routehost + ":80:" + podIP + curlCmd = []string{"-n", ns, clientPodName, "--", "curl", "http://" + routehost + "/headers", "-sv", "-H", "X-Forwarded-Host: " + exampleXForwardedHost, "--resolve", toDst, "--connect-timeout", "10"} + output, _ = repeatCmdOnClient(oc, curlCmd, "200 OK", 60, 1) + o.Expect(output).To(o.ContainSubstring(fmt.Sprintf(`"X-Forwarded-Host": "%s"`, exampleXForwardedHost))) + + // For if-none, if the http request has not the X-Forwarded-Host header, cluster will create it + compat_otp.By("9.0: Curl the http route with a specified X-Forwarded-Proto-Version, then check the header value is added") + curlCmd = []string{"-n", ns, clientPodName, "--", "curl", "http://" + routehost + "/headers", "-sv", "-H", "X-Forwarded-Proto-Version: http2", "--resolve", toDst, "--connect-timeout", "10"} + output, _ = repeatCmdOnClient(oc, curlCmd, "200 OK", 60, 1) + o.Expect(output).To(o.ContainSubstring(fmt.Sprintf(`"X-Forwarded-Proto-Version": "%s"`, "http2"))) + + // OCP-34235(Configure Ingresscontroller to never set the headers and preserve existing with forwardedHeaderPolicy set to Never) + compat_otp.By("10.0 Patch the httpHeaders with forwardedHeaderPolicy Never") + patchPath = `{"spec":{"httpHeaders":{"forwardedHeaderPolicy":"Never"}}}` + patchResourceAsAdmin(oc, ingctrl.namespace, ingctrlResource, patchPath) + ensureRouterDeployGenerationIs(oc, ingctrl.name, "3") + + compat_otp.By("11.0: Check httpCaptureHeaders configuration in haproxy") + routerpod = getOneNewRouterPodFromRollingUpdate(oc, ingctrl.name) + pollReadPodData(oc, "openshift-ingress", routerpod, "/usr/bin/env", `ROUTER_SET_FORWARDED_HEADERS=never`) + + compat_otp.By("12.0: Curl the http route with a specified X-Forwarded-Host, then check the http headers which should not contain it") + podIP = getPodv4Address(oc, routerpod, "openshift-ingress") + toDst = routehost + ":80:" + podIP + curlCmd = []string{"-n", ns, clientPodName, "--", "curl", "http://" + routehost + "/headers", "-sv", "--resolve", toDst, "--connect-timeout", "10"} + output, _ = repeatCmdOnClient(oc, curlCmd, "200 OK", 60, 1) + o.Expect(output).NotTo(o.ContainSubstring(`X-Forwarded-Host`)) + }) + + // bug: 1816540 1803001 1816544 + g.It("Author:shudili-High-57012-Forwarded header includes empty quoted proto-version parameter", func() { + compat_otp.By("Check haproxy-config.template file in a router pod and make sure proto-version is removed from the Forwarded header") + podname := getOneRouterPodNameByIC(oc, "default") + templateConfig := readRouterPodData(oc, podname, "cat haproxy-config.template", "http-request add-header Forwarded") + o.Expect(templateConfig).To(o.ContainSubstring("proto")) + o.Expect(templateConfig).NotTo(o.ContainSubstring("proto-version")) + + compat_otp.By("Check proto-version is also removed from the haproxy.config file in a router pod") + haproxyConfig := readRouterPodData(oc, podname, "cat haproxy.config", "proto") + o.Expect(haproxyConfig).NotTo(o.ContainSubstring("proto-version")) + }) + + // author: shudili@redhat.com + g.It("Author:shudili-ROSA-OSD_CCS-ARO-ConnectedOnly-High-62528-adding/deleting http headers to an edge route by a router owner", func() { + var ( + buildPruningBaseDir = testdata.FixturePath("router") + customTemp = filepath.Join(buildPruningBaseDir, "ingresscontroller-np.yaml") + testPodSvc = filepath.Join(buildPruningBaseDir, "httpbin-deploy.yaml") + clientPod = filepath.Join(buildPruningBaseDir, "test-client-pod.yaml") + unsecsvcName = "httpbin-svc-insecure" + clientPodName = "hello-pod" + clientPodLabel = "app=hello-pod" + srv = "gunicorn" + ingctrl = ingressControllerDescription{ + name: "ocp62528", + namespace: "openshift-ingress-operator", + domain: "", + template: customTemp, + } + ingctrlResource = "ingresscontroller/" + ingctrl.name + podFileDir = "/data/OCP-62528-ca" + podCaCert = podFileDir + "/62528-ca.pem" + podCustomKey = podFileDir + "/user62528.key" + podCustomCert = podFileDir + "/user62528.pem" + fileDir = "/tmp/OCP-62528-ca" + dirname = "/tmp/OCP-62528-ca/" + name = dirname + "62528" + validity = 30 + caSubj = "/CN=NE-Test-Root-CA" + userCert = dirname + "user62528" + customKey = userCert + ".key" + customCert = userCert + ".pem" + ) + defer os.RemoveAll(dirname) + err := os.MkdirAll(dirname, 0755) + o.Expect(err).NotTo(o.HaveOccurred()) + baseDomain := getBaseDomain(oc) + ingctrl.domain = ingctrl.name + "." + baseDomain + + compat_otp.By("Try to create custom key and custom certification by openssl, create a new self-signed CA at first, creating the CA key") + opensslCmd := fmt.Sprintf(`openssl genrsa -out %s-ca.key 2048`, name) + _, err = exec.Command("bash", "-c", opensslCmd).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + + compat_otp.By("Create the CA certificate") + opensslCmd = fmt.Sprintf(`openssl req -x509 -new -nodes -key %s-ca.key -sha256 -days %d -out %s-ca.pem -subj %s`, name, validity, name, caSubj) + _, err = exec.Command("bash", "-c", opensslCmd).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + + compat_otp.By("Create a new user certificate, crearing the user CSR with the private user key") + userSubj := "/CN=example-ne.com" + opensslCmd = fmt.Sprintf(`openssl req -nodes -newkey rsa:2048 -keyout %s.key -subj %s -out %s.csr`, userCert, userSubj, userCert) + _, err = exec.Command("bash", "-c", opensslCmd).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + + compat_otp.By("Sign the user CSR and generate the certificate") + opensslCmd = fmt.Sprintf(`openssl x509 -extfile <(printf "subjectAltName = DNS:*.`+ingctrl.domain+`") -req -in %s.csr -CA %s-ca.pem -CAkey %s-ca.key -CAcreateserial -out %s.pem -days %d -sha256`, userCert, name, name, userCert, validity) + _, err = exec.Command("bash", "-c", opensslCmd).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + + compat_otp.By("Create a custom ingresscontroller") + defer ingctrl.delete(oc) + ingctrl.create(oc) + ensureRouterDeployGenerationIs(oc, ingctrl.name, "1") + + compat_otp.By("create configmap client-ca-xxxxx in namespace openshift-config") + cmFile := "ca-bundle.pem=" + name + "-ca.pem" + defer deleteConfigMap(oc, "openshift-config", "client-ca-"+ingctrl.name) + createConfigMapFromFile(oc, "openshift-config", "client-ca-"+ingctrl.name, cmFile) + + compat_otp.By("patch the ingresscontroller to enable client certificate with required policy") + patchResourceAsAdmin(oc, ingctrl.namespace, ingctrlResource, "{\"spec\":{\"clientTLS\":{\"clientCA\":{\"name\":\"client-ca-"+ingctrl.name+"\"},\"clientCertificatePolicy\":\"Required\"}}}") + ensureRouterDeployGenerationIs(oc, ingctrl.name, "2") + + compat_otp.By("Create a client pod, a backend pod and its service resources") + ns := oc.Namespace() + err = oc.AsAdmin().WithoutNamespace().Run("create").Args("-n", ns, "-f", clientPod).Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + ensurePodWithLabelReady(oc, ns, clientPodLabel) + err = oc.AsAdmin().WithoutNamespace().Run("cp").Args("-n", ns, fileDir, ns+"/"+clientPodName+":"+podFileDir).Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + createResourceFromFile(oc, ns, testPodSvc) + ensurePodWithLabelReady(oc, ns, "name=httpbin-pod") + + compat_otp.By("create an edge route") + routerpod := getOneNewRouterPodFromRollingUpdate(oc, ingctrl.name) + podIP := getPodv4Address(oc, routerpod, "openshift-ingress") + edgeRouteHost := "r3-edge62528." + ingctrl.domain + lowHostEdge := strings.ToLower(edgeRouteHost) + base64HostEdge := base64.StdEncoding.EncodeToString([]byte(edgeRouteHost)) + edgeRouteDst := edgeRouteHost + ":443:" + podIP + err = oc.AsAdmin().WithoutNamespace().Run("create").Args("-n", ns, "route", "edge", "r3-edge", "--service="+unsecsvcName, "--cert="+customCert, "--key="+customKey, "--ca-cert="+name+"-ca.pem", "--hostname="+edgeRouteHost).Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + + compat_otp.By("Patch the edge route with added/deleted http request/response headers under the spec") + patchHeaders := "{\"spec\": {\"httpHeaders\": {\"actions\": {\"request\": [" + + "{\"name\": \"X-SSL-Client-Cert\", \"action\": {\"type\": \"Set\", \"set\": {\"value\": \"%{+Q}[ssl_c_der,base64]\"}}}," + + "{\"name\": \"X-Target\", \"action\": {\"type\": \"Set\", \"set\": {\"value\": \"%[req.hdr(host),lower]\"}}}," + + "{\"name\": \"reqTestHost1\", \"action\": {\"type\": \"Set\", \"set\": {\"value\": \"%[req.hdr(host),lower]\"}}}," + + "{\"name\": \"reqTestHost2\", \"action\": {\"type\": \"Set\", \"set\": {\"value\": \"%[req.hdr(host),base64]\"}}}," + + "{\"name\": \"reqTestHost3\", \"action\": {\"type\": \"Set\", \"set\": {\"value\": \"%[req.hdr(Host)]\"}}}," + + "{\"name\": \"X-Forwarded-For\", \"action\": {\"type\": \"Set\", \"set\": {\"value\": \"11.22.33.44\"}}}," + + "{\"name\": \"x-forwarded-client-cert\", \"action\": {\"type\": \"Set\", \"set\": {\"value\": \"%{+Q}[ssl_c_der,base64]\"}}}," + + "{\"name\": \"reqTestHeader\", \"action\": {\"type\": \"Set\", \"set\": {\"value\": \"bbb\"}}}," + + "{\"name\": \"cache-control\", \"action\": {\"type\": \"Set\", \"set\": {\"value\": \"private\"}}}," + + "{\"name\": \"x-ssl-client-der\", \"action\": {\"type\": \"Delete\"}}" + + "]," + + "\"response\": [" + + "{\"name\": \"X-SSL-Server-Cert\", \"action\": {\"type\": \"Set\", \"set\": {\"value\": \"%{+Q}[ssl_c_der,base64]\"}}}," + + "{\"name\": \"X-XSS-Protection\", \"action\": {\"type\": \"Set\", \"set\": {\"value\": \"1; mode=block\"}}}," + + "{\"name\": \"X-Content-Type-Options\", \"action\": {\"type\": \"Set\", \"set\": {\"value\": \"nosniff`\"}}}," + + "{\"name\": \"X-Frame-Options\", \"action\": {\"type\": \"Set\", \"set\": {\"value\": \"SAMEORIGIN\"}}}," + + "{\"name\": \"resTestServer1\", \"action\": {\"type\": \"Set\", \"set\": {\"value\": \"%[res.hdr(server),lower]\"}}}," + + "{\"name\": \"resTestServer2\", \"action\": {\"type\": \"Set\", \"set\": {\"value\": \"%[res.hdr(server),base64]\"}}}," + + "{\"name\": \"resTestServer3\", \"action\": {\"type\": \"Set\", \"set\": {\"value\": \"%[res.hdr(server)]\"}}}," + + "{\"name\": \"cache-control\", \"action\": {\"type\": \"Set\", \"set\": {\"value\": \"private\"}}}," + + "{\"name\": \"server\", \"action\": {\"type\": \"Delete\"}}" + + "]}}}}" + + output, err := oc.AsAdmin().WithoutNamespace().Run("patch").Args("route/r3-edge", "-p", patchHeaders, "--type=merge", "-n", ns).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(output).To(o.ContainSubstring("patched")) + + compat_otp.By("check backend edge route in haproxy that headers to be set or deleted") + routeBackendCfg := ensureHaproxyBlockConfigContains(oc, routerpod, "be_edge_http:"+ns+":r3-edge", []string{"X-SSL-Client-Cert"}) + o.Expect(strings.Contains(routeBackendCfg, "http-request set-header 'X-SSL-Client-Cert' '%{+Q}[ssl_c_der,base64]'")).To(o.BeTrue()) + o.Expect(strings.Contains(routeBackendCfg, "http-request set-header 'X-Target' '%[req.hdr(host),lower]'")).To(o.BeTrue()) + o.Expect(strings.Contains(routeBackendCfg, "http-request set-header 'reqTestHost1' '%[req.hdr(host),lower]'")).To(o.BeTrue()) + o.Expect(strings.Contains(routeBackendCfg, "http-request set-header 'reqTestHost2' '%[req.hdr(host),base64]'")).To(o.BeTrue()) + o.Expect(strings.Contains(routeBackendCfg, "http-request set-header 'X-Forwarded-For' '11.22.33.44'")).To(o.BeTrue()) + o.Expect(strings.Contains(routeBackendCfg, "http-request set-header 'x-forwarded-client-cert' '%{+Q}[ssl_c_der,base64]'")).To(o.BeTrue()) + o.Expect(strings.Contains(routeBackendCfg, "http-request set-header 'reqTestHeader' 'bbb'")).To(o.BeTrue()) + o.Expect(strings.Contains(routeBackendCfg, "http-request set-header 'cache-control' 'private'")).To(o.BeTrue()) + o.Expect(strings.Contains(routeBackendCfg, "http-request del-header 'x-ssl-client-der'")).To(o.BeTrue()) + o.Expect(strings.Contains(routeBackendCfg, "http-request del-header 'x-ssl-client-der'")).To(o.BeTrue()) + + o.Expect(strings.Contains(routeBackendCfg, "http-response set-header 'X-SSL-Server-Cert' '%{+Q}[ssl_c_der,base64]'")).To(o.BeTrue()) + o.Expect(strings.Contains(routeBackendCfg, "http-response set-header 'X-XSS-Protection' '1; mode=block'")).To(o.BeTrue()) + o.Expect(strings.Contains(routeBackendCfg, "http-response set-header 'X-Content-Type-Options' 'nosniff`'")).To(o.BeTrue()) + o.Expect(strings.Contains(routeBackendCfg, "http-response set-header 'X-Frame-Options' 'SAMEORIGIN'")).To(o.BeTrue()) + o.Expect(strings.Contains(routeBackendCfg, "http-response set-header 'resTestServer1' '%[res.hdr(server),lower]'")).To(o.BeTrue()) + o.Expect(strings.Contains(routeBackendCfg, "http-response set-header 'resTestServer2' '%[res.hdr(server),base64]'")).To(o.BeTrue()) + o.Expect(strings.Contains(routeBackendCfg, "http-response set-header 'resTestServer3' '%[res.hdr(server)]'")).To(o.BeTrue()) + o.Expect(strings.Contains(routeBackendCfg, "http-response set-header 'cache-control' 'private'")).To(o.BeTrue()) + o.Expect(strings.Contains(routeBackendCfg, "http-response del-header 'server'")).To(o.BeTrue()) + + compat_otp.By("send traffic to the edge route, then check http headers in the request or response message") + curlEdgeRouteReq := []string{"-n", ns, clientPodName, "--", "curl", "https://" + edgeRouteHost + "/headers", "-v", "--cacert", podCaCert, "--cert", podCustomCert, "--key", podCustomKey, "--resolve", edgeRouteDst, "--connect-timeout", "10"} + curlEdgeRouteRes := []string{"-n", ns, clientPodName, "--", "curl", "https://" + edgeRouteHost + "/headers", "-I", "--cacert", podCaCert, "--cert", podCustomCert, "--key", podCustomKey, "--resolve", edgeRouteDst, "--connect-timeout", "10"} + lowSrv := strings.ToLower(srv) + base64Srv := base64.StdEncoding.EncodeToString([]byte(srv)) + repeatCmdOnClient(oc, curlEdgeRouteRes, "200", 60, 1) + reqHeaders, _ := oc.AsAdmin().Run("exec").Args(curlEdgeRouteReq...).Output() + e2e.Logf("reqHeaders is: %v", reqHeaders) + o.Expect(len(regexp.MustCompile("\"X-Ssl-Client-Cert\": \"([0-9a-zA-Z]+)").FindStringSubmatch(reqHeaders)) > 0).To(o.BeTrue()) + o.Expect(strings.Contains(reqHeaders, "\"X-Target\": \""+edgeRouteHost+"\"")).To(o.BeTrue()) + o.Expect(strings.Contains(reqHeaders, "\"Reqtesthost1\": \""+lowHostEdge+"\"")).To(o.BeTrue()) + o.Expect(strings.Contains(reqHeaders, "\"Reqtesthost2\": \""+base64HostEdge+"\"")).To(o.BeTrue()) + o.Expect(strings.Contains(reqHeaders, "\"Reqtesthost3\": \""+edgeRouteHost+"\"")).To(o.BeTrue()) + o.Expect(strings.Contains(reqHeaders, "\"Reqtestheader\": \"bbb\"")).To(o.BeTrue()) + o.Expect(strings.Contains(reqHeaders, "\"Cache-Control\": \"private\"")).To(o.BeTrue()) + o.Expect(strings.Contains(reqHeaders, "x-ssl-client-der")).NotTo(o.BeTrue()) + + resHeaders, _ := oc.AsAdmin().Run("exec").Args(curlEdgeRouteRes...).Output() + e2e.Logf("resHeaders is: %v", resHeaders) + o.Expect(len(regexp.MustCompile("x-ssl-server-cert: ([0-9a-zA-Z]+)").FindStringSubmatch(resHeaders)) > 0).To(o.BeTrue()) + o.Expect(strings.Contains(resHeaders, "x-xss-protection: 1; mode=block")).To(o.BeTrue()) + o.Expect(strings.Contains(resHeaders, "x-content-type-options: nosniff")).To(o.BeTrue()) + o.Expect(strings.Contains(resHeaders, "x-frame-options: SAMEORIGIN")).To(o.BeTrue()) + o.Expect(strings.Contains(resHeaders, "restestserver1: "+lowSrv)).To(o.BeTrue()) + o.Expect(strings.Contains(resHeaders, "restestserver2: "+base64Srv)).To(o.BeTrue()) + o.Expect(strings.Contains(resHeaders, "restestserver3: "+srv)).To(o.BeTrue()) + o.Expect(strings.Contains(resHeaders, "cache-control: private")).To(o.BeTrue()) + o.Expect(strings.Contains(reqHeaders, "server:")).NotTo(o.BeTrue()) + }) + + // author: shudili@redhat.com + g.It("Author:shudili-ROSA-OSD_CCS-ARO-ConnectedOnly-High-66560-adding/deleting http headers to a http route by a router owner", func() { + var ( + buildPruningBaseDir = testdata.FixturePath("router") + customTemp = filepath.Join(buildPruningBaseDir, "ingresscontroller-np.yaml") + testPodSvc = filepath.Join(buildPruningBaseDir, "httpbin-deploy.yaml") + unsecsvcName = "httpbin-svc-insecure" + clientPod = filepath.Join(buildPruningBaseDir, "test-client-pod.yaml") + clientPodName = "hello-pod" + clientPodLabel = "app=hello-pod" + srv = "gunicorn" + ingctrl = ingressControllerDescription{ + name: "ocp66560", + namespace: "openshift-ingress-operator", + domain: "", + template: customTemp, + } + ) + + compat_otp.By("Create a custom ingresscontroller") + baseDomain := getBaseDomain(oc) + ingctrl.domain = ingctrl.name + "." + baseDomain + defer ingctrl.delete(oc) + ingctrl.create(oc) + ensureCustomIngressControllerAvailable(oc, ingctrl.name) + + compat_otp.By("Create a client pod, a backend pod and its service resources") + ns := oc.Namespace() + createResourceFromFile(oc, ns, clientPod) + ensurePodWithLabelReady(oc, ns, clientPodLabel) + createResourceFromFile(oc, ns, testPodSvc) + ensurePodWithLabelReady(oc, ns, "name=httpbin-pod") + + compat_otp.By("Expose a route with the unsecure service inside the namespace") + routeHost := "service-unsecure66560" + "." + ingctrl.domain + lowHost := strings.ToLower(routeHost) + base64Host := base64.StdEncoding.EncodeToString([]byte(routeHost)) + err := oc.Run("expose").Args("svc/"+unsecsvcName, "--hostname="+routeHost).Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + routeOutput := getRoutes(oc, ns) + o.Expect(routeOutput).To(o.ContainSubstring(unsecsvcName)) + + compat_otp.By("Patch the route with added/deleted http request/response headers under the spec") + patchHeaders := "{\"spec\": {\"httpHeaders\": {\"actions\": {\"request\": [" + + "{\"name\": \"X-SSL-Client-Cert\", \"action\": {\"type\": \"Set\", \"set\": {\"value\": \"%{+Q}[ssl_c_der,base64]\"}}}," + + "{\"name\": \"X-Target\", \"action\": {\"type\": \"Set\", \"set\": {\"value\": \"%[req.hdr(host),lower]\"}}}," + + "{\"name\": \"reqTestHost1\", \"action\": {\"type\": \"Set\", \"set\": {\"value\": \"%[req.hdr(host),lower]\"}}}," + + "{\"name\": \"reqTestHost2\", \"action\": {\"type\": \"Set\", \"set\": {\"value\": \"%[req.hdr(host),base64]\"}}}," + + "{\"name\": \"reqTestHost3\", \"action\": {\"type\": \"Set\", \"set\": {\"value\": \"%[req.hdr(Host)]\"}}}," + + "{\"name\": \"X-Forwarded-For\", \"action\": {\"type\": \"Set\", \"set\": {\"value\": \"11.22.33.44\"}}}," + + "{\"name\": \"x-forwarded-client-cert\", \"action\": {\"type\": \"Set\", \"set\": {\"value\": \"%{+Q}[ssl_c_der,base64]\"}}}," + + "{\"name\": \"reqTestHeader\", \"action\": {\"type\": \"Set\", \"set\": {\"value\": \"bbb\"}}}," + + "{\"name\": \"cache-control\", \"action\": {\"type\": \"Set\", \"set\": {\"value\": \"private\"}}}," + + "{\"name\": \"Referer\", \"action\": {\"type\": \"Delete\"}}" + + "]," + + "\"response\": [" + + "{\"name\": \"X-SSL-Server-Cert\", \"action\": {\"type\": \"Set\", \"set\": {\"value\": \"%{+Q}[ssl_c_der,base64]\"}}}," + + "{\"name\": \"X-XSS-Protection\", \"action\": {\"type\": \"Set\", \"set\": {\"value\": \"1; mode=block\"}}}," + + "{\"name\": \"X-Content-Type-Options\", \"action\": {\"type\": \"Set\", \"set\": {\"value\": \"nosniff`\"}}}," + + "{\"name\": \"X-Frame-Options\", \"action\": {\"type\": \"Set\", \"set\": {\"value\": \"SAMEORIGIN\"}}}," + + "{\"name\": \"resTestServer1\", \"action\": {\"type\": \"Set\", \"set\": {\"value\": \"%[res.hdr(server),lower]\"}}}," + + "{\"name\": \"resTestServer2\", \"action\": {\"type\": \"Set\", \"set\": {\"value\": \"%[res.hdr(server),base64]\"}}}," + + "{\"name\": \"resTestServer3\", \"action\": {\"type\": \"Set\", \"set\": {\"value\": \"%[res.hdr(server)]\"}}}," + + "{\"name\": \"cache-control\", \"action\": {\"type\": \"Set\", \"set\": {\"value\": \"private\"}}}," + + "{\"name\": \"server\", \"action\": {\"type\": \"Delete\"}}" + + "]}}}}" + + output, err := oc.AsAdmin().WithoutNamespace().Run("patch").Args("route/"+unsecsvcName, "-p", patchHeaders, "--type=merge", "-n", ns).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(output).To(o.ContainSubstring("patched")) + + compat_otp.By("check backend edge route in haproxy that headers to be set or deleted") + routerpod := getOneRouterPodNameByIC(oc, ingctrl.name) + routeBackendCfg := ensureHaproxyBlockConfigContains(oc, routerpod, "be_http:"+ns+":"+unsecsvcName, []string{"X-SSL-Client-Cert"}) + o.Expect(strings.Contains(routeBackendCfg, "http-request set-header 'X-SSL-Client-Cert' '%{+Q}[ssl_c_der,base64]'")).To(o.BeTrue()) + o.Expect(strings.Contains(routeBackendCfg, "http-request set-header 'X-Target' '%[req.hdr(host),lower]'")).To(o.BeTrue()) + o.Expect(strings.Contains(routeBackendCfg, "http-request set-header 'reqTestHost1' '%[req.hdr(host),lower]'")).To(o.BeTrue()) + o.Expect(strings.Contains(routeBackendCfg, "http-request set-header 'reqTestHost2' '%[req.hdr(host),base64]'")).To(o.BeTrue()) + o.Expect(strings.Contains(routeBackendCfg, "http-request set-header 'X-Forwarded-For' '11.22.33.44'")).To(o.BeTrue()) + o.Expect(strings.Contains(routeBackendCfg, "http-request set-header 'x-forwarded-client-cert' '%{+Q}[ssl_c_der,base64]'")).To(o.BeTrue()) + o.Expect(strings.Contains(routeBackendCfg, "http-request set-header 'reqTestHeader' 'bbb'")).To(o.BeTrue()) + o.Expect(strings.Contains(routeBackendCfg, "http-request set-header 'cache-control' 'private'")).To(o.BeTrue()) + o.Expect(strings.Contains(routeBackendCfg, "http-request del-header 'Referer'")).To(o.BeTrue()) + + o.Expect(strings.Contains(routeBackendCfg, "http-response set-header 'X-SSL-Server-Cert' '%{+Q}[ssl_c_der,base64]'")).To(o.BeTrue()) + o.Expect(strings.Contains(routeBackendCfg, "http-response set-header 'X-XSS-Protection' '1; mode=block'")).To(o.BeTrue()) + o.Expect(strings.Contains(routeBackendCfg, "http-response set-header 'X-Content-Type-Options' 'nosniff`'")).To(o.BeTrue()) + o.Expect(strings.Contains(routeBackendCfg, "http-response set-header 'X-Frame-Options' 'SAMEORIGIN'")).To(o.BeTrue()) + o.Expect(strings.Contains(routeBackendCfg, "http-response set-header 'resTestServer1' '%[res.hdr(server),lower]'")).To(o.BeTrue()) + o.Expect(strings.Contains(routeBackendCfg, "http-response set-header 'resTestServer2' '%[res.hdr(server),base64]'")).To(o.BeTrue()) + o.Expect(strings.Contains(routeBackendCfg, "http-response set-header 'resTestServer3' '%[res.hdr(server)]'")).To(o.BeTrue()) + o.Expect(strings.Contains(routeBackendCfg, "http-response set-header 'cache-control' 'private'")).To(o.BeTrue()) + o.Expect(strings.Contains(routeBackendCfg, "http-response del-header 'server'")).To(o.BeTrue()) + + compat_otp.By("send traffic to the edge route, then check http headers in the request or response message") + podIP := getPodv4Address(oc, routerpod, "openshift-ingress") + toDst := routeHost + ":80:" + podIP + curlHTTPRouteReq := []string{"-n", ns, clientPodName, "--", "curl", "http://" + routeHost + "/headers", "-v", "-e", "www.qe-test.com", "--resolve", toDst, "--connect-timeout", "10"} + curlHTTPRouteRes := []string{"-n", ns, clientPodName, "--", "curl", "http://" + routeHost + "/headers", "-I", "-e", "www.qe-test.com", "--resolve", toDst, "--connect-timeout", "10"} + lowSrv := strings.ToLower(srv) + base64Srv := base64.StdEncoding.EncodeToString([]byte(srv)) + repeatCmdOnClient(oc, curlHTTPRouteRes, "200", 60, 1) + reqHeaders, _ := oc.AsAdmin().Run("exec").Args(curlHTTPRouteReq...).Output() + e2e.Logf("reqHeaders is: %v", reqHeaders) + o.Expect(strings.Contains(reqHeaders, "\"X-Ssl-Client-Cert\": \"\"")).To(o.BeTrue()) + o.Expect(strings.Contains(reqHeaders, "\"X-Target\": \""+routeHost+"\"")).To(o.BeTrue()) + o.Expect(strings.Contains(reqHeaders, "\"Reqtesthost1\": \""+lowHost+"\"")).To(o.BeTrue()) + o.Expect(strings.Contains(reqHeaders, "\"Reqtesthost2\": \""+base64Host+"\"")).To(o.BeTrue()) + o.Expect(strings.Contains(reqHeaders, "\"Reqtesthost3\": \""+routeHost+"\"")).To(o.BeTrue()) + o.Expect(strings.Contains(reqHeaders, "\"Reqtestheader\": \"bbb\"")).To(o.BeTrue()) + o.Expect(strings.Contains(reqHeaders, "\"Cache-Control\": \"private\"")).To(o.BeTrue()) + o.Expect(strings.Contains(reqHeaders, "x-ssl-client-der")).NotTo(o.BeTrue()) + + resHeaders, _ := oc.AsAdmin().Run("exec").Args(curlHTTPRouteRes...).Output() + e2e.Logf("resHeaders is: %v", resHeaders) + o.Expect(strings.Contains(resHeaders, "x-ssl-server-cert: ")).To(o.BeTrue()) + o.Expect(strings.Contains(resHeaders, "x-xss-protection: 1; mode=block")).To(o.BeTrue()) + o.Expect(strings.Contains(resHeaders, "x-content-type-options: nosniff")).To(o.BeTrue()) + o.Expect(strings.Contains(resHeaders, "x-frame-options: SAMEORIGIN")).To(o.BeTrue()) + o.Expect(strings.Contains(resHeaders, "restestserver1: "+lowSrv)).To(o.BeTrue()) + o.Expect(strings.Contains(resHeaders, "restestserver2: "+base64Srv)).To(o.BeTrue()) + o.Expect(strings.Contains(resHeaders, "restestserver3: "+srv)).To(o.BeTrue()) + o.Expect(strings.Contains(resHeaders, "cache-control: private")).To(o.BeTrue()) + o.Expect(strings.Contains(reqHeaders, "server:")).NotTo(o.BeTrue()) + }) + + // author: shudili@redhat.com + g.It("Author:shudili-ROSA-OSD_CCS-ARO-ConnectedOnly-Medium-66566-supported max http headers, max length of a http header name, max length value of a http header", func() { + var ( + buildPruningBaseDir = testdata.FixturePath("router") + customTemp = filepath.Join(buildPruningBaseDir, "ingresscontroller-np.yaml") + testPodSvc = filepath.Join(buildPruningBaseDir, "httpbin-deploy.yaml") + unsecsvcName = "httpbin-svc-insecure" + clientPod = filepath.Join(buildPruningBaseDir, "test-client-pod.yaml") + clientPodName = "hello-pod" + clientPodLabel = "app=hello-pod" + maxHTTPHeaders = 20 + maxLengthHTTPHeaderName = 255 + maxLengthHTTPHeaderValue = 16384 + ingctrl = ingressControllerDescription{ + name: "ocp66566", + namespace: "openshift-ingress-operator", + domain: "", + template: customTemp, + } + ingctrlResource = "ingresscontroller/" + ingctrl.name + ) + + compat_otp.By("Create a custom ingresscontroller") + baseDomain := getBaseDomain(oc) + ingctrl.domain = ingctrl.name + "." + baseDomain + defer ingctrl.delete(oc) + ingctrl.create(oc) + ensureRouterDeployGenerationIs(oc, ingctrl.name, "1") + + compat_otp.By("Create a client pod, a backend pod and its service resources") + ns := oc.Namespace() + createResourceFromFile(oc, ns, clientPod) + ensurePodWithLabelReady(oc, ns, clientPodLabel) + createResourceFromFile(oc, ns, testPodSvc) + ensurePodWithLabelReady(oc, ns, "name=httpbin-pod") + + compat_otp.By("Expose a route with the unsecure service inside the namespace") + routehost := "service-unsecure66566" + "." + "apps." + baseDomain + err := oc.Run("expose").Args("svc/"+unsecsvcName, "--hostname="+routehost).Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + routeOutput := getRoutes(oc, ns) + o.Expect(routeOutput).To(o.ContainSubstring(unsecsvcName)) + + compat_otp.By("patch max number of http headers to a route") + var maxCfg strings.Builder + negMaxCfg := maxCfg + patchHeadersPart1 := "{\"spec\": {\"httpHeaders\": {\"actions\": {\"request\": [" + patchHeadersPart2 := "]}}}}" + maxCfg.WriteString(patchHeadersPart1) + negMaxCfg.WriteString(patchHeadersPart1) + for i := 0; i < maxHTTPHeaders-1; i++ { + maxCfg.WriteString("{\"name\": \"ocp66566testheader" + strconv.Itoa(i) + "\", \"action\": {\"type\": \"Set\", \"set\": {\"value\": \"value123abc\"}}}, ") + negMaxCfg.WriteString("{\"name\": \"ocp66566testheader" + strconv.Itoa(i) + "\", \"action\": {\"type\": \"Set\", \"set\": {\"value\": \"value123abc\"}}}, ") + } + maxCfg.WriteString("{\"name\": \"ocp66566testheader" + strconv.Itoa(maxHTTPHeaders) + "\", \"action\": {\"type\": \"Set\", \"set\": {\"value\": \"value123abc\"}}}" + patchHeadersPart2) + negMaxCfg.WriteString("{\"name\": \"ocp66566testheader" + strconv.Itoa(maxHTTPHeaders) + "\", \"action\": {\"type\": \"Set\", \"set\": {\"value\": \"value123abc\"}}}") + patchHeaders := maxCfg.String() + negMaxCfg.WriteString(", {\"name\": \"test123abc\", \"action\": {\"type\": \"Set\", \"set\": {\"value\": \"value123abc\"}}}" + patchHeadersPart2) + negPatchHeaders := negMaxCfg.String() + patchResourceAsAdmin(oc, ns, "route/"+unsecsvcName, patchHeaders) + routeBackend := "be_http:" + ns + ":" + unsecsvcName + routerpod := getOneNewRouterPodFromRollingUpdate(oc, ingctrl.name) + podIP := getPodv4Address(oc, routerpod, "openshift-ingress") + toDst := routehost + ":80:" + podIP + routeBackendCfg := ensureHaproxyBlockConfigContains(oc, routerpod, routeBackend, []string{"testheader1"}) + o.Expect(strings.Count(routeBackendCfg, "ocp66566testheader")).To(o.Equal(maxHTTPHeaders)) + + compat_otp.By("send traffic and check the max http headers specified in a route") + cmdOnPod := []string{"-n", ns, clientPodName, "--", "curl", "-Is", "http://" + routehost + "/headers", "--resolve", toDst, "--connect-timeout", "10"} + repeatCmdOnClient(oc, cmdOnPod, "200", 60, 1) + resHeaders, err := oc.Run("exec").Args("-n", ns, clientPodName, "--", "curl", "-s", "http://"+routehost+"/headers", "--resolve", toDst, "--connect-timeout", "10").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(strings.Count(strings.ToLower(resHeaders), "ocp66566testheader")).To(o.Equal(maxHTTPHeaders)) + + compat_otp.By("try to patch the exceeded max headers to a route") + patchResourceAsAdmin(oc, ns, "route/"+unsecsvcName, "{\"spec\": {\"httpHeaders\": null}}") + output, err := oc.AsAdmin().WithoutNamespace().Run("patch").Args("route/"+unsecsvcName, "-p", negPatchHeaders, "--type=merge", "-n", ns).Output() + o.Expect(err).To(o.HaveOccurred()) + o.Expect(output).To(o.ContainSubstring("request headers list can't exceed 20 items")) + + compat_otp.By("patch a http header with max header name to a route") + maxHeaderName := strings.ToLower(getFixedLengthRandomString(maxLengthHTTPHeaderName)) + negHeaderName := maxHeaderName + "a" + maxCfg.Reset() + negMaxCfg.Reset() + maxCfg.WriteString(patchHeadersPart1 + "{\"name\": \"") + maxCfg.WriteString(maxHeaderName) + maxCfg.WriteString("\", \"action\": {\"type\": \"Set\", \"set\": {\"value\": \"value123abc\"}}}" + patchHeadersPart2) + patchHeaders = maxCfg.String() + negMaxCfg.WriteString(patchHeadersPart1 + "{\"name\": \"") + negMaxCfg.WriteString(negHeaderName) + negMaxCfg.WriteString("\", \"action\": {\"type\": \"Set\", \"set\": {\"value\": \"value123abc\"}}}" + patchHeadersPart2) + negPatchHeaders = negMaxCfg.String() + patchResourceAsAdmin(oc, ns, "route/"+unsecsvcName, patchHeaders) + ensureHaproxyBlockConfigContains(oc, routerpod, routeBackend, []string{maxHeaderName}) + + compat_otp.By("send traffic and check the max header name specified in a route") + resHeaders, err = oc.Run("exec").Args(clientPodName, "--", "curl", "-s", "http://"+routehost+"/headers", "--resolve", toDst, "--connect-timeout", "10").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(strings.Contains(strings.ToLower(resHeaders), maxHeaderName+"\": \"value123abc\"")).To(o.BeTrue()) + + compat_otp.By("try to patch the header to a route with its name exceeded the max length") + patchResourceAsAdmin(oc, ns, "route/"+unsecsvcName, "{\"spec\": {\"httpHeaders\": null}}") + output, err = oc.AsAdmin().WithoutNamespace().Run("patch").Args("route/"+unsecsvcName, "-p", negPatchHeaders, "--type=merge", "-n", ns).Output() + o.Expect(err).To(o.HaveOccurred()) + o.Expect(output).To(o.ContainSubstring("exceeds the maximum length, which is 255")) + + compat_otp.By("patch a http header with max header value to a route") + maxHeaderValue := getFixedLengthRandomString(maxLengthHTTPHeaderValue) + negMaxHeaderValue := maxHeaderValue + "a" + maxCfg.Reset() + negMaxCfg.Reset() + maxCfg.WriteString(patchHeadersPart1 + "{\"name\": \"header123abc\", \"action\": {\"type\": \"Set\", \"set\": {\"value\": \"") + maxCfg.WriteString(maxHeaderValue) + maxCfg.WriteString("\"}}}" + patchHeadersPart2) + patchHeaders = maxCfg.String() + negMaxCfg.WriteString(patchHeadersPart1 + "{\"name\": \"header123abc\", \"action\": {\"type\": \"Set\", \"set\": {\"value\": \"") + negMaxCfg.WriteString(negMaxHeaderValue) + negMaxCfg.WriteString("\"}}}" + patchHeadersPart2) + negPatchHeaders = negMaxCfg.String() + + patchResourceAsAdmin(oc, ns, "route/"+unsecsvcName, patchHeaders) + haproxyHeaderName := ensureHaproxyBlockConfigContains(oc, routerpod, routeBackend, []string{"header123abc"}) + o.Expect(strings.Contains(haproxyHeaderName, "http-request set-header 'header123abc' '"+maxHeaderValue+"'")).To(o.BeTrue()) + + compat_otp.By("try to patch the header to a route with its value exceeded the max length") + patchResourceAsAdmin(oc, ns, "route/"+unsecsvcName, "{\"spec\": {\"httpHeaders\": null}}") + output, err = oc.AsAdmin().WithoutNamespace().Run("patch").Args("route/"+unsecsvcName, "-p", negPatchHeaders, "--type=merge", "-n", ns).Output() + o.Expect(err).To(o.HaveOccurred()) + o.Expect(output).To(o.ContainSubstring("exceeds the maximum length, which is 16384")) + + compat_otp.By("patch max number of http headers to an ingress controller") + patchHeadersPart1 = "{\"spec\": {\"httpHeaders\": {\"actions\": {\"response\": [" + maxCfg.Reset() + negMaxCfg.Reset() + maxCfg.WriteString(patchHeadersPart1) + negMaxCfg.WriteString(patchHeadersPart1) + for i := 0; i < maxHTTPHeaders-1; i++ { + maxCfg.WriteString("{\"name\": \"ocp66566testheader" + strconv.Itoa(i) + "\", \"action\": {\"type\": \"Set\", \"set\": {\"value\": \"value123abc\"}}}, ") + negMaxCfg.WriteString("{\"name\": \"ocp66566testheader" + strconv.Itoa(i) + "\", \"action\": {\"type\": \"Set\", \"set\": {\"value\": \"value123abc\"}}}, ") + } + maxCfg.WriteString("{\"name\": \"ocp66566testheader" + strconv.Itoa(maxHTTPHeaders) + "\", \"action\": {\"type\": \"Set\", \"set\": {\"value\": \"value123abc\"}}}" + patchHeadersPart2) + negMaxCfg.WriteString("{\"name\": \"ocp66566testheader" + strconv.Itoa(maxHTTPHeaders) + "\", \"action\": {\"type\": \"Set\", \"set\": {\"value\": \"value123abc\"}}}") + patchHeaders = maxCfg.String() + negMaxCfg.WriteString(", {\"name\": \"test123abc\", \"action\": {\"type\": \"Set\", \"set\": {\"value\": \"value123abc\"}}}" + patchHeadersPart2) + negPatchHeaders = negMaxCfg.String() + patchResourceAsAdmin(oc, ingctrl.namespace, ingctrlResource, patchHeaders) + ensureRouterDeployGenerationIs(oc, ingctrl.name, "2") + routerpod = getOneNewRouterPodFromRollingUpdate(oc, ingctrl.name) + podIP = getPodv4Address(oc, routerpod, "openshift-ingress") + toDst = routehost + ":80:" + podIP + ensureHaproxyBlockConfigContains(oc, routerpod, "frontend fe_sni", []string{"testheader1"}) + routeBackendCfg = getBlockConfig(oc, routerpod, "frontend public") + o.Expect(strings.Count(routeBackendCfg, "ocp66566testheader")).To(o.Equal(maxHTTPHeaders)) + routeBackendCfg = getBlockConfig(oc, routerpod, "frontend fe_sni") + o.Expect(strings.Count(routeBackendCfg, "ocp66566testheader")).To(o.Equal(maxHTTPHeaders)) + routeBackendCfg = getBlockConfig(oc, routerpod, "frontend fe_no_sni") + o.Expect(strings.Count(routeBackendCfg, "ocp66566testheader")).To(o.Equal(maxHTTPHeaders)) + + compat_otp.By("send traffic and check the max http headers specified in an ingress controller") + icResHeaders, err := oc.Run("exec").Args(clientPodName, "--", "curl", "-Is", "http://"+routehost+"/headers", "--resolve", toDst, "--connect-timeout", "10").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(strings.Count(strings.ToLower(icResHeaders), "ocp66566testheader") == maxHTTPHeaders).To(o.BeTrue()) + output, err = oc.AsAdmin().WithoutNamespace().Run("patch").Args(ingctrlResource, "-p", negPatchHeaders, "--type=merge", "-n", ingctrl.namespace).Output() + o.Expect(err).To(o.HaveOccurred()) + o.Expect(output).To(o.ContainSubstring("Too many: 21: must have at most 20 items")) + + compat_otp.By("patch a http header with max header name to an ingress controller") + maxCfg.Reset() + negMaxCfg.Reset() + maxCfg.WriteString(patchHeadersPart1 + "{\"name\": \"") + maxCfg.WriteString(maxHeaderName) + maxCfg.WriteString("\", \"action\": {\"type\": \"Set\", \"set\": {\"value\": \"value123abc\"}}}" + patchHeadersPart2) + patchHeaders = maxCfg.String() + negMaxCfg.WriteString(patchHeadersPart1 + "{\"name\": \"") + negMaxCfg.WriteString(negHeaderName) + negMaxCfg.WriteString("\", \"action\": {\"type\": \"Set\", \"set\": {\"value\": \"value123abc\"}}}" + patchHeadersPart2) + negPatchHeaders = negMaxCfg.String() + patchResourceAsAdmin(oc, ingctrl.namespace, ingctrlResource, patchHeaders) + ensureRouterDeployGenerationIs(oc, ingctrl.name, "3") + routerpod = getOneNewRouterPodFromRollingUpdate(oc, ingctrl.name) + podIP = getPodv4Address(oc, routerpod, "openshift-ingress") + toDst = routehost + ":80:" + podIP + ensureHaproxyBlockConfigContains(oc, routerpod, "frontend fe_sni", []string{maxHeaderName}) + routeBackendCfg = getBlockConfig(oc, routerpod, "frontend public") + o.Expect(strings.Contains(routeBackendCfg, maxHeaderName)).To(o.BeTrue()) + routeBackendCfg = getBlockConfig(oc, routerpod, "frontend fe_sni") + o.Expect(strings.Contains(routeBackendCfg, maxHeaderName)).To(o.BeTrue()) + routeBackendCfg = getBlockConfig(oc, routerpod, "frontend fe_no_sni") + o.Expect(strings.Contains(routeBackendCfg, maxHeaderName)).To(o.BeTrue()) + + compat_otp.By("send traffic and check the header with max length name specified in an ingress controller") + icResHeaders, err = oc.Run("exec").Args(clientPodName, "--", "curl", "-Is", "http://"+routehost+"/headers", "--resolve", toDst, "--connect-timeout", "10").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(strings.Contains(strings.ToLower(icResHeaders), maxHeaderName+": value123abc")).To(o.BeTrue()) + + compat_otp.By("try to patch the header to an ingress controller with its name exceeded the max length") + output, err = oc.AsAdmin().WithoutNamespace().Run("patch").Args(ingctrlResource, "-p", negPatchHeaders, "--type=merge", "-n", ingctrl.namespace).Output() + o.Expect(err).To(o.HaveOccurred()) + o.Expect(output).To(o.MatchRegexp("Too long:.+than 255")) + + compat_otp.By("patch a http header with max header value to an ingress controller") + maxCfg.Reset() + negMaxCfg.Reset() + maxCfg.WriteString(patchHeadersPart1 + "{\"name\": \"header123abc\", \"action\": {\"type\": \"Set\", \"set\": {\"value\": \"") + maxCfg.WriteString(maxHeaderValue) + maxCfg.WriteString("\"}}}" + patchHeadersPart2) + patchHeaders = maxCfg.String() + negMaxCfg.WriteString(patchHeadersPart1 + "{\"name\": \"header123abc\", \"action\": {\"type\": \"Set\", \"set\": {\"value\": \"") + negMaxCfg.WriteString(negMaxHeaderValue) + negMaxCfg.WriteString("\"}}}" + patchHeadersPart2) + negPatchHeaders = negMaxCfg.String() + patchResourceAsAdmin(oc, ingctrl.namespace, ingctrlResource, patchHeaders) + ensureRouterDeployGenerationIs(oc, ingctrl.name, "4") + routerpod = getOneNewRouterPodFromRollingUpdate(oc, ingctrl.name) + ensureHaproxyBlockConfigContains(oc, routerpod, "frontend fe_sni", []string{"header123abc"}) + routeBackendCfg = getBlockConfig(oc, routerpod, "frontend public") + o.Expect(strings.Contains(routeBackendCfg, "http-response set-header 'header123abc' '"+maxHeaderValue+"'")).To(o.BeTrue()) + routeBackendCfg = getBlockConfig(oc, routerpod, "frontend fe_sni") + o.Expect(strings.Contains(routeBackendCfg, "http-response set-header 'header123abc' '"+maxHeaderValue+"'")).To(o.BeTrue()) + routeBackendCfg = getBlockConfig(oc, routerpod, "frontend fe_no_sni") + o.Expect(strings.Contains(routeBackendCfg, "http-response set-header 'header123abc' '"+maxHeaderValue+"'")).To(o.BeTrue()) + output, err = oc.AsAdmin().WithoutNamespace().Run("patch").Args(ingctrlResource, "-p", negPatchHeaders, "--type=merge", "-n", ingctrl.namespace).Output() + o.Expect(err).To(o.HaveOccurred()) + o.Expect(output).To(o.MatchRegexp("Too long:.+than 16384")) + }) + + // author: shudili@redhat.com + g.It("Author:shudili-ROSA-OSD_CCS-ARO-ConnectedOnly-Medium-66568-negative test of adding/deleting http headers", func() { + var ( + buildPruningBaseDir = testdata.FixturePath("router") + customTemp = filepath.Join(buildPruningBaseDir, "ingresscontroller-np.yaml") + testPodSvc = filepath.Join(buildPruningBaseDir, "httpbin-deploy.yaml") + unsecsvcName = "httpbin-svc-insecure" + clientPod = filepath.Join(buildPruningBaseDir, "test-client-pod.yaml") + clientPodName = "hello-pod" + clientPodLabel = "app=hello-pod" + ingctrl = ingressControllerDescription{ + name: "ocp66568", + namespace: "openshift-ingress-operator", + domain: "", + template: customTemp, + } + ingctrlResource = "ingresscontroller/" + ingctrl.name + ) + + compat_otp.By("Create a custom ingresscontroller") + baseDomain := getBaseDomain(oc) + ingctrl.domain = ingctrl.name + "." + baseDomain + defer ingctrl.delete(oc) + ingctrl.create(oc) + ensureRouterDeployGenerationIs(oc, ingctrl.name, "1") + + compat_otp.By("Create a client pod, a backend pod and its service resources") + ns := oc.Namespace() + createResourceFromFile(oc, ns, clientPod) + ensurePodWithLabelReady(oc, ns, clientPodLabel) + createResourceFromFile(oc, ns, testPodSvc) + ensurePodWithLabelReady(oc, ns, "name=httpbin-pod") + + compat_otp.By("Expose a route with the unsecure service inside the namespace") + routehost := "service-unsecure66568" + "." + ingctrl.name + baseDomain + err := oc.Run("expose").Args("svc/"+unsecsvcName, "--hostname="+routehost).Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + routeOutput := getRoutes(oc, ns) + o.Expect(routeOutput).To(o.ContainSubstring(unsecsvcName)) + + compat_otp.By("try to patch two same headers to a route") + sameHeaders := "{\"spec\": {\"httpHeaders\": {\"actions\": {\"request\": [{\"name\": \"testheader1\", \"action\": {\"type\": \"Set\", \"set\": {\"value\": \"value1\"}}}, {\"name\": \"testheader1\", \"action\": {\"type\": \"Set\", \"set\": {\"value\": \"value1\"}}}]}}}}" + output, err := oc.AsAdmin().WithoutNamespace().Run("patch").Args("route/"+unsecsvcName, "-p", sameHeaders, "--type=merge", "-n", ns).Output() + o.Expect(err).To(o.HaveOccurred()) + o.Expect(output).To(o.MatchRegexp("Duplicate value:.+testheader1")) + + compat_otp.By("try to patch proxy header to a route") + proxyHeader := "{\"spec\": {\"httpHeaders\": {\"actions\": {\"request\": [{\"name\": \"proxy\", \"action\": {\"type\": \"Set\", \"set\": {\"value\": \"http://100.200.1.1:80\"}}}]}}}}" + output, err = oc.AsAdmin().WithoutNamespace().Run("patch").Args("route/"+unsecsvcName, "-p", proxyHeader, "--type=merge", "-n", ns).Output() + o.Expect(err).To(o.HaveOccurred()) + o.Expect(output).To(o.ContainSubstring("Forbidden: the following headers may not be modified using this API: strict-transport-security, proxy, cookie, set-cookie")) + + compat_otp.By("try to patch host header to a route") + hostHeader := `{"spec": {"httpHeaders": {"actions": {"request": [{"name": "host", "action": {"type": "Set", "set": {"value": "www.neqe-test.com"}}}]}}}}` + err = oc.AsAdmin().WithoutNamespace().Run("patch").Args("route/"+unsecsvcName, "-p", hostHeader, "--type=merge", "-n", ns).Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + jpath := `{.spec.httpHeaders.actions.request[?(@.name=="host")].action.set.value}` + host := getByJsonPath(oc, ns, "route/"+unsecsvcName, jpath) + o.Expect(host).To(o.ContainSubstring("www.neqe-test.com")) + + compat_otp.By("try to patch strict-transport-security header to a route") + hstsHeader := `{"spec": {"httpHeaders": {"actions": {"request": [{"name": "strict-transport-security", "action": {"type": "Set", "set": {"value": "max-age=31536000;includeSubDomains;preload"}}}]}}}}` + output, err = oc.AsAdmin().WithoutNamespace().Run("patch").Args("route/"+unsecsvcName, "-p", hstsHeader, "--type=merge", "-n", ns).Output() + o.Expect(err).To(o.HaveOccurred()) + o.Expect(output).To(o.ContainSubstring("Forbidden: the following headers may not be modified using this API: strict-transport-security, proxy, cookie, set-cookie")) + + compat_otp.By("try to patch cookie header to a route") + cookieHeader := "{\"spec\": {\"httpHeaders\": {\"actions\": {\"request\": [{\"name\": \"cookie\", \"action\": {\"type\": \"Set\", \"set\": {\"value\": \"cookie-test\"}}}]}}}}" + output, err = oc.AsAdmin().WithoutNamespace().Run("patch").Args("route/"+unsecsvcName, "-p", cookieHeader, "--type=merge", "-n", ns).Output() + o.Expect(err).To(o.HaveOccurred()) + o.Expect(output).To(o.ContainSubstring("Forbidden: the following headers may not be modified using this API: strict-transport-security, proxy, cookie, set-cookie")) + + compat_otp.By("try to patch set-cookie header to a route") + setCookieHeader := "{\"spec\": {\"httpHeaders\": {\"actions\": {\"request\": [{\"name\": \"set-cookie\", \"action\": {\"type\": \"Set\", \"set\": {\"value\": \"set-cookie-test\"}}}]}}}}" + output, err = oc.AsAdmin().WithoutNamespace().Run("patch").Args("route/"+unsecsvcName, "-p", setCookieHeader, "--type=merge", "-n", ns).Output() + o.Expect(err).To(o.HaveOccurred()) + o.Expect(output).To(o.ContainSubstring("Forbidden: the following headers may not be modified using this API: strict-transport-security, proxy, cookie, set-cookie")) + + compat_otp.By("try to patch two same headers to an ingress-controller") + output, err = oc.AsAdmin().WithoutNamespace().Run("patch").Args(ingctrlResource, "-p", sameHeaders, "--type=merge", "-n", ingctrl.namespace).Output() + o.Expect(err).To(o.HaveOccurred()) + o.Expect(output).To(o.MatchRegexp("Duplicate value:.+testheader1")) + + compat_otp.By("try to patch proxy header to an ingress-controller") + output, err = oc.AsAdmin().WithoutNamespace().Run("patch").Args(ingctrlResource, "-p", proxyHeader, "--type=merge", "-n", ingctrl.namespace).Output() + o.Expect(err).To(o.HaveOccurred()) + o.Expect(output).To(o.ContainSubstring("proxy header may not be modified via header actions")) + + compat_otp.By("try to patch host header to an ingress-controller") + output, err = oc.AsAdmin().WithoutNamespace().Run("patch").Args(ingctrlResource, "-p", hostHeader, "--type=merge", "-n", ingctrl.namespace).Output() + o.Expect(err).To(o.HaveOccurred()) + o.Expect(output).To(o.ContainSubstring("host header may not be modified via header actions")) + + compat_otp.By("try to patch strict-transport-security header to an ingress-controller") + output, err = oc.AsAdmin().WithoutNamespace().Run("patch").Args(ingctrlResource, "-p", hstsHeader, "--type=merge", "-n", ingctrl.namespace).Output() + o.Expect(err).To(o.HaveOccurred()) + o.Expect(output).To(o.ContainSubstring("strict-transport-security header may not be modified via header actions")) + + compat_otp.By("try to patch cookie header to an ingress-controller") + output, err = oc.AsAdmin().WithoutNamespace().Run("patch").Args(ingctrlResource, "-p", cookieHeader, "--type=merge", "-n", ingctrl.namespace).Output() + o.Expect(err).To(o.HaveOccurred()) + o.Expect(output).To(o.ContainSubstring("cookie header may not be modified via header actions")) + + compat_otp.By("try to patch set-cookie header to an ingress-controller") + output, err = oc.AsAdmin().WithoutNamespace().Run("patch").Args(ingctrlResource, "-p", setCookieHeader, "--type=merge", "-n", ingctrl.namespace).Output() + o.Expect(err).To(o.HaveOccurred()) + o.Expect(output).To(o.ContainSubstring("set-cookie header may not be modified via header actions")) + + compat_otp.By("patch a request and a response headers to a route, while patch the same headers with the same header names but with different header values to an ingress-controller") + routeHeaders := "{\"spec\": {\"httpHeaders\": {\"actions\": {\"request\": [{\"name\": \"reqtestheader\", \"action\": {\"type\": \"Set\", \"set\": {\"value\": \"req111\"}}}], \"response\": [{\"name\": \"restestheader\", \"action\": {\"type\": \"Set\", \"set\": {\"value\": \"resaaa\"}}}]}}}}" + icHeaders := "{\"spec\": {\"httpHeaders\": {\"actions\": {\"request\": [{\"name\": \"reqtestheader\", \"action\": {\"type\": \"Set\", \"set\": {\"value\": \"req222\"}}}], \"response\": [{\"name\": \"restestheader\", \"action\": {\"type\": \"Set\", \"set\": {\"value\": \"resbbb\"}}}]}}}}" + err = oc.AsAdmin().WithoutNamespace().Run("patch").Args("route/"+unsecsvcName, "-p", routeHeaders, "--type=merge", "-n", ns).Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + patchResourceAsAdmin(oc, ingctrl.namespace, ingctrlResource, icHeaders) + ensureRouterDeployGenerationIs(oc, ingctrl.name, "2") + + compat_otp.By("send traffic, check the request header reqtestheader which should be set to req111 by the route") + routerpod := getOneNewRouterPodFromRollingUpdate(oc, ingctrl.name) + podIP := getPodv4Address(oc, routerpod, "openshift-ingress") + toDst := routehost + ":80:" + podIP + cmdOnPod := []string{"-n", ns, clientPodName, "--", "curl", "-I", "http://" + routehost + "/headers", "--resolve", toDst, "--connect-timeout", "10"} + repeatCmdOnClient(oc, cmdOnPod, "200", 60, 1) + reqHeaders, err := oc.Run("exec").Args("-n", ns, clientPodName, "--", "curl", "http://"+routehost+"/headers", "--resolve", toDst, "--connect-timeout", "10").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(strings.Contains(strings.ToLower(reqHeaders), "\"reqtestheader\": \"req111\"")).To(o.BeTrue()) + + compat_otp.By("send traffic, check the response header restestheader which should be set to resbbb by the ingress-controller") + resHeaders, err := oc.Run("exec").Args(clientPodName, "--", "curl", "http://"+routehost+"/headers", "-I", "--resolve", toDst, "--connect-timeout", "10").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(strings.Contains(resHeaders, "restestheader: resbbb")).To(o.BeTrue()) + }) + + // author: shudili@redhat.com + g.It("Author:shudili-ROSA-OSD_CCS-ARO-NonPreRelease-Longduration-ConnectedOnly-Medium-66569-set different type of values for a http header name and its value", func() { + var ( + buildPruningBaseDir = testdata.FixturePath("router") + customTemp = filepath.Join(buildPruningBaseDir, "ingresscontroller-np.yaml") + testPodSvc = filepath.Join(buildPruningBaseDir, "httpbin-deploy.yaml") + unsecsvcName = "httpbin-svc-insecure" + ingctrl = ingressControllerDescription{ + name: "ocp66569", + namespace: "openshift-ingress-operator", + domain: "", + template: customTemp, + } + ingctrlResource = "ingresscontroller/" + ingctrl.name + routeResource = "route/" + unsecsvcName + ) + + compat_otp.By("Create a custom ingresscontroller") + baseDomain := getBaseDomain(oc) + ingctrl.domain = ingctrl.name + "." + baseDomain + defer ingctrl.delete(oc) + ingctrl.create(oc) + ensureRouterDeployGenerationIs(oc, ingctrl.name, "1") + + compat_otp.By("Create a backend pod and its service resources") + ns := oc.Namespace() + createResourceFromFile(oc, ns, testPodSvc) + ensurePodWithLabelReady(oc, ns, "name=httpbin-pod") + + compat_otp.By("Expose a route with the unsecure service inside the namespace") + routehost := "service-unsecure66569" + "." + ingctrl.name + baseDomain + err := oc.Run("expose").Args("svc/"+unsecsvcName, "--hostname="+routehost).Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + routeOutput := getRoutes(oc, ns) + o.Expect(routeOutput).To(o.ContainSubstring(unsecsvcName)) + + compat_otp.By("patch http headers with valid number, alphabet, a combination of both header names and header values to a route, and then check the added headers in haproxy.conf") + validHeaders := "{\"spec\": {\"httpHeaders\": {\"actions\": {\"request\": [{\"name\": \"001\", \"action\": {\"type\": \"Set\", \"set\": {\"value\": \"20230906\"}}}, {\"name\": \"aBc\", \"action\": {\"type\": \"Set\", \"set\": {\"value\": \"Wednesday\"}}}, {\"name\": \"test01\", \"action\": {\"type\": \"Set\", \"set\": {\"value\": \"value01\"}}}]}}}}" + patchResourceAsAdmin(oc, ns, routeResource, validHeaders) + routerpod := getOneNewRouterPodFromRollingUpdate(oc, ingctrl.name) + routeBackend := "be_http:" + ns + ":" + unsecsvcName + routeBackendCfg := ensureHaproxyBlockConfigContains(oc, routerpod, routeBackend, []string{"test01"}) + o.Expect(strings.Contains(routeBackendCfg, "http-request set-header '001' '20230906'")).To(o.BeTrue()) + o.Expect(strings.Contains(routeBackendCfg, "http-request set-header 'aBc' 'Wednesday'")).To(o.BeTrue()) + o.Expect(strings.Contains(routeBackendCfg, "http-request set-header 'test01' 'value01'")).To(o.BeTrue()) + + compat_otp.By("try to patch http header with blank value in the header name to a route") + blankHeaderName := "{\"spec\": {\"httpHeaders\": {\"actions\": {\"request\": [{\"name\": \"aa bb\", \"action\": {\"type\": \"Set\", \"set\": {\"value\": \"abc\"}}}]}}}}" + output, err := oc.AsAdmin().WithoutNamespace().Run("patch").Args("route/"+unsecsvcName, "-p", blankHeaderName, "--type=merge", "-n", ns).Output() + o.Expect(err).To(o.HaveOccurred()) + o.Expect(output).To(o.ContainSubstring("Invalid value: \"aa bb\": name must be a valid HTTP header name as defined in RFC 2616 section 4.2")) + + compat_otp.By("patch http header with #$* in the header name to a route, and then check it in haproxy.config") + specialHeaderName1 := "{\"spec\": {\"httpHeaders\": {\"actions\": {\"request\": [{\"name\": \"aabbccdd#$*ee\", \"action\": {\"type\": \"Set\", \"set\": {\"value\": \"abc\"}}}]}}}}" + patchResourceAsAdmin(oc, ns, routeResource, specialHeaderName1) + ensureHaproxyBlockConfigContains(oc, routerpod, routeBackend, []string{"http-request set-header 'aabbccdd#$*ee' 'abc'"}) + + compat_otp.By("patch http header with ' in the header name to a route, and then check it in haproxy.config") + specialHeaderName2 := "{\"spec\": {\"httpHeaders\": {\"actions\": {\"request\": [{\"name\": \"aabbccdd'ee\", \"action\": {\"type\": \"Set\", \"set\": {\"value\": \"abc\"}}}]}}}}" + patchResourceAsAdmin(oc, ns, routeResource, specialHeaderName2) + ensureHaproxyBlockConfigContains(oc, routerpod, routeBackend, []string{"http-request set-header 'aabbccdd'\\''ee' 'abc'"}) + + compat_otp.By("try to patch http header with \" in the header name to a route") + specialHeaderName3 := "{\"spec\": {\"httpHeaders\": {\"actions\": {\"request\": [{\"name\": \"aabbccdd\\\"ee\", \"action\": {\"type\": \"Set\", \"set\": {\"value\": \"abc\"}}}]}}}}" + output, err = oc.AsAdmin().WithoutNamespace().Run("patch").Args("route/"+unsecsvcName, "-p", specialHeaderName3, "--type=merge", "-n", ns).Output() + o.Expect(err).To(o.HaveOccurred()) + o.Expect(output).To(o.ContainSubstring("Invalid value: \"aabbccdd\\\"ee\": name must be a valid HTTP header name")) + + compat_otp.By("patch http header with specical characters in header value to a route") + specialHeaderValues := "{\"spec\": {\"httpHeaders\": {\"actions\": {\"request\": [{\"name\": \"aabbccddee\", \"action\": {\"type\": \"Set\", \"set\": {\"value\": \"vlalueabc #$*'\\\"cc\"}}}]}}}}" + patchResourceAsAdmin(oc, ns, routeResource, specialHeaderValues) + ensureHaproxyBlockConfigContains(oc, routerpod, routeBackend, []string{"http-request set-header 'aabbccddee' 'vlalueabc #$*'\\''\"cc'"}) + + compat_otp.By("patch http headers with valid number, alphabet, a combination of both header names and header values to an ingress controller, then check the added headers in haproxy.conf") + patchResourceAsAdmin(oc, ingctrl.namespace, ingctrlResource, validHeaders) + ensureRouterDeployGenerationIs(oc, ingctrl.name, "2") + routerpod = getOneNewRouterPodFromRollingUpdate(oc, ingctrl.name) + ensureHaproxyBlockConfigContains(oc, routerpod, "frontend fe_sni", []string{"test01"}) + for _, backend := range []string{"frontend public", "frontend fe_sni", "frontend fe_no_sni"} { + routeBackendCfg = getBlockConfig(oc, routerpod, backend) + o.Expect(strings.Contains(routeBackendCfg, "http-request set-header '001' '20230906'")).To(o.BeTrue()) + o.Expect(strings.Contains(routeBackendCfg, "http-request set-header 'aBc' 'Wednesday'")).To(o.BeTrue()) + o.Expect(strings.Contains(routeBackendCfg, "http-request set-header 'test01' 'value01'")).To(o.BeTrue()) + } + + compat_otp.By("patch http header with blank value in the header name to an ingress controller") + output, err = oc.AsAdmin().WithoutNamespace().Run("patch").Args(ingctrlResource, "-p", blankHeaderName, "--type=merge", "-n", ingctrl.namespace).Output() + o.Expect(err).To(o.HaveOccurred()) + e2e.Logf("blanck output is: %v", output) + o.Expect(output).To(o.ContainSubstring("Invalid value: \"aa bb\"")) + + compat_otp.By("patch http header with #$* in the header name to an ingress controller") + patchResourceAsAdmin(oc, ingctrl.namespace, ingctrlResource, specialHeaderName1) + ensureRouterDeployGenerationIs(oc, ingctrl.name, "3") + routerpod = getOneNewRouterPodFromRollingUpdate(oc, ingctrl.name) + ensureHaproxyBlockConfigContains(oc, routerpod, "frontend fe_sni", []string{"aabbccdd"}) + for _, backend := range []string{"frontend public", "frontend fe_sni", "frontend fe_no_sni"} { + routeBackendCfg = getBlockConfig(oc, routerpod, backend) + o.Expect(strings.Contains(routeBackendCfg, "http-request set-header 'aabbccdd#$*ee' 'abc'")).To(o.BeTrue()) + } + + compat_otp.By("patch http header with ' in the header name to an ingress controller") + patchResourceAsAdmin(oc, ingctrl.namespace, ingctrlResource, specialHeaderName2) + ensureRouterDeployGenerationIs(oc, ingctrl.name, "4") + routerpod = getOneNewRouterPodFromRollingUpdate(oc, ingctrl.name) + ensureHaproxyBlockConfigContains(oc, routerpod, "frontend fe_sni", []string{"aabbccdd"}) + for _, backend := range []string{"frontend public", "frontend fe_sni", "frontend fe_no_sni"} { + routeBackendCfg = getBlockConfig(oc, routerpod, backend) + o.Expect(strings.Contains(routeBackendCfg, "http-request set-header 'aabbccdd'\\''ee' 'abc'")).To(o.BeTrue()) + } + + compat_otp.By("patch http header with \" in the header name to an ingress controller") + output, err = oc.AsAdmin().WithoutNamespace().Run("patch").Args("route/"+unsecsvcName, "-p", specialHeaderName3, "--type=merge", "-n", ns).Output() + o.Expect(err).To(o.HaveOccurred()) + o.Expect(output).To(o.ContainSubstring("Invalid value: \"aabbccdd\\\"ee\"")) + + compat_otp.By("patch http header with specical characters in header value to an ingress controller") + patchResourceAsAdmin(oc, ingctrl.namespace, ingctrlResource, specialHeaderValues) + ensureRouterDeployGenerationIs(oc, ingctrl.name, "5") + routerpod = getOneNewRouterPodFromRollingUpdate(oc, ingctrl.name) + ensureHaproxyBlockConfigContains(oc, routerpod, "frontend fe_sni", []string{"aabbccdd"}) + for _, backend := range []string{"frontend public", "frontend fe_sni", "frontend fe_no_sni"} { + routeBackendCfg = getBlockConfig(oc, routerpod, backend) + o.Expect(strings.Contains(routeBackendCfg, "http-request set-header 'aabbccddee' 'vlalueabc #$*'\\''\"cc'")).To(o.BeTrue()) + } + }) + + // author: shudili@redhat.com + g.It("Author:shudili-ROSA-OSD_CCS-ARO-ConnectedOnly-High-66572-adding/deleting http headers to a http route by an ingress-controller as a cluster administrator", func() { + var ( + buildPruningBaseDir = testdata.FixturePath("router") + customTemp = filepath.Join(buildPruningBaseDir, "ingresscontroller-np.yaml") + testPodSvc = filepath.Join(buildPruningBaseDir, "httpbin-deploy.yaml") + unsecsvcName = "httpbin-svc-insecure" + clientPod = filepath.Join(buildPruningBaseDir, "test-client-pod.yaml") + clientPodName = "hello-pod" + clientPodLabel = "app=hello-pod" + srv = "gunicorn" + ingctrl = ingressControllerDescription{ + name: "ocp66572", + namespace: "openshift-ingress-operator", + domain: "", + template: customTemp, + } + ingctrlResource = "ingresscontroller/" + ingctrl.name + ) + + compat_otp.By("Create a custom ingresscontroller") + baseDomain := getBaseDomain(oc) + ingctrl.domain = ingctrl.name + "." + baseDomain + defer ingctrl.delete(oc) + ingctrl.create(oc) + ensureRouterDeployGenerationIs(oc, ingctrl.name, "1") + + compat_otp.By("Create a client pod, a backend pod and its service resources") + ns := oc.Namespace() + createResourceFromFile(oc, ns, clientPod) + ensurePodWithLabelReady(oc, ns, clientPodLabel) + createResourceFromFile(oc, ns, testPodSvc) + ensurePodWithLabelReady(oc, ns, "name=httpbin-pod") + + compat_otp.By("Expose a route with the unsecure service inside the namespace") + routeHost := "service-unsecure66572" + "." + ingctrl.domain + lowHost := strings.ToLower(routeHost) + base64Host := base64.StdEncoding.EncodeToString([]byte(routeHost)) + err := oc.Run("expose").Args("svc/"+unsecsvcName, "--hostname="+routeHost).Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + routeOutput := getRoutes(oc, ns) + o.Expect(routeOutput).To(o.ContainSubstring(unsecsvcName)) + + compat_otp.By("Patch added/deleted http request/response headers to the custom ingress-controller") + patchHeaders := "{\"spec\": {\"httpHeaders\": {\"actions\": {\"request\": [" + + "{\"name\": \"X-SSL-Client-Cert\", \"action\": {\"type\": \"Set\", \"set\": {\"value\": \"%{+Q}[ssl_c_der,base64]\"}}}," + + "{\"name\": \"X-Target\", \"action\": {\"type\": \"Set\", \"set\": {\"value\": \"%[req.hdr(host),lower]\"}}}," + + "{\"name\": \"reqTestHost1\", \"action\": {\"type\": \"Set\", \"set\": {\"value\": \"%[req.hdr(host),lower]\"}}}," + + "{\"name\": \"reqTestHost2\", \"action\": {\"type\": \"Set\", \"set\": {\"value\": \"%[req.hdr(host),base64]\"}}}," + + "{\"name\": \"reqTestHost3\", \"action\": {\"type\": \"Set\", \"set\": {\"value\": \"%[req.hdr(Host)]\"}}}," + + "{\"name\": \"X-Forwarded-For\", \"action\": {\"type\": \"Set\", \"set\": {\"value\": \"11.22.33.44\"}}}," + + "{\"name\": \"x-forwarded-client-cert\", \"action\": {\"type\": \"Set\", \"set\": {\"value\": \"%{+Q}[ssl_c_der,base64]\"}}}," + + "{\"name\": \"reqTestHeader\", \"action\": {\"type\": \"Set\", \"set\": {\"value\": \"bbb\"}}}," + + "{\"name\": \"cache-control\", \"action\": {\"type\": \"Set\", \"set\": {\"value\": \"private\"}}}," + + "{\"name\": \"Referer\", \"action\": {\"type\": \"Delete\"}}" + + "]," + + "\"response\": [" + + "{\"name\": \"X-SSL-Server-Cert\", \"action\": {\"type\": \"Set\", \"set\": {\"value\": \"%{+Q}[ssl_c_der,base64]\"}}}," + + "{\"name\": \"X-XSS-Protection\", \"action\": {\"type\": \"Set\", \"set\": {\"value\": \"1; mode=block\"}}}," + + "{\"name\": \"X-Content-Type-Options\", \"action\": {\"type\": \"Set\", \"set\": {\"value\": \"nosniff`\"}}}," + + "{\"name\": \"X-Frame-Options\", \"action\": {\"type\": \"Set\", \"set\": {\"value\": \"SAMEORIGIN\"}}}," + + "{\"name\": \"resTestServer1\", \"action\": {\"type\": \"Set\", \"set\": {\"value\": \"%[res.hdr(server),lower]\"}}}," + + "{\"name\": \"resTestServer2\", \"action\": {\"type\": \"Set\", \"set\": {\"value\": \"%[res.hdr(server),base64]\"}}}," + + "{\"name\": \"resTestServer3\", \"action\": {\"type\": \"Set\", \"set\": {\"value\": \"%[res.hdr(server)]\"}}}," + + "{\"name\": \"cache-control\", \"action\": {\"type\": \"Set\", \"set\": {\"value\": \"private\"}}}," + + "{\"name\": \"server\", \"action\": {\"type\": \"Delete\"}}" + + "]}}}}" + patchResourceAsAdmin(oc, ingctrl.namespace, ingctrlResource, patchHeaders) + ensureRouterDeployGenerationIs(oc, ingctrl.name, "2") + + compat_otp.By("check the configured added/deleted headers under public frontend, fe_sni frontend and fe_no_sni frontend in haproxy") + routerpod := getOneNewRouterPodFromRollingUpdate(oc, ingctrl.name) + ensureHaproxyBlockConfigContains(oc, routerpod, "frontend fe_sni", []string{"X-SSL-Client-Cert"}) + for _, backend := range []string{"frontend public", "frontend fe_sni", "frontend fe_no_sni"} { + haproxyBackendCfg := getBlockConfig(oc, routerpod, backend) + o.Expect(strings.Contains(haproxyBackendCfg, "http-request set-header 'X-SSL-Client-Cert' '%{+Q}[ssl_c_der,base64]'")).To(o.BeTrue()) + o.Expect(strings.Contains(haproxyBackendCfg, "http-request set-header 'X-Target' '%[req.hdr(host),lower]'")).To(o.BeTrue()) + o.Expect(strings.Contains(haproxyBackendCfg, "http-request set-header 'reqTestHost1' '%[req.hdr(host),lower]'")).To(o.BeTrue()) + o.Expect(strings.Contains(haproxyBackendCfg, "http-request set-header 'reqTestHost2' '%[req.hdr(host),base64]'")).To(o.BeTrue()) + o.Expect(strings.Contains(haproxyBackendCfg, "http-request set-header 'X-Forwarded-For' '11.22.33.44'")).To(o.BeTrue()) + o.Expect(strings.Contains(haproxyBackendCfg, "http-request set-header 'x-forwarded-client-cert' '%{+Q}[ssl_c_der,base64]'")).To(o.BeTrue()) + o.Expect(strings.Contains(haproxyBackendCfg, "http-request set-header 'reqTestHeader' 'bbb'")).To(o.BeTrue()) + o.Expect(strings.Contains(haproxyBackendCfg, "http-request set-header 'cache-control' 'private'")).To(o.BeTrue()) + o.Expect(strings.Contains(haproxyBackendCfg, "http-request del-header 'Referer'")).To(o.BeTrue()) + + o.Expect(strings.Contains(haproxyBackendCfg, "http-response set-header 'X-SSL-Server-Cert' '%{+Q}[ssl_c_der,base64]'")).To(o.BeTrue()) + o.Expect(strings.Contains(haproxyBackendCfg, "http-response set-header 'X-XSS-Protection' '1; mode=block'")).To(o.BeTrue()) + o.Expect(strings.Contains(haproxyBackendCfg, "http-response set-header 'X-Content-Type-Options' 'nosniff`'")).To(o.BeTrue()) + o.Expect(strings.Contains(haproxyBackendCfg, "http-response set-header 'X-Frame-Options' 'SAMEORIGIN'")).To(o.BeTrue()) + o.Expect(strings.Contains(haproxyBackendCfg, "http-response set-header 'resTestServer1' '%[res.hdr(server),lower]'")).To(o.BeTrue()) + o.Expect(strings.Contains(haproxyBackendCfg, "http-response set-header 'resTestServer2' '%[res.hdr(server),base64]'")).To(o.BeTrue()) + o.Expect(strings.Contains(haproxyBackendCfg, "http-response set-header 'resTestServer3' '%[res.hdr(server)]'")).To(o.BeTrue()) + o.Expect(strings.Contains(haproxyBackendCfg, "http-response set-header 'cache-control' 'private'")).To(o.BeTrue()) + o.Expect(strings.Contains(haproxyBackendCfg, "http-response del-header 'server'")).To(o.BeTrue()) + } + + compat_otp.By("send traffic to the edge route, then check http headers in the request or response message") + podIP := getPodv4Address(oc, routerpod, "openshift-ingress") + routeDst := routeHost + ":80:" + podIP + curlHTTPRouteReq := []string{"-n", ns, clientPodName, "--", "curl", "http://" + routeHost + "/headers", "-v", "-e", "www.qe-test.com", "--resolve", routeDst, "--connect-timeout", "10"} + curlHTTPRouteRes := []string{"-n", ns, clientPodName, "--", "curl", "http://" + routeHost + "/headers", "-I", "-e", "www.qe-test.com", "--resolve", routeDst, "--connect-timeout", "10"} + lowSrv := strings.ToLower(srv) + base64Srv := base64.StdEncoding.EncodeToString([]byte(srv)) + repeatCmdOnClient(oc, curlHTTPRouteRes, "200", 60, 1) + reqHeaders, err := oc.AsAdmin().Run("exec").Args(curlHTTPRouteReq...).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + e2e.Logf("reqHeaders is: %v", reqHeaders) + o.Expect(strings.Contains(reqHeaders, "\"X-Ssl-Client-Cert\": \"\"")).To(o.BeTrue()) + o.Expect(strings.Contains(reqHeaders, "\"X-Target\": \""+routeHost+"\"")).To(o.BeTrue()) + o.Expect(strings.Contains(reqHeaders, "\"Reqtesthost1\": \""+lowHost+"\"")).To(o.BeTrue()) + o.Expect(strings.Contains(reqHeaders, "\"Reqtesthost2\": \""+base64Host+"\"")).To(o.BeTrue()) + o.Expect(strings.Contains(reqHeaders, "\"Reqtesthost3\": \""+routeHost+"\"")).To(o.BeTrue()) + o.Expect(strings.Contains(reqHeaders, "\"Reqtestheader\": \"bbb\"")).To(o.BeTrue()) + o.Expect(strings.Contains(reqHeaders, "\"Cache-Control\": \"private\"")).To(o.BeTrue()) + o.Expect(strings.Contains(reqHeaders, "x-ssl-client-der")).NotTo(o.BeTrue()) + + resHeaders, err := oc.AsAdmin().Run("exec").Args(curlHTTPRouteRes...).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + e2e.Logf("resHeaders is: %v", resHeaders) + o.Expect(strings.Contains(resHeaders, "x-ssl-server-cert: ")).To(o.BeTrue()) + o.Expect(strings.Contains(resHeaders, "x-xss-protection: 1; mode=block")).To(o.BeTrue()) + o.Expect(strings.Contains(resHeaders, "x-content-type-options: nosniff")).To(o.BeTrue()) + o.Expect(strings.Contains(resHeaders, "x-frame-options: SAMEORIGIN")).To(o.BeTrue()) + o.Expect(strings.Contains(resHeaders, "restestserver1: "+lowSrv)).To(o.BeTrue()) + o.Expect(strings.Contains(resHeaders, "restestserver2: "+base64Srv)).To(o.BeTrue()) + o.Expect(strings.Contains(resHeaders, "restestserver3: "+srv)).To(o.BeTrue()) + o.Expect(strings.Contains(resHeaders, "cache-control: private")).To(o.BeTrue()) + o.Expect(strings.Contains(reqHeaders, "server:")).NotTo(o.BeTrue()) + }) + + // author: shudili@redhat.com + g.It("Author:shudili-ROSA-OSD_CCS-ARO-ConnectedOnly-High-66662-adding/deleting http headers to a reen route by a router owner", func() { + var ( + buildPruningBaseDir = testdata.FixturePath("router") + customTemp = filepath.Join(buildPruningBaseDir, "ingresscontroller-np.yaml") + serverPod = filepath.Join(buildPruningBaseDir, "httpbin-pod-withprivilege.json") + secsvc = filepath.Join(buildPruningBaseDir, "httpbin-service_secure.json") + clientPod = filepath.Join(buildPruningBaseDir, "test-client-pod.yaml") + secsvcName = "httpbin-svc-secure" + clientPodName = "hello-pod" + clientPodLabel = "app=hello-pod" + srv = "gunicorn" + srvCert = "/src/example_wildcard_chain.pem" + srvKey = "/src/example_wildcard.key" + ingctrl = ingressControllerDescription{ + name: "ocp66662", + namespace: "openshift-ingress-operator", + domain: "", + template: customTemp, + } + ingctrlResource = "ingresscontroller/" + ingctrl.name + podFileDir = "/data/OCP-66662-ca" + podCaCert = podFileDir + "/66662-ca.pem" + podCustomKey = podFileDir + "/user66662.key" + podCustomCert = podFileDir + "/user66662.pem" + fileDir = "/tmp/OCP-66662-ca" + dirname = "/tmp/OCP-66662-ca/" + name = dirname + "66662" + validity = 30 + caSubj = "/CN=NE-Test-Root-CA" + userCert = dirname + "user66662" + customKey = userCert + ".key" + customCert = userCert + ".pem" + destSubj = "/CN=*.edge.example.com" + destCA = dirname + "dst.pem" + destKey = dirname + "dst.key" + destCsr = dirname + "dst.csr" + destCnf = dirname + "openssl.cnf" + ) + + defer os.RemoveAll(dirname) + err := os.MkdirAll(dirname, 0755) + o.Expect(err).NotTo(o.HaveOccurred()) + baseDomain := getBaseDomain(oc) + ingctrl.domain = ingctrl.name + "." + baseDomain + ns := oc.Namespace() + compat_otp.SetNamespacePrivileged(oc, ns) + + compat_otp.By("Try to create custom key and custom certification by openssl, create a new self-signed CA at first, creating the CA key") + opensslCmd := fmt.Sprintf(`openssl genrsa -out %s-ca.key 2048`, name) + _, err = exec.Command("bash", "-c", opensslCmd).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + + compat_otp.By("Create the CA certificate") + opensslCmd = fmt.Sprintf(`openssl req -x509 -new -nodes -key %s-ca.key -sha256 -days %d -out %s-ca.pem -subj %s`, name, validity, name, caSubj) + _, err = exec.Command("bash", "-c", opensslCmd).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + + compat_otp.By("Create a new user certificate, crearing the user CSR with the private user key") + userSubj := "/CN=example-ne.com" + opensslCmd = fmt.Sprintf(`openssl req -nodes -newkey rsa:2048 -keyout %s.key -subj %s -out %s.csr`, userCert, userSubj, userCert) + _, err = exec.Command("bash", "-c", opensslCmd).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + + compat_otp.By("Sign the user CSR and generate the certificate") + opensslCmd = fmt.Sprintf(`openssl x509 -extfile <(printf "subjectAltName = DNS:*.`+ingctrl.domain+`") -req -in %s.csr -CA %s-ca.pem -CAkey %s-ca.key -CAcreateserial -out %s.pem -days %d -sha256`, userCert, name, name, userCert, validity) + _, err = exec.Command("bash", "-c", opensslCmd).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + + compat_otp.By("Create the destination Certification for the reencrypt route, create the key") + opensslCmd = fmt.Sprintf(`openssl genrsa -out %s 2048`, destKey) + _, err = exec.Command("bash", "-c", opensslCmd).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + + compat_otp.By("Create the csr for the destination Certification") + opensslCmd = fmt.Sprintf(`openssl req -new -key %s -subj %s -out %s`, destKey, destSubj, destCsr) + _, err = exec.Command("bash", "-c", opensslCmd).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + + compat_otp.By("1.3: Create the extension file, then create the destination certification") + sanCfg := fmt.Sprintf(` +[ v3_req ] +subjectAltName = @alt_names + +[ alt_names ] +DNS.1 = *.edge.example.com +DNS.2 = *.%s.%s.svc +`, secsvcName, ns) + + cmd := fmt.Sprintf(`echo "%s" > %s`, sanCfg, destCnf) + _, err = exec.Command("bash", "-c", cmd).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + opensslCmd = fmt.Sprintf(`openssl x509 -extfile %s -extensions v3_req -req -in %s -signkey %s -days %d -sha256 -out %s`, destCnf, destCsr, destKey, validity, destCA) + _, err = exec.Command("bash", "-c", opensslCmd).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + + compat_otp.By("Create a custom ingresscontroller") + defer ingctrl.delete(oc) + ingctrl.create(oc) + ensureRouterDeployGenerationIs(oc, ingctrl.name, "1") + + compat_otp.By("create configmap client-ca-xxxxx in namespace openshift-config") + cmFile := "ca-bundle.pem=" + name + "-ca.pem" + defer deleteConfigMap(oc, "openshift-config", "client-ca-"+ingctrl.name) + createConfigMapFromFile(oc, "openshift-config", "client-ca-"+ingctrl.name, cmFile) + + compat_otp.By("patch the ingresscontroller to enable client certificate with required policy") + patchResourceAsAdmin(oc, ingctrl.namespace, ingctrlResource, "{\"spec\":{\"clientTLS\":{\"clientCA\":{\"name\":\"client-ca-"+ingctrl.name+"\"},\"clientCertificatePolicy\":\"Required\"}}}") + ensureRouterDeployGenerationIs(oc, ingctrl.name, "2") + routerpod := getOneNewRouterPodFromRollingUpdate(oc, ingctrl.name) + podIP := getPodv4Address(oc, routerpod, "openshift-ingress") + + compat_otp.By("Create a client pod, a backend pod and its service resources") + err = oc.AsAdmin().WithoutNamespace().Run("create").Args("-n", ns, "-f", clientPod).Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + ensurePodWithLabelReady(oc, ns, clientPodLabel) + err = oc.AsAdmin().WithoutNamespace().Run("cp").Args("-n", ns, fileDir, ns+"/"+clientPodName+":"+podFileDir).Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + operateResourceFromFile(oc, "create", ns, serverPod) + ensurePodWithLabelReady(oc, ns, "name=httpbin-pod") + createResourceFromFile(oc, ns, secsvc) + + compat_otp.By("Update the certification and key in the server pod") + podName := getPodListByLabel(oc, ns, "name=httpbin-pod") + newSrvCert := ns + "/" + podName[0] + ":" + srvCert + newSrvKey := ns + "/" + podName[0] + ":" + srvKey + _, err = oc.AsAdmin().WithoutNamespace().Run("exec").Args("-n", ns, podName[0], "-c", "httpbin-https", "--", "bash", "-c", "rm -f "+srvCert).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + _, err = oc.AsAdmin().WithoutNamespace().Run("exec").Args("-n", ns, podName[0], "-c", "httpbin-https", "--", "bash", "-c", "rm -f "+srvKey).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + err = oc.AsAdmin().WithoutNamespace().Run("cp").Args("-n", ns, destCA, "-c", "httpbin-https", newSrvCert).Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + err = oc.AsAdmin().WithoutNamespace().Run("cp").Args("-n", ns, destKey, "-c", "httpbin-https", newSrvKey).Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + + compat_otp.By("create a reen route") + reenRouteHost := "r2-reen66662." + ingctrl.domain + lowHostReen := strings.ToLower(reenRouteHost) + base64HostReen := base64.StdEncoding.EncodeToString([]byte(reenRouteHost)) + reenRouteDst := reenRouteHost + ":443:" + podIP + err = oc.AsAdmin().WithoutNamespace().Run("create").Args("-n", ns, "route", "reencrypt", "r2-reen", "--service="+secsvcName, "--cert="+customCert, "--key="+customKey, "--ca-cert="+name+"-ca.pem", "--dest-ca-cert="+destCA, "--hostname="+reenRouteHost).Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + + compat_otp.By("Patch the reen route with added/deleted http request/response headers under the spec") + patchHeaders := "{\"spec\": {\"httpHeaders\": {\"actions\": {\"request\": [" + + "{\"name\": \"X-SSL-Client-Cert\", \"action\": {\"type\": \"Set\", \"set\": {\"value\": \"%{+Q}[ssl_c_der,base64]\"}}}," + + "{\"name\": \"X-Target\", \"action\": {\"type\": \"Set\", \"set\": {\"value\": \"%[req.hdr(host),lower]\"}}}," + + "{\"name\": \"reqTestHost1\", \"action\": {\"type\": \"Set\", \"set\": {\"value\": \"%[req.hdr(host),lower]\"}}}," + + "{\"name\": \"reqTestHost2\", \"action\": {\"type\": \"Set\", \"set\": {\"value\": \"%[req.hdr(host),base64]\"}}}," + + "{\"name\": \"reqTestHost3\", \"action\": {\"type\": \"Set\", \"set\": {\"value\": \"%[req.hdr(Host)]\"}}}," + + "{\"name\": \"X-Forwarded-For\", \"action\": {\"type\": \"Set\", \"set\": {\"value\": \"11.22.33.44\"}}}," + + "{\"name\": \"x-forwarded-client-cert\", \"action\": {\"type\": \"Set\", \"set\": {\"value\": \"%{+Q}[ssl_c_der,base64]\"}}}," + + "{\"name\": \"reqTestHeader\", \"action\": {\"type\": \"Set\", \"set\": {\"value\": \"bbb\"}}}," + + "{\"name\": \"cache-control\", \"action\": {\"type\": \"Set\", \"set\": {\"value\": \"private\"}}}," + + "{\"name\": \"x-ssl-client-der\", \"action\": {\"type\": \"Delete\"}}" + + "]," + + "\"response\": [" + + "{\"name\": \"X-SSL-Server-Cert\", \"action\": {\"type\": \"Set\", \"set\": {\"value\": \"%{+Q}[ssl_c_der,base64]\"}}}," + + "{\"name\": \"X-XSS-Protection\", \"action\": {\"type\": \"Set\", \"set\": {\"value\": \"1; mode=block\"}}}," + + "{\"name\": \"X-Content-Type-Options\", \"action\": {\"type\": \"Set\", \"set\": {\"value\": \"nosniff`\"}}}," + + "{\"name\": \"X-Frame-Options\", \"action\": {\"type\": \"Set\", \"set\": {\"value\": \"SAMEORIGIN\"}}}," + + "{\"name\": \"resTestServer1\", \"action\": {\"type\": \"Set\", \"set\": {\"value\": \"%[res.hdr(server),lower]\"}}}," + + "{\"name\": \"resTestServer2\", \"action\": {\"type\": \"Set\", \"set\": {\"value\": \"%[res.hdr(server),base64]\"}}}," + + "{\"name\": \"resTestServer3\", \"action\": {\"type\": \"Set\", \"set\": {\"value\": \"%[res.hdr(server)]\"}}}," + + "{\"name\": \"cache-control\", \"action\": {\"type\": \"Set\", \"set\": {\"value\": \"private\"}}}," + + "{\"name\": \"server\", \"action\": {\"type\": \"Delete\"}}" + + "]}}}}" + + output, err := oc.AsAdmin().WithoutNamespace().Run("patch").Args("route/r2-reen", "-p", patchHeaders, "--type=merge", "-n", ns).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(output).To(o.ContainSubstring("patched")) + + compat_otp.By("check backend reen route in haproxy that headers to be set or deleted") + routeBackendCfg := ensureHaproxyBlockConfigContains(oc, routerpod, "be_secure:"+ns+":r2-reen", []string{"X-SSL-Client-Cert"}) + o.Expect(strings.Contains(routeBackendCfg, "http-request set-header 'X-SSL-Client-Cert' '%{+Q}[ssl_c_der,base64]'")).To(o.BeTrue()) + o.Expect(strings.Contains(routeBackendCfg, "http-request set-header 'X-Target' '%[req.hdr(host),lower]'")).To(o.BeTrue()) + o.Expect(strings.Contains(routeBackendCfg, "http-request set-header 'reqTestHost1' '%[req.hdr(host),lower]'")).To(o.BeTrue()) + o.Expect(strings.Contains(routeBackendCfg, "http-request set-header 'reqTestHost2' '%[req.hdr(host),base64]'")).To(o.BeTrue()) + o.Expect(strings.Contains(routeBackendCfg, "http-request set-header 'X-Forwarded-For' '11.22.33.44'")).To(o.BeTrue()) + o.Expect(strings.Contains(routeBackendCfg, "http-request set-header 'x-forwarded-client-cert' '%{+Q}[ssl_c_der,base64]'")).To(o.BeTrue()) + o.Expect(strings.Contains(routeBackendCfg, "http-request set-header 'reqTestHeader' 'bbb'")).To(o.BeTrue()) + o.Expect(strings.Contains(routeBackendCfg, "http-request set-header 'cache-control' 'private'")).To(o.BeTrue()) + o.Expect(strings.Contains(routeBackendCfg, "http-request del-header 'x-ssl-client-der'")).To(o.BeTrue()) + o.Expect(strings.Contains(routeBackendCfg, "http-request del-header 'x-ssl-client-der'")).To(o.BeTrue()) + + o.Expect(strings.Contains(routeBackendCfg, "http-response set-header 'X-SSL-Server-Cert' '%{+Q}[ssl_c_der,base64]'")).To(o.BeTrue()) + o.Expect(strings.Contains(routeBackendCfg, "http-response set-header 'X-XSS-Protection' '1; mode=block'")).To(o.BeTrue()) + o.Expect(strings.Contains(routeBackendCfg, "http-response set-header 'X-Content-Type-Options' 'nosniff`'")).To(o.BeTrue()) + o.Expect(strings.Contains(routeBackendCfg, "http-response set-header 'X-Frame-Options' 'SAMEORIGIN'")).To(o.BeTrue()) + o.Expect(strings.Contains(routeBackendCfg, "http-response set-header 'resTestServer1' '%[res.hdr(server),lower]'")).To(o.BeTrue()) + o.Expect(strings.Contains(routeBackendCfg, "http-response set-header 'resTestServer2' '%[res.hdr(server),base64]'")).To(o.BeTrue()) + o.Expect(strings.Contains(routeBackendCfg, "http-response set-header 'resTestServer3' '%[res.hdr(server)]'")).To(o.BeTrue()) + o.Expect(strings.Contains(routeBackendCfg, "http-response set-header 'cache-control' 'private'")).To(o.BeTrue()) + o.Expect(strings.Contains(routeBackendCfg, "http-response del-header 'server'")).To(o.BeTrue()) + + compat_otp.By("send traffic to the reen route, then check http headers in the request or response message") + curlReenRouteReq := []string{"-n", ns, clientPodName, "--", "curl", "https://" + reenRouteHost + "/headers", "-v", "--cacert", podCaCert, "--cert", podCustomCert, "--key", podCustomKey, "--resolve", reenRouteDst, "--connect-timeout", "10"} + curlReenRouteRes := []string{"-n", ns, clientPodName, "--", "curl", "https://" + reenRouteHost + "/headers", "-I", "--cacert", podCaCert, "--cert", podCustomCert, "--key", podCustomKey, "--resolve", reenRouteDst, "--connect-timeout", "10"} + lowSrv := strings.ToLower(srv) + base64Srv := base64.StdEncoding.EncodeToString([]byte(srv)) + e2e.Logf("curlReenRouteRes is: %v", curlReenRouteRes) + repeatCmdOnClient(oc, curlReenRouteRes, "200", 60, 1) + reqHeaders, _ := oc.AsAdmin().Run("exec").Args(curlReenRouteReq...).Output() + e2e.Logf("reqHeaders is: %v", reqHeaders) + o.Expect(len(regexp.MustCompile("\"X-Ssl-Client-Cert\": \"([0-9a-zA-Z]+)").FindStringSubmatch(reqHeaders)) > 0).To(o.BeTrue()) + o.Expect(reqHeaders).To(o.ContainSubstring("\"X-Target\": \"" + reenRouteHost + "\"")) + o.Expect(reqHeaders).To(o.ContainSubstring("\"Reqtesthost1\": \"" + lowHostReen + "\"")) + o.Expect(reqHeaders).To(o.ContainSubstring("\"Reqtesthost2\": \"" + base64HostReen + "\"")) + o.Expect(reqHeaders).To(o.ContainSubstring("\"Reqtesthost3\": \"" + reenRouteHost + "\"")) + o.Expect(reqHeaders).To(o.ContainSubstring("\"Reqtestheader\": \"bbb\"")) + o.Expect(reqHeaders).To(o.ContainSubstring("\"Cache-Control\": \"private\"")) + o.Expect(strings.Contains(reqHeaders, "x-ssl-client-der")).NotTo(o.BeTrue()) + + resHeaders, _ := oc.AsAdmin().Run("exec").Args(curlReenRouteRes...).Output() + e2e.Logf("resHeaders is: %v", resHeaders) + o.Expect(len(regexp.MustCompile("x-ssl-server-cert: ([0-9a-zA-Z]+)").FindStringSubmatch(resHeaders)) > 0).To(o.BeTrue()) + o.Expect(resHeaders).To(o.ContainSubstring("x-xss-protection: 1; mode=block")) + o.Expect(resHeaders).To(o.ContainSubstring("x-content-type-options: nosniff")) + o.Expect(resHeaders).To(o.ContainSubstring("x-frame-options: SAMEORIGIN")) + o.Expect(resHeaders).To(o.ContainSubstring("restestserver1: " + lowSrv)) + o.Expect(resHeaders).To(o.ContainSubstring("restestserver2: " + base64Srv)) + o.Expect(resHeaders).To(o.ContainSubstring("restestserver3: " + srv)) + o.Expect(resHeaders).To(o.ContainSubstring("cache-control: private")) + o.Expect(strings.Contains(reqHeaders, "server:")).NotTo(o.BeTrue()) + }) + + // author: shudili@redhat.com + g.It("Author:shudili-ROSA-OSD_CCS-ARO-ConnectedOnly-High-67009-adding/deleting http headers to an edge route by an ingress-controller as a cluster administrator", func() { + var ( + buildPruningBaseDir = testdata.FixturePath("router") + customTemp = filepath.Join(buildPruningBaseDir, "ingresscontroller-np.yaml") + testPodSvc = filepath.Join(buildPruningBaseDir, "httpbin-deploy.yaml") + clientPod = filepath.Join(buildPruningBaseDir, "test-client-pod.yaml") + unsecsvcName = "httpbin-svc-insecure" + clientPodName = "hello-pod" + clientPodLabel = "app=hello-pod" + srv = "gunicorn" + ingctrl = ingressControllerDescription{ + name: "ocp67009", + namespace: "openshift-ingress-operator", + domain: "", + template: customTemp, + } + ingctrlResource = "ingresscontroller/" + ingctrl.name + podFileDir = "/data/OCP-67009-ca" + podCaCert = podFileDir + "/67009-ca.pem" + podCustomKey = podFileDir + "/user67009.key" + podCustomCert = podFileDir + "/user67009.pem" + fileDir = "/tmp/OCP-67009-ca" + dirname = "/tmp/OCP-67009-ca/" + name = dirname + "67009" + validity = 30 + caSubj = "/CN=NE-Test-Root-CA" + userCert = dirname + "user67009" + customKey = userCert + ".key" + customCert = userCert + ".pem" + ) + defer os.RemoveAll(dirname) + err := os.MkdirAll(dirname, 0755) + o.Expect(err).NotTo(o.HaveOccurred()) + baseDomain := getBaseDomain(oc) + ingctrl.domain = ingctrl.name + "." + baseDomain + + compat_otp.By("Try to create custom key and custom certification by openssl, create a new self-signed CA at first, creating the CA key") + opensslCmd := fmt.Sprintf(`openssl genrsa -out %s-ca.key 2048`, name) + _, err = exec.Command("bash", "-c", opensslCmd).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + + compat_otp.By("Create the CA certificate") + opensslCmd = fmt.Sprintf(`openssl req -x509 -new -nodes -key %s-ca.key -sha256 -days %d -out %s-ca.pem -subj %s`, name, validity, name, caSubj) + _, err = exec.Command("bash", "-c", opensslCmd).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + + compat_otp.By("Create a new user certificate, crearing the user CSR with the private user key") + userSubj := "/CN=example-ne.com" + opensslCmd = fmt.Sprintf(`openssl req -nodes -newkey rsa:2048 -keyout %s.key -subj %s -out %s.csr`, userCert, userSubj, userCert) + _, err = exec.Command("bash", "-c", opensslCmd).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + + compat_otp.By("Sign the user CSR and generate the certificate") + opensslCmd = fmt.Sprintf(`openssl x509 -extfile <(printf "subjectAltName = DNS:*.`+ingctrl.domain+`") -req -in %s.csr -CA %s-ca.pem -CAkey %s-ca.key -CAcreateserial -out %s.pem -days %d -sha256`, userCert, name, name, userCert, validity) + _, err = exec.Command("bash", "-c", opensslCmd).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + + compat_otp.By("Create a custom ingresscontroller") + defer ingctrl.delete(oc) + ingctrl.create(oc) + ensureRouterDeployGenerationIs(oc, ingctrl.name, "1") + + compat_otp.By("create configmap client-ca-xxxxx in namespace openshift-config") + cmFile := "ca-bundle.pem=" + name + "-ca.pem" + defer deleteConfigMap(oc, "openshift-config", "client-ca-"+ingctrl.name) + createConfigMapFromFile(oc, "openshift-config", "client-ca-"+ingctrl.name, cmFile) + + compat_otp.By("patch the ingresscontroller to enable client certificate with required policy") + patchResourceAsAdmin(oc, ingctrl.namespace, ingctrlResource, "{\"spec\":{\"clientTLS\":{\"clientCA\":{\"name\":\"client-ca-"+ingctrl.name+"\"},\"clientCertificatePolicy\":\"Required\"}}}") + ensureRouterDeployGenerationIs(oc, ingctrl.name, "2") + + compat_otp.By("Create a client pod, a backend pod and its service resources") + ns := oc.Namespace() + err = oc.AsAdmin().WithoutNamespace().Run("create").Args("-n", ns, "-f", clientPod).Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + ensurePodWithLabelReady(oc, ns, clientPodLabel) + err = oc.AsAdmin().WithoutNamespace().Run("cp").Args("-n", ns, fileDir, ns+"/"+clientPodName+":"+podFileDir).Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + createResourceFromFile(oc, ns, testPodSvc) + ensurePodWithLabelReady(oc, ns, "name=httpbin-pod") + + compat_otp.By("create an edge route") + edgeRouteHost := "r3-edge67009." + ingctrl.domain + lowHostEdge := strings.ToLower(edgeRouteHost) + base64HostEdge := base64.StdEncoding.EncodeToString([]byte(edgeRouteHost)) + err = oc.AsAdmin().WithoutNamespace().Run("create").Args("-n", ns, "route", "edge", "r3-edge", "--service="+unsecsvcName, "--cert="+customCert, "--key="+customKey, "--ca-cert="+name+"-ca.pem", "--hostname="+edgeRouteHost).Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + + compat_otp.By("Patch added/deleted http request/response headers to the custom ingress-controller") + patchHeaders := "{\"spec\": {\"httpHeaders\": {\"actions\": {\"request\": [" + + "{\"name\": \"X-SSL-Client-Cert\", \"action\": {\"type\": \"Set\", \"set\": {\"value\": \"%{+Q}[ssl_c_der,base64]\"}}}," + + "{\"name\": \"X-Target\", \"action\": {\"type\": \"Set\", \"set\": {\"value\": \"%[req.hdr(host),lower]\"}}}," + + "{\"name\": \"reqTestHost1\", \"action\": {\"type\": \"Set\", \"set\": {\"value\": \"%[req.hdr(host),lower]\"}}}," + + "{\"name\": \"reqTestHost2\", \"action\": {\"type\": \"Set\", \"set\": {\"value\": \"%[req.hdr(host),base64]\"}}}," + + "{\"name\": \"reqTestHost3\", \"action\": {\"type\": \"Set\", \"set\": {\"value\": \"%[req.hdr(Host)]\"}}}," + + "{\"name\": \"X-Forwarded-For\", \"action\": {\"type\": \"Set\", \"set\": {\"value\": \"11.22.33.44\"}}}," + + "{\"name\": \"x-forwarded-client-cert\", \"action\": {\"type\": \"Set\", \"set\": {\"value\": \"%{+Q}[ssl_c_der,base64]\"}}}," + + "{\"name\": \"reqTestHeader\", \"action\": {\"type\": \"Set\", \"set\": {\"value\": \"bbb\"}}}," + + "{\"name\": \"cache-control\", \"action\": {\"type\": \"Set\", \"set\": {\"value\": \"private\"}}}," + + "{\"name\": \"x-ssl-client-der\", \"action\": {\"type\": \"Delete\"}}" + + "]," + + "\"response\": [" + + "{\"name\": \"X-SSL-Server-Cert\", \"action\": {\"type\": \"Set\", \"set\": {\"value\": \"%{+Q}[ssl_c_der,base64]\"}}}," + + "{\"name\": \"X-XSS-Protection\", \"action\": {\"type\": \"Set\", \"set\": {\"value\": \"1; mode=block\"}}}," + + "{\"name\": \"X-Content-Type-Options\", \"action\": {\"type\": \"Set\", \"set\": {\"value\": \"nosniff`\"}}}," + + "{\"name\": \"X-Frame-Options\", \"action\": {\"type\": \"Set\", \"set\": {\"value\": \"SAMEORIGIN\"}}}," + + "{\"name\": \"resTestServer1\", \"action\": {\"type\": \"Set\", \"set\": {\"value\": \"%[res.hdr(server),lower]\"}}}," + + "{\"name\": \"resTestServer2\", \"action\": {\"type\": \"Set\", \"set\": {\"value\": \"%[res.hdr(server),base64]\"}}}," + + "{\"name\": \"resTestServer3\", \"action\": {\"type\": \"Set\", \"set\": {\"value\": \"%[res.hdr(server)]\"}}}," + + "{\"name\": \"cache-control\", \"action\": {\"type\": \"Set\", \"set\": {\"value\": \"private\"}}}," + + "{\"name\": \"server\", \"action\": {\"type\": \"Delete\"}}" + + "]}}}}" + + patchResourceAsAdmin(oc, ingctrl.namespace, ingctrlResource, patchHeaders) + ensureRouterDeployGenerationIs(oc, ingctrl.name, "3") + + compat_otp.By("check the configured added/deleted headers under public frontend, fe_sni frontend and fe_no_sni frontend in haproxy") + routerpod := getOneNewRouterPodFromRollingUpdate(oc, ingctrl.name) + ensureHaproxyBlockConfigContains(oc, routerpod, "frontend fe_sni", []string{"X-SSL-Client-Cert"}) + for _, backend := range []string{"frontend public", "frontend fe_sni", "frontend fe_no_sni"} { + haproxyBackendCfg := getBlockConfig(oc, routerpod, backend) + o.Expect(strings.Contains(haproxyBackendCfg, "http-request set-header 'X-SSL-Client-Cert' '%{+Q}[ssl_c_der,base64]'")).To(o.BeTrue()) + o.Expect(strings.Contains(haproxyBackendCfg, "http-request set-header 'X-Target' '%[req.hdr(host),lower]'")).To(o.BeTrue()) + o.Expect(strings.Contains(haproxyBackendCfg, "http-request set-header 'reqTestHost1' '%[req.hdr(host),lower]'")).To(o.BeTrue()) + o.Expect(strings.Contains(haproxyBackendCfg, "http-request set-header 'reqTestHost2' '%[req.hdr(host),base64]'")).To(o.BeTrue()) + o.Expect(strings.Contains(haproxyBackendCfg, "http-request set-header 'X-Forwarded-For' '11.22.33.44'")).To(o.BeTrue()) + o.Expect(strings.Contains(haproxyBackendCfg, "http-request set-header 'x-forwarded-client-cert' '%{+Q}[ssl_c_der,base64]'")).To(o.BeTrue()) + o.Expect(strings.Contains(haproxyBackendCfg, "http-request set-header 'reqTestHeader' 'bbb'")).To(o.BeTrue()) + o.Expect(strings.Contains(haproxyBackendCfg, "http-request set-header 'cache-control' 'private'")).To(o.BeTrue()) + o.Expect(strings.Contains(haproxyBackendCfg, "http-request del-header 'x-ssl-client-der'")).To(o.BeTrue()) + o.Expect(strings.Contains(haproxyBackendCfg, "http-request del-header 'x-ssl-client-der'")).To(o.BeTrue()) + + o.Expect(strings.Contains(haproxyBackendCfg, "http-response set-header 'X-SSL-Server-Cert' '%{+Q}[ssl_c_der,base64]'")).To(o.BeTrue()) + o.Expect(strings.Contains(haproxyBackendCfg, "http-response set-header 'X-XSS-Protection' '1; mode=block'")).To(o.BeTrue()) + o.Expect(strings.Contains(haproxyBackendCfg, "http-response set-header 'X-Content-Type-Options' 'nosniff`'")).To(o.BeTrue()) + o.Expect(strings.Contains(haproxyBackendCfg, "http-response set-header 'X-Frame-Options' 'SAMEORIGIN'")).To(o.BeTrue()) + o.Expect(strings.Contains(haproxyBackendCfg, "http-response set-header 'resTestServer1' '%[res.hdr(server),lower]'")).To(o.BeTrue()) + o.Expect(strings.Contains(haproxyBackendCfg, "http-response set-header 'resTestServer2' '%[res.hdr(server),base64]'")).To(o.BeTrue()) + o.Expect(strings.Contains(haproxyBackendCfg, "http-response set-header 'resTestServer3' '%[res.hdr(server)]'")).To(o.BeTrue()) + o.Expect(strings.Contains(haproxyBackendCfg, "http-response set-header 'cache-control' 'private'")).To(o.BeTrue()) + o.Expect(strings.Contains(haproxyBackendCfg, "http-response del-header 'server'")).To(o.BeTrue()) + } + + compat_otp.By("send traffic to the edge route, then check http headers in the request or response message") + podIP := getPodv4Address(oc, routerpod, "openshift-ingress") + edgeRouteDst := edgeRouteHost + ":443:" + podIP + curlEdgeRouteReq := []string{"-n", ns, clientPodName, "--", "curl", "https://" + edgeRouteHost + "/headers", "-v", "--cacert", podCaCert, "--cert", podCustomCert, "--key", podCustomKey, "--resolve", edgeRouteDst, "--connect-timeout", "10"} + curlEdgeRouteRes := []string{"-n", ns, clientPodName, "--", "curl", "https://" + edgeRouteHost + "/headers", "-I", "--cacert", podCaCert, "--cert", podCustomCert, "--key", podCustomKey, "--resolve", edgeRouteDst, "--connect-timeout", "10"} + lowSrv := strings.ToLower(srv) + base64Srv := base64.StdEncoding.EncodeToString([]byte(srv)) + repeatCmdOnClient(oc, curlEdgeRouteRes, "200", 60, 1) + reqHeaders, err := oc.AsAdmin().Run("exec").Args(curlEdgeRouteReq...).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + e2e.Logf("reqHeaders is: %v", reqHeaders) + o.Expect(len(regexp.MustCompile("\"X-Ssl-Client-Cert\": \"([0-9a-zA-Z]+)").FindStringSubmatch(reqHeaders)) > 0).To(o.BeTrue()) + o.Expect(strings.Contains(reqHeaders, "\"X-Target\": \""+edgeRouteHost+"\"")).To(o.BeTrue()) + o.Expect(strings.Contains(reqHeaders, "\"Reqtesthost1\": \""+lowHostEdge+"\"")).To(o.BeTrue()) + o.Expect(strings.Contains(reqHeaders, "\"Reqtesthost2\": \""+base64HostEdge+"\"")).To(o.BeTrue()) + o.Expect(strings.Contains(reqHeaders, "\"Reqtesthost3\": \""+edgeRouteHost+"\"")).To(o.BeTrue()) + o.Expect(strings.Contains(reqHeaders, "\"Reqtestheader\": \"bbb\"")).To(o.BeTrue()) + o.Expect(strings.Contains(reqHeaders, "\"Cache-Control\": \"private\"")).To(o.BeTrue()) + o.Expect(strings.Contains(reqHeaders, "x-ssl-client-der")).NotTo(o.BeTrue()) + + resHeaders, err := oc.AsAdmin().Run("exec").Args(curlEdgeRouteRes...).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + e2e.Logf("resHeaders is: %v", resHeaders) + o.Expect(len(regexp.MustCompile("x-ssl-server-cert: ([0-9a-zA-Z]+)").FindStringSubmatch(resHeaders)) > 0).To(o.BeTrue()) + o.Expect(strings.Contains(resHeaders, "x-xss-protection: 1; mode=block")).To(o.BeTrue()) + o.Expect(strings.Contains(resHeaders, "x-content-type-options: nosniff")).To(o.BeTrue()) + o.Expect(strings.Contains(resHeaders, "x-frame-options: SAMEORIGIN")).To(o.BeTrue()) + o.Expect(strings.Contains(resHeaders, "restestserver1: "+lowSrv)).To(o.BeTrue()) + o.Expect(strings.Contains(resHeaders, "restestserver2: "+base64Srv)).To(o.BeTrue()) + o.Expect(strings.Contains(resHeaders, "restestserver3: "+srv)).To(o.BeTrue()) + o.Expect(strings.Contains(resHeaders, "cache-control: private")).To(o.BeTrue()) + o.Expect(strings.Contains(reqHeaders, "server:")).NotTo(o.BeTrue()) + }) + + // author: shudili@redhat.com + g.It("Author:shudili-ROSA-OSD_CCS-ARO-ConnectedOnly-High-67010-adding/deleting http headers to a reen route by an ingress-controller as a cluster administrator", func() { + var ( + buildPruningBaseDir = testdata.FixturePath("router") + customTemp = filepath.Join(buildPruningBaseDir, "ingresscontroller-np.yaml") + serverPod = filepath.Join(buildPruningBaseDir, "httpbin-pod-withprivilege.json") + secsvc = filepath.Join(buildPruningBaseDir, "httpbin-service_secure.json") + clientPod = filepath.Join(buildPruningBaseDir, "test-client-pod.yaml") + secsvcName = "httpbin-svc-secure" + clientPodName = "hello-pod" + clientPodLabel = "app=hello-pod" + srv = "gunicorn" + srvCert = "/src/example_wildcard_chain.pem" + srvKey = "/src/example_wildcard.key" + ingctrl = ingressControllerDescription{ + name: "ocp67010", + namespace: "openshift-ingress-operator", + domain: "", + template: customTemp, + } + ingctrlResource = "ingresscontroller/" + ingctrl.name + podFileDir = "/data/OCP-67010-ca" + podCaCert = podFileDir + "/67010-ca.pem" + podCustomKey = podFileDir + "/user67010.key" + podCustomCert = podFileDir + "/user67010.pem" + fileDir = "/tmp/OCP-67010-ca" + dirname = "/tmp/OCP-67010-ca/" + name = dirname + "67010" + validity = 30 + caSubj = "/CN=NE-Test-Root-CA" + userCert = dirname + "user67010" + customKey = userCert + ".key" + customCert = userCert + ".pem" + destSubj = "/CN=*.edge.example.com" + destCA = dirname + "dst.pem" + destKey = dirname + "dst.key" + destCsr = dirname + "dst.csr" + destCnf = dirname + "openssl.cnf" + ) + + defer os.RemoveAll(dirname) + err := os.MkdirAll(dirname, 0755) + o.Expect(err).NotTo(o.HaveOccurred()) + baseDomain := getBaseDomain(oc) + ingctrl.domain = ingctrl.name + "." + baseDomain + ns := oc.Namespace() + compat_otp.SetNamespacePrivileged(oc, ns) + + compat_otp.By("Try to create custom key and custom certification by openssl, create a new self-signed CA at first, creating the CA key") + opensslCmd := fmt.Sprintf(`openssl genrsa -out %s-ca.key 2048`, name) + _, err = exec.Command("bash", "-c", opensslCmd).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + + compat_otp.By("Create the CA certificate") + opensslCmd = fmt.Sprintf(`openssl req -x509 -new -nodes -key %s-ca.key -sha256 -days %d -out %s-ca.pem -subj %s`, name, validity, name, caSubj) + _, err = exec.Command("bash", "-c", opensslCmd).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + + compat_otp.By("Create a new user certificate, crearing the user CSR with the private user key") + userSubj := "/CN=example-ne.com" + opensslCmd = fmt.Sprintf(`openssl req -nodes -newkey rsa:2048 -keyout %s.key -subj %s -out %s.csr`, userCert, userSubj, userCert) + _, err = exec.Command("bash", "-c", opensslCmd).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + + compat_otp.By("Sign the user CSR and generate the certificate") + opensslCmd = fmt.Sprintf(`openssl x509 -extfile <(printf "subjectAltName = DNS:*.`+ingctrl.domain+`") -req -in %s.csr -CA %s-ca.pem -CAkey %s-ca.key -CAcreateserial -out %s.pem -days %d -sha256`, userCert, name, name, userCert, validity) + _, err = exec.Command("bash", "-c", opensslCmd).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + + compat_otp.By("Create the destination Certification for the reencrypt route, create the key") + opensslCmd = fmt.Sprintf(`openssl genrsa -out %s 2048`, destKey) + _, err = exec.Command("bash", "-c", opensslCmd).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + + compat_otp.By("Create the csr for the destination Certification") + opensslCmd = fmt.Sprintf(`openssl req -new -key %s -subj %s -out %s`, destKey, destSubj, destCsr) + _, err = exec.Command("bash", "-c", opensslCmd).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + + compat_otp.By("1.3: Create the extension file, then create the destination certification") + sanCfg := fmt.Sprintf(` +[ v3_req ] +subjectAltName = @alt_names + +[ alt_names ] +DNS.1 = *.edge.example.com +DNS.2 = *.%s.%s.svc +`, secsvcName, ns) + + cmd := fmt.Sprintf(`echo "%s" > %s`, sanCfg, destCnf) + _, err = exec.Command("bash", "-c", cmd).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + opensslCmd = fmt.Sprintf(`openssl x509 -extfile %s -extensions v3_req -req -in %s -signkey %s -days %d -sha256 -out %s`, destCnf, destCsr, destKey, validity, destCA) + _, err = exec.Command("bash", "-c", opensslCmd).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + + compat_otp.By("Create a custom ingresscontroller") + defer ingctrl.delete(oc) + ingctrl.create(oc) + ensureRouterDeployGenerationIs(oc, ingctrl.name, "1") + + compat_otp.By("create configmap client-ca-xxxxx in namespace openshift-config") + cmFile := "ca-bundle.pem=" + name + "-ca.pem" + defer deleteConfigMap(oc, "openshift-config", "client-ca-"+ingctrl.name) + createConfigMapFromFile(oc, "openshift-config", "client-ca-"+ingctrl.name, cmFile) + + compat_otp.By("patch the ingresscontroller to enable client certificate with required policy") + patchResourceAsAdmin(oc, ingctrl.namespace, ingctrlResource, "{\"spec\":{\"clientTLS\":{\"clientCA\":{\"name\":\"client-ca-"+ingctrl.name+"\"},\"clientCertificatePolicy\":\"Required\"}}}") + ensureRouterDeployGenerationIs(oc, ingctrl.name, "2") + + compat_otp.By("Create a client pod, a backend pod and its service resources") + err = oc.AsAdmin().WithoutNamespace().Run("create").Args("-n", ns, "-f", clientPod).Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + ensurePodWithLabelReady(oc, ns, clientPodLabel) + err = oc.AsAdmin().WithoutNamespace().Run("cp").Args("-n", ns, fileDir, ns+"/"+clientPodName+":"+podFileDir).Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + operateResourceFromFile(oc, "create", ns, serverPod) + ensurePodWithLabelReady(oc, ns, "name=httpbin-pod") + createResourceFromFile(oc, ns, secsvc) + + compat_otp.By("Update the certification and key in the server pod") + podName := getPodListByLabel(oc, ns, "name=httpbin-pod") + newSrvCert := ns + "/" + podName[0] + ":" + srvCert + newSrvKey := ns + "/" + podName[0] + ":" + srvKey + _, err = oc.AsAdmin().WithoutNamespace().Run("exec").Args("-n", ns, podName[0], "-c", "httpbin-https", "--", "bash", "-c", "rm -f "+srvCert).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + _, err = oc.AsAdmin().WithoutNamespace().Run("exec").Args("-n", ns, podName[0], "-c", "httpbin-https", "--", "bash", "-c", "rm -f "+srvKey).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + err = oc.AsAdmin().WithoutNamespace().Run("cp").Args("-n", ns, destCA, "-c", "httpbin-https", newSrvCert).Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + err = oc.AsAdmin().WithoutNamespace().Run("cp").Args("-n", ns, destKey, "-c", "httpbin-https", newSrvKey).Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + + compat_otp.By("create a reen route") + reenRouteHost := "r2-reen67010." + ingctrl.domain + lowHostReen := strings.ToLower(reenRouteHost) + base64HostReen := base64.StdEncoding.EncodeToString([]byte(reenRouteHost)) + err = oc.AsAdmin().WithoutNamespace().Run("create").Args("-n", ns, "route", "reencrypt", "r2-reen", "--service="+secsvcName, "--cert="+customCert, "--key="+customKey, "--ca-cert="+name+"-ca.pem", "--dest-ca-cert="+destCA, "--hostname="+reenRouteHost).Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + + compat_otp.By("Patch added/deleted http request/response headers to the custom ingress-controller") + patchHeaders := "{\"spec\": {\"httpHeaders\": {\"actions\": {\"request\": [" + + "{\"name\": \"X-SSL-Client-Cert\", \"action\": {\"type\": \"Set\", \"set\": {\"value\": \"%{+Q}[ssl_c_der,base64]\"}}}," + + "{\"name\": \"X-Target\", \"action\": {\"type\": \"Set\", \"set\": {\"value\": \"%[req.hdr(host),lower]\"}}}," + + "{\"name\": \"reqTestHost1\", \"action\": {\"type\": \"Set\", \"set\": {\"value\": \"%[req.hdr(host),lower]\"}}}," + + "{\"name\": \"reqTestHost2\", \"action\": {\"type\": \"Set\", \"set\": {\"value\": \"%[req.hdr(host),base64]\"}}}," + + "{\"name\": \"reqTestHost3\", \"action\": {\"type\": \"Set\", \"set\": {\"value\": \"%[req.hdr(Host)]\"}}}," + + "{\"name\": \"X-Forwarded-For\", \"action\": {\"type\": \"Set\", \"set\": {\"value\": \"11.22.33.44\"}}}," + + "{\"name\": \"x-forwarded-client-cert\", \"action\": {\"type\": \"Set\", \"set\": {\"value\": \"%{+Q}[ssl_c_der,base64]\"}}}," + + "{\"name\": \"reqTestHeader\", \"action\": {\"type\": \"Set\", \"set\": {\"value\": \"bbb\"}}}," + + "{\"name\": \"cache-control\", \"action\": {\"type\": \"Set\", \"set\": {\"value\": \"private\"}}}," + + "{\"name\": \"x-ssl-client-der\", \"action\": {\"type\": \"Delete\"}}" + + "]," + + "\"response\": [" + + "{\"name\": \"X-SSL-Server-Cert\", \"action\": {\"type\": \"Set\", \"set\": {\"value\": \"%{+Q}[ssl_c_der,base64]\"}}}," + + "{\"name\": \"X-XSS-Protection\", \"action\": {\"type\": \"Set\", \"set\": {\"value\": \"1; mode=block\"}}}," + + "{\"name\": \"X-Content-Type-Options\", \"action\": {\"type\": \"Set\", \"set\": {\"value\": \"nosniff`\"}}}," + + "{\"name\": \"X-Frame-Options\", \"action\": {\"type\": \"Set\", \"set\": {\"value\": \"SAMEORIGIN\"}}}," + + "{\"name\": \"resTestServer1\", \"action\": {\"type\": \"Set\", \"set\": {\"value\": \"%[res.hdr(server),lower]\"}}}," + + "{\"name\": \"resTestServer2\", \"action\": {\"type\": \"Set\", \"set\": {\"value\": \"%[res.hdr(server),base64]\"}}}," + + "{\"name\": \"resTestServer3\", \"action\": {\"type\": \"Set\", \"set\": {\"value\": \"%[res.hdr(server)]\"}}}," + + "{\"name\": \"cache-control\", \"action\": {\"type\": \"Set\", \"set\": {\"value\": \"private\"}}}," + + "{\"name\": \"server\", \"action\": {\"type\": \"Delete\"}}" + + "]}}}}" + + patchResourceAsAdmin(oc, ingctrl.namespace, ingctrlResource, patchHeaders) + ensureRouterDeployGenerationIs(oc, ingctrl.name, "3") + + compat_otp.By("check the configured added/deleted headers under public frontend, fe_sni frontend and fe_no_sni frontend in haproxy") + routerpod := getOneNewRouterPodFromRollingUpdate(oc, ingctrl.name) + ensureHaproxyBlockConfigContains(oc, routerpod, "frontend fe_sni", []string{"X-SSL-Client-Cert"}) + for _, backend := range []string{"frontend public", "frontend fe_sni", "frontend fe_no_sni"} { + haproxyBackendCfg := getBlockConfig(oc, routerpod, backend) + o.Expect(strings.Contains(haproxyBackendCfg, "http-request set-header 'X-SSL-Client-Cert' '%{+Q}[ssl_c_der,base64]'")).To(o.BeTrue()) + o.Expect(strings.Contains(haproxyBackendCfg, "http-request set-header 'X-Target' '%[req.hdr(host),lower]'")).To(o.BeTrue()) + o.Expect(strings.Contains(haproxyBackendCfg, "http-request set-header 'reqTestHost1' '%[req.hdr(host),lower]'")).To(o.BeTrue()) + o.Expect(strings.Contains(haproxyBackendCfg, "http-request set-header 'reqTestHost2' '%[req.hdr(host),base64]'")).To(o.BeTrue()) + o.Expect(strings.Contains(haproxyBackendCfg, "http-request set-header 'X-Forwarded-For' '11.22.33.44'")).To(o.BeTrue()) + o.Expect(strings.Contains(haproxyBackendCfg, "http-request set-header 'x-forwarded-client-cert' '%{+Q}[ssl_c_der,base64]'")).To(o.BeTrue()) + o.Expect(strings.Contains(haproxyBackendCfg, "http-request set-header 'reqTestHeader' 'bbb'")).To(o.BeTrue()) + o.Expect(strings.Contains(haproxyBackendCfg, "http-request set-header 'cache-control' 'private'")).To(o.BeTrue()) + o.Expect(strings.Contains(haproxyBackendCfg, "http-request del-header 'x-ssl-client-der'")).To(o.BeTrue()) + o.Expect(strings.Contains(haproxyBackendCfg, "http-request del-header 'x-ssl-client-der'")).To(o.BeTrue()) + + o.Expect(strings.Contains(haproxyBackendCfg, "http-response set-header 'X-SSL-Server-Cert' '%{+Q}[ssl_c_der,base64]'")).To(o.BeTrue()) + o.Expect(strings.Contains(haproxyBackendCfg, "http-response set-header 'X-XSS-Protection' '1; mode=block'")).To(o.BeTrue()) + o.Expect(strings.Contains(haproxyBackendCfg, "http-response set-header 'X-Content-Type-Options' 'nosniff`'")).To(o.BeTrue()) + o.Expect(strings.Contains(haproxyBackendCfg, "http-response set-header 'X-Frame-Options' 'SAMEORIGIN'")).To(o.BeTrue()) + o.Expect(strings.Contains(haproxyBackendCfg, "http-response set-header 'resTestServer1' '%[res.hdr(server),lower]'")).To(o.BeTrue()) + o.Expect(strings.Contains(haproxyBackendCfg, "http-response set-header 'resTestServer2' '%[res.hdr(server),base64]'")).To(o.BeTrue()) + o.Expect(strings.Contains(haproxyBackendCfg, "http-response set-header 'resTestServer3' '%[res.hdr(server)]'")).To(o.BeTrue()) + o.Expect(strings.Contains(haproxyBackendCfg, "http-response set-header 'cache-control' 'private'")).To(o.BeTrue()) + o.Expect(strings.Contains(haproxyBackendCfg, "http-response del-header 'server'")).To(o.BeTrue()) + } + + compat_otp.By("send traffic to the reen route, then check http headers in the request or response message") + podIP := getPodv4Address(oc, routerpod, "openshift-ingress") + reenRouteDst := reenRouteHost + ":443:" + podIP + curlReenRouteReq := []string{"-n", ns, clientPodName, "--", "curl", "https://" + reenRouteHost + "/headers", "-v", "--cacert", podCaCert, "--cert", podCustomCert, "--key", podCustomKey, "--resolve", reenRouteDst, "--connect-timeout", "10"} + curlReenRouteRes := []string{"-n", ns, clientPodName, "--", "curl", "https://" + reenRouteHost + "/headers", "-I", "--cacert", podCaCert, "--cert", podCustomCert, "--key", podCustomKey, "--resolve", reenRouteDst, "--connect-timeout", "10"} + lowSrv := strings.ToLower(srv) + base64Srv := base64.StdEncoding.EncodeToString([]byte(srv)) + repeatCmdOnClient(oc, curlReenRouteRes, "200", 60, 1) + reqHeaders, err := oc.AsAdmin().Run("exec").Args(curlReenRouteReq...).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + e2e.Logf("reqHeaders is: %v", reqHeaders) + o.Expect(len(regexp.MustCompile("\"X-Ssl-Client-Cert\": \"([0-9a-zA-Z]+)").FindStringSubmatch(reqHeaders)) > 0).To(o.BeTrue()) + o.Expect(strings.Contains(reqHeaders, "\"X-Target\": \""+reenRouteHost+"\"")).To(o.BeTrue()) + o.Expect(strings.Contains(reqHeaders, "\"Reqtesthost1\": \""+lowHostReen+"\"")).To(o.BeTrue()) + o.Expect(strings.Contains(reqHeaders, "\"Reqtesthost2\": \""+base64HostReen+"\"")).To(o.BeTrue()) + o.Expect(strings.Contains(reqHeaders, "\"Reqtesthost3\": \""+reenRouteHost+"\"")).To(o.BeTrue()) + o.Expect(strings.Contains(reqHeaders, "\"Reqtestheader\": \"bbb\"")).To(o.BeTrue()) + o.Expect(strings.Contains(reqHeaders, "\"Cache-Control\": \"private\"")).To(o.BeTrue()) + o.Expect(strings.Contains(reqHeaders, "x-ssl-client-der")).NotTo(o.BeTrue()) + + resHeaders, err := oc.AsAdmin().Run("exec").Args(curlReenRouteRes...).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + e2e.Logf("resHeaders is: %v", resHeaders) + o.Expect(len(regexp.MustCompile("x-ssl-server-cert: ([0-9a-zA-Z]+)").FindStringSubmatch(resHeaders)) > 0).To(o.BeTrue()) + o.Expect(strings.Contains(resHeaders, "x-xss-protection: 1; mode=block")).To(o.BeTrue()) + o.Expect(strings.Contains(resHeaders, "x-content-type-options: nosniff")).To(o.BeTrue()) + o.Expect(strings.Contains(resHeaders, "x-frame-options: SAMEORIGIN")).To(o.BeTrue()) + o.Expect(strings.Contains(resHeaders, "restestserver1: "+lowSrv)).To(o.BeTrue()) + o.Expect(strings.Contains(resHeaders, "restestserver2: "+base64Srv)).To(o.BeTrue()) + o.Expect(strings.Contains(resHeaders, "restestserver3: "+srv)).To(o.BeTrue()) + o.Expect(strings.Contains(resHeaders, "cache-control: private")).To(o.BeTrue()) + o.Expect(strings.Contains(reqHeaders, "server:")).NotTo(o.BeTrue()) + }) + + // incorporate OCPBUGS-40850 and OCPBUGS-43095 into one + // [OCPBUGS-40850](https://issues.redhat.com/browse/OCPBUGS-40850) + // [OCPBUGS-43095](https://issues.redhat.com/browse/OCPBUGS-43095) + // author: shudili@redhat.com + g.It("Author:shudili-ROSA-OSD_CCS-ARO-ConnectedOnly-High-77284-http request with duplicated headers should not cause disruption to a router pod", func() { + var ( + buildPruningBaseDir = testdata.FixturePath("router") + customTemp = filepath.Join(buildPruningBaseDir, "ingresscontroller-np.yaml") + testPodSvc = filepath.Join(buildPruningBaseDir, "httpbin-deploy.yaml") + unsecsvcName = "httpbin-svc-insecure" + clientPod = filepath.Join(buildPruningBaseDir, "test-client-pod.yaml") + clientPodName = "hello-pod" + clientPodLabel = "app=hello-pod" + ingctrl = ingressControllerDescription{ + name: "77284", + namespace: "openshift-ingress-operator", + domain: "", + template: customTemp, + } + ) + + compat_otp.By("1.0 Create a custom ingresscontroller") + baseDomain := getBaseDomain(oc) + ingctrl.domain = ingctrl.name + "." + baseDomain + defer ingctrl.delete(oc) + ingctrl.create(oc) + ensureCustomIngressControllerAvailable(oc, ingctrl.name) + + compat_otp.By("2.0 Create a client pod, a backend pod and its service resources") + ns := oc.Namespace() + createResourceFromFile(oc, ns, clientPod) + ensurePodWithLabelReady(oc, ns, clientPodLabel) + createResourceFromFile(oc, ns, testPodSvc) + ensurePodWithLabelReady(oc, ns, "name=httpbin-pod") + + compat_otp.By("3.0 Create a HTTP route inside the namespace") + routehost := "service-unsecure77284" + "." + ingctrl.domain + createRoute(oc, ns, "http", unsecsvcName, unsecsvcName, []string{"--hostname=" + routehost}) + ensureRouteIsAdmittedByIngressController(oc, ns, unsecsvcName, "default") + + compat_otp.By("4.0: Curl the http route with two same headers in the http request, expect to get a 400 bad request if the backend server does not support such an invalid http request") + routerpod := getOneRouterPodNameByIC(oc, ingctrl.name) + podIP := getPodv4Address(oc, routerpod, "openshift-ingress") + toDst := routehost + ":80:" + podIP + cmdOnPod := []string{"-n", ns, clientPodName, "--", "curl", "-I", "http://" + routehost + "/headers", "-H", `"transfer-encoding: chunked"`, "-H", `"transfer-encoding: chunked"`, "--resolve", toDst, "--connect-timeout", "10"} + repeatCmdOnClient(oc, cmdOnPod, "400", 60, 1) + + compat_otp.By("5.0: Check that the custom router pod is Running, not Terminating") + output := getByJsonPath(oc, "openshift-ingress", "pods/"+routerpod, "{.status.phase}") + o.Expect(output).To(o.ContainSubstring("Running")) + }) +}) diff --git a/tests-extension/test/e2e/ingress-operator.go b/tests-extension/test/e2e/ingress-operator.go new file mode 100644 index 000000000..4faecb27d --- /dev/null +++ b/tests-extension/test/e2e/ingress-operator.go @@ -0,0 +1,2287 @@ +package router + +import ( + "fmt" + "net" + "os" + "os/exec" + "path/filepath" + "regexp" + "strconv" + "strings" + "time" + + g "github.com/onsi/ginkgo/v2" + o "github.com/onsi/gomega" + compat_otp "github.com/openshift/origin/test/extended/util/compat_otp" + "github.com/openshift/router-tests-extension/test/testdata" + "k8s.io/apimachinery/pkg/util/wait" + e2e "k8s.io/kubernetes/test/e2e/framework" +) + +var _ = g.Describe("[sig-network-edge] Network_Edge Component_Router", func() { + defer g.GinkgoRecover() + + var oc = compat_otp.NewCLI("ingress-operator", compat_otp.KubeConfigPath()) + + // author: hongli@redhat.com + // Includes OCP-27560: support NodePortService for custom Ingresscontroller + g.It("Author:hongli-ROSA-OSD_CCS-ARO-Critical-21873-The replicas of router deployment is controlled by ingresscontroller", func() { + buildPruningBaseDir := testdata.FixturePath("router") + customTemp := filepath.Join(buildPruningBaseDir, "ingresscontroller-np.yaml") + var ( + ingctrl = ingressControllerDescription{ + name: "ocp21873", + namespace: "openshift-ingress-operator", + domain: "", + template: customTemp, + } + ) + + compat_otp.By("Create one custom ingresscontroller") + baseDomain := getBaseDomain(oc) + ingctrl.domain = ingctrl.name + "." + baseDomain + defer ingctrl.delete(oc) + ingctrl.create(oc) + ensureCustomIngressControllerAvailable(oc, ingctrl.name) + routerpod := getOneRouterPodNameByIC(oc, ingctrl.name) + + compat_otp.By("Ensure NodePort service is created") + serviceType := getByJsonPath(oc, "openshift-ingress", "svc/router-nodeport-"+ingctrl.name, "{.spec.type}") + o.Expect(serviceType).To(o.ContainSubstring(`NodePort`)) + + compat_otp.By("Scale the replicas to 0 and ensure router pod is deleted") + patchResourceAsAdmin(oc, ingctrl.namespace, "ingresscontroller/"+ingctrl.name, `{"spec":{"replicas":0}}`) + // no status.readyReplicas filed in deployment if set replicas=0, so just wait for pod to disappear + err := waitForResourceToDisappear(oc, "openshift-ingress", "pod/"+routerpod) + compat_otp.AssertWaitPollNoErr(err, fmt.Sprintf("The router pod %v does not disapper", routerpod)) + + compat_otp.By("Scale the replicas to 1 and ensure deployment is updated") + patchResourceAsAdmin(oc, ingctrl.namespace, "ingresscontroller/"+ingctrl.name, `{"spec":{"replicas":1}}`) + waitForOutputEquals(oc, "openshift-ingress", "deployment/router-"+ingctrl.name, "{.status.readyReplicas}", "1") + }) + + // author: hongli@redhat.com + // Includes OCP-23169: The toleration of router deployment is controlled by ingresscontroller + // No control-plane/master node on HCP + g.It("Author:hongli-NonHyperShiftHOST-ROSA-OSD_CCS-ARO-Medium-22633-The nodeSelector and tolerations of router deployment are controlled by ingresscontrolle", func() { + // skip if ingress.config .status.defaultPlacement == ControlPlane + defaultPlacement := getByJsonPath(oc, "default", "ingress.config/cluster", "{.status.defaultPlacement}") + if defaultPlacement == "ControlPlane" { + g.Skip("Skip since nodeSelector is set to ControlPlane by default on this cluster") + } + + buildPruningBaseDir := testdata.FixturePath("router") + customTemp := filepath.Join(buildPruningBaseDir, "ingresscontroller-np.yaml") + var ( + ingctrl = ingressControllerDescription{ + name: "ocp22633", + namespace: "openshift-ingress-operator", + domain: "", + template: customTemp, + } + ) + + compat_otp.By("Create one custom ingresscontroller") + baseDomain := getBaseDomain(oc) + ingctrl.domain = ingctrl.name + "." + baseDomain + defer ingctrl.delete(oc) + ingctrl.create(oc) + ensureRouterDeployGenerationIs(oc, ingctrl.name, "1") + + compat_otp.By("Check the default nodeSelector and tolerations") + nodeSelector := getByJsonPath(oc, "openshift-ingress", "deployment/router-"+ingctrl.name, "{.spec.template.spec.nodeSelector}") + o.Expect(nodeSelector).To(o.ContainSubstring(`node-role.kubernetes.io/worker`)) + // note: tolerations might be empty in old release (<4.18) + tolerations := getByJsonPath(oc, "openshift-ingress", "deployment/router-"+ingctrl.name, "{.spec.template.spec.tolerations}") + o.Expect(tolerations).NotTo(o.ContainSubstring(`NoSchedule`)) + + compat_otp.By("Update the ingresscontroller nodeSelector and tolerations to deploy router pod to control-plane node") + jsonpath := `{"spec": {"nodePlacement": {"nodeSelector": {"matchLabels": {"node-role.kubernetes.io/control-plane": ""}}, "tolerations": [{"effect": "NoSchedule", "operator": "Exists"}]}}}` + patchResourceAsAdmin(oc, ingctrl.namespace, "ingresscontroller/"+ingctrl.name, jsonpath) + ensureRouterDeployGenerationIs(oc, ingctrl.name, "2") + + compat_otp.By("Ensure the nodeSelector and tolerations of router deployment is updated") + nodeSelector = getByJsonPath(oc, "openshift-ingress", "deployment/router-"+ingctrl.name, "{.spec.template.spec.nodeSelector}") + o.Expect(nodeSelector).To(o.ContainSubstring(`node-role.kubernetes.io/control-plane`)) + tolerations = getByJsonPath(oc, "openshift-ingress", "deployment/router-"+ingctrl.name, "{.spec.template.spec.tolerations}") + o.Expect(tolerations).To(o.ContainSubstring(`"effect":"NoSchedule","operator":"Exists"`)) + }) + + // Test case creater: hongli@redhat.com + g.It("Author:mjoseph-ROSA-OSD_CCS-ARO-Critical-22636-The namespaceSelector of router is controlled by ingresscontroller", func() { + var ( + buildPruningBaseDir = testdata.FixturePath("router") + customTemp = filepath.Join(buildPruningBaseDir, "ingresscontroller-np.yaml") + testPodSvc = filepath.Join(buildPruningBaseDir, "web-server-deploy.yaml") + srvName = "service-unsecure" + ingctrl = ingressControllerDescription{ + name: "ocp22636", + namespace: "openshift-ingress-operator", + domain: "", + template: customTemp, + } + ingctrlResource = "ingresscontroller/" + ingctrl.name + ) + + compat_otp.By("1. Create one custom ingresscontroller") + ns := oc.Namespace() + baseDomain := getBaseDomain(oc) + routehost := srvName + ".ocp22636." + baseDomain + ingctrl.domain = ingctrl.name + "." + baseDomain + defer ingctrl.delete(oc) + ingctrl.create(oc) + ensureRouterDeployGenerationIs(oc, ingctrl.name, "1") + + compat_otp.By("2. Create a server pod and expose an unsecure service") + createResourceFromFile(oc, ns, testPodSvc) + ensurePodWithLabelReady(oc, ns, "name=web-server-deploy") + err := oc.Run("expose").Args("service", srvName, "--hostname="+routehost, "-n", ns).Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + waitForOutputEquals(oc, ns, "route/"+srvName, "{.spec.host}", routehost) + + compat_otp.By("3. Label the namespace to 'namespace=router-test'") + err = oc.AsAdmin().WithoutNamespace().Run("label").Args("namespace", ns, "namespace=router-test").Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + + compat_otp.By("4. Patch the custom ingresscontroller with the namespaceSelector") + patchNamespaceSelector := "{\"spec\":{\"namespaceSelector\":{\"matchLabels\":{\"namespace\": \"router-test\"}}}}" + patchResourceAsAdmin(oc, ingctrl.namespace, ingctrlResource, patchNamespaceSelector) + ensureRouterDeployGenerationIs(oc, ingctrl.name, "2") + newCustContPod := getOneNewRouterPodFromRollingUpdate(oc, ingctrl.name) + + compat_otp.By("5. Check the haproxy config on the custom router pod to find the backend details of the " + ns + " route") + checkoutput := readRouterPodData(oc, newCustContPod, "cat haproxy.config", "service-unsecure") + o.Expect(checkoutput).To(o.ContainSubstring("backend be_http:" + ns + ":service-unsecure")) + + compat_otp.By("6. Check the haproxy config on the custom router to confirm no backend details of other routes are present") + output, err := oc.AsAdmin().WithoutNamespace().Run("exec").Args("-n", "openshift-ingress", newCustContPod, "--", "bash", "-c", "cat haproxy.config | grep canary").Output() + o.Expect(err).To(o.HaveOccurred()) + o.Expect(output).To(o.ContainSubstring("command terminated with exit code 1")) + }) + + // Test case creater: hongli@redhat.com + g.It("Author:mjoseph-ROSA-OSD_CCS-ARO-High-22637-The routeSelector of router is controlled by ingresscontroller", func() { + var ( + buildPruningBaseDir = testdata.FixturePath("router") + customTemp = filepath.Join(buildPruningBaseDir, "ingresscontroller-np.yaml") + testPodSvc = filepath.Join(buildPruningBaseDir, "web-server-deploy.yaml") + srvName = "service-unsecure" + ingctrl = ingressControllerDescription{ + name: "ocp22637", + namespace: "openshift-ingress-operator", + domain: "", + template: customTemp, + } + ingctrlResource = "ingresscontroller/" + ingctrl.name + ) + + compat_otp.By("1. Create one custom ingresscontroller") + ns := oc.Namespace() + baseDomain := getBaseDomain(oc) + routehost := srvName + ".ocp22637." + baseDomain + ingctrl.domain = ingctrl.name + "." + baseDomain + defer ingctrl.delete(oc) + ingctrl.create(oc) + ensureRouterDeployGenerationIs(oc, ingctrl.name, "1") + + compat_otp.By("2. Create a server pod and expose an unsecure service") + createResourceFromFile(oc, ns, testPodSvc) + ensurePodWithLabelReady(oc, ns, "name=web-server-deploy") + err := oc.Run("expose").Args("service", srvName, "--hostname="+routehost, "-n", ns).Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + waitForOutputEquals(oc, ns, "route/"+srvName, "{.spec.host}", routehost) + + compat_otp.By("3. Label the route to 'route=router-test'") + err = oc.AsAdmin().WithoutNamespace().Run("label").Args("route", "service-unsecure", "route=router-test", "-n", ns).Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + + compat_otp.By("4. Patch the custom ingresscontroller with the namespaceSelector") + patchNamespaceSelector := "{\"spec\":{\"routeSelector\":{\"matchLabels\":{\"route\": \"router-test\"}}}}" + patchResourceAsAdmin(oc, ingctrl.namespace, ingctrlResource, patchNamespaceSelector) + ensureRouterDeployGenerationIs(oc, ingctrl.name, "2") + newCustContPod := getOneNewRouterPodFromRollingUpdate(oc, ingctrl.name) + + compat_otp.By("5. Check the haproxy config on the custom router pod to find the backend details of the route for service-unsecure") + checkoutput := readRouterPodData(oc, newCustContPod, "cat haproxy.config", "service-unsecure") + o.Expect(checkoutput).To(o.ContainSubstring("backend be_http:" + ns + ":service-unsecure")) + + compat_otp.By("6. Check the haproxy config on the custom router to confirm no backend details of other routes are present") + output, err := oc.AsAdmin().WithoutNamespace().Run("exec").Args("-n", "openshift-ingress", newCustContPod, "--", "bash", "-c", "cat haproxy.config | grep canary").Output() + o.Expect(err).To(o.HaveOccurred()) + o.Expect(output).To(o.ContainSubstring("command terminated with exit code 1")) + }) + + // author: shudili@redhat.com + // Includes OCP-26150: Integrate ingress operator metrics with Prometheus + g.It("Author:shudili-ROSA-OSD_CCS-ARO-Medium-26150-misc tests for ingress operator", func() { + var ( + namespace = "openshift-ingress-operator" + servicemonitorName = "ingress-operator" + rolebindingName = "prometheus-k8s" + ) + + compat_otp.By(fmt.Sprintf("Check whether servicemonitor %s exists or not", servicemonitorName)) + jsonPath := "{.items[*].metadata.name}" + servicemonitorList := getByJsonPath(oc, namespace, "servicemonitor", jsonPath) + o.Expect(servicemonitorList).To(o.ContainSubstring(servicemonitorName)) + + compat_otp.By(fmt.Sprintf("Check whether rolebinding %s exists or not", rolebindingName)) + rolebindingList := getByJsonPath(oc, namespace, "rolebinding", jsonPath) + o.Expect(rolebindingList).To(o.ContainSubstring(rolebindingName)) + + compat_otp.By(fmt.Sprintf("check the openshift.io/cluster-monitoring label of the namespace %s, which should be true", namespace)) + jsonPath = `{.metadata.labels.openshift\.io/cluster-monitoring}` + value := getByJsonPath(oc, "default", "namespace/"+namespace, jsonPath) + o.Expect(value).To(o.ContainSubstring("true")) + }) + + // author: iamin@redhat.com + g.It("Author:iamin-ROSA-OSD_CCS-ARO-High-29207-the PROXY protocol is disabled in non-AWS platform", func() { + platforms := map[string]bool{ + "aws": true, + } + if platforms[compat_otp.CheckPlatform(oc)] { + g.Skip("Skip for AWS platform") + } + + compat_otp.By("1.0 Get default ingress pod") + routerpod := getOneRouterPodNameByIC(oc, "default") + + compat_otp.By("2.0 Check HaProxy for PROXY protocol") + cmd := fmt.Sprintf(`cat haproxy.config | grep -E 'bind :{1,3}(80|443)'`) + output, err := oc.AsAdmin().WithoutNamespace().Run("exec").Args("-n", "openshift-ingress", routerpod, "--", "bash", "-c", cmd).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(output).NotTo(o.ContainSubstring("accept-proxy")) + }) + + // author: shudili@redhat.com + g.It("Author:shudili-ROSA-OSD_CCS-ARO-Critical-30059-NetworkEdge Create an ingresscontroller that logs to a sidecar container", func() { + var ( + buildPruningBaseDir = testdata.FixturePath("router") + customTemp = filepath.Join(buildPruningBaseDir, "ingresscontroller-sidecar.yaml") + testPodSvc = filepath.Join(buildPruningBaseDir, "web-server-deploy.yaml") + srvrcInfo = "web-server-deploy" + srvName = "service-unsecure" + clientPod = filepath.Join(buildPruningBaseDir, "test-client-pod.yaml") + clientPodName = "hello-pod" + clientPodLabel = "app=hello-pod" + ingctrl = ingressControllerDescription{ + name: "ocp30059", + namespace: "openshift-ingress-operator", + domain: "", + template: customTemp, + } + ) + + compat_otp.By("1. Create a sidecar-log ingresscontroller") + baseDomain := getBaseDomain(oc) + ingctrl.domain = ingctrl.name + "." + baseDomain + defer ingctrl.delete(oc) + ingctrl.create(oc) + ensureRouterDeployGenerationIs(oc, ingctrl.name, "1") + + compat_otp.By("2. Check if the a sidecar container with name 'logs' is deployed") + routerpod := getOneNewRouterPodFromRollingUpdate(oc, ingctrl.name) + containerName := getByJsonPath(oc, "openshift-ingress", "pod/"+routerpod, "{.spec.containers[1].name}") + o.Expect(containerName).To(o.ContainSubstring("logs")) + + // check the function of logs to a sidecar container in the following steps by curl a route and check the router logs + compat_otp.By("3. Create a client pod, a server pod and an unsecure service") + ns := oc.Namespace() + createResourceFromFile(oc, ns, clientPod) + ensurePodWithLabelReady(oc, ns, clientPodLabel) + + createResourceFromFile(oc, ns, testPodSvc) + ensurePodWithLabelReady(oc, ns, "name="+srvrcInfo) + + compat_otp.By("4. Create an route") + routehost := srvName + "-" + ns + "." + ingctrl.domain + err := oc.Run("expose").Args("service", srvName, "--hostname="+routehost, "-n", ns).Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + waitForOutputEquals(oc, ns, "route", "{.items[0].metadata.name}", srvName) + + compat_otp.By("5. Curl the route, and then check the router pod's logs which should contain the new http request") + podIP := getPodv4Address(oc, routerpod, "openshift-ingress") + toDst := routehost + ":80:" + podIP + jsonPath := fmt.Sprintf(`{.items[?(@.metadata.generateName=="%s-")].metadata.name}`, srvName) + epSliceName := getByJsonPath(oc, ns, "EndpointSlice", jsonPath) + jsonPath = "{.endpoints[0].addresses[0]}:{.ports[0].port}" + ep := getByJsonPath(oc, ns, "EndpointSlice/"+epSliceName, jsonPath) + cmdOnPod := []string{"-n", ns, clientPodName, "--", "curl", "-I", "http://" + routehost, "--resolve", toDst, "--connect-timeout", "10"} + result, _ := repeatCmdOnClient(oc, cmdOnPod, "200", 30, 1) + o.Expect(result).To(o.ContainSubstring("200")) + output := waitRouterLogsAppear(oc, routerpod, ep) + log := regexp.MustCompile("haproxy.+" + ep + ".+HTTP/1.1").FindStringSubmatch(output)[0] + o.Expect(len(log) > 1).To(o.BeTrue()) + }) + + // author: shudili@redhat.com + g.It("Author:shudili-ROSA-OSD_CCS-ARO-Critical-30060-NetworkEdge Create an ingresscontroller that logs to external rsyslog instance", func() { + var ( + buildPruningBaseDir = testdata.FixturePath("router") + baseTemp = filepath.Join(buildPruningBaseDir, "ingresscontroller-np.yaml") + syslogPod = filepath.Join(buildPruningBaseDir, "rsyslogd-pod.yaml") + testPodSvc = filepath.Join(buildPruningBaseDir, "web-server-deploy.yaml") + clientPod = filepath.Join(buildPruningBaseDir, "test-client-pod.yaml") + syslogPodName = "rsyslogd-pod" + syslogPodLabel = "name=rsyslogd" + srvrcInfo = "web-server-deploy" + srvName = "service-unsecure" + clientPodName = "hello-pod" + clientPodLabel = "app=hello-pod" + ) + + compat_otp.By("1. Create a syslog pod for the log receiver") + ns := oc.Namespace() + compat_otp.SetNamespacePrivileged(oc, ns) + err := oc.AsAdmin().WithoutNamespace().Run("create").Args("-n", ns, "-f", syslogPod).Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + ensurePodWithLabelReady(oc, ns, syslogPodLabel) + + compat_otp.By("2. Create a syslog ingresscontroller") + syslogPodIP := getPodv4Address(oc, syslogPodName, ns) + extraParas := " logging:\n access:\n destination:\n type: Syslog\n syslog:\n address: " + syslogPodIP + "\n port: 514\n" + customTemp := addExtraParametersToYamlFile(baseTemp, "spec:", extraParas) + defer os.Remove(customTemp) + ingctrl := ingressControllerDescription{ + name: "ocp30060", + namespace: "openshift-ingress-operator", + domain: "", + template: customTemp, + } + baseDomain := getBaseDomain(oc) + ingctrl.domain = ingctrl.name + "." + baseDomain + defer ingctrl.delete(oc) + ingctrl.create(oc) + ensureRouterDeployGenerationIs(oc, ingctrl.name, "1") + + compat_otp.By("3. Check ROUTER_SYSLOG_ADDRESS env in a router pod") + routerpod := getOneNewRouterPodFromRollingUpdate(oc, ingctrl.name) + syslogEnv := readRouterPodEnv(oc, routerpod, "ROUTER_SYSLOG_ADDRESS") + syslogEp := syslogPodIP + ":514" + if strings.Contains(syslogPodIP, ":") { + syslogEp = "[" + syslogPodIP + "]" + ":514" + } + o.Expect(syslogEnv).To(o.ContainSubstring(syslogEp)) + + compat_otp.By("4. Create a client pod, a server pod and an unsecure service") + createResourceFromFile(oc, ns, clientPod) + ensurePodWithLabelReady(oc, ns, clientPodLabel) + + createResourceFromFile(oc, ns, testPodSvc) + ensurePodWithLabelReady(oc, ns, "name="+srvrcInfo) + + compat_otp.By("5. Create a route") + routehost := srvName + "-" + ns + "." + ingctrl.domain + createRoute(oc, ns, "http", "route-http", srvName, []string{"--hostname=" + routehost}) + ensureRouteIsAdmittedByIngressController(oc, ns, "route-http", ingctrl.name) + + compat_otp.By("6. Curl the route, and then check the rsyslogd pod's logs which should contain the new http request") + podIP := getPodv4Address(oc, routerpod, "openshift-ingress") + toDst := routehost + ":80:" + podIP + jsonPath := fmt.Sprintf(`{.items[?(@.metadata.generateName=="%s-")].metadata.name}`, srvName) + epSliceName := getByJsonPath(oc, ns, "EndpointSlice", jsonPath) + jsonPath = "{.endpoints[0].addresses[0]}:{.ports[0].port}" + ep := getByJsonPath(oc, ns, "EndpointSlice/"+epSliceName, jsonPath) + cmdOnPod := []string{"-n", ns, clientPodName, "--", "curl", "-I", "http://" + routehost, "--resolve", toDst, "--connect-timeout", "10"} + result, _ := repeatCmdOnClient(oc, cmdOnPod, "200", 30, 5) + o.Expect(result).To(o.ContainSubstring("200")) + output, _ := oc.AsAdmin().WithoutNamespace().Run("logs").Args("-n", ns, syslogPodName).Output() + o.Expect(output).To(o.MatchRegexp("haproxy.+" + ep + ".+HTTP/1.1")) + }) + + // author: iamin@redhat.com + g.It("Author:iamin-ROSA-OSD_CCS-ARO-Medium-30264-ROUTER_SYSLOG_ADDRESS changes according to the logging configuration", func() { + var ( + buildPruningBaseDir = testdata.FixturePath("router") + customTemp = filepath.Join(buildPruningBaseDir, "ingresscontroller-sidecar.yaml") + ingctrl = ingressControllerDescription{ + name: "ocp30264", + namespace: "openshift-ingress-operator", + domain: "", + template: customTemp, + } + ) + + compat_otp.By("1. Create a sidecar-log ingresscontroller") + baseDomain := getBaseDomain(oc) + ingctrl.domain = ingctrl.name + "." + baseDomain + defer ingctrl.delete(oc) + ingctrl.create(oc) + ensureRouterDeployGenerationIs(oc, ingctrl.name, "1") + + compat_otp.By("2. check router deployment") + err := oc.AsAdmin().WithoutNamespace().Run("get").Args("deployment", "-n", "openshift-ingress", "router-"+ingctrl.name).Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + + compat_otp.By("3. Verify presence of pod with label") + ingPod := getOnePodNameByLabel(oc, "openshift-ingress", "ingresscontroller.operator.openshift.io/deployment-ingresscontroller="+ingctrl.name) + o.Expect(ingPod).To(o.ContainSubstring(ingctrl.name)) + err = waitForPodWithLabelReady(oc, "openshift-ingress", "ingresscontroller.operator.openshift.io/deployment-ingresscontroller="+ingctrl.name) + o.Expect(err).NotTo(o.HaveOccurred()) + + // OCP-30066 Enable log-send-hostname in HAproxy configuration by default + compat_otp.By("4. check haproxy for output") + ensureHaproxyBlockConfigContains(oc, ingPod, "global", []string{"log-send-hostname", `/var/lib/haproxy/`}) + + compat_otp.By("5. check custom ingress pod for ROUTER_SYSLOG_ADDRESS") + grepCmd := "ROUTER_SYSLOG_ADDRESS" + cmd := fmt.Sprintf(`env | grep "%s"`, grepCmd) + output, err := oc.AsAdmin().WithoutNamespace().Run("exec").Args("-n", "openshift-ingress", ingPod, "--", "bash", "-lc", cmd).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(output).To(o.ContainSubstring("ROUTER_SYSLOG_ADDRESS=/var/lib/rsyslog/rsyslog.sock")) + + compat_otp.By("6. Patch the ingress controller with a different address and port") + patchResourceAsAdmin(oc, ingctrl.namespace, "ingresscontroller/ocp30264", `{"spec":{"logging":{"access":{"destination":{"type": "Syslog", "syslog":{"address": "1.2.3.4", "port": 514}}}}}}`) + + compat_otp.By("7. Wait for previous pod to be deleted") + err = waitForResourceToDisappear(oc, "openshift-ingress", "pod/"+ingPod) + o.Expect(err).NotTo(o.HaveOccurred()) + + compat_otp.By("8. Retrieve new pod name with label") + ingPod = getOnePodNameByLabel(oc, "openshift-ingress", "ingresscontroller.operator.openshift.io/deployment-ingresscontroller="+ingctrl.name) + o.Expect(ingPod).To(o.ContainSubstring(ingctrl.name)) + + compat_otp.By("9. check custom ingress pod for ROUTER_SYSLOG_ADDRESS") + output, err = oc.AsAdmin().WithoutNamespace().Run("exec").Args("-n", "openshift-ingress", ingPod, "--", "bash", "-lc", cmd).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(output).To(o.ContainSubstring("ROUTER_SYSLOG_ADDRESS=1.2.3.4:514")) + }) + + // author: iamin@redhat.com + g.It("Author:iamin-ROSA-OSD_CCS-ARO-Critical-33763-ingresscontroller supports AWS NLB", func() { + var ( + buildPruningBaseDir = testdata.FixturePath("router") + customTemp = filepath.Join(buildPruningBaseDir, "ingresscontroller-clb.yaml") + ingctrl = ingressControllerDescription{ + name: "ocp33763", + namespace: "openshift-ingress-operator", + domain: "", + template: customTemp, + } + ) + + compat_otp.By("Check the Platform type") + compat_otp.SkipIfPlatformTypeNot(oc, "AWS") + + compat_otp.By("1. Create a nlb ingress controller") + baseDomain := getBaseDomain(oc) + ingctrl.domain = ingctrl.name + "." + baseDomain + // Updating LB scope `External` to `Internal` and `Classic`` to `NLB` in the yaml file + sedCmd := fmt.Sprintf(`sed -i'' -e 's|External|Internal|g; s|Classic|NLB|g' %s`, customTemp) + _, err := exec.Command("bash", "-c", sedCmd).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + defer ingctrl.delete(oc) + ingctrl.create(oc) + ensureRouterDeployGenerationIs(oc, ingctrl.name, "1") + + compat_otp.By("2. Verify presence of pod with label") + ensurePodWithLabelReady(oc, "openshift-ingress", "ingresscontroller.operator.openshift.io/deployment-ingresscontroller="+ingctrl.name) + + compat_otp.By("3. check if LB service exists") + output, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("service", "-n", "openshift-ingress", "router-"+ingctrl.name).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(output).To(o.And(o.ContainSubstring(`router-`+ingctrl.name), o.ContainSubstring(`LoadBalancer`))) + + compat_otp.By("4. Check LB service annotations") + annotation := getAnnotation(oc, "openshift-ingress", "service", `router-`+ingctrl.name) + o.Expect(annotation).To(o.And(o.ContainSubstring(`service.beta.kubernetes.io/aws-load-balancer-type":"nlb`), o.ContainSubstring(`service.beta.kubernetes.io/aws-load-balancer-internal":"true`))) + }) + + // Due to bug https://issues.redhat.com/browse/OCPBUGS-43431, this case may not run on HCP cluster. + g.It("Author:mjoseph-NonHyperShiftHOST-High-38674-hard-stop-after annotation can be applied globally on all ingresscontroller [Disruptive]", func() { + buildPruningBaseDir := testdata.FixturePath("router") + customTemp := filepath.Join(buildPruningBaseDir, "ingresscontroller-np.yaml") + var ( + ingctrl = ingressControllerDescription{ + name: "ocp38674", + namespace: "openshift-ingress-operator", + domain: "", + template: customTemp, + } + ) + + compat_otp.By("Create a custom ingresscontroller") + baseDomain := getBaseDomain(oc) + ingctrl.domain = ingctrl.name + "." + baseDomain + defer ingctrl.delete(oc) + ingctrl.create(oc) + ensureRouterDeployGenerationIs(oc, ingctrl.name, "1") + defaultRouterpod := getOneRouterPodNameByIC(oc, "default") + + compat_otp.By("Annotate the ingresses.config/cluster with ingress.operator.openshift.io/hard-stop-after globally") + defer oc.AsAdmin().WithoutNamespace().Run("annotate").Args( + "-n", ingctrl.namespace, "ingresses.config/cluster", "ingress.operator.openshift.io/hard-stop-after-").Execute() + err0 := oc.AsAdmin().WithoutNamespace().Run("annotate").Args( + "-n", ingctrl.namespace, "ingresses.config/cluster", "ingress.operator.openshift.io/hard-stop-after=30m", "--overwrite").Execute() + o.Expect(err0).NotTo(o.HaveOccurred()) + err := waitForResourceToDisappear(oc, "openshift-ingress", "pod/"+defaultRouterpod) + compat_otp.AssertWaitPollNoErr(err, fmt.Sprintf("resource %v does not disapper", "pod/"+defaultRouterpod)) + ensureRouterDeployGenerationIs(oc, ingctrl.name, "2") + + compat_otp.By("Verify the annotation presence in the cluster gloabl config") + newRouterpod := getOneNewRouterPodFromRollingUpdate(oc, ingctrl.name) + newDefaultRouterpod := getOneNewRouterPodFromRollingUpdate(oc, "default") + findAnnotation := getAnnotation(oc, oc.Namespace(), "ingress.config.openshift.io", "cluster") + o.Expect(findAnnotation).To(o.ContainSubstring(`"ingress.operator.openshift.io/hard-stop-after":"30m"`)) + + compat_otp.By("Check the env variable of the custom router pod to verify the hard stop duration is 30m") + env := readRouterPodEnv(oc, newRouterpod, "ROUTER_HARD_STOP_AFTER") + o.Expect(env).To(o.ContainSubstring(`30m`)) + + compat_otp.By("Check the env variable of the default router pod to verify the hard stop duration is 30m") + env1 := readRouterPodEnv(oc, newDefaultRouterpod, "ROUTER_HARD_STOP_AFTER") + o.Expect(env1).To(o.ContainSubstring(`30m`)) + + compat_otp.By("Annotate the ingresses.config/cluster with ingress.operator.openshift.io/hard-stop-after per ingresscontroller basis") + err2 := oc.AsAdmin().WithoutNamespace().Run("annotate").Args( + "-n", ingctrl.namespace, "ingresscontrollers/"+ingctrl.name, "ingress.operator.openshift.io/hard-stop-after=45m", "--overwrite").Execute() + o.Expect(err2).NotTo(o.HaveOccurred()) + ensureRouterDeployGenerationIs(oc, ingctrl.name, "3") + + compat_otp.By("Verify the annotation presence in the ocp38674 controller config") + newRouterpod1 := getOneNewRouterPodFromRollingUpdate(oc, ingctrl.name) + findAnnotation2 := getAnnotation(oc, ingctrl.namespace, "ingresscontroller.operator.openshift.io", ingctrl.name) + o.Expect(findAnnotation2).To(o.ContainSubstring(`"ingress.operator.openshift.io/hard-stop-after":"45m"`)) + + compat_otp.By("Check the haproxy config on the defualt router pod to verify the hard stop value is still 30m") + checkoutput := readRouterPodData(oc, newDefaultRouterpod, "cat haproxy.config", "hard") + o.Expect(checkoutput).To(o.ContainSubstring(`hard-stop-after 30m`)) + + compat_otp.By("Check the haproxy config on the router pod to verify the hard stop value is changed to 45m") + checkoutput1 := readRouterPodData(oc, newRouterpod1, "cat haproxy.config", "hard") + o.Expect(checkoutput1).To(o.ContainSubstring(`hard-stop-after 45m`)) + }) + + g.It("Author:mjoseph-High-38675-hard-stop-after annotation can be applied on per ingresscontroller", func() { + buildPruningBaseDir := testdata.FixturePath("router") + customTemp := filepath.Join(buildPruningBaseDir, "ingresscontroller-np.yaml") + var ( + ingctrl38675one = ingressControllerDescription{ + name: "ocp38675-1", + namespace: "openshift-ingress-operator", + domain: "", + template: customTemp, + } + + ingctrl38675two = ingressControllerDescription{ + name: "ocp38675-2", + namespace: "openshift-ingress-operator", + domain: "", + template: customTemp, + } + ) + + compat_otp.By("1. Create two custom ingresscontrollers") + baseDomain := getBaseDomain(oc) + ingctrl38675one.domain = ingctrl38675one.name + "." + baseDomain + ingctrl38675two.domain = ingctrl38675two.name + "." + baseDomain + defer ingctrl38675one.delete(oc) + ingctrl38675one.create(oc) + defer ingctrl38675two.delete(oc) + ingctrl38675two.create(oc) + ensureCustomIngressControllerAvailable(oc, ingctrl38675one.name) + ensureCustomIngressControllerAvailable(oc, ingctrl38675two.name) + + compat_otp.By("2. Annotate the router of " + ingctrl38675one.name + " with 15m hardstop value") + setAnnotationAsAdmin(oc, ingctrl38675one.namespace, "ingresscontroller/"+ingctrl38675one.name, `ingress.operator.openshift.io/hard-stop-after=15m`) + ensureRouterDeployGenerationIs(oc, ingctrl38675one.name, "2") + + compat_otp.By("3. Annotate the router of " + ingctrl38675two.name + " with 30m hardstop value") + setAnnotationAsAdmin(oc, ingctrl38675two.namespace, "ingresscontroller/"+ingctrl38675two.name, `ingress.operator.openshift.io/hard-stop-after=30m`) + ensureRouterDeployGenerationIs(oc, ingctrl38675two.name, "2") + + compat_otp.By("4. Check the haproxy config on the " + ingctrl38675one.name + " router pod to verify the hard stop value is 15m") + oneRouterpod := getOneNewRouterPodFromRollingUpdate(oc, ingctrl38675one.name) + checkoutput := pollReadPodData(oc, "openshift-ingress", oneRouterpod, "cat haproxy.config", "hard") + o.Expect(checkoutput).To(o.ContainSubstring(`hard-stop-after 15m`)) + + compat_otp.By("5. Check the haproxy config on the " + ingctrl38675two.name + " router pod to verify the hard stop value is 30m") + twoRouterpod := getOneNewRouterPodFromRollingUpdate(oc, ingctrl38675two.name) + checkoutput1 := pollReadPodData(oc, "openshift-ingress", twoRouterpod, "cat haproxy.config", "hard") + o.Expect(checkoutput1).To(o.ContainSubstring(`hard-stop-after 30m`)) + }) + + // author: iamin@redhat.com + g.It("Author:iamin-ROSA-OSD_CCS-ARO-NonPreRelease-PreChkUpgrade-High-38812-upgrade with router shards", func() { + var ( + buildPruningBaseDir = testdata.FixturePath("router") + testPodSvc = filepath.Join(buildPruningBaseDir, "web-server-deploy.yaml") + ns = "ingress-upgrade" + customTemp = filepath.Join(buildPruningBaseDir, "ingresscontroller-shard.yaml") + ingctrl = ingressControllerDescription{ + name: "shards", + namespace: "openshift-ingress-operator", + domain: "", + shard: "alpha", + template: customTemp, + } + ) + + compat_otp.By("1.0: Confirm that the ingress co is in normal status") + jsonPath := `{.status.conditions[?(@.type=="Available")].status}{.status.conditions[?(@.type=="Progressing")].status}{.status.conditions[?(@.type=="Degraded")].status}` + waitForOutputEquals(oc, "default", "co/ingress", jsonPath, "TrueFalseFalse") + + compat_otp.By("2.0: Create a new project and a web-server application") + oc.CreateSpecifiedNamespaceAsAdmin(ns) + baseDomain := getBaseDomain(oc) + err := oc.AsAdmin().WithoutNamespace().Run("create").Args("-f", testPodSvc, "-n", ns).Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + ensurePodWithLabelReady(oc, ns, "name=web-server-deploy") + podName := getOnePodNameByLabel(oc, ns, "name=web-server-deploy") + podIP, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("pod", "-n", ns, podName, "-o=jsonpath={.status.podIPs[0].ip}").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + + compat_otp.By("3.0: Create an edge route and label the route") + _, err = oc.AsAdmin().WithoutNamespace().Run("create").Args("route", "edge", "-n", ns, "route-edge", "--service=service-unsecure", "--hostname=ingress-upgrade.example.com").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + ensureRouteIsAdmittedByIngressController(oc, ns, "route-edge", "default") + + _, err = oc.AsAdmin().WithoutNamespace().Run("label").Args("route", "route-edge", "-n", ns, "shard=alpha").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + + compat_otp.By("4.0: Get default ingresscontroller information and create shard ingresscontroller") + ingctrl.domain = ingctrl.name + "." + baseDomain + ingctrl.create(oc) + ensureRouterDeployGenerationIs(oc, ingctrl.name, "1") + ensurePodWithLabelReady(oc, "openshift-ingress", "ingresscontroller.operator.openshift.io/deployment-ingresscontroller=shards") + + compat_otp.By("5.0: Ensure the route in the matched namespace is loaded") + shardpod := getOneNewRouterPodFromRollingUpdate(oc, ingctrl.name) + ensureHaproxyBlockConfigContains(oc, shardpod, podIP, []string{podName + ":service-unsecure"}) + + compat_otp.By("6.0: Confirm that the ingress co is in normal status") + waitForOutputEquals(oc, "default", "co/ingress", jsonPath, "TrueFalseFalse") + }) + + // author: iamin@redhat.com + g.It("Author:iamin-ROSA-OSD_CCS-ARO-NonPreRelease-PstChkUpgrade-High-38812-upgrade with router shards", func() { + var ( + buildPruningBaseDir = testdata.FixturePath("router") + clientPod = filepath.Join(buildPruningBaseDir, "test-client-pod.yaml") + clientPodName = "hello-pod" + ns = "ingress-upgrade" + ) + + compat_otp.By("1.0: Confirm that the ingress co is in normal status") + jsonPath := `{.status.conditions[?(@.type=="Available")].status}{.status.conditions[?(@.type=="Progressing")].status}{.status.conditions[?(@.type=="Degraded")].status}` + waitForOutputEquals(oc, "default", "co/ingress", jsonPath, "TrueFalseFalse") + + compat_otp.By("2.0: Confirm that the custom ingress router pod is still available and get its IP") + podName := getOnePodNameByLabel(oc, "openshift-ingress", "ingresscontroller.operator.openshift.io/deployment-ingresscontroller=shards") + podIP := getPodv4Address(oc, podName, "openshift-ingress") + + compat_otp.By("3.0: Ensure that the route served by shards is still accessible after upgrade") + err := oc.AsAdmin().WithoutNamespace().Run("create").Args("-f", clientPod, "-n", ns).Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + ensurePodWithLabelReady(oc, ns, "app=hello-pod") + routehost := "ingress-upgrade.example.com" + toDst := routehost + ":443:" + podIP + curlCmd := []string{"-n", ns, clientPodName, "--", "curl", "https://" + routehost, "-ks", "--resolve", toDst, "--connect-timeout", "10"} + repeatCmdOnClient(oc, curlCmd, "Hello-OpenShift web-server-deploy", 120, 1) + }) + + // author: hongli@redhat.com + // Bug: 1960284 + g.It("Author:hongli-Critical-42276-enable annotation traffic-policy.network.alpha.openshift.io/local-with-fallback on LB and nodePort service", func() { + buildPruningBaseDir := testdata.FixturePath("router") + customTemp := filepath.Join(buildPruningBaseDir, "ingresscontroller-np.yaml") + var ( + ingctrl = ingressControllerDescription{ + name: "ocp42276", + namespace: "openshift-ingress-operator", + domain: "", + template: customTemp, + } + ) + + compat_otp.By("Create one custom ingresscontroller") + baseDomain := getBaseDomain(oc) + ingctrl.domain = ingctrl.name + "." + baseDomain + defer ingctrl.delete(oc) + ingctrl.create(oc) + ensureRouterDeployGenerationIs(oc, ingctrl.name, "1") + + compat_otp.By("check the annotation of nodeport service") + annotation := getByJsonPath(oc, "openshift-ingress", "svc/router-nodeport-ocp42276", "{.metadata.annotations}") + o.Expect(annotation).To(o.ContainSubstring(`traffic-policy.network.alpha.openshift.io/local-with-fallback`)) + + // In IBM cloud and PowerVS the externalTrafficPolicy will be 'Cluster' for default LB service, so skipping the same + platformtype := compat_otp.CheckPlatform(oc) + platforms := map[string]bool{ + "ibmcloud": true, + "powervs": true, + } + if !platforms[platformtype] { + compat_otp.By("check the annotation of default LoadBalancer service if it is available") + output, _ := oc.AsAdmin().WithoutNamespace().Run("get").Args("-n", "openshift-ingress", "service", "router-default", "-o=jsonpath={.spec.type}").Output() + // LB service is supported on public cloud platform like aws, gcp, azure and alibaba + if strings.Contains(output, "LoadBalancer") { + annotation = getByJsonPath(oc, "openshift-ingress", "svc/router-default", "{.metadata.annotations}") + o.Expect(annotation).To(o.ContainSubstring(`traffic-policy.network.alpha.openshift.io/local-with-fallback`)) + } else { + e2e.Logf("skip the default LB service checking part, since it is not supported on this cluster") + } + } + }) + + // author: mjoseph@redhat.com + g.It("Author:mjoseph-High-46287-ingresscontroller supports to update maxlength for syslog message", func() { + buildPruningBaseDir := testdata.FixturePath("router") + customTemp := filepath.Join(buildPruningBaseDir, "ingresscontroller-syslog.yaml") + var ( + ingctrl = ingressControllerDescription{ + name: "ocp46287", + namespace: "openshift-ingress-operator", + domain: "", + template: customTemp, + } + ) + + compat_otp.By("Create one custom ingresscontroller") + baseDomain := getBaseDomain(oc) + ingctrl.domain = ingctrl.name + "." + baseDomain + defer ingctrl.delete(oc) + ingctrl.create(oc) + ensureRouterDeployGenerationIs(oc, ingctrl.name, "1") + + compat_otp.By("check the env variable of the router pod to verify the default log length") + newrouterpod := getOneNewRouterPodFromRollingUpdate(oc, ingctrl.name) + logLength := readRouterPodEnv(oc, newrouterpod, "ROUTER_LOG_MAX_LENGTH") + o.Expect(logLength).To(o.ContainSubstring(`ROUTER_LOG_MAX_LENGTH=1024`)) + + compat_otp.By("check the haproxy config on the router pod to verify the default log length is enabled") + checkoutput := readRouterPodData(oc, newrouterpod, "cat haproxy.config", "1024") + o.Expect(checkoutput).To(o.ContainSubstring(`log 1.2.3.4:514 len 1024 local1 info`)) + + compat_otp.By("patch the existing custom ingress controller with minimum log length value") + patchResourceAsAdmin(oc, ingctrl.namespace, "ingresscontroller/ocp46287", "{\"spec\":{\"logging\":{\"access\":{\"destination\":{\"syslog\":{\"maxLength\":480}}}}}}") + ensureRouterDeployGenerationIs(oc, ingctrl.name, "2") + + compat_otp.By("check the env variable of the router pod to verify the minimum log length") + newrouterpod = getOneNewRouterPodFromRollingUpdate(oc, "ocp46287") + minimumlogLength := readRouterPodEnv(oc, newrouterpod, "ROUTER_LOG_MAX_LENGTH") + o.Expect(minimumlogLength).To(o.ContainSubstring(`ROUTER_LOG_MAX_LENGTH=480`)) + + compat_otp.By("patch the existing custom ingress controller with maximum log length value") + patchResourceAsAdmin(oc, ingctrl.namespace, "ingresscontroller/ocp46287", "{\"spec\":{\"logging\":{\"access\":{\"destination\":{\"syslog\":{\"maxLength\":4096}}}}}}") + ensureRouterDeployGenerationIs(oc, ingctrl.name, "3") + + compat_otp.By("check the env variable of the router pod to verify the maximum log length") + newrouterpod = getOneNewRouterPodFromRollingUpdate(oc, "ocp46287") + maximumlogLength := readRouterPodEnv(oc, newrouterpod, "ROUTER_LOG_MAX_LENGTH") + o.Expect(maximumlogLength).To(o.ContainSubstring(`ROUTER_LOG_MAX_LENGTH=4096`)) + }) + + // author: mjoseph@redhat.com + g.It("Author:mjoseph-Low-46288-ingresscontroller should deny invalid maxlengh value for syslog message", func() { + buildPruningBaseDir := testdata.FixturePath("router") + customTemp := filepath.Join(buildPruningBaseDir, "ingresscontroller-syslog.yaml") + var ( + ingctrl = ingressControllerDescription{ + name: "ocp46288", + namespace: "openshift-ingress-operator", + domain: "", + template: customTemp, + } + ) + + compat_otp.By("Create one custom ingresscontroller") + baseDomain := getBaseDomain(oc) + ingctrl.domain = ingctrl.name + "." + baseDomain + defer ingctrl.delete(oc) + ingctrl.create(oc) + ensureRouterDeployGenerationIs(oc, ingctrl.name, "1") + + compat_otp.By("patch the existing custom ingress controller with log length value less than minimum threshold") + output1, _ := oc.AsAdmin().WithoutNamespace().Run("patch").Args("ingresscontroller/ocp46288", "-p", "{\"spec\":{\"logging\":{\"access\":{\"destination\":{\"syslog\":{\"maxLength\":479}}}}}}", "--type=merge", "-n", ingctrl.namespace).Output() + o.Expect(output1).To(o.ContainSubstring("Invalid value: 479: spec.logging.access.destination.syslog.maxLength in body should be greater than or equal to 480")) + + compat_otp.By("patch the existing custom ingress controller with log length value more than maximum threshold") + output2, _ := oc.AsAdmin().WithoutNamespace().Run("patch").Args("ingresscontroller/ocp46288", "-p", "{\"spec\":{\"logging\":{\"access\":{\"destination\":{\"syslog\":{\"maxLength\":4097}}}}}}", "--type=merge", "-n", ingctrl.namespace).Output() + o.Expect(output2).To(o.ContainSubstring("Invalid value: 4097: spec.logging.access.destination.syslog.maxLength in body should be less than or equal to 4096")) + + compat_otp.By("check the haproxy config on the router pod to verify the default log length is enabled") + routerpod := getOneNewRouterPodFromRollingUpdate(oc, ingctrl.name) + checkoutput := readRouterPodData(oc, routerpod, "cat haproxy.config", "1024") + o.Expect(checkoutput).To(o.ContainSubstring(`log 1.2.3.4:514 len 1024 local1 info`)) + }) + + g.It("Author:mjoseph-LEVEL0-Critical-51255-cluster-ingress-operator can set AWS ELB idle Timeout on per controller basis", func() { + compat_otp.By("Pre-flight check for the platform type") + compat_otp.SkipIfPlatformTypeNot(oc, "AWS") + + buildPruningBaseDir := testdata.FixturePath("router") + customTemp := filepath.Join(buildPruningBaseDir, "ingresscontroller-clb.yaml") + var ( + ingctrl = ingressControllerDescription{ + name: "ocp51255", + namespace: "openshift-ingress-operator", + domain: "", + template: customTemp, + } + ) + + compat_otp.By("Create a custom ingresscontroller") + baseDomain := getBaseDomain(oc) + ingctrl.domain = ingctrl.name + "." + baseDomain + defer ingctrl.delete(oc) + ingctrl.create(oc) + ensureRouterDeployGenerationIs(oc, ingctrl.name, "1") + + compat_otp.By("Patch the new custom ingress controller with connectionIdleTimeout as 2m") + patchResourceAsAdmin(oc, ingctrl.namespace, "ingresscontroller/"+ingctrl.name, "{\"spec\":{\"endpointPublishingStrategy\":{\"loadBalancer\":{\"providerParameters\":{\"aws\":{\"classicLoadBalancer\":{\"connectionIdleTimeout\":\"2m\"}}}}}}}") + + compat_otp.By("Check the LB service and ensure the annotations are updated") + waitForOutputContains(oc, "openshift-ingress", "svc/router-"+ingctrl.name, "{.metadata.annotations}", `"service.beta.kubernetes.io/aws-load-balancer-connection-idle-timeout":"120"`) + + compat_otp.By("Check the connectionIdleTimeout value in the controller status") + waitForOutputContains(oc, ingctrl.namespace, "ingresscontroller/"+ingctrl.name, "{.status.endpointPublishingStrategy.loadBalancer.providerParameters.aws.classicLoadBalancer.connectionIdleTimeout}", "2m0s") + }) + + g.It("Author:mjoseph-Medium-51256-cluster-ingress-operator does not accept negative value of AWS ELB idle Timeout option", func() { + compat_otp.By("Pre-flight check for the platform type") + compat_otp.SkipIfPlatformTypeNot(oc, "AWS") + + buildPruningBaseDir := testdata.FixturePath("router") + customTemp := filepath.Join(buildPruningBaseDir, "ingresscontroller-clb.yaml") + var ( + ingctrl = ingressControllerDescription{ + name: "ocp51256", + namespace: "openshift-ingress-operator", + domain: "", + template: customTemp, + } + ) + + compat_otp.By("Create a custom ingresscontroller") + baseDomain := getBaseDomain(oc) + ingctrl.domain = ingctrl.name + "." + baseDomain + defer ingctrl.delete(oc) + ingctrl.create(oc) + ensureRouterDeployGenerationIs(oc, ingctrl.name, "1") + + compat_otp.By("Patch the new custom ingress controller with connectionIdleTimeout with a negative value") + patchResourceAsAdmin(oc, ingctrl.namespace, "ingresscontroller/"+ingctrl.name, "{\"spec\":{\"endpointPublishingStrategy\":{\"loadBalancer\":{\"providerParameters\":{\"aws\":{\"classicLoadBalancer\":{\"connectionIdleTimeout\":\"-2m\"}}}}}}}") + + compat_otp.By("Check the LB service and ensure the annotation is not added") + findAnnotation := getAnnotation(oc, "openshift-ingress", "svc", "router-"+ingctrl.name) + o.Expect(findAnnotation).NotTo(o.ContainSubstring("service.beta.kubernetes.io/aws-load-balancer-connection-idle-timeout")) + + compat_otp.By("Check the connectionIdleTimeout value is '0s' in the controller status") + waitForOutputContains(oc, ingctrl.namespace, "ingresscontroller/"+ingctrl.name, "{.status.endpointPublishingStrategy.loadBalancer.providerParameters.aws.classicLoadBalancer.connectionIdleTimeout}", "0s") + }) + + // author: shudili@redhat.com + g.It("Author:shudili-High-54868-Configurable dns Management for LoadBalancerService Ingress Controllers on AWS", func() { + var ( + buildPruningBaseDir = testdata.FixturePath("router") + customTemp = filepath.Join(buildPruningBaseDir, "ingresscontroller-external.yaml") + ingctrl1 = ingressControllerDescription{ + name: "ocp54868cus11", + namespace: "openshift-ingress-operator", + domain: "", + template: customTemp, + } + ingctrl2 = ingressControllerDescription{ + name: "ocp54868cus22", + namespace: "openshift-ingress-operator", + domain: "ocp54868cus22.test.com", + template: customTemp, + } + ingctrlResource1 = "ingresscontrollers/" + ingctrl1.name + dnsrecordResource1 = "dnsrecords/" + ingctrl1.name + "-wildcard" + ingctrlResource2 = "ingresscontrollers/" + ingctrl2.name + dnsrecordResource2 = "dnsrecords/" + ingctrl2.name + "-wildcard" + ) + + // skip if platform is not AWS + compat_otp.SkipIfPlatformTypeNot(oc, "AWS") + // skip if private cluster in 4.19+ + if isInternalLBScopeInDefaultIngresscontroller(oc) { + g.Skip("Skip for private cluster since Internal LB scope in default ingresscontroller") + } + + // skip if the AWS platform has NOT zones and thus the feature is not supported on this cluster + dnsZone, _ := oc.AsAdmin().WithoutNamespace().Run("get").Args("dnses.config", "cluster", "-o=jsonpath={.spec.privateZone}").Output() + if len(dnsZone) < 1 { + jsonPath := "{.status.conditions[?(@.type==\"DNSManaged\")].status}: {.status.conditions[?(@.type==\"DNSManaged\")].reason}" + output, _ := oc.AsAdmin().WithoutNamespace().Run("get").Args("ingresscontrollers/default", "-n", "openshift-ingress-operator", "-o=jsonpath="+jsonPath).Output() + o.Expect(output).To(o.ContainSubstring("False: NoDNSZones")) + g.Skip("Skip for this AWS platform has NOT DNS zones, which means this case is not supported on this AWS platform") + } + + compat_otp.By("Create two custom ingresscontrollers, one matches the cluster's base domain, the other doesn't") + baseDomain := getBaseDomain(oc) + ingctrl1.domain = ingctrl1.name + "." + baseDomain + defer ingctrl1.delete(oc) + ingctrl1.create(oc) + defer ingctrl2.delete(oc) + ingctrl2.create(oc) + ensureCustomIngressControllerAvailable(oc, ingctrl1.name) + ensureCustomIngressControllerAvailable(oc, ingctrl2.name) + + compat_otp.By("check the default dnsManagementPolicy value of ingress-controller1 matching the base domain, which should be Managed") + output, _ := oc.AsAdmin().WithoutNamespace().Run("get").Args(ingctrlResource1, "-n", ingctrl1.namespace, "-o=jsonpath={.spec.endpointPublishingStrategy.loadBalancer.dnsManagementPolicy}").Output() + o.Expect(output).To(o.ContainSubstring("Managed")) + + compat_otp.By("check ingress-controller1's status") + output, _ = oc.AsAdmin().WithoutNamespace().Run("get").Args(ingctrlResource1, "-n", ingctrl1.namespace, "-o=jsonpath={.status.conditions[?(@.type==\"DNSManaged\")].status}{.status.conditions[?(@.type==\"DNSReady\")].status}").Output() + o.Expect(output).To(o.ContainSubstring("TrueTrue")) + + compat_otp.By("check the default dnsManagementPolicy value of dnsrecord ocp54868cus1, which should be Managed, too") + output, _ = oc.AsAdmin().WithoutNamespace().Run("get").Args(dnsrecordResource1, "-n", ingctrl1.namespace, "-o=jsonpath={.spec.dnsManagementPolicy}").Output() + o.Expect(output).To(o.ContainSubstring("Managed")) + + compat_otp.By("check dnsrecord ocp54868cus1's status") + output, _ = oc.AsAdmin().WithoutNamespace().Run("get").Args(dnsrecordResource1, "-n", ingctrl1.namespace, "-o=jsonpath={.status.zones[0].conditions[0].status}{.status.zones[0].conditions[0].reason}").Output() + o.Expect(output).To(o.ContainSubstring("TrueProviderSuccess")) + + compat_otp.By("patch custom ingress-controller1 with dnsManagementPolicy Unmanaged") + defer func() { + jsonpath := `{.status.conditions[?(@.type=="DNSManaged")].status}{.status.conditions[?(@.type=="DNSReady")].status}` + patchResourceAsAdmin(oc, ingctrl1.namespace, ingctrlResource1, `{"spec":{"endpointPublishingStrategy":{"loadBalancer":{"dnsManagementPolicy":"Managed"}}}}`) + waitForOutputEquals(oc, ingctrl1.namespace, ingctrlResource1, jsonpath, "TrueTrue") + jsonpath = "{.spec.dnsManagementPolicy}" + waitForOutputEquals(oc, ingctrl1.namespace, dnsrecordResource1, jsonpath, "Managed") + }() + patchResourceAsAdmin(oc, ingctrl1.namespace, ingctrlResource1, `{"spec":{"endpointPublishingStrategy":{"loadBalancer":{"dnsManagementPolicy":"Unmanaged"}}}}`) + + compat_otp.By("check the dnsManagementPolicy value of ingress-controller1, which should be Unmanaged") + jpath := "{.spec.endpointPublishingStrategy.loadBalancer.dnsManagementPolicy}" + waitForOutputEquals(oc, ingctrl1.namespace, ingctrlResource1, jpath, "Unmanaged") + + compat_otp.By("check ingress-controller1's status") + jpath = `{.status.conditions[?(@.type=="DNSManaged")].status}{.status.conditions[?(@.type=="DNSReady")].status}` + waitForOutputEquals(oc, ingctrl1.namespace, ingctrlResource1, jpath, "FalseUnknown") + + compat_otp.By("check the dnsManagementPolicy value of dnsrecord ocp54868cus1, which should be Unmanaged, too") + jpath = "{.spec.dnsManagementPolicy}" + waitForOutputEquals(oc, ingctrl1.namespace, dnsrecordResource1, jpath, "Unmanaged") + + compat_otp.By("check dnsrecord ocp54868cus1's status") + jpath = "{.status.zones[0].conditions[0].status}{.status.zones[0].conditions[0].reason}" + waitForOutputEquals(oc, ingctrl1.namespace, dnsrecordResource1, jpath, "UnknownUnmanagedDNS") + + // there was a bug OCPBUGS-2247 in the below test step + // compat_otp.By("check the default dnsManagementPolicy value of ingress-controller2 not matching the base domain, which should be Unmanaged") + // output, _ = oc.AsAdmin().WithoutNamespace().Run("get").Args(ingctrlResource2, "-n", ingctrl2.namespace, "-o=jsonpath={.spec.endpointPublishingStrategy.loadBalancer.dnsManagementPolicy}").Output() + // o.Expect(output).To(o.ContainSubstring("Unmanaged")) + + compat_otp.By("check ingress-controller2's status") + output, _ = oc.AsAdmin().WithoutNamespace().Run("get").Args(ingctrlResource2, "-n", ingctrl2.namespace, "-o=jsonpath={.status.conditions[?(@.type==\"DNSManaged\")].status}{.status.conditions[?(@.type==\"DNSReady\")].status}").Output() + o.Expect(output).To(o.ContainSubstring("FalseUnknown")) + + // there was a bug OCPBUGS-2247 in the below test step + // compat_otp.By("check the default dnsManagementPolicy value of dnsrecord ocp54868cus2, which should be Unmanaged, too") + // output, _ = oc.AsAdmin().WithoutNamespace().Run("get").Args(dnsrecordResource2, "-n", ingctrl2.namespace, "-o=jsonpath={.spec.dnsManagementPolicy}").Output() + // o.Expect(output).To(o.ContainSubstring("Unmanaged")) + + compat_otp.By("check dnsrecord ocp54868cus2's status") + output, _ = oc.AsAdmin().WithoutNamespace().Run("get").Args(dnsrecordResource2, "-n", ingctrl2.namespace, "-o=jsonpath={.status.zones[0].conditions[0].status}{.status.zones[0].conditions[0].reason}").Output() + o.Expect(output).To(o.ContainSubstring("UnknownUnmanagedDNS")) + + }) + + // author: shudili@redhat.com + g.It("Author:shudili-Low-54995-Negative Test of Configurable dns Management for LoadBalancerService Ingress Controllers on AWS", func() { + var ( + buildPruningBaseDir = testdata.FixturePath("router") + customTemp = filepath.Join(buildPruningBaseDir, "ingresscontroller-external.yaml") + ingctrl = ingressControllerDescription{ + name: "ocp54995", + namespace: "openshift-ingress-operator", + domain: "ocp54995.test.com", + template: customTemp, + } + ingctrlResource = "ingresscontrollers/" + ingctrl.name + ) + + // skip if platform is not AWS + compat_otp.SkipIfPlatformTypeNot(oc, "AWS") + + compat_otp.By("Create a custom ingresscontrollers") + defer ingctrl.delete(oc) + ingctrl.create(oc) + ensureRouterDeployGenerationIs(oc, ingctrl.name, "1") + + compat_otp.By("try to patch the custom ingress-controller with dnsManagementPolicy unmanaged") + patch := "{\"spec\":{\"endpointPublishingStrategy\":{\"loadBalancer\":{\"dnsManagementPolicy\":\"unmanaged\"}}}}" + output, err := oc.AsAdmin().WithoutNamespace().Run("patch").Args(ingctrlResource, "-p", patch, "--type=merge", "-n", ingctrl.namespace).Output() + o.Expect(err).To(o.HaveOccurred()) + o.Expect(output).To(o.ContainSubstring("dnsManagementPolicy: Unsupported value: \"unmanaged\": supported values: \"Managed\", \"Unmanaged\"")) + + compat_otp.By("try to patch the custom ingress-controller with dnsManagementPolicy abc") + patch = "{\"spec\":{\"endpointPublishingStrategy\":{\"loadBalancer\":{\"dnsManagementPolicy\":\"abc\"}}}}" + output, err = oc.AsAdmin().WithoutNamespace().Run("patch").Args(ingctrlResource, "-p", patch, "--type=merge", "-n", ingctrl.namespace).Output() + o.Expect(err).To(o.HaveOccurred()) + o.Expect(output).To(o.ContainSubstring("dnsManagementPolicy: Unsupported value: \"abc\": supported values: \"Managed\", \"Unmanaged\"")) + + compat_otp.By("try to patch the custom ingress-controller with dnsManagementPolicy 123") + patch = "{\"spec\":{\"endpointPublishingStrategy\":{\"loadBalancer\":{\"dnsManagementPolicy\":123}}}}" + output, err = oc.AsAdmin().WithoutNamespace().Run("patch").Args(ingctrlResource, "-p", patch, "--type=merge", "-n", ingctrl.namespace).Output() + o.Expect(err).To(o.HaveOccurred()) + o.Expect(output).To(o.ContainSubstring("dnsManagementPolicy: Unsupported value: 123: supported values: \"Managed\", \"Unmanaged\"")) + }) + + // author: mjoseph@redhat.com + g.It("Author:mjoseph-LEVEL0-Critical-55223-Configuring list of IP address ranges using allowedSourceRanges in LoadBalancerService", func() { + var ( + buildPruningBaseDir = testdata.FixturePath("router") + customTemp = filepath.Join(buildPruningBaseDir, "ingresscontroller-external.yaml") + ingctrl = ingressControllerDescription{ + name: "ocp55223", + namespace: "openshift-ingress-operator", + domain: "", + template: customTemp, + } + ingctrlResource = "ingresscontrollers/ocp55223" + ) + + // skip if platform is not AWS, GCP, AZURE or IBM + compat_otp.By("Pre-flight check for the platform type") + platformtype := compat_otp.CheckPlatform(oc) + platforms := map[string]bool{ + "aws": true, + "azure": true, + "gcp": true, + "ibmcloud": true, + } + if !platforms[platformtype] { + g.Skip("Skip for non-supported platform") + } + + compat_otp.By("Create a custom ingresscontroller") + baseDomain := getBaseDomain(oc) + ingctrl.domain = ingctrl.name + "." + baseDomain + defer ingctrl.delete(oc) + ingctrl.create(oc) + ensureRouterDeployGenerationIs(oc, ingctrl.name, "1") + + compat_otp.By("Patch the custom ingress-controller with IP address ranges to which access to the load balancer should be restricted") + output, errCfg := patchResourceAsAdminAndGetLog(oc, ingctrl.namespace, ingctrlResource, + "{\"spec\":{\"endpointPublishingStrategy\":{\"loadBalancer\":{\"allowedSourceRanges\":[\"10.0.0.0/8\"]}}}}") + o.Expect(errCfg).NotTo(o.HaveOccurred()) + o.Expect(output).To(o.ContainSubstring("ingresscontroller.operator.openshift.io/ocp55223 patched")) + + compat_otp.By("Check the LB svc of the custom controller") + jsonPath := "{.spec.loadBalancerSourceRanges}" + waitForOutputContains(oc, "openshift-ingress", "svc/router-ocp55223", jsonPath, `10.0.0.0/8`) + + compat_otp.By("Patch the custom ingress-controller with more 'allowedSourceRanges' value") + output, errCfg = patchResourceAsAdminAndGetLog(oc, ingctrl.namespace, ingctrlResource, + "{\"spec\":{\"endpointPublishingStrategy\":{\"loadBalancer\":{\"allowedSourceRanges\":[\"20.0.0.0/8\", \"50.0.0.0/16\", \"3dee:ef5::/12\"]}}}}") + o.Expect(errCfg).NotTo(o.HaveOccurred()) + o.Expect(output).To(o.ContainSubstring("ingresscontroller.operator.openshift.io/ocp55223 patched")) + + compat_otp.By("Check the LB svc of the custom controller for additional values") + waitForOutputContains(oc, "openshift-ingress", "svc/router-ocp55223", jsonPath, `20.0.0.0/8`) + waitForOutputContains(oc, "openshift-ingress", "svc/router-ocp55223", jsonPath, `50.0.0.0/16`) + waitForOutputContains(oc, "openshift-ingress", "svc/router-ocp55223", jsonPath, `3dee:ef5::/12`) + }) + + // author: mjoseph@redhat.com + g.It("Author:mjoseph-High-55341-configuring list of IP address ranges using load-balancer-source-ranges annotation", func() { + buildPruningBaseDir := testdata.FixturePath("router") + customTemp := filepath.Join(buildPruningBaseDir, "ingresscontroller-external.yaml") + var ( + ingctrl = ingressControllerDescription{ + name: "ocp55341", + namespace: "openshift-ingress-operator", + domain: "", + template: customTemp, + } + ) + + // skip if platform is not AWS, GCP, AZURE or IBM + compat_otp.By("Pre-flight check for the platform type") + platformtype := compat_otp.CheckPlatform(oc) + platforms := map[string]bool{ + "aws": true, + "azure": true, + "gcp": true, + "ibmcloud": true, + } + if !platforms[platformtype] { + g.Skip("Skip for non-supported platform") + } + + compat_otp.By("Create one custom ingresscontroller") + baseDomain := getBaseDomain(oc) + ingctrl.domain = ingctrl.name + "." + baseDomain + defer ingctrl.delete(oc) + ingctrl.create(oc) + ensureRouterDeployGenerationIs(oc, ingctrl.name, "1") + + compat_otp.By("Add the IP address ranges for an custom IngressController using annotation") + err1 := oc.AsAdmin().WithoutNamespace().Run("annotate").Args( + "-n", "openshift-ingress", "svc/router-ocp55341", "service.beta.kubernetes.io/load-balancer-source-ranges=10.0.0.0/8", "--overwrite").Execute() + o.Expect(err1).NotTo(o.HaveOccurred()) + + compat_otp.By("Verify the annotation presence") + findAnnotation := getAnnotation(oc, "openshift-ingress", "svc", "router-ocp55341") + o.Expect(findAnnotation).To(o.ContainSubstring("service.beta.kubernetes.io/load-balancer-source-ranges")) + o.Expect(findAnnotation).To(o.ContainSubstring("10.0.0.0/8")) + + compat_otp.By("Check the annotation value in the allowedSourceRanges in the controller status") + waitForOutputContains(oc, "openshift-ingress-operator", "ingresscontroller/ocp55341", "{.status.endpointPublishingStrategy.loadBalancer.allowedSourceRanges}", `10.0.0.0/8`) + + compat_otp.By("Patch the loadBalancerSourceRanges in the LB service") + patchResourceAsAdmin(oc, "openshift-ingress", "svc/router-ocp55341", "{\"spec\":{\"loadBalancerSourceRanges\":[\"30.0.0.0/16\"]}}") + + compat_otp.By("Check the annotation value and sourcerange value in LB svc") + findAnnotation = getAnnotation(oc, "openshift-ingress", "svc", "router-ocp55341") + o.Expect(findAnnotation).To(o.ContainSubstring("service.beta.kubernetes.io/load-balancer-source-ranges")) + o.Expect(findAnnotation).To(o.ContainSubstring("10.0.0.0/8")) + waitForOutputContains(oc, "openshift-ingress", "svc/router-ocp55341", "{.spec.loadBalancerSourceRanges}", `30.0.0.0/16`) + + compat_otp.By("Check the controller status and confirm the sourcerange value's precedence over the annotation") + waitForOutputContains(oc, "openshift-ingress-operator", "ingresscontroller/ocp55341", "{.status.endpointPublishingStrategy.loadBalancer.allowedSourceRanges}", `30.0.0.0/16`) + }) + + // author: mjoseph@redhat.com + g.It("Author:mjoseph-Medium-55381-Configuring wrong data for allowedSourceRanges in LoadBalancerService", func() { + var ( + buildPruningBaseDir = testdata.FixturePath("router") + customTemp = filepath.Join(buildPruningBaseDir, "ingresscontroller-external.yaml") + ingctrl = ingressControllerDescription{ + name: "ocp55381", + namespace: "openshift-ingress-operator", + domain: "", + template: customTemp, + } + ingctrlResource = "ingresscontrollers/ocp55381" + ) + + // skip if platform is not AWS, GCP, AZURE or IBM + compat_otp.By("Pre-flight check for the platform type") + platformtype := compat_otp.CheckPlatform(oc) + platforms := map[string]bool{ + "aws": true, + "azure": true, + "gcp": true, + "ibmcloud": true, + } + if !platforms[platformtype] { + g.Skip("Skip for non-supported platform") + } + + compat_otp.By("Create a custom ingresscontroller") + baseDomain := getBaseDomain(oc) + ingctrl.domain = ingctrl.name + "." + baseDomain + defer ingctrl.delete(oc) + ingctrl.create(oc) + ensureRouterDeployGenerationIs(oc, ingctrl.name, "1") + + compat_otp.By("Patch the custom ingress-controller with only IP address") + output, errCfg := patchResourceAsAdminAndGetLog(oc, ingctrl.namespace, ingctrlResource, + "{\"spec\":{\"endpointPublishingStrategy\":{\"loadBalancer\":{\"allowedSourceRanges\":[\"10.0.0.0\"]}}}}") + o.Expect(errCfg).To(o.HaveOccurred()) + o.Expect(output).To(o.ContainSubstring("The IngressController \"ocp55381\" is invalid")) + + compat_otp.By("Patch the custom ingress-controller with a invalid IPv6 address") + output, errCfg = patchResourceAsAdminAndGetLog(oc, ingctrl.namespace, ingctrlResource, + "{\"spec\":{\"endpointPublishingStrategy\":{\"loadBalancer\":{\"allowedSourceRanges\":[\"3dee:ef5:/12\"]}}}}") + o.Expect(errCfg).To(o.HaveOccurred()) + o.Expect(output).To(o.ContainSubstring("The IngressController \"ocp55381\" is invalid")) + + compat_otp.By("Patch the custom ingress-controller with IP address ranges") + patchResourceAsAdmin(oc, ingctrl.namespace, ingctrlResource, "{\"spec\":{\"endpointPublishingStrategy\":{\"loadBalancer\":{\"allowedSourceRanges\":[\"10.0.0.0/8\"]}}}}") + + compat_otp.By("Delete the allowedSourceRanges from custom controller") + patchResourceAsAdmin(oc, ingctrl.namespace, ingctrlResource, "{\"spec\":{\"endpointPublishingStrategy\":{\"loadBalancer\":{\"allowedSourceRanges\":[]}}}}") + + compat_otp.By("Check the ingress operator status to confirm whether it is still in Progress") + ensureClusterOperatorProgress(oc, "ingress") + + compat_otp.By("Patch the same loadBalancerSourceRanges value in the LB service to remove the Progressing from the ingress operator") + patchResourceAsAdmin(oc, "openshift-ingress", "svc/router-ocp55381", "{\"spec\":{\"loadBalancerSourceRanges\":[]}}") + }) + + // bug: 2007246 + g.It("Author:shudili-Medium-56772-Ingress Controller does not set allowPrivilegeEscalation in the router deployment [Serial]", func() { + var ( + buildPruningBaseDir = testdata.FixturePath("router") + customTemp = filepath.Join(buildPruningBaseDir, "ingresscontroller-np.yaml") + scc = filepath.Join(buildPruningBaseDir, "scc-bug2007246.json") + ingctrl = ingressControllerDescription{ + name: "ocp56772", + namespace: "openshift-ingress-operator", + domain: "", + template: customTemp, + } + ) + + compat_otp.By("Create a custom ingresscontrollers") + baseDomain := getBaseDomain(oc) + ingctrl.domain = ingctrl.name + "." + baseDomain + defer ingctrl.delete(oc) + ingctrl.create(oc) + ensureCustomIngressControllerAvailable(oc, ingctrl.name) + + compat_otp.By("Create the custom-restricted SecurityContextConstraints") + defer operateResourceFromFile(oc, "delete", "openshift-ingress", scc) + operateResourceFromFile(oc, "create", "openshift-ingress", scc) + + compat_otp.By("check the allowPrivilegeEscalation in the router deployment, which should be true") + jsonPath := "{.spec.template.spec.containers..securityContext.allowPrivilegeEscalation}" + value := getByJsonPath(oc, "openshift-ingress", "deployment/router-"+ingctrl.name, jsonPath) + o.Expect(value).To(o.ContainSubstring("true")) + + compat_otp.By("get router pods and then delete one router pod") + podList1, err1 := oc.AsAdmin().WithoutNamespace().Run("get").Args("pods", "-l", "ingresscontroller.operator.openshift.io/deployment-ingresscontroller="+ingctrl.name, "-o=jsonpath={.items[*].metadata.name}", "-n", "openshift-ingress").Output() + o.Expect(err1).NotTo(o.HaveOccurred()) + routerpod := getOneRouterPodNameByIC(oc, ingctrl.name) + err := oc.AsAdmin().WithoutNamespace().Run("delete").Args("pod", routerpod, "-n", "openshift-ingress").Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + err = waitForResourceToDisappear(oc, "openshift-ingress", "pod/"+routerpod) + compat_otp.AssertWaitPollNoErr(err, fmt.Sprintf("resource %v does not disapper", "pod/"+routerpod)) + + compat_otp.By("get router pods again, and check if it is different with the previous router pod list") + podList2, err2 := oc.AsAdmin().WithoutNamespace().Run("get").Args("pods", "-l", "ingresscontroller.operator.openshift.io/deployment-ingresscontroller="+ingctrl.name, "-o=jsonpath={.items[*].metadata.name}", "-n", "openshift-ingress").Output() + o.Expect(err2).NotTo(o.HaveOccurred()) + o.Expect(len(podList1)).To(o.Equal(len(podList2))) + o.Expect(strings.Compare(podList1, podList2)).NotTo(o.Equal(0)) + }) + + // author: shudili@redhat.com + g.It("Author:shudili-ROSA-OSD_CCS-ARO-NonPreRelease-Medium-60012-matchExpressions for routeSelector defined in an ingress-controller", func() { + var ( + buildPruningBaseDir = testdata.FixturePath("router") + customTemp = filepath.Join(buildPruningBaseDir, "ingresscontroller-np.yaml") + testPodSvc = filepath.Join(buildPruningBaseDir, "web-server-deploy.yaml") + srvrcInfo = "web-server-deploy" + srvName = "service-unsecure" + ingctrl = ingressControllerDescription{ + name: "ocp60012", + namespace: "openshift-ingress-operator", + domain: "", + template: customTemp, + } + ingctrlResource = "ingresscontroller/" + ingctrl.name + ) + + compat_otp.By("Create one custom ingresscontroller") + baseDomain := getBaseDomain(oc) + ingctrl.domain = ingctrl.name + "." + baseDomain + defer ingctrl.delete(oc) + ingctrl.create(oc) + ensureRouterDeployGenerationIs(oc, ingctrl.name, "1") + + compat_otp.By("Create an unsecure service and its backend pod") + ns := oc.Namespace() + createResourceFromFile(oc, ns, testPodSvc) + ensurePodWithLabelReady(oc, ns, "name="+srvrcInfo) + + compat_otp.By("Create 4 routes for the testing") + err := oc.WithoutNamespace().Run("expose").Args("service", srvName, "--name=unsrv-1", "-n", ns).Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + err = oc.WithoutNamespace().Run("expose").Args("service", srvName, "--name=unsrv-2", "-n", ns).Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + err = oc.WithoutNamespace().Run("expose").Args("service", srvName, "--name=unsrv-3", "-n", ns).Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + err = oc.WithoutNamespace().Run("expose").Args("service", srvName, "--name=unsrv-4", "-n", ns).Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + waitForOutputEquals(oc, ns, "route/unsrv-1", "{.metadata.name}", "unsrv-1") + waitForOutputEquals(oc, ns, "route/unsrv-2", "{.metadata.name}", "unsrv-2") + waitForOutputEquals(oc, ns, "route/unsrv-3", "{.metadata.name}", "unsrv-3") + waitForOutputEquals(oc, ns, "route/unsrv-4", "{.metadata.name}", "unsrv-4") + + compat_otp.By("Add labels to 3 routes") + err = oc.WithoutNamespace().Run("label").Args("route", "unsrv-1", "test=aaa", "-n", ns).Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + err = oc.WithoutNamespace().Run("label").Args("route", "unsrv-2", "test=bbb", "-n", ns).Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + err = oc.WithoutNamespace().Run("label").Args("route", "unsrv-3", "test=ccc", "-n", ns).Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + + compat_otp.By("Patch the custom ingress-controllers with In matchExpressions routeSelector") + routerpod := getOneNewRouterPodFromRollingUpdate(oc, ingctrl.name) + patchRouteSelector := "{\"spec\":{\"routeSelector\":{\"matchExpressions\":[{\"key\": \"test\", \"operator\": \"In\", \"values\":[\"aaa\", \"bbb\"]}]}}}" + err = oc.AsAdmin().WithoutNamespace().Run("patch").Args(ingctrlResource, "-p", patchRouteSelector, "--type=merge", "-n", ingctrl.namespace).Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + err = waitForResourceToDisappear(oc, "openshift-ingress", "pod/"+routerpod) + compat_otp.AssertWaitPollNoErr(err, fmt.Sprintf("resource %v does not disapper", "pod/"+routerpod)) + + compat_otp.By("Check if route unsrv-1 and unsrv-2 are admitted by the custom IC with In matchExpressions routeSelector, while route unsrv-3 and unsrv-4 not") + jsonPath := "{.status.ingress[?(@.routerName==\"" + ingctrl.name + "\")].conditions[?(@.type==\"Admitted\")].status}" + ensureRouteIsAdmittedByIngressController(oc, ns, "unsrv-1", ingctrl.name) + ensureRouteIsAdmittedByIngressController(oc, ns, "unsrv-2", ingctrl.name) + waitForOutputEquals(oc, ns, "route/unsrv-3", jsonPath, "") + waitForOutputEquals(oc, ns, "route/unsrv-4", jsonPath, "") + + compat_otp.By("Patch the custom ingress-controllers with NotIn matchExpressions routeSelector") + routerpod = getOneRouterPodNameByIC(oc, ingctrl.name) + patchRouteSelector = "{\"spec\":{\"routeSelector\":{\"matchExpressions\":[{\"key\": \"test\", \"operator\": \"NotIn\", \"values\":[\"aaa\", \"bbb\"]}]}}}" + err = oc.AsAdmin().WithoutNamespace().Run("patch").Args(ingctrlResource, "-p", patchRouteSelector, "--type=merge", "-n", ingctrl.namespace).Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + err = waitForResourceToDisappear(oc, "openshift-ingress", "pod/"+routerpod) + compat_otp.AssertWaitPollNoErr(err, fmt.Sprintf("resource %v does not disapper", "pod/"+routerpod)) + + compat_otp.By("Check if route unsrv-3 and unsrv-4 are admitted by the custom IC with NotIn matchExpressions routeSelector, while route unsrv-1 and unsrv-2 not") + ensureRouteIsAdmittedByIngressController(oc, ns, "unsrv-3", ingctrl.name) + waitForOutputEquals(oc, ns, "route/unsrv-1", jsonPath, "") + waitForOutputEquals(oc, ns, "route/unsrv-2", jsonPath, "") + ensureRouteIsAdmittedByIngressController(oc, ns, "unsrv-4", ingctrl.name) + + compat_otp.By("Patch the custom ingress-controllers with Exists matchExpressions routeSelector") + routerpod = getOneRouterPodNameByIC(oc, ingctrl.name) + patchRouteSelector = "{\"spec\":{\"routeSelector\":{\"matchExpressions\":[{\"key\": \"test\", \"operator\": \"Exists\"}]}}}" + err = oc.AsAdmin().WithoutNamespace().Run("patch").Args(ingctrlResource, "-p", patchRouteSelector, "--type=merge", "-n", ingctrl.namespace).Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + err = waitForResourceToDisappear(oc, "openshift-ingress", "pod/"+routerpod) + compat_otp.AssertWaitPollNoErr(err, fmt.Sprintf("resource %v does not disapper", "pod/"+routerpod)) + + compat_otp.By("Check if route unsrv-1, unsrv-2 and unsrv-3 are admitted by the custom IC with Exists matchExpressions routeSelector, while route unsrv-4 not") + ensureRouteIsAdmittedByIngressController(oc, ns, "unsrv-1", ingctrl.name) + ensureRouteIsAdmittedByIngressController(oc, ns, "unsrv-2", ingctrl.name) + ensureRouteIsAdmittedByIngressController(oc, ns, "unsrv-3", ingctrl.name) + waitForOutputEquals(oc, ns, "route/unsrv-4", jsonPath, "") + + compat_otp.By("Patch the custom ingress-controllers with DoesNotExist matchExpressions routeSelector") + routerpod = getOneRouterPodNameByIC(oc, ingctrl.name) + patchRouteSelector = "{\"spec\":{\"routeSelector\":{\"matchExpressions\":[{\"key\": \"test\", \"operator\": \"DoesNotExist\"}]}}}" + err = oc.AsAdmin().WithoutNamespace().Run("patch").Args(ingctrlResource, "-p", patchRouteSelector, "--type=merge", "-n", ingctrl.namespace).Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + err = waitForResourceToDisappear(oc, "openshift-ingress", "pod/"+routerpod) + compat_otp.AssertWaitPollNoErr(err, fmt.Sprintf("resource %v does not disapper", "pod/"+routerpod)) + + compat_otp.By("Check if route unsrv-4 is admitted by the custom IC with DoesNotExist matchExpressions routeSelector, while route unsrv-1, unsrv-2 and unsrv-3 not") + ensureRouteIsAdmittedByIngressController(oc, ns, "unsrv-4", ingctrl.name) + waitForOutputEquals(oc, ns, "route/unsrv-1", jsonPath, "") + waitForOutputEquals(oc, ns, "route/unsrv-2", jsonPath, "") + waitForOutputEquals(oc, ns, "route/unsrv-3", jsonPath, "") + }) + + // author: shudili@redhat.com + g.It("Author:shudili-ROSA-OSD_CCS-ARO-NonPreRelease-Medium-60013-matchExpressions for namespaceSelector defined in an ingress-controller", func() { + var ( + buildPruningBaseDir = testdata.FixturePath("router") + customTemp = filepath.Join(buildPruningBaseDir, "ingresscontroller-np.yaml") + testPodSvc = filepath.Join(buildPruningBaseDir, "web-server-deploy.yaml") + srvrcInfo = "web-server-deploy" + srvName = "service-unsecure" + ingctrl = ingressControllerDescription{ + name: "ocp60013", + namespace: "openshift-ingress-operator", + domain: "", + template: customTemp, + } + ingctrlResource = "ingresscontroller/" + ingctrl.name + ) + + compat_otp.By("Create one custom ingresscontroller") + baseDomain := getBaseDomain(oc) + ingctrl.domain = ingctrl.name + "." + baseDomain + defer ingctrl.delete(oc) + ingctrl.create(oc) + ensureRouterDeployGenerationIs(oc, ingctrl.name, "1") + + compat_otp.By("create 3 more projects") + project1 := oc.Namespace() + oc.SetupProject() + project2 := oc.Namespace() + oc.SetupProject() + project3 := oc.Namespace() + oc.SetupProject() + project4 := oc.Namespace() + + compat_otp.By("Create an unsecure service and its backend pod, create the route in each of the 4 projects, then wait for some time until the backend pod and route are available") + for index, ns := range []string{project1, project2, project3, project4} { + nsSeq := index + 1 + err := oc.AsAdmin().WithoutNamespace().Run("create").Args("-f", testPodSvc, "-n", ns).Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + err = oc.AsAdmin().WithoutNamespace().Run("expose").Args("service", srvName, "--name=shard-ns"+strconv.Itoa(nsSeq), "-n", ns).Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + } + for indexWt, nsWt := range []string{project1, project2, project3, project4} { + nsSeqWt := indexWt + 1 + ensurePodWithLabelReady(oc, nsWt, "name="+srvrcInfo) + waitForOutputEquals(oc, nsWt, "route/shard-ns"+strconv.Itoa(nsSeqWt), "{.metadata.name}", "shard-ns"+strconv.Itoa(nsSeqWt)) + } + + compat_otp.By("Add labels to 3 projects") + err := oc.AsAdmin().WithoutNamespace().Run("label").Args("namespace", project1, "test=aaa").Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + err = oc.AsAdmin().WithoutNamespace().Run("label").Args("namespace", project2, "test=bbb").Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + err = oc.AsAdmin().WithoutNamespace().Run("label").Args("namespace", project3, "test=ccc").Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + + compat_otp.By("Patch the custom ingresscontroller with In matchExpressions namespaceSelector") + routerpod := getOneNewRouterPodFromRollingUpdate(oc, ingctrl.name) + patchNamespaceSelector := "{\"spec\":{\"namespaceSelector\":{\"matchExpressions\":[{\"key\": \"test\", \"operator\": \"In\", \"values\":[\"aaa\", \"bbb\"]}]}}}" + err = oc.AsAdmin().WithoutNamespace().Run("patch").Args(ingctrlResource, "-p", patchNamespaceSelector, "--type=merge", "-n", ingctrl.namespace).Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + err = waitForResourceToDisappear(oc, "openshift-ingress", "pod/"+routerpod) + compat_otp.AssertWaitPollNoErr(err, fmt.Sprintf("resource %v does not disapper", "pod/"+routerpod)) + + compat_otp.By("Check if route shard-ns1 and shard-ns2 are admitted by the custom IC with In matchExpressions namespaceSelector, while route shard-ns3 and shard-ns4 not") + jsonPath := "{.status.ingress[?(@.routerName==\"" + ingctrl.name + "\")].conditions[?(@.type==\"Admitted\")].status}" + ensureRouteIsAdmittedByIngressController(oc, project1, "shard-ns1", ingctrl.name) + ensureRouteIsAdmittedByIngressController(oc, project2, "shard-ns2", ingctrl.name) + waitForOutputEquals(oc, project3, "route/shard-ns3", jsonPath, "") + waitForOutputEquals(oc, project4, "route/shard-ns4", jsonPath, "") + + compat_otp.By("Patch the custom ingresscontroller with NotIn matchExpressions namespaceSelector") + routerpod = getOneRouterPodNameByIC(oc, ingctrl.name) + patchNamespaceSelector = "{\"spec\":{\"namespaceSelector\":{\"matchExpressions\":[{\"key\": \"test\", \"operator\": \"NotIn\", \"values\":[\"aaa\", \"bbb\"]}]}}}" + err = oc.AsAdmin().WithoutNamespace().Run("patch").Args(ingctrlResource, "-p", patchNamespaceSelector, "--type=merge", "-n", ingctrl.namespace).Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + err = waitForResourceToDisappear(oc, "openshift-ingress", "pod/"+routerpod) + compat_otp.AssertWaitPollNoErr(err, fmt.Sprintf("resource %v does not disapper", "pod/"+routerpod)) + + compat_otp.By("Check if route shard-ns3 and shard-ns4 are admitted by the custom IC with NotIn matchExpressions namespaceSelector, while route shard-ns1 and shard-ns2 not") + ensureRouteIsAdmittedByIngressController(oc, project3, "shard-ns3", ingctrl.name) + waitForOutputEquals(oc, project1, "route/shard-ns1", jsonPath, "") + waitForOutputEquals(oc, project2, "route/shard-ns2", jsonPath, "") + ensureRouteIsAdmittedByIngressController(oc, project4, "shard-ns4", ingctrl.name) + + compat_otp.By("Patch the custom ingresscontroller with Exists matchExpressions namespaceSelector") + routerpod = getOneRouterPodNameByIC(oc, ingctrl.name) + patchNamespaceSelector = "{\"spec\":{\"namespaceSelector\":{\"matchExpressions\":[{\"key\": \"test\", \"operator\": \"Exists\"}]}}}" + err = oc.AsAdmin().WithoutNamespace().Run("patch").Args(ingctrlResource, "-p", patchNamespaceSelector, "--type=merge", "-n", ingctrl.namespace).Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + err = waitForResourceToDisappear(oc, "openshift-ingress", "pod/"+routerpod) + compat_otp.AssertWaitPollNoErr(err, fmt.Sprintf("resource %v does not disapper", "pod/"+routerpod)) + + compat_otp.By("Check if route shard-ns1, shard-ns2 and shard-ns3 are admitted by the custom IC with Exists matchExpressions namespaceSelector, while route shard-ns4 not") + ensureRouteIsAdmittedByIngressController(oc, project1, "shard-ns1", ingctrl.name) + ensureRouteIsAdmittedByIngressController(oc, project2, "shard-ns2", ingctrl.name) + ensureRouteIsAdmittedByIngressController(oc, project3, "shard-ns3", ingctrl.name) + waitForOutputEquals(oc, project4, "route/shard-ns4", jsonPath, "") + + compat_otp.By("Patch the custom ingresscontroller with DoesNotExist matchExpressions namespaceSelector") + routerpod = getOneRouterPodNameByIC(oc, ingctrl.name) + patchNamespaceSelector = "{\"spec\":{\"namespaceSelector\":{\"matchExpressions\":[{\"key\": \"test\", \"operator\": \"DoesNotExist\"}]}}}" + err = oc.AsAdmin().WithoutNamespace().Run("patch").Args(ingctrlResource, "-p", patchNamespaceSelector, "--type=merge", "-n", ingctrl.namespace).Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + err = waitForResourceToDisappear(oc, "openshift-ingress", "pod/"+routerpod) + compat_otp.AssertWaitPollNoErr(err, fmt.Sprintf("resource %v does not disapper", "pod/"+routerpod)) + + compat_otp.By("Check if route shard-ns4 is admitted by the custom IC with DoesNotExist matchExpressions namespaceSelector, while route shard-ns1, shard-ns2 and shard-ns3 not") + ensureRouteIsAdmittedByIngressController(oc, project4, "shard-ns4", ingctrl.name) + waitForOutputEquals(oc, project1, "route/shard-ns1", jsonPath, "") + waitForOutputEquals(oc, project2, "route/shard-ns2", jsonPath, "") + waitForOutputEquals(oc, project3, "route/shard-ns3", jsonPath, "") + }) + + // OCPBUGS-853 + g.It("Author:shudili-ROSA-OSD_CCS-ARO-Critical-62530-openshift ingress operator is failing to update router-certs [Serial]", func() { + var ( + buildPruningBaseDir = testdata.FixturePath("router") + customTemp = filepath.Join(buildPruningBaseDir, "ingresscontroller-np.yaml") + ingctrl = ingressControllerDescription{ + name: "ocp62530", + namespace: "openshift-ingress-operator", + domain: "", + template: customTemp, + } + ingctrlResource = "ingresscontroller/" + ingctrl.name + ingctrlCert = "custom-cert-62530" + dirname = "/tmp/-OCP-62530-ca/" + name = dirname + "custom62530" + validity = 3650 + caSubj = "/CN=NE-Test-Root-CA" + userCert = dirname + "test" + userSubj = "/CN=*.ocp62530.example.com" + customKey = userCert + ".key" + customCert = userCert + ".crt" + ) + + defer os.RemoveAll(dirname) + err := os.MkdirAll(dirname, 0755) + o.Expect(err).NotTo(o.HaveOccurred()) + + compat_otp.By("Try to create custom key and custom certification by openssl, create a new self-signed CA at first, creating the CA key") + // Generation of a new self-signed CA, in case a corporate or another CA is already existing can be used. + opensslCmd := fmt.Sprintf(`openssl genrsa -out %s-ca.key 4096`, name) + _, err = exec.Command("bash", "-c", opensslCmd).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + + compat_otp.By("Create the CA certificate") + opensslCmd = fmt.Sprintf(`openssl req -x509 -new -nodes -key %s-ca.key -sha256 -days %d -out %s-ca.crt -subj %s`, name, validity, name, caSubj) + _, err = exec.Command("bash", "-c", opensslCmd).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + + compat_otp.By("Create a new user certificate, crearing the user CSR with the private user key") + opensslCmd = fmt.Sprintf(`openssl req -nodes -newkey rsa:2048 -keyout %s.key -subj %s -out %s.csr`, userCert, userSubj, userCert) + _, err = exec.Command("bash", "-c", opensslCmd).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + + compat_otp.By("Sign the user CSR and generate the certificate") + opensslCmd = fmt.Sprintf(`openssl x509 -extfile <(printf "subjectAltName = DNS:*.ocp62530.example.com") -req -in %s.csr -CA %s-ca.crt -CAkey %s-ca.key -CAcreateserial -out %s.crt -days %d -sha256`, userCert, name, name, userCert, validity) + _, err = exec.Command("bash", "-c", opensslCmd).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + + compat_otp.By("Create a tls secret in openshift-ingress ns") + defer oc.AsAdmin().WithoutNamespace().Run("delete").Args("-n", "openshift-ingress", "secret", ingctrlCert).Execute() + err = oc.AsAdmin().WithoutNamespace().Run("create").Args("-n", "openshift-ingress", "secret", "tls", ingctrlCert, "--cert="+customCert, "--key="+customKey).Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + + compat_otp.By("Create a custom ingresscontroller") + ingctrl.domain = ingctrl.name + ".example.com" + defer ingctrl.delete(oc) + ingctrl.create(oc) + ensureRouterDeployGenerationIs(oc, ingctrl.name, "1") + + compat_otp.By("Patch defaultCertificate with custom secret to the IC") + patchResourceAsAdmin(oc, ingctrl.namespace, ingctrlResource, "{\"spec\":{\"defaultCertificate\":{\"name\": \""+ingctrlCert+"\"}}}") + ensureRouterDeployGenerationIs(oc, ingctrl.name, "2") + secret := getByJsonPath(oc, ingctrl.namespace, ingctrlResource, "{.spec.defaultCertificate.name}") + o.Expect(secret).To(o.ContainSubstring(ingctrlCert)) + + compat_otp.By("Check the router-certs in the openshift-config-managed namespace, the data is 1, which should not be increased") + output, err2 := oc.AsAdmin().WithoutNamespace().Run("get").Args("-n", "openshift-config-managed", "secret", "router-certs", "-o=go-template='{{len .data}}'").Output() + o.Expect(err2).NotTo(o.HaveOccurred()) + o.Expect(strings.Trim(output, "'")).To(o.Equal("1")) + }) + + //author: asood@redhat.com + //bug: https://issues.redhat.com/browse/OCPBUGS-6013 + g.It("Author:asood-NonHyperShiftHOST-ConnectedOnly-ROSA-OSD_CCS-Medium-63832-Cluster ingress health checks and routes fail on swapping application router between public and private", func() { + var ( + namespace = "openshift-ingress" + operatorNamespace = "openshift-ingress-operator" + caseID = "63832" + curlURL = "" + strategyScope []string + ) + platform := compat_otp.CheckPlatform(oc) + acceptedPlatform := strings.Contains(platform, "aws") + if !acceptedPlatform { + g.Skip("Test cases should be run on AWS cluster with ovn network plugin, skip for other platforms or other network plugin!!") + } + compat_otp.By("0. Create a custom ingress controller") + buildPruningBaseDir := testdata.FixturePath("router") + customIngressControllerTemp := filepath.Join(buildPruningBaseDir, "ingresscontroller-clb.yaml") + var ( + ingctrl = ingressControllerDescription{ + name: "ocp" + caseID, + namespace: operatorNamespace, + domain: caseID + ".test.com", + template: customIngressControllerTemp, + } + ) + + defer ingctrl.delete(oc) + ingctrl.create(oc) + ensureRouterDeployGenerationIs(oc, ingctrl.name, "1") + + compat_otp.By("1. Annotate ingress controller") + addAnnotationPatch := `{"metadata":{"annotations":{"ingress.operator.openshift.io/auto-delete-load-balancer":""}}}` + errAnnotate := oc.AsAdmin().WithoutNamespace().Run("patch").Args("-n", ingctrl.namespace, "ingresscontrollers/"+ingctrl.name, "--type=merge", "-p", addAnnotationPatch).Execute() + o.Expect(errAnnotate).NotTo(o.HaveOccurred()) + + strategyScope = append(strategyScope, `{"spec":{"endpointPublishingStrategy":{"loadBalancer":{"scope":"Internal"},"type":"LoadBalancerService"}}}`) + strategyScope = append(strategyScope, `{"spec":{"endpointPublishingStrategy":{"loadBalancer":{"scope":"External"},"type":"LoadBalancerService"}}}`) + + compat_otp.By("2. Get the health check node port") + prevHealthCheckNodePort, err := oc.AsAdmin().Run("get").Args("svc", "router-"+ingctrl.name, "-n", namespace, "-o=jsonpath={.spec.healthCheckNodePort}").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + for i := 0; i < len(strategyScope); i++ { + compat_otp.By("3. Change the endpoint publishing strategy") + changeScope := strategyScope[i] + changeScopeErr := oc.AsAdmin().WithoutNamespace().Run("patch").Args("-n", ingctrl.namespace, "ingresscontrollers/"+ingctrl.name, "--type=merge", "-p", changeScope).Execute() + o.Expect(changeScopeErr).NotTo(o.HaveOccurred()) + + compat_otp.By("3.1 Check the state of custom ingress operator") + ensureCustomIngressControllerAvailable(oc, ingctrl.name) + + compat_otp.By("3.2 Check the pods are in running state") + podList, podListErr := compat_otp.GetAllPodsWithLabel(oc, namespace, "ingresscontroller.operator.openshift.io/deployment-ingresscontroller="+ingctrl.name) + o.Expect(podListErr).NotTo(o.HaveOccurred()) + o.Expect(len(podList)).ShouldNot(o.Equal(0)) + podName := podList[0] + + compat_otp.By("3.3 Get node name of one of the pod") + nodeName, nodeNameErr := compat_otp.GetPodNodeName(oc, namespace, podName) + o.Expect(nodeNameErr).NotTo(o.HaveOccurred()) + + compat_otp.By("3.4. Get new health check node port") + err = wait.Poll(30*time.Second, 5*time.Minute, func() (bool, error) { + healthCheckNodePort, healthCheckNPErr := oc.AsAdmin().Run("get").Args("svc", "router-"+ingctrl.name, "-n", namespace, "-o=jsonpath={.spec.healthCheckNodePort}").Output() + o.Expect(healthCheckNPErr).NotTo(o.HaveOccurred()) + if healthCheckNodePort == prevHealthCheckNodePort { + return false, nil + } + curlURL = net.JoinHostPort(nodeName, healthCheckNodePort) + prevHealthCheckNodePort = healthCheckNodePort + return true, nil + }) + compat_otp.AssertWaitPollNoErr(err, fmt.Sprintf("Failed to get health check node port %s", err)) + + compat_otp.By("3.5. Check endpoint is 1") + cmd := fmt.Sprintf("curl %s -s --connect-timeout 5", curlURL) + output, err := compat_otp.DebugNode(oc, nodeName, "bash", "-c", cmd) + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(strings.Contains(output, "\"localEndpoints\": 1")).To(o.BeTrue()) + } + }) + + g.It("Author:mjoseph-NonHyperShiftHOST-Critical-64611-Ingress operator support for private hosted zones in Shared VPC clusters", func() { + compat_otp.By("Pre-flight check for the platform type") + compat_otp.SkipIfPlatformTypeNot(oc, "AWS") + + compat_otp.By("Pre-flight check for the shared VPC platform") + // privateZoneIAMRole needs to be present for shared vpc cluster + privateZoneIAMRole, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("dns.config", "cluster", "-o=jsonpath={.spec.platform.aws}").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + if !strings.Contains(privateZoneIAMRole, "privateZoneIAMRole") { + g.Skip("Skip since this is not a shared vpc cluster") + } + + buildPruningBaseDir := testdata.FixturePath("router") + customTemp1 := filepath.Join(buildPruningBaseDir, "ingresscontroller-external.yaml") + customTemp2 := filepath.Join(buildPruningBaseDir, "ingresscontroller-clb.yaml") + var ( + ingctrl1 = ingressControllerDescription{ + name: "ocp64611external", + namespace: "openshift-ingress-operator", + domain: "", + template: customTemp1, + } + ingctrl2 = ingressControllerDescription{ + name: "ocp64611clb", + namespace: "openshift-ingress-operator", + domain: "", + template: customTemp2, + } + ) + + compat_otp.By("1. Check the STS Role in the cluster") + output, ouputErr := oc.AsAdmin().WithoutNamespace().Run("get").Args("CredentialsRequest/openshift-ingress", "-n", "openshift-cloud-credential-operator", "-o=jsonpath={.spec.providerSpec.statementEntries[0].action}").Output() + o.Expect(ouputErr).NotTo(o.HaveOccurred()) + o.Expect(output).To(o.ContainSubstring("sts:AssumeRole")) + + compat_otp.By("2. Check whether the privateZoneIAMRole is created using the ARN") + arn, getArnErr := oc.AsAdmin().WithoutNamespace().Run("get").Args("dns.config", "cluster", "-o=jsonpath={.spec.platform.aws.privateZoneIAMRole}").Output() + o.Expect(getArnErr).NotTo(o.HaveOccurred()) + privateZoneIAMRoleRegex := regexp.MustCompile("arn:(aws|aws-cn|aws-us-gov):iam::[0-9]{12}:role/.*") + privateZoneIAMRoleMatch := privateZoneIAMRoleRegex.FindStringSubmatch(arn) + o.Expect(arn).To(o.ContainSubstring(privateZoneIAMRoleMatch[0])) + + compat_otp.By("3. Check the default DNS management status") + dnsManagementPolicy := getByJsonPath(oc, "openshift-ingress-operator", "dnsrecords/default-wildcard", "{.spec.dnsManagementPolicy}") + o.Expect("Managed").To(o.ContainSubstring(dnsManagementPolicy)) + + compat_otp.By("4. Collecting the public zone and private zone id from dns config") + privateZoneId := getByJsonPath(oc, "openshift-dns", "dns.config/cluster", "{.spec.privateZone.id}") + publicZoneId := getByJsonPath(oc, "openshift-dns", "dns.config/cluster", "{.spec.publicZone.id}") + + compat_otp.By("5. Collecting zone details from default ingress controller and cross checking it wih dns config details") + checkDnsRecordsInIngressOperator(oc, "default-wildcard", privateZoneId, publicZoneId) + + compat_otp.By("6. Check the default dnsrecord of the ingress operator to confirm there is no degardes") + checkDnsRecordStatusOfIngressOperator(oc, "default-wildcard", "status", "True") + checkDnsRecordStatusOfIngressOperator(oc, "default-wildcard", "reason", "ProviderSuccess") + + compat_otp.By("7. Create two custom ingresscontrollers") + baseDomain := getBaseDomain(oc) + ingctrl1.domain = ingctrl1.name + "." + baseDomain + ingctrl2.domain = ingctrl2.name + "." + baseDomain + defer ingctrl1.delete(oc) + ingctrl1.create(oc) + defer ingctrl2.delete(oc) + ingctrl2.create(oc) + ensureCustomIngressControllerAvailable(oc, ingctrl1.name) + ensureCustomIngressControllerAvailable(oc, ingctrl2.name) + + compat_otp.By("8. Check the custom DNS management status") + dnsManagementPolicy1 := getByJsonPath(oc, "openshift-ingress-operator", "dnsrecords/ocp64611external-wildcard", "{.spec.dnsManagementPolicy}") + o.Expect("Managed").To(o.ContainSubstring(dnsManagementPolicy1)) + dnsManagementPolicy2 := getByJsonPath(oc, "openshift-ingress-operator", "dnsrecords/ocp64611clb-wildcard", "{.spec.dnsManagementPolicy}") + o.Expect("Managed").To(o.ContainSubstring(dnsManagementPolicy2)) + + compat_otp.By("9. Collecting zone details from custom ingress controller and cross checking it wih dns zone details") + checkDnsRecordsInIngressOperator(oc, "ocp64611external-wildcard", privateZoneId, publicZoneId) + checkDnsRecordsInIngressOperator(oc, "ocp64611clb-wildcard", privateZoneId, publicZoneId) + + compat_otp.By("10. Check the custom dnsrecord of the ingress operator to confirm there is no degardes") + checkDnsRecordStatusOfIngressOperator(oc, "ocp64611external-wildcard", "status", "True") + checkDnsRecordStatusOfIngressOperator(oc, "ocp64611external-wildcard", "reason", "ProviderSuccess") + checkDnsRecordStatusOfIngressOperator(oc, "ocp64611clb-wildcard", "status", "True") + checkDnsRecordStatusOfIngressOperator(oc, "ocp64611clb-wildcard", "reason", "ProviderSuccess") + }) + + // author: mjoseph@redhat.com + g.It("Author:mjoseph-High-65827-allow Ingress to modify the HAProxy Log Length when using a Sidecar", func() { + buildPruningBaseDir := testdata.FixturePath("router") + customTemp := filepath.Join(buildPruningBaseDir, "ingresscontroller-sidecar.yaml") + var ( + ingctrl = ingressControllerDescription{ + name: "ocp65827", + namespace: "openshift-ingress-operator", + domain: "", + template: customTemp, + } + ) + + compat_otp.By("1. Create one custom ingresscontroller") + baseDomain := getBaseDomain(oc) + ingctrl.domain = ingctrl.name + "." + baseDomain + defer ingctrl.delete(oc) + ingctrl.create(oc) + ensureRouterDeployGenerationIs(oc, ingctrl.name, "1") + + compat_otp.By("2. Check the env variable of the router pod to verify the default log length") + newrouterpod := getOneNewRouterPodFromRollingUpdate(oc, ingctrl.name) + logLength := readRouterPodEnv(oc, newrouterpod, "ROUTER_LOG_MAX_LENGTH") + o.Expect(logLength).To(o.ContainSubstring(`ROUTER_LOG_MAX_LENGTH=1024`)) + + compat_otp.By("3. Check the haproxy config on the router pod to verify the default log length is enabled") + checkoutput := readRouterPodData(oc, newrouterpod, "cat haproxy.config", "1024") + o.Expect(checkoutput).To(o.ContainSubstring(`log /var/lib/rsyslog/rsyslog.sock len 1024 local1 info`)) + + compat_otp.By("4. Patch the existing custom ingress controller with minimum log length value") + patchResourceAsAdmin(oc, ingctrl.namespace, "ingresscontroller/ocp65827", "{\"spec\":{\"logging\":{\"access\":{\"destination\":{\"container\":{\"maxLength\":480}}}}}}") + ensureRouterDeployGenerationIs(oc, ingctrl.name, "2") + + compat_otp.By("5. Check the env variable of the router pod to verify the minimum log length") + newrouterpod = getOneNewRouterPodFromRollingUpdate(oc, ingctrl.name) + minimumlogLength := readRouterPodEnv(oc, newrouterpod, "ROUTER_LOG_MAX_LENGTH") + o.Expect(minimumlogLength).To(o.ContainSubstring(`ROUTER_LOG_MAX_LENGTH=480`)) + + compat_otp.By("6. Patch the existing custom ingress controller with maximum log length value") + patchResourceAsAdmin(oc, ingctrl.namespace, "ingresscontroller/ocp65827", "{\"spec\":{\"logging\":{\"access\":{\"destination\":{\"container\":{\"maxLength\":8192}}}}}}") + ensureRouterDeployGenerationIs(oc, ingctrl.name, "3") + + compat_otp.By("7. Check the env variable of the router pod to verify the maximum log length") + newrouterpod = getOneNewRouterPodFromRollingUpdate(oc, ingctrl.name) + maximumlogLength := readRouterPodEnv(oc, newrouterpod, "ROUTER_LOG_MAX_LENGTH") + o.Expect(maximumlogLength).To(o.ContainSubstring(`ROUTER_LOG_MAX_LENGTH=8192`)) + }) + + // author: mjoseph@redhat.com + g.It("Author:mjoseph-Low-65903-ingresscontroller should deny invalid maxlengh value when using a Sidecar", func() { + buildPruningBaseDir := testdata.FixturePath("router") + customTemp1 := filepath.Join(buildPruningBaseDir, "ingresscontroller-syslog.yaml") + customTemp2 := filepath.Join(buildPruningBaseDir, "ingresscontroller-sidecar.yaml") + + var ( + ingctrl1 = ingressControllerDescription{ + name: "ocp65903syslog", + namespace: "openshift-ingress-operator", + domain: "", + template: customTemp1, + } + ingctrl2 = ingressControllerDescription{ + name: "ocp65903sidecar", + namespace: "openshift-ingress-operator", + domain: "", + template: customTemp2, + } + ) + compat_otp.By("1. Create two custom ingresscontrollers") + baseDomain := getBaseDomain(oc) + ingctrl1.domain = ingctrl1.name + "." + baseDomain + ingctrl2.domain = ingctrl2.name + "." + baseDomain + defer ingctrl1.delete(oc) + ingctrl1.create(oc) + defer ingctrl2.delete(oc) + ingctrl2.create(oc) + ensureRouterDeployGenerationIs(oc, ingctrl1.name, "1") + ensureRouterDeployGenerationIs(oc, ingctrl2.name, "1") + + compat_otp.By("2. Check the haproxy config on the syslog router pod to verify the default log length") + syslogRouterpod := getOneNewRouterPodFromRollingUpdate(oc, ingctrl1.name) + checkoutput1 := readRouterPodData(oc, syslogRouterpod, "cat haproxy.config", "1024") + o.Expect(checkoutput1).To(o.ContainSubstring(`log 1.2.3.4:514 len 1024 local1 info`)) + + compat_otp.By("3. Check the haproxy config on the sidecar router pod to verify the default log length") + sidecarRouterpod := getOneNewRouterPodFromRollingUpdate(oc, ingctrl2.name) + checkoutput2 := readRouterPodData(oc, sidecarRouterpod, "cat haproxy.config", "1024") + o.Expect(checkoutput2).To(o.ContainSubstring(`log /var/lib/rsyslog/rsyslog.sock len 1024 local1 info`)) + + compat_otp.By("4. Patch the existing sidecar ingress controller with log length value less than minimum threshold") + output1, _ := oc.AsAdmin().WithoutNamespace().Run("patch").Args("ingresscontroller/ocp65903sidecar", "-p", "{\"spec\":{\"logging\":{\"access\":{\"destination\":{\"container\":{\"maxLength\":479}}}}}}", "--type=merge", "-n", ingctrl2.namespace).Output() + o.Expect(output1).To(o.ContainSubstring("spec.logging.access.destination.container.maxLength: Invalid value: 479: spec.logging.access.destination.container.maxLength in body should be greater than or equal to 480")) + + compat_otp.By("5. Patch the existing sidecar ingress controller with log length value more than maximum threshold") + output2, _ := oc.AsAdmin().WithoutNamespace().Run("patch").Args("ingresscontroller/ocp65903sidecar", "-p", "{\"spec\":{\"logging\":{\"access\":{\"destination\":{\"container\":{\"maxLength\":8193}}}}}}", "--type=merge", "-n", ingctrl2.namespace).Output() + o.Expect(output2).To(o.ContainSubstring("spec.logging.access.destination.container.maxLength: Invalid value: 8193: spec.logging.access.destination.container.maxLength in body should be less than or equal to 8192")) + }) + + // OCPBUGS-33657(including OCPBUGS-35027 and OCPBUGS-35454 in OCP-75907) + // guest hypershift cluster had not the ingress-operator pod, skipped on it + g.It("Author:shudili-NonHyperShiftHOST-ROSA-OSD_CCS-ARO-High-75907-Ingress Operator should not always remain in the progressing state [Disruptive]", func() { + buildPruningBaseDir := testdata.FixturePath("router") + privateTemp := filepath.Join(buildPruningBaseDir, "ingresscontroller-private.yaml") + hostnetworkTemp := filepath.Join(buildPruningBaseDir, "ingresscontroller-hostnetwork-only.yaml") + + workerNodeCount, _ := exactNodeDetails(oc) + if workerNodeCount < 1 { + g.Skip("Skipping as we at least need one Linux worker node") + } + + // make sure the ingress operator is normal + jsonPath := `-o=jsonpath={.status.conditions[?(@.type=="Available")].status}{.status.conditions[?(@.type=="Progressing")].status}{.status.conditions[?(@.type=="Degraded")].status}` + output := getByJsonPath(oc, "default", "clusteroperator/ingress", jsonPath) + if !strings.Contains(output, "TrueFalseFalse") { + jsonPath = `-o=jsonpath={.status}` + output = getByJsonPath(oc, "openshift-ingress-operator", "ingresscontroller/default", jsonPath) + e2e.Logf("check the status of the default ingresscontroller:\n%v", output) + ensureClusterOperatorNormal(oc, "ingress", 1, 120) + } + + // OCPBUGS-35027 + extraParas := " clientTLS:\n clientCA:\n name: client-ca-cert\n clientCertificatePolicy: Required\n" + customTemp35027 := addExtraParametersToYamlFile(privateTemp, "spec:", extraParas) + defer os.Remove(customTemp35027) + var ( + ingctrl35027 = ingressControllerDescription{ + name: "bug35027", + namespace: "openshift-ingress-operator", + domain: "", + template: customTemp35027, + } + ) + + compat_otp.By("1. Create a configmap with empty configration") + output, err := oc.AsAdmin().WithoutNamespace().Run("create").Args("-n", "openshift-config", "configmap", "custom-ca35027").Output() + defer oc.AsAdmin().WithoutNamespace().Run("delete").Args("-n", "openshift-config", "configmap", "custom-ca35027").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(output).To(o.ContainSubstring("configmap/custom-ca35027 created")) + + compat_otp.By("2. Create a ingresscontroller for OCPBUGS-35027") + baseDomain := getBaseDomain(oc) + ingctrl35027.domain = ingctrl35027.name + "." + baseDomain + defer ingctrl35027.delete(oc) + ingctrl35027.create(oc) + err = waitForCustomIngressControllerAvailable(oc, ingctrl35027.name) + o.Expect(err).To(o.HaveOccurred()) + + compat_otp.By("3. Check the custom router pod should not be created for the custom ingresscontroller was abnormal") + wait.PollImmediate(5*time.Second, 30*time.Second, func() (bool, error) { + _, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("pods", "-l", "ingresscontroller.operator.openshift.io/deployment-ingresscontroller="+ingctrl35027.name, "-o=jsonpath={.items[0].metadata.name}", "-n", "openshift-ingress").Output() + o.Expect(err).To(o.HaveOccurred()) + return false, nil + }) + + compat_otp.By("4. Delete the custom ingress controller, and then check the logs that clientca-configmap finalizer log should not appear") + ingctrl35027.delete(oc) + output, err = oc.AsAdmin().WithoutNamespace().Run("logs").Args("-n", "openshift-ingress-operator", "-c", "ingress-operator", "deployments/ingress-operator", "--tail=20").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(strings.Contains(output, `failed to add custom-ca35027-configmap finalizer: IngressController.operator.openshift.io \"custom-ca35027-configmap\" is invalid`)).NotTo(o.BeTrue()) + + // OCPBUGS-35454 + var ( + ingctrlhp35454 = ingctrlHostPortDescription{ + name: "bug35454", + namespace: "openshift-ingress-operator", + domain: "", + httpport: 22080, + httpsport: 22443, + statsport: 22936, + template: hostnetworkTemp, + } + ) + + compat_otp.By("5. Create the custom ingress controller for OCPBUGS-35454") + ingctrlhp35454.domain = ingctrlhp35454.name + "." + baseDomain + defer ingctrlhp35454.delete(oc) + ingctrlhp35454.create(oc) + ensureCustomIngressControllerAvailable(oc, ingctrlhp35454.name) + + compat_otp.By(`6. Check the service's spec ports of http/https/metrics`) + output = getByJsonPath(oc, "openshift-ingress", "service/router-internal-"+ingctrlhp35454.name, "{.spec.ports}") + o.Expect(output).Should(o.And( + o.ContainSubstring(`{"name":"http","port":80,"protocol":"TCP","targetPort":"http"}`), + o.ContainSubstring(`{"name":"https","port":443,"protocol":"TCP","targetPort":"https"}`), + o.ContainSubstring(`{"name":"metrics","port":1936,"protocol":"TCP","targetPort":"metrics"}`))) + + compat_otp.By(`7. Check the service's ep of http/https/metrics`) + output = getByJsonPath(oc, "openshift-ingress", "ep/router-internal-"+ingctrlhp35454.name, "{.subsets[0].ports}") + o.Expect(output).Should(o.And( + o.ContainSubstring(`{"name":"http","port":22080,"protocol":"TCP"}`), + o.ContainSubstring(`{"name":"https","port":22443,"protocol":"TCP"}`), + o.ContainSubstring(`{"name":"metrics","port":22936,"protocol":"TCP"}`))) + + compat_otp.By(`8. Check the configuration update for the custom router deployment and the internal service`) + output = getByJsonPath(oc, "openshift-ingress", "deployment/router-"+ingctrlhp35454.name, "{..livenessProbe}") + o.Expect(output).Should(o.And( + o.ContainSubstring(`"failureThreshold":3`), + o.ContainSubstring(`"scheme":"HTTP"`), + o.ContainSubstring(`"periodSeconds":10`), + o.ContainSubstring(`"successThreshold":1`), + o.ContainSubstring(`"timeoutSeconds":1`))) + + output = getByJsonPath(oc, "openshift-ingress", "deployment/router-"+ingctrlhp35454.name, "{..readinessProbe}") + o.Expect(output).Should(o.And( + o.ContainSubstring(`"failureThreshold":3`), + o.ContainSubstring(`"scheme":"HTTP"`), + o.ContainSubstring(`"periodSeconds":10`), + o.ContainSubstring(`"successThreshold":1`), + o.ContainSubstring(`"timeoutSeconds":1`))) + + output = getByJsonPath(oc, "openshift-ingress", "deployment/router-"+ingctrlhp35454.name, "{..startupProbe}") + o.Expect(output).Should(o.And( + o.ContainSubstring(`"scheme":"HTTP"`), + o.ContainSubstring(`"successThreshold":1`), + o.ContainSubstring(`"timeoutSeconds":1`))) + + output = getByJsonPath(oc, "openshift-ingress", "deployment/router-"+ingctrlhp35454.name, "{..configMap.defaultMode}") + o.Expect(output).To(o.ContainSubstring("420")) + + compat_otp.By(`9. Check the service's sessionAffinity, which should be None`) + output = getByJsonPath(oc, "openshift-ingress", "service/router-internal-"+ingctrlhp35454.name, "{.spec.sessionAffinity}") + o.Expect(output).To(o.ContainSubstring("None")) + + compat_otp.By(`10. Patch the custom ingress controller with other http/https/metrics ports`) + jsonpath := `{"spec": { "endpointPublishingStrategy": { "hostNetwork": {"httpPort": 23080, "httpsPort": 23443, "statsPort": 23936 }}}}` + patchResourceAsAdmin(oc, ingctrlhp35454.namespace, "ingresscontroller/"+ingctrlhp35454.name, jsonpath) + ensureRouterDeployGenerationIs(oc, ingctrlhp35454.name, "2") + + compat_otp.By(`11. Check the service's ep of http/https/metrics, which should be updated to the specified ports`) + output = getByJsonPath(oc, "openshift-ingress", "ep/router-internal-"+ingctrlhp35454.name, "{.subsets[0].ports}") + o.Expect(output).Should(o.And( + o.ContainSubstring(`{"name":"http","port":23080,"protocol":"TCP"}`), + o.ContainSubstring(`{"name":"https","port":23443,"protocol":"TCP"}`), + o.ContainSubstring(`{"name":"metrics","port":23936,"protocol":"TCP"}`))) + }) + + // OCPBUGS-29373 + g.It("Author:shudili-ROSA-OSD_CCS-ARO-High-75908-http2 connection coalescing component routing should not be broken with single certificate [Disruptive]", func() { + var ( + dirname = "/tmp/OCP-75908-ca/" + validity = 30 + caSubj = "/CN=NE-Test-Root-CA" + caCrt = dirname + "75908-ca.crt" + caKey = dirname + "75908-ca.key" + usrCrt = dirname + "75908-usr.crt" + usrKey = dirname + "75908-usr.key" + usrCsr = dirname + "75908-usr.csr" + ) + + compat_otp.By("1.0: skip for http2 not enabled clusters") + routerpod := getOneRouterPodNameByIC(oc, "default") + disableHttp2 := readRouterPodEnv(oc, routerpod, "ROUTER_DISABLE_HTTP2") + if !strings.Contains(disableHttp2, "false") { + g.Skip("OCPBUGS-29373 occur on ROSA/OSD cluster, skip for http2 not enabled clusters!") + } + + compat_otp.By("2.0: Get some info including hostnames of console/oauth route for the testing") + appsDomain := "apps." + getBaseDomain(oc) + consoleRoute := getByJsonPath(oc, "openshift-console", "route/console", "{.spec.host}") + oauthRoute := getByJsonPath(oc, "openshift-authentication", "route/oauth-openshift", "{.spec.host}") + defaultRoute := "foo." + appsDomain + ingressOperatorPod := getPodListByLabel(oc, "openshift-ingress-operator", "name=ingress-operator")[0] + backupComponentRoutes := getByJsonPath(oc, "default", "ingresses.config.openshift.io/cluster", "{.spec.componentRoutes}") + defaultComponentRoutes := fmt.Sprintf(`[{"op":"replace", "path":"/spec/componentRoutes", "value":%s}]`, backupComponentRoutes) + + compat_otp.By("3.0: Use openssl to create the certification and key") + defer os.RemoveAll(dirname) + err := os.MkdirAll(dirname, 0755) + o.Expect(err).NotTo(o.HaveOccurred()) + + compat_otp.By("3.1: Create a new self-signed CA including the ca certification and ca key") + opensslCmd := fmt.Sprintf(`openssl req -x509 -newkey rsa:2048 -days %d -keyout %s -out %s -nodes -subj %s`, validity, caKey, caCrt, caSubj) + _, err = exec.Command("bash", "-c", opensslCmd).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + + compat_otp.By("3.2: Create the user CSR and the user key") + opensslCmd = fmt.Sprintf(`openssl req -newkey rsa:2048 -nodes -keyout %s -out %s -subj "/CN=*.%s"`, usrKey, usrCsr, appsDomain) + _, err = exec.Command("bash", "-c", opensslCmd).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + + compat_otp.By("3.3: Create the user certification") + opensslCmd = fmt.Sprintf(`openssl x509 -extfile <(printf "subjectAltName = DNS.1:%s,DNS.2:%s") -req -in %s -CA %s -CAkey %s -CAcreateserial -out %s -days %d`, consoleRoute, oauthRoute, usrCsr, caCrt, caKey, usrCrt, validity) + _, err = exec.Command("bash", "-c", opensslCmd).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + + compat_otp.By("4.0: Create the custom secret on the cluster with the created user certification and user key") + defer oc.AsAdmin().WithoutNamespace().Run("delete").Args("-n", "openshift-config", "secret", "custom-cert75908").Output() + output, err := oc.AsAdmin().WithoutNamespace().Run("create").Args("-n", "openshift-config", "secret", "tls", "custom-cert75908", "--cert="+usrCrt, "--key="+usrKey).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(output).To(o.ContainSubstring("created")) + + compat_otp.By("5.0: path the ingress with the console route host and the custom secret") + patchContent := fmt.Sprintf(` +spec: + componentRoutes: + - hostname: %s + name: console + namespace: openshift-console + servingCertKeyPairSecret: + name: custom-cert75908 +`, consoleRoute) + defer patchGlobalResourceAsAdmin(oc, "ingresses.config.openshift.io/cluster", defaultComponentRoutes) + patchResourceAsAdminAnyType(oc, "default", "ingresses.config.openshift.io/cluster", patchContent, "merge") + + compat_otp.By("6.0: Check the console route has HTTP/2 enabled") + output, err = oc.AsAdmin().WithoutNamespace().Run("exec").Args("-n", "openshift-ingress", routerpod, "--", "bash", "-c", "cat cert_config.map").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(output).To(o.ContainSubstring(`console.pem [alpn h2,http/1.1] ` + consoleRoute)) + + compat_otp.By("7.0: Check console certificate has different SHA1 Fingerprint with OAuth certificate and default certificate, by using openssl command") + curlCmd := []string{"-n", "openshift-ingress-operator", "-c", "ingress-operator", ingressOperatorPod, "--", "curl", "https://" + consoleRoute + "/headers", "-kI", "--connect-timeout", "10"} + opensslConnectCmd := fmt.Sprintf(`openssl s_client -connect %s:443 /dev/null | openssl x509 -sha1 -in /dev/stdin -noout -fingerprint`, consoleRoute) + repeatCmdOnClient(oc, curlCmd, "200", 30, 1) + consoleOutput, err := oc.AsAdmin().WithoutNamespace().Run("exec").Args("-n", "openshift-ingress-operator", "-c", "ingress-operator", ingressOperatorPod, "--", "bash", "-c", opensslConnectCmd).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + + curlCmd = []string{"-n", "openshift-ingress-operator", "-c", "ingress-operator", ingressOperatorPod, "--", "curl", "https://" + oauthRoute + "/headers", "-kI", "--connect-timeout", "10"} + opensslConnectCmd = fmt.Sprintf(`openssl s_client -connect %s:443 /dev/null | openssl x509 -sha1 -in /dev/stdin -noout -fingerprint`, oauthRoute) + repeatCmdOnClient(oc, curlCmd, "403", 30, 1) + oauthOutput, err := oc.AsAdmin().WithoutNamespace().Run("exec").Args("-n", "openshift-ingress-operator", "-c", "ingress-operator", ingressOperatorPod, "--", "bash", "-c", opensslConnectCmd).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + + curlCmd = []string{"-n", "openshift-ingress-operator", "-c", "ingress-operator", ingressOperatorPod, "--", "curl", "https://" + defaultRoute + "/headers", "-kI", "--connect-timeout", "10"} + opensslConnectCmd = fmt.Sprintf(`openssl s_client -connect %s:443 /dev/null | openssl x509 -sha1 -in /dev/stdin -noout -fingerprint`, defaultRoute) + repeatCmdOnClient(oc, curlCmd, "503", 30, 1) + defaultOutput, err := oc.AsAdmin().WithoutNamespace().Run("exec").Args("-n", "openshift-ingress-operator", "-c", "ingress-operator", ingressOperatorPod, "--", "bash", "-c", opensslConnectCmd).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + + o.Expect(consoleOutput).NotTo(o.ContainSubstring(oauthOutput)) + o.Expect(oauthOutput).To(o.ContainSubstring(defaultOutput)) + }) + + // OCPBUGS-33657(including OCPBUGS-34757, OCPBUGS-34110 and OCPBUGS-34888 in OCP-75909) + // guest hypershift cluster had not the ingress-operator pod, skipped on it + g.It("Author:shudili-NonHyperShiftHOST-ROSA-OSD_CCS-ARO-High-75909-Ingress Operator should not always remain in the progressing state [Disruptive]", func() { + buildPruningBaseDir := testdata.FixturePath("router") + nodePortTemp := filepath.Join(buildPruningBaseDir, "ingresscontroller-np.yaml") + privateTemp := filepath.Join(buildPruningBaseDir, "ingresscontroller-private.yaml") + hostnetworkTemp := filepath.Join(buildPruningBaseDir, "ingresscontroller-hostnetwork-only.yaml") + + workerNodeCount, _ := exactNodeDetails(oc) + if workerNodeCount < 1 { + g.Skip("Skipping as we at least need one Linux worker node") + } + + // OCPBUGS-34757 + var ( + ingctrl34757one = ingressControllerDescription{ + name: "bug34757one", + namespace: "openshift-ingress-operator", + domain: "", + template: nodePortTemp, + } + + ingctrl34757two = ingressControllerDescription{ + name: "bug34757two", + namespace: "openshift-ingress-operator", + domain: "", + template: privateTemp, + } + ) + + compat_otp.By("1. after the cluster is ready, check openshift-ingress-operator logs, which should not contain updated internal service") + output, err := oc.AsAdmin().WithoutNamespace().Run("logs").Args("-n", "openshift-ingress-operator", "deployment/ingress-operator").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(strings.Contains(output, "updated internal service")).NotTo(o.BeTrue()) + + compat_otp.By("2. Create two custom ingresscontrollers for OCPBUGS-34757") + baseDomain := getBaseDomain(oc) + ingctrl34757one.domain = ingctrl34757one.name + "." + baseDomain + ingctrl34757two.domain = ingctrl34757two.name + "." + baseDomain + defer ingctrl34757one.delete(oc) + ingctrl34757one.create(oc) + defer ingctrl34757two.delete(oc) + ingctrl34757two.create(oc) + ensureCustomIngressControllerAvailable(oc, ingctrl34757one.name) + ensureCustomIngressControllerAvailable(oc, ingctrl34757two.name) + + compat_otp.By("3. Check the logs again after the custom ingresscontrollers are ready, which should not contain updated internal service") + output, err = oc.AsAdmin().WithoutNamespace().Run("logs").Args("-n", "openshift-ingress-operator", "deployment/ingress-operator").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(strings.Contains(output, "updated internal service")).NotTo(o.BeTrue()) + ingctrl34757one.delete(oc) + ingctrl34757two.delete(oc) + + // OCPBUGS-34110 + compat_otp.By("4. delete the ingress-operator pod") + _, err = oc.AsAdmin().WithoutNamespace().Run("delete").Args("-n", "openshift-ingress-operator", "pods", "-l", "name=ingress-operator").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + ensurePodWithLabelReady(oc, "openshift-ingress-operator", "name=ingress-operator") + ensureClusterOperatorNormal(oc, "ingress", 1, 120) + output, err = oc.AsAdmin().WithoutNamespace().Run("logs").Args("-n", "openshift-ingress-operator", "deployment/ingress-operator").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(strings.Contains(output, "updated IngressClass")).NotTo(o.BeTrue()) + + // OCPBUGS-34888 + var ( + ingctrlhp34888one = ingctrlHostPortDescription{ + name: "bug34888one", + namespace: "openshift-ingress-operator", + domain: "", + httpport: 10080, + httpsport: 10443, + statsport: 10936, + template: hostnetworkTemp, + } + + ingctrlhp34888two = ingctrlHostPortDescription{ + name: "bug34888two", + namespace: "openshift-ingress-operator", + domain: "", + httpport: 11080, + httpsport: 11443, + statsport: 11936, + template: hostnetworkTemp, + } + ) + + compat_otp.By("5. Create two custom ingresscontrollers for OCPBUGS-34888") + ingctrlhp34888one.domain = ingctrlhp34888one.name + "." + baseDomain + ingctrlhp34888two.domain = ingctrlhp34888two.name + "." + baseDomain + + defer ingctrlhp34888one.delete(oc) + ingctrlhp34888one.create(oc) + defer ingctrlhp34888two.delete(oc) + ingctrlhp34888two.create(oc) + ensureCustomIngressControllerAvailable(oc, ingctrlhp34888one.name) + ensureCustomIngressControllerAvailable(oc, ingctrlhp34888two.name) + + compat_otp.By("6. Check there was not the updated router deployment log") + output, err = oc.AsAdmin().WithoutNamespace().Run("logs").Args("-n", "openshift-ingress-operator", "-c", "ingress-operator", "deployment/ingress-operator").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(output).NotTo(o.ContainSubstring("updated router deployment")) + }) + + // author: shudili@redhat.com + // [OCPBUGS-42480](https://issues.redhat.com/browse/OCPBUGS-42480) + // [OCPBUGS-43063](https://issues.redhat.com/browse/OCPBUGS-43063) + g.It("Author:shudili-ROSA-OSD_CCS-ARO-Critical-77283-Router should support SHA1 CA certificates in the default certificate chain", func() { + var ( + buildPruningBaseDir = testdata.FixturePath("router") + baseTemp = filepath.Join(buildPruningBaseDir, "ingresscontroller-np.yaml") + ingctrl = ingressControllerDescription{ + name: "77283", + namespace: "openshift-ingress-operator", + domain: "", + template: "", + } + + dirname = "/tmp/OCP-77283-ca/" + validity = 30 + caSubj = "/C=US/ST=SC/L=Default City/O=Default Company Ltd/OU=Test CA/CN=www.exampleca.com/emailAddress=example@example.com" + caCrt = dirname + "77283-ca.crt" + caKey = dirname + "77283-ca.key" + usrSubj = "/CN=www.example.com/ST=SC/C=US/emailAddress=example@example.com/O=Example/OU=Example" + usrCrt = dirname + "77283-usr.crt" + usrKey = dirname + "77283-usr.key" + usrCsr = dirname + "77283-usr.csr" + ext = dirname + "77283-extfile" + combinationCrt = dirname + "77283-combo.crt" + ) + + compat_otp.By("1.0: Use openssl to create the certification and key") + defer os.RemoveAll(dirname) + err := os.MkdirAll(dirname, 0755) + o.Expect(err).NotTo(o.HaveOccurred()) + output := getByJsonPath(oc, "default", "ingresses.config/cluster", "{.spec.domain}") + wildcard := "*." + output + + compat_otp.By("1.1: Create a new self-signed sha1 root CA including the ca certification and ca key") + opensslCmd := fmt.Sprintf(`openssl req -x509 -sha1 -newkey rsa:2048 -days %d -keyout %s -out %s -nodes -subj '%s'`, validity, caKey, caCrt, caSubj) + _, err = exec.Command("bash", "-c", opensslCmd).Output() + if err != nil { + // The CI OpenSSL 3.0.7 1 Nov 2022 (Library: OpenSSL 3.0.7 1 Nov 2022) under Red Hat Enterprise Linux release doesn't support sha1 certification, skip this case if the error occur + g.Skip("Skipping as openssl under the OS doesn't support sha1 certification") + } + + compat_otp.By("1.2: Create the user CSR and the user key") + opensslCmd = fmt.Sprintf(`openssl req -newkey rsa:2048 -nodes -keyout %s -out %s -subj %s`, usrKey, usrCsr, usrSubj) + _, err = exec.Command("bash", "-c", opensslCmd).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + + compat_otp.By("1.3: Create the extension file, then create the user certification") + cmd := fmt.Sprintf(`echo $'[ext]\nbasicConstraints = CA:FALSE\nsubjectKeyIdentifier = none\nauthorityKeyIdentifier = none\nextendedKeyUsage=serverAuth,clientAuth\nkeyUsage=nonRepudiation, digitalSignature, keyEncipherment\nsubjectAltName = DNS:'%s > %s`, ext, wildcard) + _, err = exec.Command("bash", "-c", cmd).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + + opensslCmd = fmt.Sprintf(`openssl x509 -req -days %d -sha256 -in %s -CA %s -CAcreateserial -CAkey %s -extensions ext -out %s`, validity, usrCsr, caCrt, caKey, usrCrt) + _, err = exec.Command("bash", "-c", opensslCmd).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + + compat_otp.By("1.4: create the file including the sha1 certification and user certification") + catCmd := fmt.Sprintf(`cat %s %s > %s`, usrCrt, caCrt, combinationCrt) + _, err = exec.Command("bash", "-c", catCmd).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + + compat_otp.By("2.0: Create the custom secret on the cluster with the created the combination certifications and user key") + defer oc.AsAdmin().WithoutNamespace().Run("delete").Args("-n", "openshift-ingress", "secret", "custom-cert77283").Output() + output, err = oc.AsAdmin().WithoutNamespace().Run("create").Args("-n", "openshift-ingress", "secret", "tls", "custom-cert77283", "--cert="+combinationCrt, "--key="+usrKey).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(output).To(o.ContainSubstring("created")) + + compat_otp.By("3.0: Create the custom ingresscontroller for the testing") + extraParas := fmt.Sprintf(` + defaultCertificate: + name: custom-cert77283 +`) + customTemp := addExtraParametersToYamlFile(baseTemp, "spec:", extraParas) + defer os.Remove(customTemp) + ingctrl.template = customTemp + baseDomain := getBaseDomain(oc) + ingctrl.domain = ingctrl.name + "." + baseDomain + defer ingctrl.delete(oc) + ingctrl.create(oc) + ensureCustomIngressControllerAvailable(oc, ingctrl.name) + + compat_otp.By("4.0: Check the ingress co, it should be upgradable") + jsonPath := `{.status.conditions[?(@.type=="Upgradeable")].status}` + status := getByJsonPath(oc, "default", "co/ingress", jsonPath) + o.Expect(status).To(o.ContainSubstring("True")) + + compat_otp.By("5.0: The canary route is accessable") + jsonPath = "{.status.ingress[0].host}" + routehost := getByJsonPath(oc, "openshift-ingress-canary", "route/canary", jsonPath) + waitForOutsideCurlContains("https://"+routehost, "-kI", `200`) + }) + + // https://issues.redhat.com/browse/OCPBUGS-60885 + // author: shudili@redhat.com + g.It("Author:shudili-ROSA-OSD_CCS-ARO-High-86155-Supporting ClosedClientConnectionPolicy in the IngressController", func() { + var ( + buildPruningBaseDir = testdata.FixturePath("router") + customTemp = filepath.Join(buildPruningBaseDir, "ingresscontroller-np.yaml") + ingctrl = ingressControllerDescription{ + name: "ocp86155", + namespace: "openshift-ingress-operator", + domain: "", + template: customTemp, + } + ingctrlResource = "ingresscontrollers/" + ingctrl.name + ) + + compat_otp.By("1.0: Create an custom ingresscontroller for the testing") + baseDomain := getBaseDomain(oc) + ingctrl.domain = ingctrl.name + "." + baseDomain + defer ingctrl.delete(oc) + ingctrl.create(oc) + ensureRouterDeployGenerationIs(oc, ingctrl.name, "1") + + compat_otp.By("2.0: Check the default closedClientConnectionPolicy in the ingress-controller which should be Continue") + policy := getByJsonPath(oc, ingctrl.namespace, ingctrlResource, "{.spec.closedClientConnectionPolicy}") + o.Expect(policy).To(o.ContainSubstring("Continue")) + + compat_otp.By("2.1: Check in the haproxy.conf there is NOT option abortonclose") + podname := getOneNewRouterPodFromRollingUpdate(oc, ingctrl.name) + defaultsCfg := getBlockConfig(oc, podname, "defaults") + o.Expect(defaultsCfg).NotTo(o.ContainSubstring("abortonclose")) + + compat_otp.By("2.2: Check in the haproxy-config.template whether or not there is option abortonclose") + output, err := oc.AsAdmin().WithoutNamespace().Run("exec").Args("-n", "openshift-ingress", podname, "--", "bash", "-c", "cat haproxy-config.template | grep abortonclose").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(output).To(o.ContainSubstring("option abortonclose")) + + compat_otp.By("3.0: Patch the ingress-controller with Abort closedClientConnectionPolicy") + patchResourceAsAdmin(oc, ingctrl.namespace, ingctrlResource, `{"spec":{"closedClientConnectionPolicy":"Abort"}}`) + ensureRouterDeployGenerationIs(oc, ingctrl.name, "2") + + compat_otp.By("3.1: Check the spec of the ingress-controller, closedClientConnectionPolicy should be Abort") + policy = getByJsonPath(oc, ingctrl.namespace, ingctrlResource, "{.spec.closedClientConnectionPolicy}") + o.Expect(policy).To(o.ContainSubstring("Abort")) + + compat_otp.By("3.2: Check in the haproxy.config whether or not there is option abortonclose") + podname = getOneNewRouterPodFromRollingUpdate(oc, ingctrl.name) + ensureHaproxyBlockConfigContains(oc, podname, "defaults", []string{"option abortonclose"}) + + compat_otp.By("3.3: Check ROUTER_ABORT_ON_CLOSE env in a route pod which should be true") + policyEnv := readRouterPodEnv(oc, podname, "ROUTER_ABORT_ON_CLOSE") + o.Expect(policyEnv).To(o.ContainSubstring("ROUTER_ABORT_ON_CLOSE=true")) + }) +}) diff --git a/tests-extension/test/e2e/ingress.go b/tests-extension/test/e2e/ingress.go new file mode 100644 index 000000000..ab67528b9 --- /dev/null +++ b/tests-extension/test/e2e/ingress.go @@ -0,0 +1,704 @@ +package router + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + + g "github.com/onsi/ginkgo/v2" + o "github.com/onsi/gomega" + compat_otp "github.com/openshift/origin/test/extended/util/compat_otp" + "github.com/openshift/router-tests-extension/test/testdata" + e2e "k8s.io/kubernetes/test/e2e/framework" +) + +var _ = g.Describe("[sig-network-edge] Network_Edge Component_Router", func() { + defer g.GinkgoRecover() + + var oc = compat_otp.NewCLI("router-ingressclass", compat_otp.KubeConfigPath()) + + // incorporate OCP-33960, OCP-33961, OCP-33962 and OCP-33986 into one + // Test case creater: aiyengar@redhat.com - OCP-33960 Setting "route.openshift.io/termination" annotation to "Edge" in ingress resource deploys "Edge" terminated route object + // Test case creater: aiyengar@redhat.com - OCP-33961 Setting "route.openshift.io/termination" annotation to "Passthrough" in ingress resource deploys "passthrough" terminated route object + // Test case creater: aiyengar@redhat.com - OCP-33962 Setting "route.openshift.io/termination" annotation to "Reencrypt" in ingress resource deploys "reen" terminated route object + // Test case creater: aiyengar@redhat.com - OCP-33986 Setting values other than "edge/passthrough/reencrypt" for "route.openshift.io/termination" annotation are ignored by ingress object + g.It("Author:mjoseph-ROSA-OSD_CCS-ARO-Critical-33960-Setting 'route.openshift.io/termination' annotation to Edge/Passthrough/Reencrypt in ingress resource", func() { + var ( + buildPruningBaseDir = testdata.FixturePath("router") + testPodSvc = filepath.Join(buildPruningBaseDir, "web-server-signed-deploy.yaml") + testIngress = filepath.Join(buildPruningBaseDir, "ingress-resource.yaml") + tmpdir = "/tmp/OCP-33960-CA/" + caKey = tmpdir + "ca.key" + caCrt = tmpdir + "ca.crt" + serverKey = tmpdir + "server.key" + serverCsr = tmpdir + "server.csr" + serverCrt = tmpdir + "server.crt" + ) + + compat_otp.By("1. Create a server and client pod") + baseDomain := getBaseDomain(oc) + ns := oc.Namespace() + routeEdgeHost := "ingress-edge-" + ns + ".apps." + baseDomain + routePassHost := "ingress-passth-" + ns + ".apps." + baseDomain + routeReenHost := "ingress-reen-" + ns + ".apps." + baseDomain + routeRandHost := "33960-random-" + ns + ".example.com" + createResourceFromFile(oc, ns, testPodSvc) + ensurePodWithLabelReady(oc, ns, "name=web-server-deploy") + podName := getOneRouterPodNameByIC(oc, "default") + + compat_otp.By("2. Create a secret certificate for ingress edge and reen termination") + // prepare the tmp folder and create self-signed cerfitcate and a secret + defer os.RemoveAll(tmpdir) + err := os.MkdirAll(tmpdir, 0755) + o.Expect(err).NotTo(o.HaveOccurred()) + opensslNewCa(caKey, caCrt, "/CN=ne-root-ca") + opensslNewCsr(serverKey, serverCsr, "/CN=ne-server-cert") + // san just contains edge route host but not reen route host + san := "subjectAltName=DNS:" + routeEdgeHost + opensslSignCsr(san, serverCsr, caCrt, caKey, serverCrt) + err = oc.AsAdmin().WithoutNamespace().Run("create").Args("-n", ns, "secret", "tls", "ingress-secret", "--cert="+serverCrt, "--key="+serverKey).Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + + compat_otp.By("3. Create edge, passthroguh and reen routes") + sedCmd := fmt.Sprintf(`sed -i'' -e 's@edgehostname@%s@g;s@passhostname@%s@g;s@reenhostname@%s@g;s@randomhostname@%s@g' %s`, routeEdgeHost, routePassHost, routeReenHost, routeRandHost, testIngress) + _, err = exec.Command("bash", "-c", sedCmd).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + createResourceFromFile(oc, ns, testIngress) + routeOutput := getRoutes(oc, ns) + + // OCP-33960: Setting "route.openshift.io/termination" annotation to "Edge" in ingress resource deploys "Edge" terminated route object + compat_otp.By("4. Verify the haproxy configuration to ensure the ingress edge route is configured") + edgeBackendName := "be_edge_http:" + ns + ":ingress-edge" + ensureHaproxyBlockConfigContains(oc, podName, edgeBackendName, []string{":service-unsecure:http:"}) + + compat_otp.By("5. Check the reachability of the edge route") + waitForOutsideCurlContains("https://"+routeEdgeHost, "-kI", `200`) + + // OCP-33961: Setting "route.openshift.io/termination" annotation to "Passthrough" in ingress resource deploys "passthrough" terminated route object + compat_otp.By("6. Verify the haproxy configuration to ensure the ingress passthrough route is configured") + passthroughBackendName := "be_tcp:" + ns + ":ingress-passth" + ensureHaproxyBlockConfigContains(oc, podName, passthroughBackendName, []string{":service-secure:https:"}) + + compat_otp.By("7. Check the reachability of the passthrough route") + waitForOutsideCurlContains("https://"+routePassHost, "-kI", `200`) + + // OCP-33962: Setting "route.openshift.io/termination" annotation to "Reencrypt" in ingress resource deploys "reen" terminated route object + compat_otp.By("8. Verify the haproxy configuration to ensure the ingress reen route is configured") + reenBackendName := "be_secure:" + ns + ":ingress-reencrypt" + reenBackendOutput := ensureHaproxyBlockConfigContains(oc, podName, reenBackendName, []string{":service-secure:https:"}) + o.Expect(reenBackendOutput).To(o.ContainSubstring(`/var/run/configmaps/service-ca/service-ca.crt`)) + + compat_otp.By("9. Check the reachability of the reen route") + waitForOutsideCurlContains("https://"+routeReenHost, "-kI", `200`) + + // OCP-33986: Setting values other than "edge/passthrough/reencrypt" for "route.openshift.io/termination" annotation are ignored by ingress object + compat_otp.By("10. Verify the random route type's annotation are ignored") + o.Expect(routeOutput).NotTo(o.ContainSubstring("abcd")) + }) + + // bug: 1820075 + // author: hongli@redhat.com + g.It("Author:hongli-Critical-41109-use IngressClass controller for ingress-to-route", func() { + var ( + output string + buildPruningBaseDir = testdata.FixturePath("router") + testPodSvc = filepath.Join(buildPruningBaseDir, "web-server-deploy.yaml") + testIngress = filepath.Join(buildPruningBaseDir, "ingress-with-class.yaml") + ) + + compat_otp.By("Create pod, svc, and ingress that mismatch with default ingressclass") + ns := oc.Namespace() + createResourceFromFile(oc, ns, testPodSvc) + ensurePodWithLabelReady(oc, ns, "name=web-server-deploy") + createResourceFromFile(oc, ns, testIngress) + + compat_otp.By("ensure no route is created from the ingress") + output, err := oc.Run("get").Args("route").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(output).NotTo(o.ContainSubstring("ingress-with-class")) + + compat_otp.By("patch the ingress to use default ingressclass") + patchResourceAsUser(oc, ns, "ingress/ingress-with-class", "{\"spec\":{\"ingressClassName\": \"openshift-default\"}}") + compat_otp.By("ensure one route is created from the ingress") + waitForOutputContains(oc, ns, "route", "{.items[*].metadata.name}", "ingress-with-class") + + // bug:- 1820075 + compat_otp.By("Confirm the address field is getting populated with the Router domain details") + baseDomain := getBaseDomain(oc) + ingressOut := getIngress(oc, ns) + o.Expect(ingressOut).To(o.ContainSubstring("router-default.apps." + baseDomain)) + }) + + // author: hongli@redhat.com + g.It("Author:hongli-Critical-41117-ingress operator manages the IngressClass for each ingresscontroller", func() { + buildPruningBaseDir := testdata.FixturePath("router") + customTemp := filepath.Join(buildPruningBaseDir, "ingresscontroller-np.yaml") + var ( + ingctrl = ingressControllerDescription{ + name: "ocp41117", + namespace: "openshift-ingress-operator", + domain: "", + template: customTemp, + } + ) + + compat_otp.By("check the ingress class created by default ingresscontroller") + output, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("ingressclass/openshift-default").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(output).To(o.ContainSubstring("openshift.io/ingress-to-route")) + + compat_otp.By("create another custom ingresscontroller") + baseDomain := getBaseDomain(oc) + ingctrl.domain = ingctrl.name + "." + baseDomain + defer ingctrl.delete(oc) + ingctrl.create(oc) + ensureRouterDeployGenerationIs(oc, ingctrl.name, "1") + + compat_otp.By("check the ingressclass is created by custom ingresscontroller") + ingressclassname := "openshift-" + ingctrl.name + output, err = oc.AsAdmin().WithoutNamespace().Run("get").Args("ingressclass", ingressclassname).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(output).To(o.ContainSubstring("openshift.io/ingress-to-route")) + + compat_otp.By("delete the custom ingresscontroller and ensure the ingresscalsss is removed") + ingctrl.delete(oc) + output, err = oc.AsAdmin().WithoutNamespace().Run("get").Args("ingressclass", ingressclassname).Output() + o.Expect(err).To(o.HaveOccurred()) + o.Expect(output).To(o.ContainSubstring("NotFound")) + }) + + // author: mjoseph@redhat.com + g.It("Author:mjoseph-Critical-51148-host name of the route depends on the subdomain if provided", func() { + var ( + buildPruningBaseDir = testdata.FixturePath("router") + customTemp = filepath.Join(buildPruningBaseDir, "subdomain-routes/ocp51148-route.yaml") + testPodSvc = filepath.Join(buildPruningBaseDir, "web-server-deploy.yaml") + rut = routeDescription{ + namespace: "", + domain: "", + subDomain: "foo", + template: customTemp, + } + ) + + compat_otp.By("Create a pod") + ns := oc.Namespace() + createResourceFromFile(oc, ns, testPodSvc) + ensurePodWithLabelReady(oc, ns, "name=web-server-deploy") + podName := getPodListByLabel(oc, ns, "name=web-server-deploy") + baseDomain := getBaseDomain(oc) + rut.domain = "apps" + "." + baseDomain + rut.namespace = ns + + compat_otp.By("create routes and get the details") + rut.create(oc) + // to show the route details + getRoutes(oc, ns) + + compat_otp.By("check the domain name is present in 'foo-unsecure1' route details") + output := getByJsonPath(oc, ns, "route/foo-unsecure1", "{.spec}") + o.Expect(output).Should(o.ContainSubstring(`"subdomain":"foo"`)) + + compat_otp.By("check the domain name is not present in 'foo-unsecure2' route details") + output = getByJsonPath(oc, ns, "route/foo-unsecure2", "{.spec}") + o.Expect(output).NotTo(o.ContainSubstring("subdomain")) + + compat_otp.By("check the domain name is present in 'foo-unsecure3' route details") + output = getByJsonPath(oc, ns, "route/foo-unsecure3", "{.spec}") + o.Expect(output).Should(o.ContainSubstring(`"subdomain":"foo"`)) + + compat_otp.By("check the domain name is not present in 'foo-unsecure4' route details") + output = getByJsonPath(oc, ns, "route/foo-unsecure4", "{.spec}") + o.Expect(output).NotTo(o.ContainSubstring("subdomain")) + + // curling through default controller will not work for proxy cluster. + if checkProxy(oc) { + e2e.Logf("This is proxy cluster, skiping the curling part.") + } else { + compat_otp.By("check the reachability of the 'foo-unsecure1' host") + waitForCurl(oc, podName[0], baseDomain, "foo.apps.", "Hello-OpenShift", "") + + compat_otp.By("check the reachability of the 'foo-unsecure2' host") + waitForCurl(oc, podName[0], baseDomain, "foo-unsecure2-"+ns+".apps.", "Hello-OpenShift", "") + + compat_otp.By("check the reachability of the 'foo-unsecure3' host") + waitForCurl(oc, podName[0], baseDomain, "man-"+ns+".apps.", "Hello-OpenShift", "") + + compat_otp.By("check the reachability of the 'foo-unsecure4' host") + waitForCurl(oc, podName[0], baseDomain, "bar-"+ns+".apps.", "Hello-OpenShift", "") + } + }) + + // author: mjoseph@redhat.com + g.It("Author:mjoseph-High-51429-different router deployment with same route using subdomain", func() { + var ( + buildPruningBaseDir = testdata.FixturePath("router") + customTemp2 = filepath.Join(buildPruningBaseDir, "subdomain-routes/route.yaml") + testPodSvc = filepath.Join(buildPruningBaseDir, "web-server-deploy.yaml") + customTemp = filepath.Join(buildPruningBaseDir, "ingresscontroller-np.yaml") + ingctrl = ingressControllerDescription{ + name: "ocp51429", + namespace: "openshift-ingress-operator", + domain: "", + template: customTemp, + } + rut = routeDescription{ + namespace: "", + domain: "", + subDomain: "foobar", + template: customTemp2, + } + ) + + compat_otp.By("Create a pod") + baseDomain := getBaseDomain(oc) + ns := oc.Namespace() + createResourceFromFile(oc, ns, testPodSvc) + ensurePodWithLabelReady(oc, ns, "name=web-server-deploy") + podName := getPodListByLabel(oc, ns, "name=web-server-deploy") + rut.domain = "apps" + "." + baseDomain + rut.namespace = ns + + compat_otp.By("Create a custom ingresscontroller") + ingctrl.domain = ingctrl.name + "." + baseDomain + defer ingctrl.delete(oc) + ingctrl.create(oc) + ensureRouterDeployGenerationIs(oc, ingctrl.name, "1") + custContPod := getOneNewRouterPodFromRollingUpdate(oc, ingctrl.name) + defaultContPod := getOneRouterPodNameByIC(oc, "default") + + compat_otp.By("create routes and get the details") + rut.create(oc) + getRoutes(oc, ns) + + compat_otp.By("check whether required host is present in 'foobar-unsecure' route details") + waitForOutputContains(oc, ns, "route/foobar-unsecure", "{.status.ingress}", fmt.Sprintf(`"host":"foobar.apps.%s"`, baseDomain)) + waitForOutputContains(oc, ns, "route/foobar-unsecure", "{.status.ingress}", fmt.Sprintf(`"host":"foobar.ocp51429.%s"`, baseDomain)) + + compat_otp.By("check the router pod and ensure the routes are loaded in haproxy.config in default controller") + searchOutput1 := pollReadPodData(oc, "openshift-ingress", defaultContPod, "cat haproxy.config", "foobar-unsecure") + o.Expect(searchOutput1).To(o.ContainSubstring("backend be_http:" + ns + ":foobar-unsecure")) + + compat_otp.By("check the router pod and ensure the routes are loaded in haproxy.config of custom controller") + searchOutput2 := pollReadPodData(oc, "openshift-ingress", custContPod, "cat haproxy.config", "foobar-unsecure") + o.Expect(searchOutput2).To(o.ContainSubstring("backend be_http:" + ns + ":foobar-unsecure")) + + // curling through default controller will not work for proxy cluster. + if checkProxy(oc) { + e2e.Logf("This is proxy cluster, skiping the curling part through default controller.") + } else { + compat_otp.By("check the reachability of the 'foobar-unsecure' host in default controller") + waitForCurl(oc, podName[0], baseDomain, "foobar.apps.", "Hello-OpenShift", "") + } + + compat_otp.By("check the reachability of the 'foobar-unsecure' host in custom controller") + custContIP := getPodv4Address(oc, custContPod, "openshift-ingress") + waitForCurl(oc, podName[0], baseDomain, "foobar.ocp51429.", "Hello-OpenShift", custContIP) + }) + + // author: mjoseph@redhat.com + g.It("Author:mjoseph-NonHyperShiftHOST-ROSA-OSD_CCS-ARO-High-51437-Router deployment using different shard with same subdomain ", func() { + var ( + buildPruningBaseDir = testdata.FixturePath("router") + customTemp2 = filepath.Join(buildPruningBaseDir, "subdomain-routes/alpha-shard-route.yaml") + testPodSvc = filepath.Join(buildPruningBaseDir, "web-server-deploy.yaml") + customTemp = filepath.Join(buildPruningBaseDir, "ingresscontroller-shard.yaml") + ingctrl1 = ingressControllerDescription{ + name: "alpha-ocp51437", + namespace: "openshift-ingress-operator", + domain: "", + shard: "alpha", + template: customTemp, + } + ingctrl2 = ingressControllerDescription{ + name: "beta-ocp51437", + namespace: "openshift-ingress-operator", + domain: "", + shard: "beta", + template: customTemp, + } + rut = routeDescription{ + namespace: "", + domain: "", + subDomain: "bar", + template: customTemp2, + } + ) + + compat_otp.By("Create a pod") + baseDomain := getBaseDomain(oc) + ns := oc.Namespace() + createResourceFromFile(oc, ns, testPodSvc) + ensurePodWithLabelReady(oc, ns, "name=web-server-deploy") + podName := getPodListByLabel(oc, ns, "name=web-server-deploy") + rut.domain = "apps" + "." + baseDomain + rut.namespace = ns + + compat_otp.By("Create first shard ingresscontroller") + ingctrl1.domain = ingctrl1.name + "." + baseDomain + defer ingctrl1.delete(oc) + ingctrl1.create(oc) + ensureRouterDeployGenerationIs(oc, ingctrl1.name, "1") + custContPod1 := getOneNewRouterPodFromRollingUpdate(oc, ingctrl1.name) + + compat_otp.By("Create second shard ingresscontroller") + ingctrl2.domain = ingctrl2.name + "." + baseDomain + defer ingctrl2.delete(oc) + ingctrl2.create(oc) + ensureRouterDeployGenerationIs(oc, ingctrl2.name, "1") + custContPod2 := getOneNewRouterPodFromRollingUpdate(oc, ingctrl2.name) + + compat_otp.By("create routes and get the details") + rut.create(oc) + getRoutes(oc, ns) + + compat_otp.By("check whether required host is present in alpha ingress controller domain") + waitForOutputContains(oc, ns, "route/bar-unsecure", "{.status.ingress}", fmt.Sprintf(`"host":"bar.apps.%s"`, baseDomain)) + waitForOutputContains(oc, ns, "route/bar-unsecure", "{.status.ingress}", fmt.Sprintf(`"host":"bar.alpha-ocp51437.%s"`, baseDomain)) + + compat_otp.By("check the router pod and ensure the routes are loaded in haproxy.config of alpha controller") + searchOutput1 := pollReadPodData(oc, "openshift-ingress", custContPod1, "cat haproxy.config", "bar-unsecure") + o.Expect(searchOutput1).To(o.ContainSubstring("backend be_http:" + ns + ":bar-unsecure")) + + // curling through default controller will not work for proxy cluster. + if checkProxy(oc) { + e2e.Logf("This is proxy cluster, skiping the curling part through default controller.") + } else { + compat_otp.By("check the reachability of the 'bar-unsecure' host in default controller") + waitForCurl(oc, podName[0], baseDomain, "bar.apps.", "Hello-OpenShift", "") + } + + compat_otp.By("check the reachability of the 'bar-unsecure' host in 'alpha shard' controller") + custContIP := getPodv4Address(oc, custContPod1, "openshift-ingress") + waitForCurl(oc, podName[0], baseDomain, "bar.alpha-ocp51437.", "Hello-OpenShift", custContIP) + + compat_otp.By("Overwrite route with beta shard") + _, err := oc.AsAdmin().WithoutNamespace().Run("label").Args("routes/bar-unsecure", "--overwrite", "shard=beta", "-n", ns).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + + compat_otp.By("check whether required host is present in beta ingress controller domain") + waitForOutputContains(oc, ns, "route/bar-unsecure", "{.status.ingress}", fmt.Sprintf(`"host":"bar.apps.%s"`, baseDomain)) + waitForOutputContains(oc, ns, "route/bar-unsecure", "{.status.ingress}", fmt.Sprintf(`"host":"bar.beta-ocp51437.%s"`, baseDomain)) + + compat_otp.By("check the router pod and ensure the routes are loaded in haproxy.config of beta controller") + searchOutput2 := pollReadPodData(oc, "openshift-ingress", custContPod2, "cat haproxy.config", "bar-unsecure") + o.Expect(searchOutput2).To(o.ContainSubstring("backend be_http:" + ns + ":bar-unsecure")) + + compat_otp.By("check the reachability of the 'bar-unsecure' host in 'beta shard' controller") + custContIP2 := getPodv4Address(oc, custContPod2, "openshift-ingress") + waitForCurl(oc, podName[0], baseDomain, "bar.beta-ocp51437.", "Hello-OpenShift", custContIP2) + }) + + // bug: 1914127 + g.It("Author:shudili-NonPreRelease-Longduration-High-56228-Deletion of default router service under the openshift ingress namespace hangs flag [Disruptive]", func() { + var ( + svcResource = "service/router-default" + namespace = "openshift-ingress" + ) + + compat_otp.By("check if the cluster has the router-default service") + output, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("service", "-n", namespace).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + if !strings.Contains(output, "router-default") { + g.Skip("This cluster has NOT the router-defaut service, skip the test.") + } + + compat_otp.By("check if all COs are in good status") + badOpList := checkAllClusterOperatorsStatus(oc) + if len(badOpList) > 0 { + g.Skip("Some cluster operators are NOT in good status, skip the test.") + } + + compat_otp.By("check the created time of svc router-default") + jsonPath := "{.metadata.creationTimestamp}" + svcCreatedTime1 := getByJsonPath(oc, namespace, svcResource, jsonPath) + o.Expect(svcCreatedTime1).NotTo(o.BeEmpty()) + + compat_otp.By("try to delete the svc router-default, should no errors") + defer ensureAllClusterOperatorsNormal(oc, 720) + err = oc.AsAdmin().WithoutNamespace().Run("delete").Args(svcResource, "-n", namespace).Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + + compat_otp.By("wait for new svc router-default is created") + jsonPath = "{.metadata.name}" + waitForOutputEquals(oc, namespace, svcResource, jsonPath, "router-default") + + compat_otp.By("check the created time of the new svc router-default") + jsonPath = "{.metadata.creationTimestamp}" + svcCreatedTime2 := getByJsonPath(oc, namespace, svcResource, jsonPath) + o.Expect(svcCreatedTime2).NotTo(o.BeEmpty()) + o.Expect(svcCreatedTime1).NotTo(o.Equal(svcCreatedTime2)) + }) + + // bug: 2013004 + g.It("ARO-Author:shudili-High-57089-Error syncing load balancer and failed to parse the VMAS ID on Azure platform", func() { + var ( + buildPruningBaseDir = testdata.FixturePath("router") + lbServices = filepath.Join(buildPruningBaseDir, "bug2013004-lb-services.yaml") + testPodSvc = filepath.Join(buildPruningBaseDir, "web-server-deploy.yaml") + srvrcInfo = "web-server-deploy" + externalSvc = "external-lb-57089" + internalSvc = "internal-lb-57089" + ) + + // skip if platform is not AZURE + compat_otp.By("Pre-flight check for the platform type") + platformtype := compat_otp.CheckPlatform(oc) + if platformtype != "azure" { + g.Skip("Skip for it not azure platform") + } + + compat_otp.By("create a server pod") + ns := oc.Namespace() + createResourceFromFile(oc, ns, testPodSvc) + ensurePodWithLabelReady(oc, ns, "name="+srvrcInfo) + + compat_otp.By("try to create an external load balancer service and an internal load balancer service") + operateResourceFromFile(oc, "create", ns, lbServices) + waitForOutputEquals(oc, ns, "service/"+externalSvc, "{.metadata.name}", externalSvc) + waitForOutputEquals(oc, ns, "service/"+internalSvc, "{.metadata.name}", internalSvc) + + compat_otp.By("check if the lb services have obtained the EXTERNAL-IPs") + regExp := "([0-9]+.[0-9]+.[0-9]+.[0-9]+)" + searchOutput1 := waitForOutputMatchRegexp(oc, ns, "service/"+externalSvc, "{.status.loadBalancer.ingress..ip}", regExp) + o.Expect(searchOutput1).NotTo(o.ContainSubstring("NotMatch")) + searchOutput2 := waitForOutputMatchRegexp(oc, ns, "service/"+internalSvc, "{.status.loadBalancer.ingress..ip}", regExp) + o.Expect(searchOutput2).NotTo(o.ContainSubstring("NotMatch")) + }) + + // incorporate OCP-57370and OCP-14059 into one + // Test case creater: mjoseph@redhat.com - OCP-57370: Hostname of componentRoutes should be RFC compliant + // bugzilla: 2039256 + // Due to bug https://issues.redhat.com/browse/OCPBUGS-43431, this case may not run on HCP cluster. + // Test case creater: zzhao@redhat.com - OCP-14059: Use the default destination CA of router if the route does not specify one for reencrypt route + g.It("Author:mjoseph-NonHyperShiftHOST-High-57370-hostname of componentRoutes should be RFC compliant", func() { + // Check whether the console operator is present or not + output, err := oc.WithoutNamespace().AsAdmin().Run("get").Args("route", "console", "-n", "openshift-console").Output() + if strings.Contains(output, "namespaces \"openshift-console\" not found") || err != nil { + g.Skip("This cluster dont have console operator, so skipping the test.") + } + var ( + resourceName = "ingress.config/cluster" + ) + + compat_otp.By("1. Create route and get the details") + removeRoute := fmt.Sprintf("[{\"op\":\"remove\", \"path\":\"/spec/componentRoutes\", \"value\":[{\"hostname\": \"1digit9.apps.%s\", \"name\": \"downloads\", \"namespace\": \"openshift-console\"}]}]}]", getBaseDomain(oc)) + addRoute := fmt.Sprintf("[{\"op\":\"add\", \"path\":\"/spec/componentRoutes\", \"value\":[{\"hostname\": \"1digit9.apps.%s\", \"name\": \"downloads\", \"namespace\": \"openshift-console\"}]}]}]", getBaseDomain(oc)) + defer patchGlobalResourceAsAdmin(oc, resourceName, removeRoute) + patchGlobalResourceAsAdmin(oc, resourceName, addRoute) + waitForOutputContains(oc, "openshift-console", "route", "{.items..metadata.name}", "downloads-custom") + + compat_otp.By("2. Check the router pod and ensure the routes are loaded in haproxy.config") + podname := getOneRouterPodNameByIC(oc, "default") + backendConfig := pollReadPodData(oc, "openshift-ingress", podname, "cat haproxy.config", "downloads-custom") + o.Expect(backendConfig).To(o.ContainSubstring("backend be_edge_http:openshift-console:downloads-custom")) + + compat_otp.By("3. Confirm from the component Route, the RFC complaint hostname") + cmd := fmt.Sprintf(`1digit9.apps.%s`, getBaseDomain(oc)) + waitForOutputContains(oc, oc.Namespace(), "ingress.config.openshift.io/cluster", "{.spec.componentRoutes[0].hostname}", cmd) + + // OCP-14059: Use the default destination CA of router if the route does not specify one for reencrypt route + // since console route is using the reencrypt route without destination CA we are using it to check + compat_otp.By("4. Confirm from the console service the 'serving-cert-secret-name'") + findAnnotation := getAnnotation(oc, "openshift-console", "svc", "console") + o.Expect(findAnnotation).To(o.Or(o.ContainSubstring(`service.alpha.openshift.io/serving-cert-secret-name":"console-serving-cert`), o.ContainSubstring(`service.beta.openshift.io/serving-cert-secret-name":"console-serving-cert`))) + + compat_otp.By("5. Confirm from the 'service-ca.crt' is present in haproxy.config for console route") + backendConfig1 := pollReadPodData(oc, "openshift-ingress", podname, "cat haproxy.config", "console.openshift-console.svc") + o.Expect(backendConfig1).To(o.ContainSubstring("required ca-file /var/run/configmaps/service-ca/service-ca.crt")) + }) + + g.It("Author:mjoseph-ROSA-OSD_CCS-ARO-Critical-73619-Checking whether the ingress operator is enabled as optional component", func() { + var ( + enabledCapabilities = "{.status.capabilities.enabledCapabilities}" + capability = "{.metadata.annotations}" + ingressCapability = `"capability.openshift.io/name":"Ingress"` + ) + + compat_otp.By("Check whether 'enabledCapabilities' is enabled in cluster version resource") + searchLine, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("clusterversion", "version", "-o=jsonpath="+enabledCapabilities).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(searchLine).To(o.ContainSubstring("Ingress")) + + compat_otp.By("Check the Ingress capability in dnsrecords crd") + searchLine, err = oc.AsAdmin().WithoutNamespace().Run("get").Args("crd", "dnsrecords.ingress.operator.openshift.io", "-o=jsonpath="+capability).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(searchLine).To(o.ContainSubstring(ingressCapability)) + + compat_otp.By("Check the Ingress capability in ingresscontrollers crd") + searchLine, err = oc.AsAdmin().WithoutNamespace().Run("get").Args("crd", "ingresscontrollers.operator.openshift.io", "-o=jsonpath="+capability).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(searchLine).To(o.ContainSubstring(ingressCapability)) + }) + + // OCPBUGS-49769 + g.It("Author:shudili-ROSA-OSD_CCS-ARO-High-85280-Haproxy router pods should validate the key and cert content before accepting", func() { + var ( + buildPruningBaseDir = testdata.FixturePath("router") + testPodSvc = filepath.Join(buildPruningBaseDir, "web-server-deploy.yaml") + unSecSvcName = "service-unsecure" + svcPort = "27017" + fileDir = "/tmp/OCP-85280" + validity = 30 + caSubj = "/CN=NE-Test-Root-CA" + caCert = fileDir + "/exampleca.pem" + caKey = fileDir + "/exampleca.key" + caCsr = fileDir + "/exampleca.Csr" + secretName = "secret85280" + ingressName = "custom85280" + ) + + compat_otp.By("1.0: Create a file folder for saving the created key pair") + defer os.RemoveAll(fileDir) + err := os.MkdirAll(fileDir, 0755) + o.Expect(err).NotTo(o.HaveOccurred()) + + compat_otp.By("1.1: Create the CA key") + opensslCmd := fmt.Sprintf(`openssl genpkey -algorithm RSA -out %s`, caKey) + _, err = exec.Command("bash", "-c", opensslCmd).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + + compat_otp.By("1.2: Create the CA Csr") + opensslCmd = fmt.Sprintf(`openssl req -new -key %s -subj %s -out %s`, caKey, caSubj, caCsr) + _, err = exec.Command("bash", "-c", opensslCmd).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + + compat_otp.By("1.3: Create the broken CA certificate by used a different signature algorithm") + opensslCmd = fmt.Sprintf(`openssl x509 -req -days %d -in %s -signkey %s -outform der -out %s -sigopt rsa_padding_mode:pss -sigopt rsa_pss_saltlen:max`, validity, caCsr, caKey, caCert) + _, err = exec.Command("bash", "-c", opensslCmd).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + + compat_otp.By("2.0: Create a pod and the serivce") + ns := oc.Namespace() + createResourceFromFile(oc, ns, testPodSvc) + + compat_otp.By("3.0 Create a secret using the broken pair") + output, err := oc.AsAdmin().WithoutNamespace().Run("create").Args("secret", "generic", secretName, "--from-file=tls.crt="+caCert, "--from-file=tls.key="+caKey, `--type=kubernetes.io/tls`, "-n", ns).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(output).To(o.ContainSubstring("created")) + + compat_otp.By("4.0 Create an ingress with the secret") + routehost := "edge85280.apps." + getBaseDomain(oc) + rule := fmt.Sprintf(`--rule=%s/*=%s:%s,tls=%s`, routehost, unSecSvcName, svcPort, secretName) + output, err = oc.AsAdmin().WithoutNamespace().Run("create").Args("-n", ns, "ingress", ingressName, rule).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(output).To(o.ContainSubstring("created")) + + compat_otp.By("5.0 Check the route, which should not be admitted for ExtendedValidationFailed") + output = getRoutes(oc, ns) + o.Expect(output).To(o.ContainSubstring("ExtendedValidationFailed")) + }) + + // includes OCP-85950 and OCP-86100 + // OCP-85950: Propagate ingress labels to routes + // OCP-86100: Negative tests of propagating ingress labels to routes + // author: shudili@redhat.com + g.It("Author:shudili-ROSA-OSD_CCS-ARO-High-85950-Propagate ingress labels to routes", func() { + var ( + buildPruningBaseDir = testdata.FixturePath("router") + testPodSvc = filepath.Join(buildPruningBaseDir, "web-server-deploy.yaml") + testIngress = filepath.Join(buildPruningBaseDir, "ingress-with-class.yaml") + ingressName = "unsecure-ingress85950" + ) + + compat_otp.By("1.0: Prepare the ingress file for testing") + routehost := "unsecure85950" + ".apps." + getBaseDomain(oc) + sedCmd := fmt.Sprintf(`sed -i'' -e 's@ingress-with-class@%s@g;s@mytest@openshift-default@g;s@foo.bar.com@%s@g' %s`, ingressName, routehost, testIngress) + _, err := exec.Command("bash", "-c", sedCmd).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + + compat_otp.By("2.0: Create pod, svc, and ingress") + ns := oc.Namespace() + createResourceFromFile(oc, ns, testPodSvc) + ensurePodWithLabelReady(oc, ns, "name=web-server-deploy") + createResourceFromFile(oc, ns, testIngress) + waitForOutputContains(oc, ns, "route", "{.items[0].metadata.name}", ingressName) + routeName := getByJsonPath(oc, ns, "route", "{.items[0].metadata.name}") + + compat_otp.By(`3.0: Set route.openshift.io/reconcile-labels="true" annotation to the ingress`) + setAnnotationAsAdmin(oc, ns, "ingress/"+ingressName, "route.openshift.io/reconcile-labels=true") + + compat_otp.By(`3.1: Add a label to the ingress`) + addLabelAsAdmin(oc, ns, "ingress/"+ingressName, "shard=alpha") + + compat_otp.By(`3.2: Check and make sure the annotation and the label are synced to the route`) + findAnnotation := getAnnotation(oc, ns, "route", routeName) + o.Expect(findAnnotation).To(o.ContainSubstring(`"route.openshift.io/reconcile-labels":"true"`)) + routeLabels := getByJsonPath(oc, ns, "route/"+routeName, "{.metadata.labels}") + o.Expect(routeLabels).To(o.ContainSubstring(`"shard":"alpha"`)) + + compat_otp.By(`4.0: Set route.openshift.io/reconcile-labels annotation with an invalid value ffalse to the ingress`) + setAnnotationAsAdmin(oc, ns, "ingress/"+ingressName, `route.openshift.io/reconcile-labels=ffalse`) + + compat_otp.By(`4.1: Add another label to the ingress`) + addLabelAsAdmin(oc, ns, "ingress/"+ingressName, "custom=internal") + + compat_otp.By(`4.2: Check and make sure the updated annotation is synced to the route, while the new added label isn't`) + findAnnotation = getAnnotation(oc, ns, "route", routeName) + o.Expect(findAnnotation).To(o.ContainSubstring(`"route.openshift.io/reconcile-labels":"ffalse"`)) + routeLabels = getByJsonPath(oc, ns, "route/"+routeName, "{.metadata.labels}") + o.Expect(routeLabels).To(o.ContainSubstring(`"shard":"alpha"`)) + o.Expect(routeLabels).NotTo(o.ContainSubstring(`"custom":"internal"`)) + + compat_otp.By(`5.0: Modify the first added label of the ingress`) + addLabelAsAdmin(oc, ns, "ingress/"+ingressName, "shard=beta") + + compat_otp.By(`5.1: Set route.openshift.io/reconcile-labels="true" annotation again to the ingress`) + setAnnotationAsAdmin(oc, ns, "ingress/"+ingressName, `route.openshift.io/reconcile-labels=true`) + + compat_otp.By(`5.2: Check and make sure the updated annotation and all the ingress' labels are synced to the route`) + findAnnotation = getAnnotation(oc, ns, "route", routeName) + o.Expect(findAnnotation).To(o.ContainSubstring(`"route.openshift.io/reconcile-labels":"true"`)) + routeLabels = getByJsonPath(oc, ns, "route/"+routeName, "{.metadata.labels}") + o.Expect(routeLabels).Should(o.And( + o.ContainSubstring(`"shard":"beta"`), + o.ContainSubstring(`"custom":"internal"`))) + + compat_otp.By(`6.0: Add a label with the name containing upper chars and the value containing the upper chars to the ingress`) + addLabelAsAdmin(oc, ns, "ingress/"+ingressName, "Market_01=No1-In-APEC") + + compat_otp.By(`6.1: Check and make sure all the ingress' labels are synced to the route`) + routeLabels = getByJsonPath(oc, ns, "route/"+routeName, "{.metadata.labels}") + o.Expect(routeLabels).Should(o.And( + o.ContainSubstring(`"shard":"beta"`), + o.ContainSubstring(`"custom":"internal"`), + o.ContainSubstring(`"Market_01":"No1-In-APEC"`))) + + // OCP-86100: Negative tests of propagating ingress labels to routes + compat_otp.By(`7.0: Remove the "custom" label from the ingress, then check the route`) + addLabelAsAdmin(oc, ns, "ingress/"+ingressName, "custom-") + + compat_otp.By(`7.1: Check the route that the "custom" label is removed as well`) + routeLabels = getByJsonPath(oc, ns, "route/"+routeName, "{.metadata.labels}") + o.Expect(routeLabels).NotTo(o.ContainSubstring(`custom`)) + o.Expect(routeLabels).Should(o.And( + o.ContainSubstring(`"shard":"beta"`), + o.ContainSubstring(`"Market_01":"No1-In-APEC"`))) + + compat_otp.By(`8.0: Delete the ingress route and wait for the new ingress route is created automatically`) + err = oc.AsAdmin().Run("delete").Args("-n", ns, "route", routeName).Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + waitForOutputContains(oc, ns, "route", "{.items[*].metadata.name}", ingressName) + routeName = getByJsonPath(oc, ns, "route", "{.items[0].metadata.name}") + + compat_otp.By(`8.1: Check the route that all the ingress' labels are synced to it`) + o.Expect(routeLabels).Should(o.And( + o.ContainSubstring(`"shard":"beta"`), + o.ContainSubstring(`"Market_01":"No1-In-APEC"`))) + + compat_otp.By(`9.0: Remove the "route.openshift.io/reconcile-labels" annotation from the ingress`) + setAnnotationAsAdmin(oc, ns, "ingress/"+ingressName, `route.openshift.io/reconcile-labels-`) + + compat_otp.By(`9.1: Check the route that the annotation is removed from it`) + findAnnotation = getAnnotation(oc, ns, "route", routeName) + o.Expect(findAnnotation).NotTo(o.ContainSubstring(`route.openshift.io/reconcile-labels`)) + + compat_otp.By(`9.2: Check the route that the labels are still there`) + routeLabels = getByJsonPath(oc, ns, "route/"+routeName, "{.metadata.labels}") + o.Expect(routeLabels).Should(o.And( + o.ContainSubstring(`"shard":"beta"`), + o.ContainSubstring(`"Market_01":"No1-In-APEC"`))) + }) +}) diff --git a/tests-extension/test/e2e/ipfailover.go b/tests-extension/test/e2e/ipfailover.go new file mode 100644 index 000000000..f35edee07 --- /dev/null +++ b/tests-extension/test/e2e/ipfailover.go @@ -0,0 +1,368 @@ +package router + +import ( + "path/filepath" + "strings" + "time" + + g "github.com/onsi/ginkgo/v2" + o "github.com/onsi/gomega" + + compat_otp "github.com/openshift/origin/test/extended/util/compat_otp" + "github.com/openshift/router-tests-extension/test/testdata" +) + +var _ = g.Describe("[sig-network-edge] Network_Edge", func() { + defer g.GinkgoRecover() + + var oc = compat_otp.NewCLI("router-ipfailover", compat_otp.KubeConfigPath()) + var HAInterfaces = "br-ex" + + g.BeforeEach(func() { + g.By("Check platforms") + platformtype := compat_otp.CheckPlatform(oc) + platforms := map[string]bool{ + // 'None' also for Baremetal + "none": true, + "baremetal": true, + "vsphere": true, + "openstack": true, + "nutanix": true, + } + if !platforms[platformtype] { + g.Skip("Skip for non-supported platform") + } + + g.By("check whether there are two worker nodes present for testing hostnetwork") + workerNodeCount, _ := exactNodeDetails(oc) + if workerNodeCount < 2 { + g.Skip("Skipping as we need two worker nodes") + } + + g.By("check the cluster has remote worker profile") + remoteWorkerDetails, _ := oc.AsAdmin().WithoutNamespace().Run("get").Args("nodes", "-l", "kubernetes.io/hostname").Output() + if strings.Contains(remoteWorkerDetails, "remote-worker") { + g.Skip("Skip as ipfailover currently doesn't support on remote-worker profile") + } + + g.By("check whether the cluster is not ipv6 single stack") + stacktype := compat_otp.GetIPVersionStackType(oc) + if stacktype == "ipv6single" { + g.Skip("Skip as ipfailover currently doesn't support ipv6 single stack") + } + + }) + + g.JustBeforeEach(func() { + g.By("Check network type") + networkType := compat_otp.CheckNetworkType(oc) + if strings.Contains(networkType, "openshiftsdn") { + HAInterfaces = "ens3" + } + }) + + // author: hongli@redhat.com + // might conflict with other ipfailover cases so set it as Serial + g.It("Author:hongli-NonHyperShiftHOST-ConnectedOnly-Critical-41025-support to deploy ipfailover [Serial]", func() { + buildPruningBaseDir := testdata.FixturePath("router") + customTemp := filepath.Join(buildPruningBaseDir, "ipfailover.yaml") + var ( + ipf = ipfailoverDescription{ + name: "ipf-41025", + namespace: "", + image: "", + HAInterface: HAInterfaces, + template: customTemp, + } + ) + + g.By("get pull spec of ipfailover image from payload") + ipf.image = getImagePullSpecFromPayload(oc, "keepalived-ipfailover") + ipf.namespace = oc.Namespace() + ns := oc.Namespace() + g.By("create ipfailover deployment and ensure one of pod enter MASTER state") + ipf.create(oc, ns) + unicastIPFailover(oc, ns, ipf.name) + ensurePodWithLabelReady(oc, ns, "ipfailover=hello-openshift") + podName := getPodListByLabel(oc, ns, "ipfailover=hello-openshift") + ensureIpfailoverMasterBackup(oc, ns, podName) + }) + + // author: mjoseph@redhat.com + // might conflict with other ipfailover cases so set it as Serial + g.It("Author:mjoseph-NonHyperShiftHOST-ConnectedOnly-Medium-41027-pod and service automatically switched over to standby when master fails [Disruptive]", func() { + buildPruningBaseDir := testdata.FixturePath("router") + customTemp := filepath.Join(buildPruningBaseDir, "ipfailover.yaml") + var ( + ipf = ipfailoverDescription{ + name: "ipf-41027", + namespace: "", + image: "", + HAInterface: HAInterfaces, + template: customTemp, + } + ) + g.By("1. Get pull spec of ipfailover image from payload") + ipf.image = getImagePullSpecFromPayload(oc, "keepalived-ipfailover") + ipf.namespace = oc.Namespace() + ns := oc.Namespace() + g.By("2. Create ipfailover deployment and ensure one of pod enter MASTER state") + ipf.create(oc, ns) + unicastIPFailover(oc, ns, ipf.name) + ensurePodWithLabelReady(oc, ns, "ipfailover=hello-openshift") + podNames := getPodListByLabel(oc, ns, "ipfailover=hello-openshift") + ensureIpfailoverMasterBackup(oc, ns, podNames) + + g.By("3. Set the HA virtual IP for the failover group") + ipv4Address := getPodIP(oc, ns, podNames[0]) + virtualIP := replaceIPOctet(ipv4Address, 3, "100") + setEnvVariable(oc, ns, "deploy/"+ipf.name, "OPENSHIFT_HA_VIRTUAL_IPS="+virtualIP) + + g.By("4. Verify the HA virtual ip ENV variable") + ensurePodWithLabelReady(oc, ns, "ipfailover=hello-openshift") + newPodName := getPodListByLabel(oc, ns, "ipfailover=hello-openshift") + masterNode, _ := ensureIpfailoverMasterBackup(oc, ns, newPodName) + checkenv := pollReadPodData(oc, ns, newPodName[0], "/usr/bin/env ", "OPENSHIFT_HA_VIRTUAL_IPS") + o.Expect(checkenv).To(o.ContainSubstring("OPENSHIFT_HA_VIRTUAL_IPS=" + virtualIP)) + + g.By("5. Find the primary and the secondary pod using the virtual IP") + primaryPod := getVipOwnerPod(oc, ns, newPodName, virtualIP) + o.Expect(masterNode).To(o.ContainSubstring(primaryPod)) + + g.By("6. Restarting the ipfailover primary pod") + err := oc.AsAdmin().WithoutNamespace().Run("delete").Args("-n", ns, "pod", primaryPod).Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + + g.By("7. Verify the virtual IP is floated onto the new MASTER node") + newPodName1 := getPodListByLabel(oc, ns, "ipfailover=hello-openshift") + newMasterNode, _ := ensureIpfailoverMasterBackup(oc, ns, newPodName1) + waitForPrimaryPod(oc, ns, newMasterNode, virtualIP) + }) + + // author: mjoseph@redhat.com + // might conflict with other ipfailover cases so set it as Serial + g.It("Author:mjoseph-NonHyperShiftHOST-ConnectedOnly-Medium-41028-ipfailover configuration can be customized by ENV [Serial]", func() { + buildPruningBaseDir := testdata.FixturePath("router") + customTemp := filepath.Join(buildPruningBaseDir, "ipfailover.yaml") + var ( + ipf = ipfailoverDescription{ + name: "ipf-41028", + namespace: "", + image: "", + HAInterface: HAInterfaces, + template: customTemp, + } + ) + + g.By("get pull spec of ipfailover image from payload") + ipf.image = getImagePullSpecFromPayload(oc, "keepalived-ipfailover") + ipf.namespace = oc.Namespace() + ns := oc.Namespace() + g.By("create ipfailover deployment and ensure one of pod enter MASTER state") + ipf.create(oc, ns) + unicastIPFailover(oc, ns, ipf.name) + ensurePodWithLabelReady(oc, ns, "ipfailover=hello-openshift") + + g.By("set the HA virtual IP for the failover group") + podNames := getPodListByLabel(oc, ns, "ipfailover=hello-openshift") + ipv4Address := getPodIP(oc, ns, podNames[0]) + virtualIP := replaceIPOctet(ipv4Address, 3, "100") + setEnvVariable(oc, ns, "deploy/"+ipf.name, "OPENSHIFT_HA_VIRTUAL_IPS="+virtualIP) + + g.By("set other ipfailover env varibales") + setEnvVariable(oc, ns, "deploy/"+ipf.name, "OPENSHIFT_HA_CONFIG_NAME=ipfailover") + setEnvVariable(oc, ns, "deploy/"+ipf.name, "OPENSHIFT_HA_VIP_GROUPS=4") + setEnvVariable(oc, ns, "deploy/"+ipf.name, "OPENSHIFT_HA_MONITOR_PORT=30061") + setEnvVariable(oc, ns, "deploy/"+ipf.name, "OPENSHIFT_HA_VRRP_ID_OFFSET=2") + setEnvVariable(oc, ns, "deploy/"+ipf.name, "OPENSHIFT_HA_REPLICA_COUNT=3") + setEnvVariable(oc, ns, "deploy/"+ipf.name, `OPENSHIFT_HA_USE_UNICAST=true`) + setEnvVariable(oc, ns, "deploy/"+ipf.name, `OPENSHIFT_HA_IPTABLES_CHAIN=OUTPUT`) + setEnvVariable(oc, ns, "deploy/"+ipf.name, `OPENSHIFT_HA_NOTIFY_SCRIPT=/etc/keepalive/mynotifyscript.sh`) + setEnvVariable(oc, ns, "deploy/"+ipf.name, `OPENSHIFT_HA_CHECK_SCRIPT=/etc/keepalive/mycheckscript.sh`) + setEnvVariable(oc, ns, "deploy/"+ipf.name, `OPENSHIFT_HA_PREEMPTION=preempt_delay 600`) + setEnvVariable(oc, ns, "deploy/"+ipf.name, "OPENSHIFT_HA_CHECK_INTERVAL=3") + + g.By("verify the HA virtual ip ENV variable") + ensurePodWithLabelReady(oc, ns, "ipfailover=hello-openshift") + newPodName := getPodListByLabel(oc, ns, "ipfailover=hello-openshift") + ensureIpfailoverMasterBackup(oc, ns, newPodName) + checkenv := pollReadPodData(oc, ns, newPodName[0], "/usr/bin/env ", "OPENSHIFT_HA_VIRTUAL_IPS") + o.Expect(checkenv).To(o.ContainSubstring("OPENSHIFT_HA_VIRTUAL_IPS=" + virtualIP)) + + g.By("check the ipfailover configurations and verify the other ENV variables") + result := describePodResource(oc, newPodName[0], ns) + o.Expect(result).To(o.ContainSubstring("OPENSHIFT_HA_VIP_GROUPS: 4")) + o.Expect(result).To(o.ContainSubstring("OPENSHIFT_HA_CONFIG_NAME: ipfailover")) + o.Expect(result).To(o.ContainSubstring("OPENSHIFT_HA_MONITOR_PORT: 30061")) + o.Expect(result).To(o.ContainSubstring("OPENSHIFT_HA_VRRP_ID_OFFSET: 2")) + o.Expect(result).To(o.ContainSubstring("OPENSHIFT_HA_REPLICA_COUNT: 3")) + o.Expect(result).To(o.ContainSubstring(`OPENSHIFT_HA_USE_UNICAST: true`)) + o.Expect(result).To(o.ContainSubstring(`OPENSHIFT_HA_IPTABLES_CHAIN: OUTPUT`)) + o.Expect(result).To(o.ContainSubstring(`OPENSHIFT_HA_NOTIFY_SCRIPT: /etc/keepalive/mynotifyscript.sh`)) + o.Expect(result).To(o.ContainSubstring(`OPENSHIFT_HA_CHECK_SCRIPT: /etc/keepalive/mycheckscript.sh`)) + o.Expect(result).To(o.ContainSubstring(`OPENSHIFT_HA_PREEMPTION: preempt_delay 600`)) + o.Expect(result).To(o.ContainSubstring("OPENSHIFT_HA_CHECK_INTERVAL: 3")) + o.Expect(result).To(o.ContainSubstring("OPENSHIFT_HA_VIRTUAL_IPS: " + virtualIP)) + }) + + // author: mjoseph@redhat.com + // might conflict with other ipfailover cases so set it as Serial + g.It("Author:mjoseph-NonHyperShiftHOST-ConnectedOnly-Medium-41029-ipfailover can support up to a maximum of 255 VIPs for the entire cluster [Serial]", func() { + if compat_otp.CheckPlatform(oc) == "nutanix" { + g.Skip("This test will not works for Nutanix") + } + buildPruningBaseDir := testdata.FixturePath("router") + customTemp := filepath.Join(buildPruningBaseDir, "ipfailover.yaml") + var ( + ipf = ipfailoverDescription{ + name: "ipf-41029", + namespace: "", + image: "", + HAInterface: HAInterfaces, + template: customTemp, + } + ) + + g.By("get pull spec of ipfailover image from payload") + ipf.image = getImagePullSpecFromPayload(oc, "keepalived-ipfailover") + ipf.namespace = oc.Namespace() + ns := oc.Namespace() + g.By("create ipfailover deployment and ensure one of pod enter MASTER state") + ipf.create(oc, ns) + unicastIPFailover(oc, ns, ipf.name) + ensurePodWithLabelReady(oc, ns, "ipfailover=hello-openshift") + podName := getPodListByLabel(oc, ns, "ipfailover=hello-openshift") + ensureIpfailoverMasterBackup(oc, ns, podName) + + g.By("add some VIP configuration for the failover group") + setEnvVariable(oc, ns, "deploy/"+ipf.name, "OPENSHIFT_HA_VRRP_ID_OFFSET=0") + setEnvVariable(oc, ns, "deploy/"+ipf.name, "OPENSHIFT_HA_VIP_GROUPS=238") + setEnvVariable(oc, ns, "deploy/"+ipf.name, `OPENSHIFT_HA_VIRTUAL_IPS=192.168.254.1-255`) + + g.By("verify from the ipfailover pod, the 255 VIPs are added") + ensurePodWithLabelReady(oc, ns, "ipfailover=hello-openshift") + newPodName := getPodListByLabel(oc, ns, "ipfailover=hello-openshift") + checkenv := pollReadPodData(oc, ns, newPodName[0], "/usr/bin/env ", "OPENSHIFT_HA_VIP_GROUPS") + o.Expect(checkenv).To(o.ContainSubstring("OPENSHIFT_HA_VIP_GROUPS=238")) + }) + + // author: mjoseph@redhat.com + // might conflict with other ipfailover cases so set it as Serial + g.It("Author:mjoseph-NonHyperShiftHOST-ConnectedOnly-High-41030-preemption strategy for keepalived ipfailover [Disruptive]", func() { + buildPruningBaseDir := testdata.FixturePath("router") + customTemp := filepath.Join(buildPruningBaseDir, "ipfailover.yaml") + var ( + ipf = ipfailoverDescription{ + name: "ipf-41030", + namespace: "", + image: "", + HAInterface: HAInterfaces, + template: customTemp, + } + ) + g.By("1. Get pull spec of ipfailover image from payload") + ipf.image = getImagePullSpecFromPayload(oc, "keepalived-ipfailover") + ipf.namespace = oc.Namespace() + ns := oc.Namespace() + g.By("2. Create ipfailover deployment and ensure one of pod enter MASTER state") + ipf.create(oc, ns) + unicastIPFailover(oc, ns, ipf.name) + ensurePodWithLabelReady(oc, ns, "ipfailover=hello-openshift") + podName := getPodListByLabel(oc, ns, "ipfailover=hello-openshift") + ensureIpfailoverMasterBackup(oc, ns, podName) + + g.By("3. Set the HA virtual IP for the failover group") + podNames := getPodListByLabel(oc, ns, "ipfailover=hello-openshift") + ipv4Address := getPodIP(oc, ns, podNames[0]) + virtualIP := replaceIPOctet(ipv4Address, 3, "100") + setEnvVariable(oc, ns, "deploy/"+ipf.name, "OPENSHIFT_HA_VIRTUAL_IPS="+virtualIP) + + g.By("4. Verify the HA virtual ip ENV variable") + ensurePodWithLabelReady(oc, ns, "ipfailover=hello-openshift") + newPodName := getPodListByLabel(oc, ns, "ipfailover=hello-openshift") + master, backup := ensureIpfailoverMasterBackup(oc, ns, newPodName) + checkenv := pollReadPodData(oc, ns, newPodName[0], "/usr/bin/env ", "OPENSHIFT_HA_VIRTUAL_IPS") + o.Expect(checkenv).To(o.ContainSubstring("OPENSHIFT_HA_VIRTUAL_IPS=" + virtualIP)) + checkenv1 := pollReadPodData(oc, ns, newPodName[0], "/usr/bin/env ", "OPENSHIFT_HA_PREEMPTION") + o.Expect(checkenv1).To(o.ContainSubstring("nopreempt")) + + g.By("5. Find the primary and the secondary pod") + primaryPod := getVipOwnerPod(oc, ns, newPodName, virtualIP) + secondaryPod := slicingElement(primaryPod, newPodName) + o.Expect(master).To(o.ContainSubstring(primaryPod)) + o.Expect(backup).To(o.ContainSubstring(secondaryPod[0])) + + g.By("6. Restarting the ipfailover primary pod") + err := oc.AsAdmin().WithoutNamespace().Run("delete").Args("-n", ns, "pod", primaryPod).Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + + g.By("7. Verify whether the other pod becomes primary and it has the VIP") + waitForPrimaryPod(oc, ns, secondaryPod[0], virtualIP) + + g.By("8. Now set the preemption delay timer of 120s for the failover group") + setEnvVariable(oc, ns, "deploy/"+ipf.name, `OPENSHIFT_HA_PREEMPTION=preempt_delay 120`) + ensurePodWithLabelReady(oc, ns, "ipfailover=hello-openshift") + newPodName1 := getPodListByLabel(oc, ns, "ipfailover=hello-openshift") + new_master, new_backup := ensureIpfailoverMasterBackup(oc, ns, newPodName1) + checkenv2 := pollReadPodData(oc, ns, newPodName1[0], "/usr/bin/env ", "OPENSHIFT_HA_PREEMPTION") + o.Expect(checkenv2).To(o.ContainSubstring("preempt_delay 120")) + + g.By("9. Again restart the ipfailover primary(master) pod") + // the below steps will make the 'new_backup' pod the master + err = oc.AsAdmin().WithoutNamespace().Run("delete").Args("-n", ns, "pod", new_master).Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + + g.By("10. Verify the newly created pod preempts the exiting primary after the delay expires") + latestpods := getPodListByLabel(oc, ns, "ipfailover=hello-openshift") + // removing the existing master pod from the latest pods + futurePrimaryPod := slicingElement(new_backup, latestpods) + // waiting till the preempt delay 120 seconds expires + time.Sleep(125 * time.Second) + // confirming the newer pod is the master by checking the VIP + waitForPrimaryPod(oc, ns, futurePrimaryPod[0], virtualIP) + }) + + // author: mjoseph@redhat.com + // might conflict with other ipfailover cases so set it as Serial + g.It("Author:mjoseph-NonHyperShiftHOST-ConnectedOnly-Medium-49214-Excluding the existing VRRP cluster ID from ipfailover deployments [Serial]", func() { + buildPruningBaseDir := testdata.FixturePath("router") + customTemp := filepath.Join(buildPruningBaseDir, "ipfailover.yaml") + var ( + ipf = ipfailoverDescription{ + name: "ipf-49214", + namespace: "", + image: "", + HAInterface: HAInterfaces, + template: customTemp, + } + ) + + g.By("get pull spec of ipfailover image from payload") + ipf.image = getImagePullSpecFromPayload(oc, "keepalived-ipfailover") + ipf.namespace = oc.Namespace() + ns := oc.Namespace() + g.By("create ipfailover deployment and ensure one of pod enter MASTER state") + ipf.create(oc, ns) + unicastIPFailover(oc, ns, ipf.name) + ensurePodWithLabelReady(oc, ns, "ipfailover=hello-openshift") + podName := getPodListByLabel(oc, ns, "ipfailover=hello-openshift") + ensureIpfailoverMasterBackup(oc, ns, podName) + + g.By("add 254 VIPs for the failover group") + setEnvVariable(oc, ns, "deploy/"+ipf.name, `OPENSHIFT_HA_VIRTUAL_IPS=192.168.254.1-254`) + + g.By("Exclude VIP '9' from the ipfailover group") + getPodListByLabel(oc, ns, "ipfailover=hello-openshift") + setEnvVariable(oc, ns, "deploy/"+ipf.name, `HA_EXCLUDED_VRRP_IDS=9`) + + g.By("verify from the ipfailover pod, the excluded VRRP_ID is configured") + ensurePodWithLabelReady(oc, ns, "ipfailover=hello-openshift") + newPodName := getPodListByLabel(oc, ns, "ipfailover=hello-openshift") + checkenv := pollReadPodData(oc, ns, newPodName[0], "/usr/bin/env ", "HA_EXCLUDED_VRRP_IDS") + o.Expect(checkenv).To(o.ContainSubstring("HA_EXCLUDED_VRRP_IDS=9")) + + g.By("verify the excluded VIP is removed from the router_ids of ipfailover pods") + routerIds := pollReadPodData(oc, ns, newPodName[0], `cat /etc/keepalived/keepalived.conf`, `virtual_router_id`) + o.Expect(routerIds).NotTo(o.ContainSubstring(`virtual_router_id 9`)) + }) +}) diff --git a/tests-extension/test/e2e/logging.go b/tests-extension/test/e2e/logging.go new file mode 100644 index 000000000..0ce2b555d --- /dev/null +++ b/tests-extension/test/e2e/logging.go @@ -0,0 +1,388 @@ +package router + +import ( + "fmt" + "os" + "path/filepath" + + g "github.com/onsi/ginkgo/v2" + o "github.com/onsi/gomega" + + compat_otp "github.com/openshift/origin/test/extended/util/compat_otp" + "github.com/openshift/router-tests-extension/test/testdata" +) + +var _ = g.Describe("[sig-network-edge] Network_Edge Component_Router", func() { + defer g.GinkgoRecover() + + var oc = compat_otp.NewCLI("router-logging", compat_otp.KubeConfigPath()) + + // author: shudili@redhat.com + g.It("Author:shudili-ROSA-OSD_CCS-ARO-High-34166-capture and log http cookies with specific prefixes via httpCaptureCookies option", func() { + buildPruningBaseDir := testdata.FixturePath("router") + baseTemp := filepath.Join(buildPruningBaseDir, "ingresscontroller-np.yaml") + extraParas := fmt.Sprintf(` + logging: + access: + destination: + type: Container + httpCaptureCookies: + - matchType: Prefix + maxLength: 100 + namePrefix: foo +`) + customTemp := addExtraParametersToYamlFile(baseTemp, "spec:", extraParas) + defer os.Remove(customTemp) + + var ( + testPodSvc = filepath.Join(buildPruningBaseDir, "web-server-deploy.yaml") + srvrcInfo = "web-server-deploy" + unsecSvcName = "service-unsecure" + clientPod = filepath.Join(buildPruningBaseDir, "test-client-pod.yaml") + clientPodName = "hello-pod" + clientPodLabel = "app=hello-pod" + ingctrl = ingressControllerDescription{ + name: "ocp34166", + namespace: "openshift-ingress-operator", + domain: "", + template: customTemp, + } + ) + + compat_otp.By("1.0 Create an custom IC for logging http cookies with the specific prefix") + baseDomain := getBaseDomain(oc) + ingctrl.domain = ingctrl.name + "." + baseDomain + defer ingctrl.delete(oc) + ingctrl.create(oc) + ensureRouterDeployGenerationIs(oc, ingctrl.name, "1") + + compat_otp.By("2.0: Create a client pod, a deployment and the services in a namespace") + ns := oc.Namespace() + createResourceFromFile(oc, ns, clientPod) + ensurePodWithLabelReady(oc, ns, clientPodLabel) + createResourceFromFile(oc, ns, testPodSvc) + ensurePodWithLabelReady(oc, ns, "name="+srvrcInfo) + + compat_otp.By("3.0: Create a http route for the testing") + routehost := "unsecure34166" + "." + ingctrl.domain + createRoute(oc, ns, "http", "route-http", unsecSvcName, []string{"--hostname=" + routehost}) + ensureRouteIsAdmittedByIngressController(oc, ns, "route-http", ingctrl.name) + + compat_otp.By("4.0: Check httpCaptureCookies configuration in haproxy") + routerpod := getOneNewRouterPodFromRollingUpdate(oc, ingctrl.name) + ensureHaproxyBlockConfigContains(oc, routerpod, "frontend public", []string{"capture cookie foo len 100"}) + + compat_otp.By("5.0: Curl the http route with cookie fo=nobar, expect to get 200 OK") + podIP := getPodv4Address(oc, routerpod, "openshift-ingress") + toDst := routehost + ":80:" + podIP + curlCmd := []string{"-n", ns, clientPodName, "--", "curl", "http://" + routehost + "/index.html", "-sI", "-b", "fo=nobar", "--resolve", toDst, "--connect-timeout", "10"} + repeatCmdOnClient(oc, curlCmd, "200 OK", 90, 6) + + compat_otp.By("6.0: Curl the http route with cookie foo=bar, expect to get 200 OK") + curlCmd = []string{"-n", ns, clientPodName, "--", "curl", "http://" + routehost + "/index.html", "-sI", "-b", "foo=bar", "--resolve", toDst, "--connect-timeout", "10"} + repeatCmdOnClient(oc, curlCmd, "200 OK", 90, 6) + + compat_otp.By("7.0: Check the router logs, which should contain both the cookie foo=bar and the url") + logs := waitRouterLogsAppear(oc, routerpod, "foo=bar") + o.Expect(logs).To(o.ContainSubstring("index.html")) + + compat_otp.By("8.0: Check the router logs, which should NOT contain the cookie fo=nobar") + logs, err := oc.AsAdmin().WithoutNamespace().Run("logs").Args("-n", "openshift-ingress", "-c", "logs", routerpod).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(logs).NotTo(o.ContainSubstring("fo=nobar")) + + compat_otp.By("9.0: Curl the http route with cookie foo22=bar22, expect to get 200 OK") + curlCmd = []string{"-n", ns, clientPodName, "--", "curl", "http://" + routehost + "/index.html", "-sI", "-b", "foo22=bar22", "--resolve", toDst, "--connect-timeout", "10"} + repeatCmdOnClient(oc, curlCmd, "200 OK", 90, 6) + + compat_otp.By("10.0: Check the router logs, which should contain both the cookie foo22=bar22 and the url") + logs = waitRouterLogsAppear(oc, routerpod, "foo22=bar22") + o.Expect(logs).To(o.ContainSubstring("index.html")) + }) + + // author: shudili@redhat.com + g.It("Author:shudili-ROSA-OSD_CCS-ARO-High-34178-capture and log http cookies with exact match via httpCaptureCookies option", func() { + buildPruningBaseDir := testdata.FixturePath("router") + baseTemp := filepath.Join(buildPruningBaseDir, "ingresscontroller-np.yaml") + extraParas := fmt.Sprintf(` + logging: + access: + destination: + type: Container + httpCaptureCookies: + - matchType: Exact + maxLength: 100 + name: foo +`) + customTemp := addExtraParametersToYamlFile(baseTemp, "spec:", extraParas) + defer os.Remove(customTemp) + + var ( + testPodSvc = filepath.Join(buildPruningBaseDir, "web-server-deploy.yaml") + srvrcInfo = "web-server-deploy" + unsecSvcName = "service-unsecure" + clientPod = filepath.Join(buildPruningBaseDir, "test-client-pod.yaml") + clientPodName = "hello-pod" + clientPodLabel = "app=hello-pod" + ingctrl = ingressControllerDescription{ + name: "ocp34178", + namespace: "openshift-ingress-operator", + domain: "", + template: customTemp, + } + ) + + compat_otp.By("1.0 Create an custom IC for logging http cookies with the exact cookie name") + baseDomain := getBaseDomain(oc) + ingctrl.domain = ingctrl.name + "." + baseDomain + defer ingctrl.delete(oc) + ingctrl.create(oc) + ensureRouterDeployGenerationIs(oc, ingctrl.name, "1") + + compat_otp.By("2.0: Create a client pod, a deployment and the services in a namespace") + ns := oc.Namespace() + createResourceFromFile(oc, ns, clientPod) + ensurePodWithLabelReady(oc, ns, clientPodLabel) + createResourceFromFile(oc, ns, testPodSvc) + ensurePodWithLabelReady(oc, ns, "name="+srvrcInfo) + + compat_otp.By("3.0: Create a http route for the testing") + routehost := "unsecure34178" + "." + ingctrl.domain + createRoute(oc, ns, "http", "route-http", unsecSvcName, []string{"--hostname=" + routehost}) + ensureRouteIsAdmittedByIngressController(oc, ns, "route-http", ingctrl.name) + + compat_otp.By("4.0: Check httpCaptureCookies configuration in haproxy") + routerpod := getOneNewRouterPodFromRollingUpdate(oc, ingctrl.name) + ensureHaproxyBlockConfigContains(oc, routerpod, "frontend public", []string{"capture cookie foo= len 100"}) + + compat_otp.By("5.0: Curl the http route with cookie fooor=nobar, expect to get 200 OK") + podIP := getPodv4Address(oc, routerpod, "openshift-ingress") + toDst := routehost + ":80:" + podIP + curlCmd := []string{"-n", ns, clientPodName, "--", "curl", "http://" + routehost + "/index.html", "-sI", "-b", "fooor=nobar", "--resolve", toDst, "--connect-timeout", "10"} + repeatCmdOnClient(oc, curlCmd, "200 OK", 90, 6) + + compat_otp.By("6.0: Curl the http route with cookie foo=bar, expect to get 200 OK") + curlCmd = []string{"-n", ns, clientPodName, "--", "curl", "http://" + routehost + "/index.html", "-sI", "-b", "foo=bar", "--resolve", toDst, "--connect-timeout", "10"} + repeatCmdOnClient(oc, curlCmd, "200 OK", 90, 6) + + compat_otp.By("7.0: Check the router logs, which should contain both the cookie foo=bar and the url") + logs := waitRouterLogsAppear(oc, routerpod, "foo=bar") + o.Expect(logs).To(o.ContainSubstring("index.html")) + + compat_otp.By("8.0: Check the router logs, which should NOT contain the cookie fooor=nobar") + logs, err := oc.AsAdmin().WithoutNamespace().Run("logs").Args("-n", "openshift-ingress", "-c", "logs", routerpod).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(logs).NotTo(o.ContainSubstring("fooor=nobar")) + }) + + // author: shudili@redhat.com + g.It("Author:shudili-ROSA-OSD_CCS-ARO-High-34188-capture and log http requests using UniqueID with custom logging format defined via httpHeader option", func() { + var ( + buildPruningBaseDir = testdata.FixturePath("router") + baseTemp = filepath.Join(buildPruningBaseDir, "ingresscontroller-np.yaml") + testPodSvc = filepath.Join(buildPruningBaseDir, "web-server-deploy.yaml") + srvrcInfo = "web-server-deploy" + unsecSvcName = "service-unsecure" + clientPod = filepath.Join(buildPruningBaseDir, "test-client-pod.yaml") + clientPodName = "hello-pod" + clientPodLabel = "app=hello-pod" + ingctrl = ingressControllerDescription{ + name: "ocp34188", + namespace: "openshift-ingress-operator", + domain: "", + template: baseTemp, + } + ingctrlResource = "ingresscontroller/" + ingctrl.name + ) + + compat_otp.By("1.0: Create an custom IC for logging http cookies with maxLength 10") + baseDomain := getBaseDomain(oc) + ingctrl.domain = ingctrl.name + "." + baseDomain + defer ingctrl.delete(oc) + ingctrl.create(oc) + ensureRouterDeployGenerationIs(oc, ingctrl.name, "1") + + compat_otp.By("1.1: Patch the hostNetwork ingresscontroller with logging http requests using the UniqueID ") + patchPath := `{"spec":{"httpHeaders":{"uniqueId":{"format":"%{+Q}b","name":"X-Request-Id"}},"logging":{"access":{"destination":{"type":"Container"},"httpLogFormat":"%ID %ci:%cp [%tr] %ft %s %TR/%Tw/%Tc/%Tr/%Ta %ST %B %CC %CS %tsc %hr %hs %{+Q}r"}}}}` + patchResourceAsAdmin(oc, ingctrl.namespace, ingctrlResource, patchPath) + ensureRouterDeployGenerationIs(oc, ingctrl.name, "2") + + compat_otp.By("2.0: Create a client pod, a deployment and the services in a namespace") + ns := oc.Namespace() + createResourceFromFile(oc, ns, clientPod) + ensurePodWithLabelReady(oc, ns, clientPodLabel) + createResourceFromFile(oc, ns, testPodSvc) + ensurePodWithLabelReady(oc, ns, "name="+srvrcInfo) + + compat_otp.By("3.0: Create a http route for the testing") + routehost := "unsecure34188" + "." + ingctrl.domain + createRoute(oc, ns, "http", "route-http", unsecSvcName, []string{"--hostname=" + routehost}) + ensureRouteIsAdmittedByIngressController(oc, ns, "route-http", ingctrl.name) + + compat_otp.By("4.0: Check the unique-id configuration in haproxy") + routerpod := getOneNewRouterPodFromRollingUpdate(oc, ingctrl.name) + ensureHaproxyBlockConfigContains(oc, routerpod, "defaults", []string{`log-format "%ID %ci:%cp [%tr] %ft %s %TR/%Tw/%Tc/%Tr/%Ta %ST %B %CC %CS %tsc %hr %hs %{+Q}r"`}) + publicCfg := getBlockConfig(oc, routerpod, "frontend public") + o.Expect(publicCfg).Should(o.And( + o.ContainSubstring(`unique-id-format "%{+Q}b"`), + o.ContainSubstring(`unique-id-header X-Request-Id`))) + + compat_otp.By("5.0: Curl the http route, expect to get 200 OK") + podIP := getPodv4Address(oc, routerpod, "openshift-ingress") + toDst := routehost + ":80:" + podIP + curlCmd := []string{"-n", ns, clientPodName, "--", "curl", "http://" + routehost + "/index.html", "-sI", "--resolve", toDst, "--connect-timeout", "10"} + repeatCmdOnClient(oc, curlCmd, "200 OK", 90, 6) + + compat_otp.By("6.0: Checking the access log verify if the UniqueID pattern is logged and matches") + pattern := fmt.Sprintf(`"be_http:%s:route-http"`, ns) + waitRouterLogsAppear(oc, routerpod, pattern) + }) + + // author: shudili@redhat.com + g.It("Author:shudili-ROSA-OSD_CCS-ARO-Medium-34189-The httpCaptureCookies option strictly adheres to the maxlength parameter", func() { + buildPruningBaseDir := testdata.FixturePath("router") + baseTemp := filepath.Join(buildPruningBaseDir, "ingresscontroller-np.yaml") + extraParas := fmt.Sprintf(` + logging: + access: + destination: + type: Container + httpCaptureCookies: + - matchType: Prefix + maxLength: 10 + namePrefix: foo +`) + customTemp := addExtraParametersToYamlFile(baseTemp, "spec:", extraParas) + defer os.Remove(customTemp) + + var ( + testPodSvc = filepath.Join(buildPruningBaseDir, "web-server-deploy.yaml") + srvrcInfo = "web-server-deploy" + unsecSvcName = "service-unsecure" + clientPod = filepath.Join(buildPruningBaseDir, "test-client-pod.yaml") + clientPodName = "hello-pod" + clientPodLabel = "app=hello-pod" + ingctrl = ingressControllerDescription{ + name: "ocp34189", + namespace: "openshift-ingress-operator", + domain: "", + template: customTemp, + } + ) + + compat_otp.By("1.0 Create an custom IC for logging http cookies with maxLength 10") + baseDomain := getBaseDomain(oc) + ingctrl.domain = ingctrl.name + "." + baseDomain + defer ingctrl.delete(oc) + ingctrl.create(oc) + ensureRouterDeployGenerationIs(oc, ingctrl.name, "1") + + compat_otp.By("2.0: Create a client pod, a deployment and the services in a namespace") + ns := oc.Namespace() + createResourceFromFile(oc, ns, clientPod) + ensurePodWithLabelReady(oc, ns, clientPodLabel) + createResourceFromFile(oc, ns, testPodSvc) + ensurePodWithLabelReady(oc, ns, "name="+srvrcInfo) + + compat_otp.By("3.0: Create a http route for the testing") + routehost := "unsecure34189" + "." + ingctrl.domain + createRoute(oc, ns, "http", "route-http", unsecSvcName, []string{"--hostname=" + routehost}) + ensureRouteIsAdmittedByIngressController(oc, ns, "route-http", ingctrl.name) + + compat_otp.By("4.0: Check httpCaptureCookies configuration in haproxy") + routerpod := getOneNewRouterPodFromRollingUpdate(oc, ingctrl.name) + ensureHaproxyBlockConfigContains(oc, routerpod, "frontend public", []string{"capture cookie foo len 10"}) + + compat_otp.By("5.0: Curl the http route with cookie foo=bar8 which length less than 10, expect to get 200 OK") + podIP := getPodv4Address(oc, routerpod, "openshift-ingress") + toDst := routehost + ":80:" + podIP + curlCmd := []string{"-n", ns, clientPodName, "--", "curl", "http://" + routehost + "/index.html", "-sI", "-b", "foo=bar8", "--resolve", toDst, "--connect-timeout", "10"} + repeatCmdOnClient(oc, curlCmd, "200 OK", 90, 6) + + compat_otp.By("6.0: Curl the http route with cookie foo2=bar9abcdefg which length larger than 10, expect to get 200 OK") + curlCmd = []string{"-n", ns, clientPodName, "--", "curl", "http://" + routehost + "/index.html", "-sI", "-b", "foo2=bar9abcdefg", "--resolve", toDst, "--connect-timeout", "10"} + repeatCmdOnClient(oc, curlCmd, "200 OK", 90, 6) + + compat_otp.By("7.0: Check the router logs for cookie foo=bar8 which length less than 10") + logs := waitRouterLogsAppear(oc, routerpod, "foo=bar8") + o.Expect(logs).To(o.ContainSubstring("index.html")) + + compat_otp.By("8.0: Check the router logs for cookie foo2=bar9abcdefg which length larger than 10") + logs = waitRouterLogsAppear(oc, routerpod, "foo2=bar9a") + o.Expect(logs).To(o.ContainSubstring("index.html")) + o.Expect(logs).NotTo(o.ContainSubstring("foo2=bar9ab")) + }) + + // author: shudili@redhat.com + g.It("Author:shudili-ROSA-OSD_CCS-ARO-Medium-34191-The httpCaptureHeaders option strictly adheres to the maxlength parameter", func() { + buildPruningBaseDir := testdata.FixturePath("router") + baseTemp := filepath.Join(buildPruningBaseDir, "ingresscontroller-np.yaml") + extraParas := fmt.Sprintf(` + logging: + access: + destination: + type: Container + httpCaptureHeaders: + request: + - maxLength: 13 + name: Host + response: + - maxLength: 5 + name: Server +`) + customTemp := addExtraParametersToYamlFile(baseTemp, "spec:", extraParas) + defer os.Remove(customTemp) + + var ( + testPodSvc = filepath.Join(buildPruningBaseDir, "web-server-deploy.yaml") + srvrcInfo = "web-server-deploy" + unsecSvcName = "service-unsecure" + clientPod = filepath.Join(buildPruningBaseDir, "test-client-pod.yaml") + clientPodName = "hello-pod" + clientPodLabel = "app=hello-pod" + server = "nginx" + ingctrl = ingressControllerDescription{ + name: "ocp34191", + namespace: "openshift-ingress-operator", + domain: "", + template: customTemp, + } + ) + + compat_otp.By("1.0 Create an custom IC for logging http headers with the specified maxLength") + baseDomain := getBaseDomain(oc) + ingctrl.domain = ingctrl.name + "." + baseDomain + defer ingctrl.delete(oc) + ingctrl.create(oc) + ensureRouterDeployGenerationIs(oc, ingctrl.name, "1") + + compat_otp.By("2.0: Create a client pod, a deployment and the services in a namespace") + ns := oc.Namespace() + createResourceFromFile(oc, ns, clientPod) + ensurePodWithLabelReady(oc, ns, clientPodLabel) + createResourceFromFile(oc, ns, testPodSvc) + ensurePodWithLabelReady(oc, ns, "name="+srvrcInfo) + + compat_otp.By("3.0: Create a http route for the testing") + routehostPrefix := "unsecure34191" + routehost := routehostPrefix + "." + ingctrl.domain + createRoute(oc, ns, "http", "route-http", unsecSvcName, []string{"--hostname=" + routehost}) + ensureRouteIsAdmittedByIngressController(oc, ns, "route-http", ingctrl.name) + + compat_otp.By("4.0: Check httpCaptureHeaders configuration in haproxy") + routerpod := getOneNewRouterPodFromRollingUpdate(oc, ingctrl.name) + ensureHaproxyBlockConfigContains(oc, routerpod, "frontend fe_sni", []string{"capture request header Host len 13", "capture response header Server len 5"}) + + compat_otp.By("5.0: Curl the http route, expect to get 200 OK") + podIP := getPodv4Address(oc, routerpod, "openshift-ingress") + toDst := routehost + ":80:" + podIP + curlCmd := []string{"-n", ns, clientPodName, "--", "curl", "http://" + routehost + "/index.html", "-sI", "--resolve", toDst, "--connect-timeout", "10"} + repeatCmdOnClient(oc, curlCmd, "200 OK", 90, 6) + + compat_otp.By("6.0: Check the router logs, which should contain the request host header and response server header with values not exceeding the maxLength") + logs := waitRouterLogsAppear(oc, routerpod, routehostPrefix) + o.Expect(logs).NotTo(o.ContainSubstring(routehostPrefix + ".")) + o.Expect(logs).To(o.ContainSubstring(server)) + o.Expect(logs).NotTo(o.ContainSubstring(server + "/")) + }) +}) diff --git a/tests-extension/test/e2e/microshift.go b/tests-extension/test/e2e/microshift.go new file mode 100644 index 000000000..c92fed997 --- /dev/null +++ b/tests-extension/test/e2e/microshift.go @@ -0,0 +1,2242 @@ +package router + +import ( + "fmt" + "math/rand" + "os" + "os/exec" + "path/filepath" + "strconv" + "strings" + "time" + + g "github.com/onsi/ginkgo/v2" + o "github.com/onsi/gomega" + compat_otp "github.com/openshift/origin/test/extended/util/compat_otp" + "github.com/openshift/router-tests-extension/test/testdata" + e2e "k8s.io/kubernetes/test/e2e/framework" +) + +var _ = g.Describe("[sig-network-edge] Network_Edge", func() { + defer g.GinkgoRecover() + + var oc = compat_otp.NewCLIWithoutNamespace("router-microshift") + + g.It("Author:mjoseph-MicroShiftOnly-High-60136-reencrypt route using Ingress resource for Microshift with destination CA certificate", func() { + var ( + e2eTestNamespace = "e2e-ne-ocp60136-" + getRandomString() + buildPruningBaseDir = testdata.FixturePath("router") + testPodSvc = filepath.Join(buildPruningBaseDir, "web-server-signed-deploy.yaml") + ingressFile = filepath.Join(buildPruningBaseDir, "microshift-ingress-destca.yaml") + ) + + compat_otp.By("create a namespace for the scenario") + defer oc.DeleteSpecifiedNamespaceAsAdmin(e2eTestNamespace) + oc.CreateSpecifiedNamespaceAsAdmin(e2eTestNamespace) + + compat_otp.By("create a web-server-deploy pod and its services") + defer operateResourceFromFile(oc, "delete", e2eTestNamespace, testPodSvc) + createResourceFromFile(oc, e2eTestNamespace, testPodSvc) + ensurePodWithLabelReady(oc, e2eTestNamespace, "name=web-server-deploy") + podName := getPodListByLabel(oc, e2eTestNamespace, "name=web-server-deploy") + ingressPod := getOneRouterPodNameByIC(oc, "default") + + compat_otp.By("create ingress using the file and get the route details") + defer operateResourceFromFile(oc, "delete", e2eTestNamespace, ingressFile) + createResourceFromFile(oc, e2eTestNamespace, ingressFile) + getIngress(oc, e2eTestNamespace) + getRoutes(oc, e2eTestNamespace) + routeNames := getResourceName(oc, e2eTestNamespace, "route") + + compat_otp.By("check whether route details are present") + waitForOutputEquals(oc, e2eTestNamespace, "route/"+routeNames[0], "{.status.ingress[0].conditions[0].type}", "Admitted") + waitForOutputEquals(oc, e2eTestNamespace, "route/"+routeNames[0], "{.status.ingress[0].host}", "service-secure-test.example.com") + waitForOutputEquals(oc, e2eTestNamespace, "route/"+routeNames[0], "{.spec.tls.termination}", "reencrypt") + + compat_otp.By("check the reachability of the host in test pod") + routerPodIP := getPodv4Address(oc, ingressPod, "openshift-ingress") + curlCmd := []string{"-n", e2eTestNamespace, podName[0], "--", "curl", "https://service-secure-test.example.com:443", "-k", "-I", "--resolve", "service-secure-test.example.com:443:" + routerPodIP, "--connect-timeout", "10"} + repeatCmdOnClient(oc, curlCmd, "200", 30, 1) + + compat_otp.By("check the router pod and ensure the routes are loaded in haproxy.config") + searchOutput := readRouterPodData(oc, ingressPod, "cat haproxy.config", "ingress-ms-reen") + o.Expect(searchOutput).To(o.ContainSubstring("backend be_secure:" + e2eTestNamespace + ":" + routeNames[0])) + }) + + g.It("Author:mjoseph-MicroShiftOnly-Critical-60149-http route using Ingress resource for Microshift", func() { + var ( + e2eTestNamespace = "e2e-ne-ocp60149-" + getRandomString() + buildPruningBaseDir = testdata.FixturePath("router") + testPodSvc = filepath.Join(buildPruningBaseDir, "web-server-deploy.yaml") + ingressFile = filepath.Join(buildPruningBaseDir, "microshift-ingress-http.yaml") + httpRoute = "service-unsecure-test.example.com" + ) + + compat_otp.By("create a namespace for the scenario") + defer oc.DeleteSpecifiedNamespaceAsAdmin(e2eTestNamespace) + oc.CreateSpecifiedNamespaceAsAdmin(e2eTestNamespace) + + compat_otp.By("create a web-server-deploy pod and its services") + defer operateResourceFromFile(oc, "delete", e2eTestNamespace, testPodSvc) + createResourceFromFile(oc, e2eTestNamespace, testPodSvc) + ensurePodWithLabelReady(oc, e2eTestNamespace, "name=web-server-deploy") + podName := getPodListByLabel(oc, e2eTestNamespace, "name=web-server-deploy") + ingressPod := getOneRouterPodNameByIC(oc, "default") + + compat_otp.By("create ingress using the file and get the route details") + defer operateResourceFromFile(oc, "delete", e2eTestNamespace, ingressFile) + createResourceFromFile(oc, e2eTestNamespace, ingressFile) + getIngress(oc, e2eTestNamespace) + getRoutes(oc, e2eTestNamespace) + routeNames := getResourceName(oc, e2eTestNamespace, "route") + + compat_otp.By("check whether http route details are present") + waitForOutputEquals(oc, e2eTestNamespace, "route/"+routeNames[0], "{.spec.port.targetPort}", "http") + waitForOutputEquals(oc, e2eTestNamespace, "route/"+routeNames[0], "{.status.ingress[0].host}", httpRoute) + waitForOutputEquals(oc, e2eTestNamespace, "route/"+routeNames[0], "{.status.ingress[0].conditions[0].type}", "Admitted") + + compat_otp.By("check the reachability of the host in test pod for http route") + routerPodIP := getPodv4Address(oc, ingressPod, "openshift-ingress") + curlCmd := []string{"-n", e2eTestNamespace, podName[0], "--", "curl", "http://service-unsecure-test.example.com:80", "-k", "-I", "--resolve", "service-unsecure-test.example.com:80:" + routerPodIP, "--connect-timeout", "10"} + repeatCmdOnClient(oc, curlCmd, "200", 30, 1) + + compat_otp.By("check the router pod and ensure the http route is loaded in haproxy.config") + searchOutput := readRouterPodData(oc, ingressPod, "cat haproxy.config", "ingress-on-microshift") + o.Expect(searchOutput).To(o.ContainSubstring("backend be_http:" + e2eTestNamespace + ":" + routeNames[0])) + }) + + g.It("Author:mjoseph-MicroShiftOnly-Critical-60266-creation of edge and passthrough routes for Microshift", func() { + var ( + e2eTestNamespace = "e2e-ne-ocp60266-" + getRandomString() + buildPruningBaseDir = testdata.FixturePath("router") + testPodSvc = filepath.Join(buildPruningBaseDir, "web-server-deploy.yaml") + edgeRouteHost = "route-edge-" + e2eTestNamespace + ".apps.example.com" + passRouteHost = "route-pass-" + e2eTestNamespace + ".apps.example.com" + ) + + compat_otp.By("create a namespace for the scenario") + defer oc.DeleteSpecifiedNamespaceAsAdmin(e2eTestNamespace) + oc.CreateSpecifiedNamespaceAsAdmin(e2eTestNamespace) + + compat_otp.By("create a web-server-deploy pod and its services") + defer operateResourceFromFile(oc, "delete", e2eTestNamespace, testPodSvc) + createResourceFromFile(oc, e2eTestNamespace, testPodSvc) + ensurePodWithLabelReady(oc, e2eTestNamespace, "name=web-server-deploy") + podName := getPodListByLabel(oc, e2eTestNamespace, "name=web-server-deploy") + ingressPod := getOneRouterPodNameByIC(oc, "default") + + compat_otp.By("create a passthrough route") + createRoute(oc, e2eTestNamespace, "passthrough", "ms-pass", "service-secure", []string{"--hostname=" + passRouteHost}) + getRoutes(oc, e2eTestNamespace) + + compat_otp.By("check whether passthrough route details are present") + waitForOutputEquals(oc, e2eTestNamespace, "route/ms-pass", "{.spec.tls.termination}", "passthrough") + waitForOutputEquals(oc, e2eTestNamespace, "route/ms-pass", "{.status.ingress[0].host}", passRouteHost) + waitForOutputEquals(oc, e2eTestNamespace, "route/ms-pass", "{.status.ingress[0].conditions[0].type}", "Admitted") + + compat_otp.By("check the reachability of the host in test pod for passthrough route") + routerPodIP := getPodv4Address(oc, ingressPod, "openshift-ingress") + passRoute := passRouteHost + ":443:" + routerPodIP + curlCmd := []string{"-n", e2eTestNamespace, podName[0], "--", "curl", "https://" + passRouteHost + ":443", "-k", "-I", "--resolve", passRoute, "--connect-timeout", "10"} + repeatCmdOnClient(oc, curlCmd, "200", 30, 1) + + compat_otp.By("check the router pod and ensure the passthrough route is loaded in haproxy.config") + searchOutput := readRouterPodData(oc, ingressPod, "cat haproxy.config", "ms-pass") + o.Expect(searchOutput).To(o.ContainSubstring("backend be_tcp:" + e2eTestNamespace + ":ms-pass")) + + compat_otp.By("create a edge route") + createRoute(oc, e2eTestNamespace, "edge", "ms-edge", "service-unsecure", []string{"--hostname=" + edgeRouteHost}) + getRoutes(oc, e2eTestNamespace) + + compat_otp.By("check whether edge route details are present") + waitForOutputEquals(oc, e2eTestNamespace, "route/ms-edge", "{.spec.tls.termination}", "edge") + waitForOutputEquals(oc, e2eTestNamespace, "route/ms-edge", "{.status.ingress[0].host}", edgeRouteHost) + waitForOutputEquals(oc, e2eTestNamespace, "route/ms-edge", "{.status.ingress[0].conditions[0].type}", "Admitted") + + compat_otp.By("check the reachability of the host in test pod for edge route") + edgeRoute := edgeRouteHost + ":443:" + routerPodIP + curlCmd1 := []string{"-n", e2eTestNamespace, podName[0], "--", "curl", "https://" + edgeRouteHost + ":443", "-k", "-I", "--resolve", edgeRoute, "--connect-timeout", "10"} + repeatCmdOnClient(oc, curlCmd1, "200", 30, 1) + + compat_otp.By("check the router pod and ensure the edge route is loaded in haproxy.config") + searchOutput1 := readRouterPodData(oc, ingressPod, "cat haproxy.config", "ms-edge") + o.Expect(searchOutput1).To(o.ContainSubstring("backend be_edge_http:" + e2eTestNamespace + ":ms-edge")) + }) + + g.It("Author:mjoseph-MicroShiftOnly-Critical-60283-creation of http and re-encrypt routes for Microshift", func() { + var ( + e2eTestNamespace = "e2e-ne-ocp60283-" + getRandomString() + buildPruningBaseDir = testdata.FixturePath("router") + testPodSvc = filepath.Join(buildPruningBaseDir, "web-server-signed-deploy.yaml") + httpRouteHost = "route-http-" + e2eTestNamespace + ".apps.example.com" + reenRouteHost = "route-reen-" + e2eTestNamespace + ".apps.example.com" + ) + + compat_otp.By("create a namespace for the scenario") + defer oc.DeleteSpecifiedNamespaceAsAdmin(e2eTestNamespace) + oc.CreateSpecifiedNamespaceAsAdmin(e2eTestNamespace) + + compat_otp.By("create a signed web-server-deploy pod and its services") + defer operateResourceFromFile(oc, "delete", e2eTestNamespace, testPodSvc) + createResourceFromFile(oc, e2eTestNamespace, testPodSvc) + ensurePodWithLabelReady(oc, e2eTestNamespace, "name=web-server-deploy") + podName := getPodListByLabel(oc, e2eTestNamespace, "name=web-server-deploy") + ingressPod := getOneRouterPodNameByIC(oc, "default") + + compat_otp.By("create a http route") + _, err := oc.WithoutNamespace().Run("expose").Args("-n", e2eTestNamespace, "--name=ms-http", "service", "service-unsecure", "--hostname="+httpRouteHost).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + getRoutes(oc, e2eTestNamespace) + + compat_otp.By("check whether http route details are present") + waitForOutputEquals(oc, e2eTestNamespace, "route/ms-http", "{.spec.port.targetPort}", "http") + waitForOutputEquals(oc, e2eTestNamespace, "route/ms-http", "{.status.ingress[0].host}", httpRouteHost) + waitForOutputEquals(oc, e2eTestNamespace, "route/ms-http", "{.status.ingress[0].conditions[0].type}", "Admitted") + + compat_otp.By("check the reachability of the host in test pod for http route") + routerPodIP := getPodv4Address(oc, ingressPod, "openshift-ingress") + httpRoute := httpRouteHost + ":80:" + routerPodIP + curlCmd := []string{"-n", e2eTestNamespace, podName[0], "--", "curl", "http://" + httpRouteHost + ":80", "-k", "-I", "--resolve", httpRoute, "--connect-timeout", "10"} + repeatCmdOnClient(oc, curlCmd, "200", 30, 1) + + compat_otp.By("check the router pod and ensure the http route is loaded in haproxy.config") + searchOutput := readRouterPodData(oc, ingressPod, "cat haproxy.config", "ms-http") + o.Expect(searchOutput).To(o.ContainSubstring("backend be_http:" + e2eTestNamespace + ":ms-http")) + + compat_otp.By("create a reen route") + createRoute(oc, e2eTestNamespace, "reencrypt", "ms-reen", "service-secure", []string{"--hostname=" + reenRouteHost}) + getRoutes(oc, e2eTestNamespace) + + compat_otp.By("check whether reen route details are present") + waitForOutputEquals(oc, e2eTestNamespace, "route/ms-reen", "{.spec.tls.termination}", "reencrypt") + waitForOutputEquals(oc, e2eTestNamespace, "route/ms-reen", "{.status.ingress[0].host}", reenRouteHost) + waitForOutputEquals(oc, e2eTestNamespace, "route/ms-reen", "{.status.ingress[0].conditions[0].type}", "Admitted") + + compat_otp.By("check the reachability of the host in test pod reen route") + reenRoute := reenRouteHost + ":443:" + routerPodIP + curlCmd1 := []string{"-n", e2eTestNamespace, podName[0], "--", "curl", "https://" + reenRouteHost + ":443", "-k", "-I", "--resolve", reenRoute, "--connect-timeout", "10"} + repeatCmdOnClient(oc, curlCmd1, "200", 30, 1) + + compat_otp.By("check the router pod and ensure the reen route is loaded in haproxy.config") + searchOutput1 := readRouterPodData(oc, ingressPod, "cat haproxy.config", "ms-reen") + o.Expect(searchOutput1).To(o.ContainSubstring("backend be_secure:" + e2eTestNamespace + ":ms-reen")) + }) + + g.It("Author:shudili-MicroShiftOnly-High-72802-make router namespace ownership check configurable for the default microshift configuration", func() { + var ( + buildPruningBaseDir = testdata.FixturePath("router") + testPodSvc = filepath.Join(buildPruningBaseDir, "web-server-deploy.yaml") + srvrcInfo = "web-server-deploy" + unSecSvcName = "service-unsecure" + secSvcName = "service-secure" + clientPod = filepath.Join(buildPruningBaseDir, "test-client-pod.yaml") + clientPodName = "hello-pod" + clientPodLabel = "app=hello-pod" + e2eTestNamespace1 = "e2e-ne-ocp72802-" + getRandomString() + e2eTestNamespace2 = "e2e-ne-ocp72802-" + getRandomString() + ) + + compat_otp.By("1. check the Env ROUTER_DISABLE_NAMESPACE_OWNERSHIP_CHECK of deployment/default-router, which should be true for the default configuration") + routerPodName := getOneRouterPodNameByIC(oc, "default") + defaultVal := readRouterPodEnv(oc, routerPodName, "ROUTER_DISABLE_NAMESPACE_OWNERSHIP_CHECK") + o.Expect(defaultVal).To(o.ContainSubstring("true")) + + compat_otp.By("2. prepare two namespaces for the following testing") + defer oc.DeleteSpecifiedNamespaceAsAdmin(e2eTestNamespace1) + oc.CreateSpecifiedNamespaceAsAdmin(e2eTestNamespace1) + defer oc.DeleteSpecifiedNamespaceAsAdmin(e2eTestNamespace2) + oc.CreateSpecifiedNamespaceAsAdmin(e2eTestNamespace2) + path1 := "/path" + path2 := "/test" + httpRoutehost := unSecSvcName + "-" + "ocp72802." + "apps.example.com" + edgeRoute := "route-edge" + "-" + "ocp72802." + "apps.example.com" + reenRoute := "route-reen" + "-" + "ocp72802." + "apps.example.com" + + compat_otp.By("3. create a client pod, a server pod and two services in one ns") + createResourceFromFile(oc, e2eTestNamespace1, clientPod) + ensurePodWithLabelReady(oc, e2eTestNamespace1, clientPodLabel) + + createResourceFromFile(oc, e2eTestNamespace1, testPodSvc) + ensurePodWithLabelReady(oc, e2eTestNamespace1, "name="+srvrcInfo) + + compat_otp.By("4. create a server pod and two services in the other ns") + createResourceFromFile(oc, e2eTestNamespace2, clientPod) + ensurePodWithLabelReady(oc, e2eTestNamespace2, clientPodLabel) + + createResourceFromFile(oc, e2eTestNamespace2, testPodSvc) + ensurePodWithLabelReady(oc, e2eTestNamespace2, "name="+srvrcInfo) + + compat_otp.By("5. expose an insecure/edge/REEN type routes with path " + path1 + " in the first ns") + err := oc.Run("expose").Args("service", unSecSvcName, "--hostname="+httpRoutehost, "--path="+path1, "-n", e2eTestNamespace1).Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + waitForOutputEquals(oc, e2eTestNamespace1, "route", "{.items[0].metadata.name}", unSecSvcName) + + _, err = oc.WithoutNamespace().Run("create").Args("route", "edge", "route-edge", "--service="+unSecSvcName, "--hostname="+edgeRoute, "--path="+path1, "-n", e2eTestNamespace1).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + output, err := oc.WithoutNamespace().Run("get").Args("route", "-n", e2eTestNamespace1).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(output).To(o.ContainSubstring("route-edge")) + + _, err = oc.WithoutNamespace().Run("create").Args("route", "reencrypt", "route-reen", "--service="+secSvcName, "--hostname="+reenRoute, "--path="+path1, "-n", e2eTestNamespace1).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + output, err = oc.WithoutNamespace().Run("get").Args("route", "-n", e2eTestNamespace1).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(output).To(o.ContainSubstring("route-reen")) + + compat_otp.By("6. expose an insecure/edge/REEN type routes with path " + path2 + " in the second ns") + err = oc.Run("expose").Args("service", unSecSvcName, "--hostname="+httpRoutehost, "--path="+path2, "-n", e2eTestNamespace2).Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + waitForOutputEquals(oc, e2eTestNamespace2, "route", "{.items[0].metadata.name}", unSecSvcName) + + _, err = oc.WithoutNamespace().Run("create").Args("route", "edge", "route-edge", "--service="+unSecSvcName, "--hostname="+edgeRoute, "--path="+path2, "-n", e2eTestNamespace2).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + output, err = oc.WithoutNamespace().Run("get").Args("route", "-n", e2eTestNamespace2).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(output).To(o.ContainSubstring("route-edge")) + + _, err = oc.WithoutNamespace().Run("create").Args("route", "reencrypt", "route-reen", "--service="+secSvcName, "--hostname="+reenRoute, "--path="+path2, "-n", e2eTestNamespace2).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + output, err = oc.WithoutNamespace().Run("get").Args("route", "-n", e2eTestNamespace2).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(output).To(o.ContainSubstring("route-reen")) + + compat_otp.By("7.1. check the http route in the first ns should be adimitted") + jpath := "{.status.ingress[0].conditions[0].status}" + adtInfo := getByJsonPath(oc, e2eTestNamespace1, "route/"+unSecSvcName, jpath) + o.Expect(adtInfo).To(o.Equal("True")) + + compat_otp.By("7.2. check the edge route in the first ns should be adimitted") + adtInfo = getByJsonPath(oc, e2eTestNamespace1, "route/route-edge", jpath) + o.Expect(adtInfo).To(o.Equal("True")) + + compat_otp.By("7.3. check the REEN route in the first ns should be adimitted") + adtInfo = getByJsonPath(oc, e2eTestNamespace1, "route/route-reen", jpath) + o.Expect(adtInfo).To(o.Equal("True")) + + compat_otp.By("8.1. check the http route in the second ns with the same hostname but with different path should be adimitted too") + adtInfo = getByJsonPath(oc, e2eTestNamespace2, "route/"+unSecSvcName, jpath) + o.Expect(adtInfo).To(o.Equal("True")) + + compat_otp.By("8.2. check the edge route in the second ns with the same hostname but with different path should be adimitted too") + adtInfo = getByJsonPath(oc, e2eTestNamespace2, "route/route-edge", jpath) + o.Expect(adtInfo).To(o.Equal("True")) + + compat_otp.By("8.3. check the REEN route in the second ns with the same hostname but with different path should be adimitted too") + adtInfo = getByJsonPath(oc, e2eTestNamespace2, "route/route-reen", jpath) + o.Expect(adtInfo).To(o.Equal("True")) + + compat_otp.By("9. curl the first HTTP route and check the result") + srvPodName := getPodListByLabel(oc, e2eTestNamespace1, "name=web-server-deploy") + routerPodIP := getPodv4Address(oc, routerPodName, "openshift-ingress") + toDst := httpRoutehost + ":80:" + routerPodIP + cmdOnPod := []string{"-n", e2eTestNamespace1, clientPodName, "--", "curl", "http://" + httpRoutehost + "/path/index.html", "--resolve", toDst, "--connect-timeout", "10"} + result, _ := repeatCmdOnClient(oc, cmdOnPod, "http-8080", 30, 1) + o.Expect(result).To(o.ContainSubstring("http-8080")) + output, err = oc.Run("exec").Args(cmdOnPod...).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(output).To(o.ContainSubstring("ocp-test " + srvPodName[0] + " http-8080")) + + compat_otp.By("10. curl the second HTTP route and check the result") + srvPodName = getPodListByLabel(oc, e2eTestNamespace2, "name=web-server-deploy") + cmdOnPod = []string{"-n", e2eTestNamespace1, clientPodName, "--", "curl", "http://" + httpRoutehost + "/test/index.html", "--resolve", toDst, "--connect-timeout", "10"} + result, _ = repeatCmdOnClient(oc, cmdOnPod, "http-8080", 60, 1) + o.Expect(result).To(o.ContainSubstring("Hello-OpenShift-Path-Test " + srvPodName[0] + " http-8080")) + }) + + g.It("Author:shudili-MicroShiftOnly-High-73152-Expose router as load balancer service type", func() { + var ( + buildPruningBaseDir = testdata.FixturePath("router") + testPodSvc = filepath.Join(buildPruningBaseDir, "web-server-signed-deploy.yaml") + clientPod = filepath.Join(buildPruningBaseDir, "test-client-pod.yaml") + unsecsvcName = "service-unsecure" + secsvcName = "service-secure" + clientPodName = "hello-pod" + clientPodLabel = "app=hello-pod" + e2eTestNamespace = "e2e-ne-ocp73152-" + getRandomString() + ) + + compat_otp.By("Check the router-default service is a load balancer and has a load balancer ip") + svcType := getByJsonPath(oc, "openshift-ingress", "service/router-default", "{.spec.type}") + o.Expect(svcType).To(o.ContainSubstring("LoadBalancer")) + lbIPs := getByJsonPath(oc, "openshift-ingress", "service/router-default", "{.status.loadBalancer.ingress[0].ip}") + o.Expect(len(lbIPs) > 4).To(o.BeTrue()) + + compat_otp.By("Deploy a project with a client pod, a backend pod and its services resources") + defer oc.DeleteSpecifiedNamespaceAsAdmin(e2eTestNamespace) + oc.CreateSpecifiedNamespaceAsAdmin(e2eTestNamespace) + createResourceFromFile(oc, e2eTestNamespace, clientPod) + ensurePodWithLabelReady(oc, e2eTestNamespace, clientPodLabel) + createResourceFromFile(oc, e2eTestNamespace, testPodSvc) + ensurePodWithLabelReady(oc, e2eTestNamespace, "name=web-server-deploy") + + compat_otp.By("Create a HTTP/Edge/Passthrough/REEN route") + httpRouteHost := unsecsvcName + "-" + "ocp73152." + "apps.example.com" + edgeRouteHost := "route-edge" + "-" + "ocp73152." + "apps.example.com" + passThRouteHost := "route-passth" + "-" + "ocp73152." + "apps.example.com" + reenRouteHost := "route-reen" + "-" + "ocp73152." + "apps.example.com" + lbIP := strings.Split(lbIPs, " ")[0] + httpRouteDst := httpRouteHost + ":80:" + lbIP + edgeRouteDst := edgeRouteHost + ":443:" + lbIP + passThRouteDst := passThRouteHost + ":443:" + lbIP + reenRouteDst := reenRouteHost + ":443:" + lbIP + createRoute(oc, e2eTestNamespace, "http", "route-http", unsecsvcName, []string{"--hostname=" + httpRouteHost}) + createRoute(oc, e2eTestNamespace, "edge", "route-edge", unsecsvcName, []string{"--hostname=" + edgeRouteHost}) + createRoute(oc, e2eTestNamespace, "passthrough", "route-passth", secsvcName, []string{"--hostname=" + passThRouteHost}) + createRoute(oc, e2eTestNamespace, "reencrypt", "route-reen", secsvcName, []string{"--hostname=" + reenRouteHost}) + ensureRouteIsAdmittedByIngressController(oc, e2eTestNamespace, "route-reen", "default") + output := getByJsonPath(oc, e2eTestNamespace, "route", "{.items[*].metadata.name}") + o.Expect(output).Should(o.And( + o.ContainSubstring("route-http"), + o.ContainSubstring("route-edge"), + o.ContainSubstring("route-passth"), + o.ContainSubstring("route-reen"))) + + compat_otp.By("Curl the HTTP route") + routeReq := []string{"-n", e2eTestNamespace, clientPodName, "--", "curl", "http://" + httpRouteHost, "-I", "--resolve", httpRouteDst, "--connect-timeout", "10"} + repeatCmdOnClient(oc, routeReq, "200", 60, 1) + + compat_otp.By("Curl the Edge route") + routeReq = []string{"-n", e2eTestNamespace, clientPodName, "--", "curl", "https://" + edgeRouteHost, "-k", "-I", "--resolve", edgeRouteDst, "--connect-timeout", "10"} + repeatCmdOnClient(oc, routeReq, "200", 60, 1) + + compat_otp.By("Curl the Passthrough route") + routeReq = []string{"-n", e2eTestNamespace, clientPodName, "--", "curl", "https://" + passThRouteHost, "-k", "-I", "--resolve", passThRouteDst, "--connect-timeout", "10"} + repeatCmdOnClient(oc, routeReq, "200", 60, 1) + + compat_otp.By("Curl the REEN route") + routeReq = []string{"-n", e2eTestNamespace, clientPodName, "--", "curl", "https://" + reenRouteHost, "-k", "-I", "--resolve", reenRouteDst, "--connect-timeout", "10"} + repeatCmdOnClient(oc, routeReq, "200", 60, 1) + }) + + g.It("Author:shudili-MicroShiftOnly-High-73202-Add configurable listening IP addresses and listening ports", func() { + var ( + buildPruningBaseDir = testdata.FixturePath("router") + testPodSvc = filepath.Join(buildPruningBaseDir, "web-server-signed-deploy.yaml") + clientPod = filepath.Join(buildPruningBaseDir, "test-client-pod.yaml") + unsecsvcName = "service-unsecure" + secsvcName = "service-secure" + clientPodName = "hello-pod" + clientPodLabel = "app=hello-pod" + findIpCmd = "ip address | grep \"inet \"" + hostIPList []string + e2eTestNamespace = "e2e-ne-ocp73202-" + getRandomString() + ) + + compat_otp.By("create a namespace for testing, then debug node and get the valid host ips") + defer oc.DeleteSpecifiedNamespaceAsAdmin(e2eTestNamespace) + oc.CreateSpecifiedNamespaceAsAdmin(e2eTestNamespace) + compat_otp.SetNamespacePrivileged(oc, e2eTestNamespace) + nodeName := getByJsonPath(oc, "default", "nodes", "{.items[0].metadata.name}") + podCIDR := getByJsonPath(oc, "default", "nodes/"+nodeName, "{.spec.podCIDR}") + if strings.Contains(podCIDR, `:`) { + findIpCmd = "ip address | grep \"inet6 \" | grep global" + hostAddresses, err := oc.AsAdmin().WithoutNamespace().Run("debug").Args("-n", e2eTestNamespace, "--quiet=true", "node/"+nodeName, "--", "chroot", "/host", "bash", "-c", findIpCmd).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + hostIPList = getValidIPv6Addresses(hostAddresses) + e2e.Logf("hostIPList is: /n%v", hostIPList) + } else { + hostAddresses, err := oc.AsAdmin().WithoutNamespace().Run("debug").Args("-n", e2eTestNamespace, "--quiet=true", "node/"+nodeName, "--", "chroot", "/host", "bash", "-c", findIpCmd).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + _, hostIPList = getValidInterfacesAndIPs(hostAddresses) + e2e.Logf("hostIPList is: /n%v", hostIPList) + } + + compat_otp.By("check the default load balancer ips of the router-default service, which should be all node's valid host ips") + lbIPs := getByJsonPath(oc, "openshift-ingress", "service/router-default", "{.status.loadBalancer.ingress[*].ip}") + lbIPs = getSortedString(lbIPs) + hostIPs := getSortedString(hostIPList) + o.Expect(lbIPs).To(o.Equal(hostIPs)) + + compat_otp.By("check the default load balancer ports of the router-default service, which should be 80 for the unsecure http port and 443 for the seccure https port") + httpPort := getByJsonPath(oc, "openshift-ingress", "service/router-default", `{.spec.ports[?(@.name=="http")].port}`) + o.Expect(httpPort).To(o.Equal("80")) + httpsPort := getByJsonPath(oc, "openshift-ingress", "service/router-default", `{.spec.ports[?(@.name=="https")].port}`) + o.Expect(httpsPort).To(o.Equal("443")) + + compat_otp.By("Deploy a backend pod and its services resources in the created ns") + createResourceFromFile(oc, e2eTestNamespace, clientPod) + ensurePodWithLabelReady(oc, e2eTestNamespace, clientPodLabel) + createResourceFromFile(oc, e2eTestNamespace, testPodSvc) + ensurePodWithLabelReady(oc, e2eTestNamespace, "name=web-server-deploy") + + compat_otp.By("Create a HTTP/Edge/Passthrough/REEN route") + httpRouteHost := unsecsvcName + "-" + "ocp73202." + "apps.example.com" + edgeRouteHost := "route-edge" + "-" + "ocp73202." + "apps.example.com" + passThRouteHost := "route-passth" + "-" + "ocp73202." + "apps.example.com" + reenRouteHost := "route-reen" + "-" + "ocp73202." + "apps.example.com" + createRoute(oc, e2eTestNamespace, "http", "route-http", unsecsvcName, []string{"--hostname=" + httpRouteHost}) + createRoute(oc, e2eTestNamespace, "edge", "route-edge", unsecsvcName, []string{"--hostname=" + edgeRouteHost}) + createRoute(oc, e2eTestNamespace, "passthrough", "route-passth", secsvcName, []string{"--hostname=" + passThRouteHost}) + createRoute(oc, e2eTestNamespace, "reencrypt", "route-reen", secsvcName, []string{"--hostname=" + reenRouteHost}) + ensureRouteIsAdmittedByIngressController(oc, e2eTestNamespace, "route-reen", "default") + output := getByJsonPath(oc, e2eTestNamespace, "route", "{.items[*].metadata.name}") + o.Expect(output).Should(o.And( + o.ContainSubstring("route-http"), + o.ContainSubstring("route-edge"), + o.ContainSubstring("route-passth"), + o.ContainSubstring("route-reen"))) + + compat_otp.By("Curl the routes with destination to each load balancer ip") + for _, lbIP := range strings.Split(lbIPs, " ") { + // config firewall for ipv6 load balancer + configFwForLB(oc, e2eTestNamespace, nodeName, lbIP) + httpRouteDst := httpRouteHost + ":80:" + lbIP + edgeRouteDst := edgeRouteHost + ":443:" + lbIP + passThRouteDst := passThRouteHost + ":443:" + lbIP + reenRouteDst := reenRouteHost + ":443:" + lbIP + + compat_otp.By("Curl the http route with destination " + lbIP) + routeReq := []string{"-n", e2eTestNamespace, clientPodName, "--", "curl", "http://" + httpRouteHost, "-I", "--resolve", httpRouteDst, "--connect-timeout", "10"} + repeatCmdOnClient(oc, routeReq, "200", 150, 1) + + compat_otp.By("Curl the Edge route with destination " + lbIP) + routeReq = []string{"-n", e2eTestNamespace, clientPodName, "--", "curl", "https://" + edgeRouteHost, "-k", "-I", "--resolve", edgeRouteDst, "--connect-timeout", "10"} + repeatCmdOnClient(oc, routeReq, "200", 60, 1) + + compat_otp.By("Curl the Pass-through route with destination " + lbIP) + routeReq = []string{"-n", e2eTestNamespace, clientPodName, "--", "curl", "https://" + passThRouteHost, "-k", "-I", "--resolve", passThRouteDst, "--connect-timeout", "10"} + repeatCmdOnClient(oc, routeReq, "200", 60, 1) + + compat_otp.By("Curl the REEN route with destination " + lbIP) + routeReq = []string{"-n", e2eTestNamespace, clientPodName, "--", "curl", "https://" + reenRouteHost, "-k", "-I", "--resolve", reenRouteDst, "--connect-timeout", "10"} + repeatCmdOnClient(oc, routeReq, "200", 60, 1) + } + }) + + g.It("Author:shudili-MicroShiftOnly-NonPreRelease-Longduration-High-73203-configuring listening IP addresses and listening Ports [Disruptive]", func() { + var ( + buildPruningBaseDir = testdata.FixturePath("router") + testPodSvc = filepath.Join(buildPruningBaseDir, "web-server-signed-deploy.yaml") + clientPod = filepath.Join(buildPruningBaseDir, "test-client-pod.yaml") + unsecsvcName = "service-unsecure" + secsvcName = "service-secure" + clientPodName = "hello-pod" + clientPodLabel = "app=hello-pod" + specifiedAddress string + randHostIP string + caseID = "73203" + e2eTestNamespace = "e2e-ne-" + caseID + "-" + getRandomString() + ) + + compat_otp.By(`1. Create a namespace for testing, then debug node and get all valid host interfaces and invalid host ips`) + defer oc.DeleteSpecifiedNamespaceAsAdmin(e2eTestNamespace) + oc.CreateSpecifiedNamespaceAsAdmin(e2eTestNamespace) + compat_otp.SetNamespacePrivileged(oc, e2eTestNamespace) + nodeName := getByJsonPath(oc, "default", "nodes", "{.items[0].metadata.name}") + podCIDR := getByJsonPath(oc, "default", "nodes/"+nodeName, "{.spec.podCIDR}") + if !strings.Contains(podCIDR, `:`) { + hostAddresses, err := oc.AsAdmin().WithoutNamespace().Run("debug").Args("--quiet=true", "node/"+nodeName, "--", "chroot", "/host", "bash", "-c", "ip address | grep \"inet \"").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + intfaceList, hostIPList := getValidInterfacesAndIPs(hostAddresses) + seed := rand.New(rand.NewSource(time.Now().UnixNano())) + index := seed.Intn(len(intfaceList)) + specifiedAddress = intfaceList[index] + randHostIP = hostIPList[index] + } else { + hostAddresses, err := oc.AsAdmin().WithoutNamespace().Run("debug").Args("--quiet=true", "node/"+nodeName, "--", "chroot", "/host", "bash", "-c", "ip address | grep \"inet6 \" | grep global").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + hostIPList := getValidIPv6Addresses(hostAddresses) + seed := rand.New(rand.NewSource(time.Now().UnixNano())) + index := seed.Intn(len(hostIPList)) + randHostIP = hostIPList[index] + specifiedAddress = randHostIP + } + + compat_otp.By("2. Debug node to backup the config.yaml, and restore it before the test finishes running") + defer restoreConfigYaml(oc, e2eTestNamespace, caseID, nodeName) + backupConfigYaml(oc, e2eTestNamespace, caseID, nodeName) + + compat_otp.By(`3. Debug node and configure ingress with the desired listening IP addresses and listening Ports`) + ingressConfig := fmt.Sprintf(` +ingress: + listenAddress: + - %s + ports: + http: 10080 + https: 10443`, specifiedAddress) + + appendIngressToConfigYaml(oc, e2eTestNamespace, caseID, nodeName, ingressConfig) + + compat_otp.By("4. Wait the check router-default service is updated and its load balancer ip is as same as configured in default.yaml") + regExp := "^" + randHostIP + "$" + searchOutput := waitForOutputMatchRegexp(oc, "openshift-ingress", "service/router-default", "{.status.loadBalancer.ingress[*].ip}", regExp, 240*time.Second) + o.Expect(searchOutput).To(o.Equal(randHostIP)) + + compat_otp.By("5. Check service router-default's http port is changed to 10080 and its https port is changed to 10443") + jpath := `{.spec.ports[?(@.name=="http")].port}` + httpPort := getByJsonPath(oc, "openshift-ingress", "svc/router-default", jpath) + o.Expect(httpPort).To(o.Equal("10080")) + jpath = `{.spec.ports[?(@.name=="https")].port}` + httpsPort := getByJsonPath(oc, "openshift-ingress", "svc/router-default", jpath) + o.Expect(httpsPort).To(o.Equal("10443")) + + compat_otp.By("6. Deploy a client pod, a backend pod and its services resources") + createResourceFromFile(oc, e2eTestNamespace, clientPod) + ensurePodWithLabelReady(oc, e2eTestNamespace, clientPodLabel) + createResourceFromFile(oc, e2eTestNamespace, testPodSvc) + ensurePodWithLabelReady(oc, e2eTestNamespace, "name=web-server-deploy") + + compat_otp.By("7. Create a HTTP/Edge/Passthrough/REEN route") + httpRouteHost := unsecsvcName + "-" + "ocp73203." + "apps.example.com" + edgeRouteHost := "route-edge" + "-" + "ocp73203." + "apps.example.com" + passThRouteHost := "route-passth" + "-" + "ocp73203." + "apps.example.com" + reenRouteHost := "route-reen" + "-" + "ocp73203." + "apps.example.com" + createRoute(oc, e2eTestNamespace, "http", "route-http", unsecsvcName, []string{"--hostname=" + httpRouteHost}) + createRoute(oc, e2eTestNamespace, "edge", "route-edge", unsecsvcName, []string{"--hostname=" + edgeRouteHost}) + createRoute(oc, e2eTestNamespace, "passthrough", "route-passth", secsvcName, []string{"--hostname=" + passThRouteHost}) + createRoute(oc, e2eTestNamespace, "reencrypt", "route-reen", secsvcName, []string{"--hostname=" + reenRouteHost}) + ensureRouteIsAdmittedByIngressController(oc, e2eTestNamespace, "route-reen", "default") + output := getByJsonPath(oc, e2eTestNamespace, "route", "{.items[*].metadata.name}") + o.Expect(output).Should(o.And( + o.ContainSubstring("route-http"), + o.ContainSubstring("route-edge"), + o.ContainSubstring("route-passth"), + o.ContainSubstring("route-reen"))) + + compat_otp.By("8. Curl the routes with destination to the the custom load balancer ip and http/https ports") + httpRouteDst := httpRouteHost + ":10080:" + randHostIP + edgeRouteDst := edgeRouteHost + ":10443:" + randHostIP + passThRouteDst := passThRouteHost + ":10443:" + randHostIP + reenRouteDst := reenRouteHost + ":10443:" + randHostIP + + compat_otp.By("9. Curl the http route") + // config firewall for ipv6 load balancer + configFwForLB(oc, e2eTestNamespace, nodeName, randHostIP) + + routeReq := []string{"-n", e2eTestNamespace, clientPodName, "--", "curl", "http://" + httpRouteHost + ":10080", "-I", "--resolve", httpRouteDst, "--connect-timeout", "10"} + repeatCmdOnClient(oc, routeReq, "200", 150, 1) + + compat_otp.By("10. Curl the Edge route") + routeReq = []string{"-n", e2eTestNamespace, clientPodName, "--", "curl", "https://" + edgeRouteHost + ":10443", "-k", "-I", "--resolve", edgeRouteDst, "--connect-timeout", "10"} + repeatCmdOnClient(oc, routeReq, "200", 60, 1) + + compat_otp.By("11. Curl the Passthrough route") + routeReq = []string{"-n", e2eTestNamespace, clientPodName, "--", "curl", "https://" + passThRouteHost + ":10443", "-k", "-I", "--resolve", passThRouteDst, "--connect-timeout", "10"} + repeatCmdOnClient(oc, routeReq, "200", 60, 1) + + compat_otp.By("12. Curl the REEN route") + routeReq = []string{"-n", e2eTestNamespace, clientPodName, "--", "curl", "https://" + reenRouteHost + ":10443", "-k", "-I", "--resolve", reenRouteDst, "--connect-timeout", "10"} + repeatCmdOnClient(oc, routeReq, "200", 60, 1) + }) + + g.It("Author:shudili-MicroShiftOnly-NonPreRelease-Longduration-High-73209-Add enable/disable option for default router [Disruptive]", func() { + var ( + caseID = "73209" + e2eTestNamespace = "e2e-ne-" + caseID + "-" + getRandomString() + ) + + compat_otp.By("1. Create a namespace for testing, then debug node and get the valid host ips") + defer oc.DeleteSpecifiedNamespaceAsAdmin(e2eTestNamespace) + oc.CreateSpecifiedNamespaceAsAdmin(e2eTestNamespace) + compat_otp.SetNamespacePrivileged(oc, e2eTestNamespace) + + nodeName := getByJsonPath(oc, "default", "nodes", "{.items[0].metadata.name}") + podCIDR := getByJsonPath(oc, "default", "nodes/"+nodeName, "{.spec.podCIDR}") + hostIPs := "" + if !strings.Contains(podCIDR, `:`) { + hostAddresses, err := oc.AsAdmin().WithoutNamespace().Run("debug").Args("-n", e2eTestNamespace, "--quiet=true", "node/"+nodeName, "--", "chroot", "/host", "bash", "-c", "ip address | grep \"inet \"").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + _, hostIPList := getValidInterfacesAndIPs(hostAddresses) + hostIPs = getSortedString(hostIPList) + } else { + hostAddresses, err := oc.AsAdmin().WithoutNamespace().Run("debug").Args("--quiet=true", "node/"+nodeName, "--", "chroot", "/host", "bash", "-c", "ip address | grep \"inet6 \" | grep global").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + hostIPList := getValidIPv6Addresses(hostAddresses) + hostIPs = getSortedString(hostIPList) + } + + compat_otp.By("2. Debug node to backup the config.yaml, and restore it before the test finishes running") + defer restoreConfigYaml(oc, e2eTestNamespace, caseID, nodeName) + backupConfigYaml(oc, e2eTestNamespace, caseID, nodeName) + + compat_otp.By("3. Debug node to disable the default router by setting ingress status to Removed") + ingressConfig := fmt.Sprintf(` +ingress: + status: Removed`) + + appendIngressToConfigYaml(oc, e2eTestNamespace, caseID, nodeName, ingressConfig) + + compat_otp.By("4. Check the openshift-ingress namespace will be deleted") + err := waitForResourceToDisappear(oc, "default", "ns/"+"openshift-ingress") + compat_otp.AssertWaitPollNoErr(err, fmt.Sprintf("resource %v does not disapper", "namespace openshift-ingress")) + + compat_otp.By(`5. Debug node to enable the default router by setting ingress status to Managed`) + ingressConfig = fmt.Sprintf(` +ingress: + status: Managed`) + + appendIngressToConfigYaml(oc, e2eTestNamespace, caseID, nodeName, ingressConfig) + + compat_otp.By("6. Check router-default load balancer is enabled") + waitForOutputEquals(oc, "openshift-ingress", "service/router-default", `{.spec.ports[?(@.name=="http")].port}`, "80", 240*time.Second) + lbIPs := getByJsonPath(oc, "openshift-ingress", "service/router-default", "{.status.loadBalancer.ingress[*].ip}") + lbIPs = getSortedString(lbIPs) + o.Expect(lbIPs).To(o.Equal(hostIPs)) + httpsPort := getByJsonPath(oc, "openshift-ingress", "svc/router-default", `{.spec.ports[?(@.name=="https")].port}`) + o.Expect(httpsPort).To(o.Equal("443")) + }) + + g.It("Author:shudili-MicroShiftOnly-NonPreRelease-Longduration-Medium-73621-Disable/Enable namespace ownership support for router [Disruptive]", func() { + var ( + buildPruningBaseDir = testdata.FixturePath("router") + testPodSvc = filepath.Join(buildPruningBaseDir, "web-server-deploy.yaml") + srvrcInfo = "web-server-deploy" + unSecSvcName = "service-unsecure" + caseID = "73621" + e2eTestNamespace1 = "e2e-ne-" + caseID + "-" + getRandomString() + e2eTestNamespace2 = "e2e-ne-" + caseID + "-" + getRandomString() + baseDomain = "apps.example.com" + httpRouteHost = unSecSvcName + "-" + caseID + "." + baseDomain + path1 = "/path" + path2 = "/path/second" + ) + + compat_otp.By("1. Prepare two namespaces for the following testing") + defer oc.DeleteSpecifiedNamespaceAsAdmin(e2eTestNamespace1) + oc.CreateSpecifiedNamespaceAsAdmin(e2eTestNamespace1) + compat_otp.SetNamespacePrivileged(oc, e2eTestNamespace1) + defer oc.DeleteSpecifiedNamespaceAsAdmin(e2eTestNamespace2) + oc.CreateSpecifiedNamespaceAsAdmin(e2eTestNamespace2) + + compat_otp.By("2: Debug node to backup the config.yaml, and restore it before the test finishes running") + nodeName := getByJsonPath(oc, "default", "nodes", "{.items[0].metadata.name}") + defer restoreConfigYaml(oc, e2eTestNamespace1, caseID, nodeName) + backupConfigYaml(oc, e2eTestNamespace1, caseID, nodeName) + actualGenerationInt := getRouterDeploymentGeneration(oc, "router-default") + + compat_otp.By(`3. Debug node to disable namespace ownership support by setting namespaceOwnership to Strict in the config.yaml file"`) + ingressConfig := fmt.Sprintf(` +ingress: + routeAdmissionPolicy: + namespaceOwnership: Strict`) + + appendIngressToConfigYaml(oc, e2eTestNamespace1, caseID, nodeName, ingressConfig) + ensureRouterDeployGenerationIs(oc, "default", strconv.Itoa(actualGenerationInt+1)) + + compat_otp.By("4. Check the Env ROUTER_DISABLE_NAMESPACE_OWNERSHIP_CHECK of deployment/default-router, which should be false") + routerPodName := getOneNewRouterPodFromRollingUpdate(oc, "default") + ownershipVal := readRouterPodEnv(oc, routerPodName, "ROUTER_DISABLE_NAMESPACE_OWNERSHIP_CHECK") + o.Expect(ownershipVal).To(o.ContainSubstring("false")) + + compat_otp.By("5. Create a server pod and the services in one ns") + createResourceFromFile(oc, e2eTestNamespace1, testPodSvc) + ensurePodWithLabelReady(oc, e2eTestNamespace1, "name="+srvrcInfo) + + compat_otp.By("6. Create a route with path " + path1 + " in the first ns, which should be admitted") + extraParas := []string{"--hostname=" + httpRouteHost, "--path=" + path1} + jpath := "{.status.ingress[0].conditions[0].status}" + createRoute(oc, e2eTestNamespace1, "http", "route-http", unSecSvcName, extraParas) + waitForOutputEquals(oc, e2eTestNamespace1, "route", "{.items[0].metadata.name}", "route-http") + ensureRouteIsAdmittedByIngressController(oc, e2eTestNamespace1, "route-http", "default") + + compat_otp.By("7. Create a server pod and the services in the other ns") + createResourceFromFile(oc, e2eTestNamespace2, testPodSvc) + ensurePodWithLabelReady(oc, e2eTestNamespace2, "name="+srvrcInfo) + + compat_otp.By("8. Create a route with path " + path2 + " in the second ns, which should NOT be admitted") + extraParas = []string{"--hostname=" + httpRouteHost, "--path=" + path2} + createRoute(oc, e2eTestNamespace2, "http", "route-http", unSecSvcName, extraParas) + waitForOutputEquals(oc, e2eTestNamespace2, "route", "{.items[0].metadata.name}", "route-http") + ensureRouteIsNotAdmittedByIngressController(oc, e2eTestNamespace2, "route-http", "default") + + compat_otp.By("9. Check the two routes with same hostname but with different path for the second time, the first one is adimitted, while the second one isn't") + adtInfo := getByJsonPath(oc, e2eTestNamespace1, "route/route-http", jpath) + o.Expect(adtInfo).To(o.ContainSubstring("True")) + adtInfo = getByJsonPath(oc, e2eTestNamespace2, "route/route-http", jpath) + o.Expect(adtInfo).To(o.ContainSubstring("False")) + + compat_otp.By("10. Confirm the second route is shown as HostAlreadyClaimed") + jpath2 := `{.status.ingress[?(@.routerName=="default")].conditions[*].reason}` + searchOutput := getByJsonPath(oc, e2eTestNamespace2, "route/route-http", jpath2) + o.Expect(searchOutput).To(o.ContainSubstring("HostAlreadyClaimed")) + + compat_otp.By("11. Debug node to enable namespace ownership support by setting namespaceOwnership to InterNamespaceAllowed in the config.yaml file") + ingressConfig = fmt.Sprintf(` +ingress: + routeAdmissionPolicy: + namespaceOwnership: InterNamespaceAllowed`) + + appendIngressToConfigYaml(oc, e2eTestNamespace1, caseID, nodeName, ingressConfig) + ensureRouterDeployGenerationIs(oc, "default", strconv.Itoa(actualGenerationInt+2)) + + compat_otp.By("12. Check the two route with same hostname but with different path, both of them should be adimitted") + ensureRouteIsAdmittedByIngressController(oc, e2eTestNamespace1, "route-http", "default") + ensureRouteIsAdmittedByIngressController(oc, e2eTestNamespace2, "route-http", "default") + + compat_otp.By("13. Confirm no route is shown as HostAlreadyClaimed") + searchOutput1 := getByJsonPath(oc, e2eTestNamespace1, "route/route-http", jpath2) + searchOutput2 := getByJsonPath(oc, e2eTestNamespace2, "route/route-http", jpath2) + o.Expect(strings.Count(searchOutput1+searchOutput2, "HostAlreadyClaimed")).To(o.Equal(0)) + }) + + g.It("Author:shudili-MicroShiftOnly-High-77349-introduce ingress controller customization with microshift config.yaml [Disruptive]", func() { + + var ( + buildPruningBaseDir = testdata.FixturePath("router") + testPodSvc = filepath.Join(buildPruningBaseDir, "web-server-signed-deploy.yaml") + unsecsvcName = "service-unsecure" + caseID = "77349" + e2eTestNamespace = "e2e-ne-" + caseID + "-" + getRandomString() + httpRouteHost = unsecsvcName + "-" + caseID + ".apps.example.com" + + // prepare the data for test, for every slice, the first element is the env name, the second is the expected default env value in the deloyment, the third is the expected custom env value in the deloyment, + // the fourth is the expected default haproxy configuration, the last is the expected custom haproxy configuration + // https://issues.redhat.com/browse/OCPBUGS-45191 for routerBackendCheckInterval, the haproxy config should be "check inter 5000ms", marked it "skip for none" in the case + // for routerSetForwardedHeaders, set expected haproxy config with "skip for none" for the haproxy hasn't such an configuration + // for routerEnableCompression, routerCompressionMime and routerDontLogNull, set expected haproxy config with "skip for none" for the default values couldn't be seen in the haproxy.config + routerBufSize = []string{`ROUTER_BUF_SIZE`, `32768`, `65536`, `tune.bufsize 32768`, `tune.bufsize 65536`} + routerMaxRewriteSize = []string{`ROUTER_MAX_REWRITE_SIZE`, `8192`, `16384`, `tune.maxrewrite 8192`, `tune.maxrewrite 16384`} + routerBackendCheckInterval = []string{`ROUTER_BACKEND_CHECK_INTERVAL`, `5s`, `10s`, `skip for none`, `skip for none`} + routerDefaultClientTimeout = []string{`ROUTER_DEFAULT_CLIENT_TIMEOUT`, `30s`, `1m`, `timeout client 30s`, `timeout client 1m`} + routerClientFinTimeout = []string{`ROUTER_CLIENT_FIN_TIMEOUT`, `1s`, `2s`, `timeout client-fin 1s`, `timeout client-fin 2s`} + routerDefaultServerTimeout = []string{`ROUTER_DEFAULT_SERVER_TIMEOUT`, `30s`, `1m`, `timeout server 30s`, `timeout server 1m`} + routerDefaultServerFinTimeout = []string{`ROUTER_DEFAULT_SERVER_FIN_TIMEOUT`, `1s`, `2s`, `timeout server-fin 1s`, `timeout server-fin 2s`} + routerDefaultTunnelTimeout = []string{`ROUTER_DEFAULT_TUNNEL_TIMEOUT`, `1h`, `2h`, `timeout tunnel 1h`, `timeout tunnel 2h`} + routerInspectDelay = []string{`ROUTER_INSPECT_DELAY`, `5s`, `10s`, `tcp-request inspect-delay 5s`, `tcp-request inspect-delay 10s`} + routerThreads = []string{`ROUTER_THREADS`, `4`, `8`, `nbthread 4`, `nbthread 8`} + routerMaxConnections = []string{`ROUTER_MAX_CONNECTIONS`, `50000`, `100000`, `maxconn 50000`, `maxconn 100000`} + routerEnableCompression = []string{`ROUTER_ENABLE_COMPRESSION`, `false`, `true`, `skip for none`, `compression algo`} + routerCompressionMime = []string{`ROUTER_COMPRESSION_MIME`, ``, `image`, `skip for none`, `compression type image`} + routerDontLogNull = []string{`ROUTER_DONT_LOG_NULL`, `false`, `true`, `skip for none`, `option dontlognull`} + routerSetForwardedHeaders = []string{`ROUTER_SET_FORWARDED_HEADERS`, `Append`, `Replace`, `skip for none`, `skip for none`} + allParas = [][]string{routerBufSize, routerMaxRewriteSize, routerBackendCheckInterval, routerDefaultClientTimeout, routerClientFinTimeout, routerDefaultServerTimeout, routerDefaultServerFinTimeout, routerDefaultTunnelTimeout, routerInspectDelay, routerThreads, routerMaxConnections, routerEnableCompression, routerCompressionMime, routerDontLogNull, routerSetForwardedHeaders} + ) + + compat_otp.By("1.0 Deploy a project with a backend pod and its services resources, then create a route") + defer oc.DeleteSpecifiedNamespaceAsAdmin(e2eTestNamespace) + oc.CreateSpecifiedNamespaceAsAdmin(e2eTestNamespace) + compat_otp.SetNamespacePrivileged(oc, e2eTestNamespace) + createResourceFromFile(oc, e2eTestNamespace, testPodSvc) + ensurePodWithLabelReady(oc, e2eTestNamespace, "name=web-server-deploy") + createRoute(oc, e2eTestNamespace, "http", "route-http", unsecsvcName, []string{"--hostname=" + httpRouteHost}) + + compat_otp.By("2.0 Check the router-default deployment that all default ENVs of tested parameters are as expected") + for _, routerEntry := range allParas { + jsonPath := fmt.Sprintf(`{.spec.template.spec.containers[0].env[?(@.name=="%s")].value}`, routerEntry[0]) + envValue := getByJsonPath(oc, "openshift-ingress", "deployment/router-default", jsonPath) + if envValue != routerEntry[1] { + e2e.Logf("the retrieved default value of env: %s is not as expected: %s", envValue, routerEntry[1]) + } + o.Expect(envValue == routerEntry[1]).To(o.BeTrue()) + } + + compat_otp.By("3.0 Check the haproxy.config that all default vaules of tested parameters are set as expected") + routerpod := getOneRouterPodNameByIC(oc, "default") + for _, routerEntry := range allParas { + if routerEntry[3] != "skip for none" { + haCfg := ensureHaproxyBlockConfigContains(oc, routerpod, routerEntry[3], []string{routerEntry[3]}) + if !strings.Contains(haCfg, routerEntry[3]) { + e2e.Logf("the retrieved default value of haproxy: %s is not as expected: %s", haCfg, routerEntry[3]) + } + } + } + + compat_otp.By("4.0: Debug node to backup the config.yaml, and restore it before the test finishes running") + nodeName := getByJsonPath(oc, "default", "nodes", "{.items[0].metadata.name}") + defer restoreConfigYaml(oc, e2eTestNamespace, caseID, nodeName) + backupConfigYaml(oc, e2eTestNamespace, caseID, nodeName) + actualGenerationInt := getRouterDeploymentGeneration(oc, "router-default") + + compat_otp.By(`5.0: Debug node to configure the ingress in the config.yaml`) + ingressConfig := fmt.Sprintf(` +ingress: + forwardedHeaderPolicy: "Replace" + httpCompression: + mimeTypes: + - "image" + logEmptyRequests: "Ignore" + tuningOptions: + clientFinTimeout: "2s" + clientTimeout: "60s" + headerBufferBytes: 65536 + headerBufferMaxRewriteBytes: 16384 + healthCheckInterval: "10s" + maxConnections: 100000 + serverFinTimeout: "2s" + serverTimeout: "60s" + threadCount: 8 + tlsInspectDelay: "10s" + tunnelTimeout: "2h"`) + + appendIngressToConfigYaml(oc, e2eTestNamespace, caseID, nodeName, ingressConfig) + ensureRouterDeployGenerationIs(oc, "default", strconv.Itoa(actualGenerationInt+1)) + + compat_otp.By("6.0 Check the router-default deployment that all updated ENVs of tested parameters are as expected") + for _, routerEntry := range allParas { + jsonPath := fmt.Sprintf(`{.spec.template.spec.containers[0].env[?(@.name=="%s")].value}`, routerEntry[0]) + envValue := getByJsonPath(oc, "openshift-ingress", "deployment/router-default", jsonPath) + if envValue != routerEntry[2] { + e2e.Logf("the retrieved updated value of env: %s is not as expected: %s", envValue, routerEntry[2]) + } + o.Expect(envValue == routerEntry[2]).To(o.BeTrue()) + } + + compat_otp.By("7.0 Check the haproxy.config that all updated vaules of tested parameters are set as expected") + routerpod = getOneRouterPodNameByIC(oc, "default") + for _, routerEntry := range allParas { + if routerEntry[4] != "skip for none" { + haCfg := ensureHaproxyBlockConfigContains(oc, routerpod, routerEntry[4], []string{routerEntry[4]}) + if !strings.Contains(haCfg, routerEntry[4]) { + e2e.Logf("the retrieved updated value of haproxy: %s is not as expected: %s", haCfg, routerEntry[4]) + } + } + } + }) + + // author: shudili@redhat.com + g.It("Author:shudili-MicroShiftOnly-High-80508-supporting customerized default certification for Ingress Controller [Disruptive]", func() { + var ( + buildPruningBaseDir = testdata.FixturePath("router") + testPodSvc = filepath.Join(buildPruningBaseDir, "web-server-deploy.yaml") + srvrcInfo = "web-server-deploy" + unSecSvcName = "service-unsecure" + clientPod = filepath.Join(buildPruningBaseDir, "test-client-pod.yaml") + clientPodName = "hello-pod" + clientPodLabel = "app=hello-pod" + caseID = "80508" + e2eTestNamespace = "e2e-ne-" + caseID + "-" + getRandomString() + baseDomain = "apps.example.com" + podDirname = "/data/OCP-" + caseID + podDefaultCaCrt = podDirname + "/" + caseID + "-ca.crt" + podDefaultUsrCrt = podDirname + "/" + caseID + "-usr.crt" + podDefaultUsrKey = podDirname + "/" + caseID + "-usr.key" + dirname = "/tmp/OCP-" + caseID + validity = 30 + defaultCaSubj = "/CN=MS-default-CA" + defaultCaCrt = dirname + "/" + caseID + "-ca.crt" + defaultCaKey = dirname + "/" + caseID + "-ca.key" + defaultCaCsr = dirname + "/" + caseID + "-usr.csr" + defaultUserSubj = "/CN=example-ne.com" + defaultUsrCrt = dirname + "/" + caseID + "-usr.crt" + defaultUsrKey = dirname + "/" + caseID + "-usr.key" + defaultUsrCsr = dirname + "/" + caseID + "-usr.csr" + defaultCnf = dirname + "openssl.cnf" + customCert = "custom-cert" + caseID + edgeRoute = "route-edge" + caseID + "." + baseDomain + ) + + compat_otp.By("1.0: Use openssl to create a certification for the ingress default certification") + defer os.RemoveAll(dirname) + err := os.MkdirAll(dirname, 0755) + o.Expect(err).NotTo(o.HaveOccurred()) + + compat_otp.By("1.1: Create a key for the ingress default certification") + opensslCmd := fmt.Sprintf(`openssl genrsa -out %s 2048`, defaultCaKey) + _, err = exec.Command("bash", "-c", opensslCmd).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + + compat_otp.By("1.2: Create a csr for the ingress default certification") + opensslCmd = fmt.Sprintf(`openssl req -new -key %s -subj %s -out %s`, defaultCaKey, defaultCaSubj, defaultCaCsr) + _, err = exec.Command("bash", "-c", opensslCmd).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + + compat_otp.By("1.3: Create the extension file, then create the customerized certification for the ingress default certification") + sanCfg := fmt.Sprintf(` +[ v3_req ] +subjectAltName = @alt_names + +[ alt_names ] +DNS.1 = *.%s +`, baseDomain) + + cmd := fmt.Sprintf(`echo "%s" > %s`, sanCfg, defaultCnf) + _, err = exec.Command("bash", "-c", cmd).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + opensslCmd = fmt.Sprintf(`openssl x509 -extfile %s -extensions v3_req -req -in %s -signkey %s -days %d -sha256 -out %s`, defaultCnf, defaultCaCsr, defaultCaKey, validity, defaultCaCrt) + _, err = exec.Command("bash", "-c", opensslCmd).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + + compat_otp.By("1.4: Create a user CSR and the user key for a route") + opensslNewCsr(defaultUsrKey, defaultUsrCsr, defaultUserSubj) + + compat_otp.By("1.5: Sign the user CSR and generate the user certificate for a route") + san := "subjectAltName = DNS.1:*." + baseDomain + ",DNS.2:" + edgeRoute + opensslSignCsr(san, defaultUsrCsr, defaultCaCrt, defaultCaKey, defaultUsrCrt) + + compat_otp.By("2.0: Create the secret in the cluster for the customerized default certification") + defer oc.AsAdmin().WithoutNamespace().Run("delete").Args("-n", "openshift-ingress", "secret", customCert).Output() + output, err := oc.AsAdmin().WithoutNamespace().Run("create").Args("-n", "openshift-ingress", "secret", "tls", customCert, "--cert="+defaultCaCrt, "--key="+defaultCaKey).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(output).To(o.ContainSubstring("created")) + + compat_otp.By("3.0: Prepare a namespace for the following testing") + defer oc.DeleteSpecifiedNamespaceAsAdmin(e2eTestNamespace) + oc.CreateSpecifiedNamespaceAsAdmin(e2eTestNamespace) + compat_otp.SetNamespacePrivileged(oc, e2eTestNamespace) + + compat_otp.By("4.0: Debug node to backup the config.yaml, and restore it before the test finishes running") + nodeName := getByJsonPath(oc, "default", "nodes", "{.items[0].metadata.name}") + defer restoreConfigYaml(oc, e2eTestNamespace, caseID, nodeName) + backupConfigYaml(oc, e2eTestNamespace, caseID, nodeName) + actualGenerationInt := getRouterDeploymentGeneration(oc, "router-default") + + compat_otp.By(`5.0: Debug node and configure the default certification for the ingress`) + ingressConfig := fmt.Sprintf(` +ingress: + certificateSecret: "%s"`, customCert) + + appendIngressToConfigYaml(oc, e2eTestNamespace, caseID, nodeName, ingressConfig) + ensureRouterDeployGenerationIs(oc, "default", strconv.Itoa(actualGenerationInt+1)) + + compat_otp.By("6.0: Create a client pod, a deployment and the services") + createResourceFromFile(oc, e2eTestNamespace, clientPod) + ensurePodWithLabelReady(oc, e2eTestNamespace, clientPodLabel) + err = oc.AsAdmin().WithoutNamespace().Run("cp").Args("-n", e2eTestNamespace, dirname, e2eTestNamespace+"/"+clientPodName+":"+podDirname).Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + createResourceFromFile(oc, e2eTestNamespace, testPodSvc) + ensurePodWithLabelReady(oc, e2eTestNamespace, "name="+srvrcInfo) + + compat_otp.By("7.0: Create an edge route for the testing") + createRoute(oc, e2eTestNamespace, "edge", "route-edge", unSecSvcName, []string{"--hostname=" + edgeRoute}) + ensureRouteIsAdmittedByIngressController(oc, e2eTestNamespace, "route-edge", "default") + + compat_otp.By("8.0: Check the router-default deployment that the volume of default certificate is updated to the custom") + output = getByJsonPath(oc, "openshift-ingress", "deployment/router-default", "{..volumes[?(@.name==\"default-certificate\")].secret.secretName}") + o.Expect(output).To(o.ContainSubstring(customCert)) + + compat_otp.By("9.0: Check the customed default certification in a router pod") + routerpod := getOneRouterPodNameByIC(oc, "default") + output, err = oc.AsAdmin().WithoutNamespace().Run("exec").Args("-n", "openshift-ingress", routerpod, "--", "bash", "-c", "openssl x509 -noout -in /etc/pki/tls/private/tls.crt -text").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(output).To(o.ContainSubstring("Issuer: CN = MS-default-CA")) + + compat_otp.By("10.0: Curl the edge route with the user certification, issued by MS-default-CA") + podIP := getPodv4Address(oc, routerpod, "openshift-ingress") + toDst := edgeRoute + ":443:" + podIP + curlCmd := []string{"-n", e2eTestNamespace, clientPodName, "--", "curl", "https://" + edgeRoute, "-sI", "--cacert", podDefaultCaCrt, "--cert", podDefaultUsrCrt, "--key", podDefaultUsrKey, "--resolve", toDst, "--connect-timeout", "10"} + repeatCmdOnClient(oc, curlCmd, "200 OK", 60, 1) + + compat_otp.By("11.0: Curl the edge route again without any certification") + curlCmd = []string{"-n", e2eTestNamespace, clientPodName, "--", "curl", "https://" + edgeRoute, "-skI", "--resolve", toDst, "--connect-timeout", "10"} + repeatCmdOnClient(oc, curlCmd, "200 OK", 60, 1) + }) + + // author: shudili@redhat.com + // incorporate OCP-80510 and OCP-80513 into one + g.It("Author:shudili-MicroShiftOnly-High-80510-supporting Old tlsSecurityProfile for the ingress controller [Disruptive]", func() { + var ( + caseID = "80510" + e2eTestNamespace = "e2e-ne-" + caseID + "-" + getRandomString() + ) + + compat_otp.By("1.0: prepare a namespace for the following testing") + defer oc.DeleteSpecifiedNamespaceAsAdmin(e2eTestNamespace) + oc.CreateSpecifiedNamespaceAsAdmin(e2eTestNamespace) + compat_otp.SetNamespacePrivileged(oc, e2eTestNamespace) + actualGenerationInt := getRouterDeploymentGeneration(oc, "router-default") + + compat_otp.By("2.0: Debug node to backup the config.yaml, and restore it before the test finishes running") + nodeName := getByJsonPath(oc, "default", "nodes", "{.items[0].metadata.name}") + defer restoreConfigYaml(oc, e2eTestNamespace, caseID, nodeName) + backupConfigYaml(oc, e2eTestNamespace, caseID, nodeName) + + // OCP-80513 - [MicroShift] supporting Intermediate tlsSecurityProfile for the ingress controller + compat_otp.By("3.0: Check default TLS env in a router pod that the SSL_MIN_VERSION, ROUTER_CIPHER and ROUTER_CIPHERS should be as same as Intermediate profile defined") + routerpod := getOneRouterPodNameByIC(oc, "default") + env := readRouterPodEnv(oc, routerpod, "SSL_MIN_VERSION") + o.Expect(env).To(o.ContainSubstring(`SSL_MIN_VERSION=TLSv1.2`)) + env = readRouterPodEnv(oc, routerpod, "ROUTER_CIPHER") + o.Expect(env).To(o.ContainSubstring(`ROUTER_CIPHERSUITES=TLS_AES_128_GCM_SHA256:TLS_AES_256_GCM_SHA384:TLS_CHACHA20_POLY1305_SHA256`)) + o.Expect(env).To(o.ContainSubstring(`ROUTER_CIPHERS=ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384`)) + + // OCP-80510 - [MicroShift] supporting Old tlsSecurityProfile for the ingress controller + compat_otp.By("4.0: Debug node and configure the Old tls profile for the ingress") + ingressConfig := fmt.Sprintf(` +ingress: + tlsSecurityProfile: + old: {} + type: Old`) + + appendIngressToConfigYaml(oc, e2eTestNamespace, caseID, nodeName, ingressConfig) + ensureRouterDeployGenerationIs(oc, "default", strconv.Itoa(actualGenerationInt+1)) + + compat_otp.By("5.0: Check the TLS env in a router pod that the SSL_MIN_VERSION, ROUTER_CIPHER and ROUTER_CIPHERS should be as same as Old profile defined") + newrouterpod := getOneNewRouterPodFromRollingUpdate(oc, "default") + env = readRouterPodEnv(oc, newrouterpod, "SSL_MIN_VERSION") + o.Expect(env).To(o.ContainSubstring(`SSL_MIN_VERSION=TLSv1.1`)) + env = readRouterPodEnv(oc, newrouterpod, "ROUTER_CIPHER") + o.Expect(env).To(o.ContainSubstring(`ROUTER_CIPHERSUITES=TLS_AES_128_GCM_SHA256:TLS_AES_256_GCM_SHA384:TLS_CHACHA20_POLY1305_SHA256`)) + o.Expect(env).To(o.ContainSubstring(`ROUTER_CIPHERS=ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384:DHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA:ECDHE-RSA-AES256-SHA:DHE-RSA-AES128-SHA256:DHE-RSA-AES256-SHA256:AES128-GCM-SHA256:AES256-GCM-SHA384:AES128-SHA256:AES256-SHA256:AES128-SHA:AES256-SHA:DES-CBC3-SHA`)) + + // OCP-80513 - [MicroShift] supporting Intermediate tlsSecurityProfile for the ingress controller + compat_otp.By("6.0: Debug node and Configure the Intermidiate tls profile for the ingress") + ingressConfig = fmt.Sprintf(` +ingress: + tlsSecurityProfile: + intermediate: {} + type: Intermediate`) + + appendIngressToConfigYaml(oc, e2eTestNamespace, caseID, nodeName, ingressConfig) + ensureRouterDeployGenerationIs(oc, "default", strconv.Itoa(actualGenerationInt+2)) + + compat_otp.By("7.0: Check TLS env in a router pod that the SSL_MIN_VERSION, ROUTER_CIPHER and ROUTER_CIPHERS should be as same as the default defined by the Intermediate profile") + newrouterpod = getOneNewRouterPodFromRollingUpdate(oc, "default") + env = readRouterPodEnv(oc, newrouterpod, "SSL_MIN_VERSION") + o.Expect(env).To(o.ContainSubstring(`SSL_MIN_VERSION=TLSv1.2`)) + env = readRouterPodEnv(oc, newrouterpod, "ROUTER_CIPHER") + o.Expect(env).To(o.ContainSubstring(`ROUTER_CIPHERSUITES=TLS_AES_128_GCM_SHA256:TLS_AES_256_GCM_SHA384:TLS_CHACHA20_POLY1305_SHA256`)) + o.Expect(env).To(o.ContainSubstring(`ROUTER_CIPHERS=ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384`)) + }) + + // author: shudili@redhat.com + // incorporate OCP-80514 and OCP-80516 into one + g.It("Author:shudili-MicroShiftOnly-High-80514-supporting Modern tlsSecurityProfile for the ingress controller [Disruptive]", func() { + var ( + caseID = "80514" + e2eTestNamespace = "e2e-ne-" + caseID + "-" + getRandomString() + ) + + compat_otp.By("1.0: Prepare a namespace for the following testing") + defer oc.DeleteSpecifiedNamespaceAsAdmin(e2eTestNamespace) + oc.CreateSpecifiedNamespaceAsAdmin(e2eTestNamespace) + compat_otp.SetNamespacePrivileged(oc, e2eTestNamespace) + actualGenerationInt := getRouterDeploymentGeneration(oc, "router-default") + + compat_otp.By("2.0: Debug node to backup the config.yaml, and restore it before the test finishes running") + nodeName := getByJsonPath(oc, "default", "nodes", "{.items[0].metadata.name}") + defer restoreConfigYaml(oc, e2eTestNamespace, caseID, nodeName) + backupConfigYaml(oc, e2eTestNamespace, caseID, nodeName) + + // OCP-80514 - [MicroShift] supporting Modern tlsSecurityProfile for the ingress controller + + compat_otp.By("3.0: Debug node and configure the Modern tls profile for the ingress") + ingressConfig := fmt.Sprintf(` +ingress: + tlsSecurityProfile: + modern: {} + type: Modern`) + + appendIngressToConfigYaml(oc, e2eTestNamespace, caseID, nodeName, ingressConfig) + ensureRouterDeployGenerationIs(oc, "default", strconv.Itoa(actualGenerationInt+1)) + + compat_otp.By("4.0: Check TLS env in a router pod that the SSL_MIN_VERSION and ROUTER_CIPHERSUITES should be as same as the Modern profile defined") + newrouterpod := getOneNewRouterPodFromRollingUpdate(oc, "default") + env := readRouterPodEnv(oc, newrouterpod, "SSL_MIN_VERSION") + o.Expect(env).To(o.ContainSubstring(`SSL_MIN_VERSION=TLSv1.3`)) + env = readRouterPodEnv(oc, newrouterpod, "ROUTER_CIPHERSUITES") + o.Expect(env).To(o.ContainSubstring(`ROUTER_CIPHERSUITES=TLS_AES_128_GCM_SHA256:TLS_AES_256_GCM_SHA384:TLS_CHACHA20_POLY1305_SHA256`)) + + compat_otp.By("5.0: Check the haproxy config on the router pod to ensure the ssl version TLSv1.3 is reflected") + tlsVersion := readRouterPodData(oc, newrouterpod, "cat haproxy.config", "ssl-min-ver") + o.Expect(tlsVersion).To(o.ContainSubstring(`ssl-default-bind-options ssl-min-ver TLSv1.3`)) + + // OCP-80516 - [MicroShift] supporting Custom tlsSecurityProfile for the ingress controller + compat_otp.By("6.0: Debug node and configure the Custom tls profile for the ingress") + ingressConfig = fmt.Sprintf(` +ingress: + tlsSecurityProfile: + custom: + ciphers: + - DHE-RSA-AES256-GCM-SHA384 + - ECDHE-ECDSA-AES256-GCM-SHA384 + minTLSVersion: VersionTLS12 + type: Custom`) + + appendIngressToConfigYaml(oc, e2eTestNamespace, caseID, nodeName, ingressConfig) + ensureRouterDeployGenerationIs(oc, "default", strconv.Itoa(actualGenerationInt+2)) + + compat_otp.By("7.0: Check TLS env in a router pod that the SSL_MIN_VERSION and ROUTER_CIPHER should be as same as the Custom profile defined") + newrouterpod = getOneNewRouterPodFromRollingUpdate(oc, "default") + env = readRouterPodEnv(oc, newrouterpod, "SSL_MIN_VERSION") + o.Expect(env).To(o.ContainSubstring(`SSL_MIN_VERSION=TLSv1.2`)) + env = readRouterPodEnv(oc, newrouterpod, "ROUTER_CIPHER") + o.Expect(env).To(o.ContainSubstring(`ROUTER_CIPHERS=DHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES256-GCM-SHA384`)) + }) + + // author: shudili@redhat.com + g.It("Author:shudili-MicroShiftOnly-NonPreRelease-High-80517-mTLS supporting client certificate with Optional or Required policy [Disruptive]", func() { + var ( + buildPruningBaseDir = testdata.FixturePath("router") + testPodSvc = filepath.Join(buildPruningBaseDir, "web-server-deploy.yaml") + srvrcInfo = "web-server-deploy" + unSecSvcName = "service-unsecure" + clientPod = filepath.Join(buildPruningBaseDir, "test-client-pod.yaml") + clientPodName = "hello-pod" + clientPodLabel = "app=hello-pod" + caseID = "80517" + e2eTestNamespace = "e2e-ne-" + caseID + "-" + getRandomString() + baseDomain = "apps.example.com" + podDirname = "/data/OCP-" + caseID + "-ca" + podCaCrt = podDirname + "/" + caseID + "-ca.crt" + podUsrCrt = podDirname + "/" + caseID + "-usr.crt" + podUsrKey = podDirname + "/" + caseID + "-usr.key" + dirname = "/tmp/OCP-" + caseID + "-ca" + caSubj = "/CN=MS-Test-Root-CA" + caCrt = dirname + "/" + caseID + "-ca.crt" + caKey = dirname + "/" + caseID + "-ca.key" + userSubj = "/CN=example-test.com" + usrCrt = dirname + "/" + caseID + "-usr.crt" + usrKey = dirname + "/" + caseID + "-usr.key" + usrCsr = dirname + "/" + caseID + "-usr.csr" + cmName = "ocp" + caseID + edgeRoute = "route-edge" + caseID + "." + baseDomain + ) + + compat_otp.By("1.0: Use openssl to create custom client certification, create a new self-signed CA including the ca certification and ca key") + defer os.RemoveAll(dirname) + err := os.MkdirAll(dirname, 0755) + o.Expect(err).NotTo(o.HaveOccurred()) + opensslNewCa(caKey, caCrt, caSubj) + + compat_otp.By("1.1: Create a user CSR and the user key for a client") + opensslNewCsr(usrKey, usrCsr, userSubj) + + compat_otp.By("1.2: Sign the user CSR and generate the user certificate") + san := "subjectAltName = DNS.1:*." + baseDomain + ",DNS.2:" + edgeRoute + opensslSignCsr(san, usrCsr, caCrt, caKey, usrCrt) + + compat_otp.By("2.0: Create a cm with date ca certification") + defer deleteConfigMap(oc, "openshift-ingress", cmName) + createConfigMapFromFile(oc, "openshift-ingress", cmName, "ca-bundle.pem="+caCrt) + + compat_otp.By("3.0: Prepare a namespace for the following testing") + defer oc.DeleteSpecifiedNamespaceAsAdmin(e2eTestNamespace) + oc.CreateSpecifiedNamespaceAsAdmin(e2eTestNamespace) + compat_otp.SetNamespacePrivileged(oc, e2eTestNamespace) + + compat_otp.By("4.0: Debug node to backup the config.yaml, and restore it before the test finishes running") + nodeName := getByJsonPath(oc, "default", "nodes", "{.items[0].metadata.name}") + defer restoreConfigYaml(oc, e2eTestNamespace, caseID, nodeName) + backupConfigYaml(oc, e2eTestNamespace, caseID, nodeName) + actualGenerationInt := getRouterDeploymentGeneration(oc, "router-default") + + compat_otp.By("5.0: Check the ROUTER_MUTUAL_TLS_AUTH env in a router pod which should be empty for the default clientCertificatePolicy") + routerpod := getOneRouterPodNameByIC(oc, "default") + env := readRouterPodEnv(oc, routerpod, "ROUTER_MUTUAL_TLS_AUTH") + o.Expect(env).To(o.ContainSubstring(`NotFound`)) + + compat_otp.By("6.0: Debug node and configure clientTLS with clientCertificatePolicy Required in the config.yaml") + ingressConfig := fmt.Sprintf(` +ingress: + clientTLS: + clientCA: + name: "%s" + clientCertificatePolicy: "Required"`, cmName) + + appendIngressToConfigYaml(oc, e2eTestNamespace, caseID, nodeName, ingressConfig) + ensureRouterDeployGenerationIs(oc, "default", strconv.Itoa(actualGenerationInt+1)) + + compat_otp.By("7.0: Create a client pod, a deployment and the services") + createResourceFromFile(oc, e2eTestNamespace, clientPod) + ensurePodWithLabelReady(oc, e2eTestNamespace, clientPodLabel) + err = oc.AsAdmin().WithoutNamespace().Run("cp").Args("-n", e2eTestNamespace, dirname, e2eTestNamespace+"/"+clientPodName+":"+podDirname).Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + createResourceFromFile(oc, e2eTestNamespace, testPodSvc) + ensurePodWithLabelReady(oc, e2eTestNamespace, "name="+srvrcInfo) + + compat_otp.By("8.0: Create an edge route for the testing") + createRoute(oc, e2eTestNamespace, "edge", "route-edge", unSecSvcName, []string{"--hostname=" + edgeRoute, "--cert=" + usrCrt, "--key=" + usrKey}) + ensureRouteIsAdmittedByIngressController(oc, e2eTestNamespace, "route-edge", "default") + + compat_otp.By("9.0: Check the ROUTER_MUTUAL_TLS_AUTH and ROUTER_MUTUAL_TLS_AUTH_CA envs in a router pod for the Required clientCertificatePolicy") + routerpod = getOneNewRouterPodFromRollingUpdate(oc, "default") + env = readRouterPodEnv(oc, routerpod, "ROUTER_MUTUAL_TLS_AUTH") + o.Expect(env).To(o.ContainSubstring(`ROUTER_MUTUAL_TLS_AUTH=required`)) + env = readRouterPodEnv(oc, routerpod, "ROUTER_MUTUAL_TLS_AUTH_CA") + o.Expect(env).To(o.ContainSubstring(`ROUTER_MUTUAL_TLS_AUTH_CA=/etc/pki/tls/client-ca/ca-bundle.pem`)) + + compat_otp.By("10.0: Curl the edge route with the user certification, expect to get 200 OK") + podIP := getPodv4Address(oc, routerpod, "openshift-ingress") + toDst := edgeRoute + ":443:" + podIP + curlCmd := []string{"-n", e2eTestNamespace, clientPodName, "--", "curl", "https://" + edgeRoute, "-sI", "--cacert", podCaCrt, "--cert", podUsrCrt, "--key", podUsrKey, "--resolve", toDst, "--connect-timeout", "10"} + repeatCmdOnClient(oc, curlCmd, "200 OK", 60, 1) + + compat_otp.By("11.0: Curl the edge route without any certifications, expect to get SSL_read error") + curlCmd = []string{"-n", e2eTestNamespace, clientPodName, "--", "curl", "https://" + edgeRoute, "-skv", "--resolve", toDst, "--connect-timeout", "10"} + waitForErrorOccur(oc, curlCmd, "SSL_read: error", 60) + + compat_otp.By("12.0: Debug node and configure clientTLS with clientCertificatePolicy Optional in the config.yaml") + ingressConfig = fmt.Sprintf(` +ingress: + clientTLS: + clientCA: + name: "%s" + clientCertificatePolicy: "Optional"`, cmName) + + appendIngressToConfigYaml(oc, e2eTestNamespace, caseID, nodeName, ingressConfig) + ensureRouterDeployGenerationIs(oc, "default", strconv.Itoa(actualGenerationInt+2)) + + compat_otp.By("13.0: Check the ROUTER_MUTUAL_TLS_AUTH env in a router pod for the Optional clientCertificatePolicy") + routerpod = getOneNewRouterPodFromRollingUpdate(oc, "default") + env = readRouterPodEnv(oc, routerpod, "ROUTER_MUTUAL_TLS_AUTH") + o.Expect(env).To(o.ContainSubstring(`ROUTER_MUTUAL_TLS_AUTH=optional`)) + + compat_otp.By("14.0: Curl the edge route with the user certification, expect to get 200 OK") + podIP = getPodv4Address(oc, routerpod, "openshift-ingress") + toDst = edgeRoute + ":443:" + podIP + curlCmd = []string{"-n", e2eTestNamespace, clientPodName, "--", "curl", "https://" + edgeRoute, "-sI", "--cacert", podCaCrt, "--cert", podUsrCrt, "--key", podUsrKey, "--resolve", toDst, "--connect-timeout", "10"} + repeatCmdOnClient(oc, curlCmd, "200 OK", 60, 1) + + compat_otp.By("15.0: Curl the edge route without any certifications, expect to get 200 OK") + curlCmd = []string{"-n", e2eTestNamespace, clientPodName, "--", "curl", "https://" + edgeRoute, "-skI", "--resolve", toDst, "--connect-timeout", "10"} + repeatCmdOnClient(oc, curlCmd, "200 OK", 60, 1) + }) + + // author: shudili@redhat.com + g.It("Author:shudili-MicroShiftOnly-High-80518-mTLS supporting client certificate with the subject filter [Disruptive]", func() { + var ( + buildPruningBaseDir = testdata.FixturePath("router") + testPodSvc = filepath.Join(buildPruningBaseDir, "web-server-deploy.yaml") + srvrcInfo = "web-server-deploy" + unSecSvcName = "service-unsecure" + clientPod = filepath.Join(buildPruningBaseDir, "test-client-pod.yaml") + clientPodName = "hello-pod" + clientPodLabel = "app=hello-pod" + caseID = "80518" + e2eTestNamespace = "e2e-ne-" + caseID + "-" + getRandomString() + baseDomain = "apps.example.com" + podDirname = "/data/OCP-" + caseID + "-ca" + podCaCrt = podDirname + "/" + caseID + "-ca.crt" + podUsrCrt = podDirname + "/" + caseID + "-usr.crt" + podUsrKey = podDirname + "/" + caseID + "-usr.key" + podUsrCrt2 = podDirname + "/" + caseID + "-usr2.crt" + podUsrKey2 = podDirname + "/" + caseID + "-usr2.key" + dirname = "/tmp/OCP-" + caseID + "-ca" + caSubj = "/CN=MS-Test-Root-CA" + caCrt = dirname + "/" + caseID + "-ca.crt" + caKey = dirname + "/" + caseID + "-ca.key" + userSubj = "/CN=example-test.com" + usrCrt = dirname + "/" + caseID + "-usr.crt" + usrKey = dirname + "/" + caseID + "-usr.key" + usrCsr = dirname + "/" + caseID + "-usr.csr" + userSubj2 = "/CN=example-test2.com" + usrCrt2 = dirname + "/" + caseID + "-usr2.crt" + usrKey2 = dirname + "/" + caseID + "-usr2.key" + usrCsr2 = dirname + "/" + caseID + "-usr2.csr" + cmName = "ocp" + caseID + filter = userSubj + edgeRoute = "route-edge" + caseID + "." + baseDomain + edgeRoute2 = "route2-edge" + caseID + "." + baseDomain + ) + + compat_otp.By("1.0: Use openssl to create custom client certification, create a new self-signed CA including the ca certification and ca key") + defer os.RemoveAll(dirname) + err := os.MkdirAll(dirname, 0755) + o.Expect(err).NotTo(o.HaveOccurred()) + opensslNewCa(caKey, caCrt, caSubj) + + compat_otp.By("1.1: Create a user CSR and the user key for a client") + opensslNewCsr(usrKey, usrCsr, userSubj) + + compat_otp.By("1.2: Sign the user CSR and generate the user certificate") + san := "subjectAltName = DNS.1:*." + baseDomain + ",DNS.2:" + edgeRoute + opensslSignCsr(san, usrCsr, caCrt, caKey, usrCrt) + + compat_otp.By("1.3: Create another user CSR and the user key for the client") + opensslNewCsr(usrKey2, usrCsr2, userSubj2) + + compat_otp.By("1.4: Sign the another user CSR and generate the user certificate") + san = "subjectAltName = DNS.1:*." + baseDomain + ",DNS.2:" + edgeRoute2 + opensslSignCsr(san, usrCsr2, caCrt, caKey, usrCrt2) + + compat_otp.By("2.0: Create a cm with date ca certification") + defer deleteConfigMap(oc, "openshift-ingress", cmName) + createConfigMapFromFile(oc, "openshift-ingress", cmName, "ca-bundle.pem="+caCrt) + + compat_otp.By("3.0: Prepare a namespace for the following testing") + defer oc.DeleteSpecifiedNamespaceAsAdmin(e2eTestNamespace) + oc.CreateSpecifiedNamespaceAsAdmin(e2eTestNamespace) + compat_otp.SetNamespacePrivileged(oc, e2eTestNamespace) + actualGenerationInt := getRouterDeploymentGeneration(oc, "router-default") + + compat_otp.By("4.0: Debug node to backup the config.yaml, and restore it before the test finishes running") + nodeName := getByJsonPath(oc, "default", "nodes", "{.items[0].metadata.name}") + defer restoreConfigYaml(oc, e2eTestNamespace, caseID, nodeName) + backupConfigYaml(oc, e2eTestNamespace, caseID, nodeName) + compat_otp.By("5.0: Debug node and configure clientTLS with allowedSubjectPatterns in the config.yaml for permitting the first route") + ingressConfig := fmt.Sprintf(` +ingress: + clientTLS: + allowedSubjectPatterns: ["%s"] + clientCA: + name: "%s" + clientCertificatePolicy: "Required"`, filter, cmName) + + appendIngressToConfigYaml(oc, e2eTestNamespace, caseID, nodeName, ingressConfig) + ensureRouterDeployGenerationIs(oc, "default", strconv.Itoa(actualGenerationInt+1)) + + compat_otp.By("6.0: Create a client pod, a deployment and the services") + createResourceFromFile(oc, e2eTestNamespace, clientPod) + ensurePodWithLabelReady(oc, e2eTestNamespace, clientPodLabel) + err = oc.AsAdmin().WithoutNamespace().Run("cp").Args("-n", e2eTestNamespace, dirname, e2eTestNamespace+"/"+clientPodName+":"+podDirname).Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + createResourceFromFile(oc, e2eTestNamespace, testPodSvc) + ensurePodWithLabelReady(oc, e2eTestNamespace, "name="+srvrcInfo) + + compat_otp.By("7.0: Create two edge route for the testing, one mathing allowedSubjectPatterns of the clientTLS") + createRoute(oc, e2eTestNamespace, "edge", "route-edge", unSecSvcName, []string{"--hostname=" + edgeRoute, "--cert=" + usrCrt, "--key=" + usrKey}) + createRoute(oc, e2eTestNamespace, "edge", "route-edge2", unSecSvcName, []string{"--hostname=" + edgeRoute2, "--cert=" + usrCrt2, "--key=" + usrKey2}) + ensureRouteIsAdmittedByIngressController(oc, e2eTestNamespace, "route-edge", "default") + ensureRouteIsAdmittedByIngressController(oc, e2eTestNamespace, "route-edge2", "default") + + compat_otp.By("8.0: Check the ROUTER_MUTUAL_TLS_AUTH env in a router pod") + routerpod := getOneNewRouterPodFromRollingUpdate(oc, "default") + env := readRouterPodEnv(oc, routerpod, "ROUTER_MUTUAL_TLS_AUTH_FILTER") + o.Expect(env).To(o.ContainSubstring(filter)) + + compat_otp.By("9.0: Curl the first edge route with the user certification, expect to get 200 OK for mathing the allowedSubjectPatterns of the clientTLS") + podIP := getPodv4Address(oc, routerpod, "openshift-ingress") + toDst := edgeRoute + ":443:" + podIP + curlCmd := []string{"-n", e2eTestNamespace, clientPodName, "--", "curl", "https://" + edgeRoute, "-sI", "--cacert", podCaCrt, "--cert", podUsrCrt, "--key", podUsrKey, "--resolve", toDst, "--connect-timeout", "10"} + repeatCmdOnClient(oc, curlCmd, "200 OK", 60, 1) + + compat_otp.By("10.0: Curl the second edge route with the user2 certification, expect to get 403 Forbidden for not mathing the allowedSubjectPatterns of the clientTLS") + toDst = edgeRoute2 + ":443:" + podIP + curlCmd = []string{"-n", e2eTestNamespace, clientPodName, "--", "curl", "https://" + edgeRoute2, "-sI", "--cacert", podCaCrt, "--cert", podUsrCrt2, "--key", podUsrKey2, "--resolve", toDst, "--connect-timeout", "10"} + repeatCmdOnClient(oc, curlCmd, "403 Forbidden", 60, 1) + }) + + // author: shudili@redhat.com + g.It("Author:shudili-MicroShiftOnly-NonPreRelease-High-80520-supporting wildcard routeAdmissionPolicy for the Ingress Controller [Disruptive]", func() { + var ( + buildPruningBaseDir = testdata.FixturePath("router") + testPodSvc = filepath.Join(buildPruningBaseDir, "web-server-deploy.yaml") + srvrcInfo = "web-server-deploy" + unsecSvcName = "service-unsecure" + clientPod = filepath.Join(buildPruningBaseDir, "test-client-pod.yaml") + clientPodName = "hello-pod" + clientPodLabel = "app=hello-pod" + caseID = "80520" + e2eTestNamespace = "e2e-ne-" + caseID + "-" + getRandomString() + baseDomain = "apps.example.com" + ) + + compat_otp.By("1.0: Prepare a namespace for the following testing") + defer oc.DeleteSpecifiedNamespaceAsAdmin(e2eTestNamespace) + oc.CreateSpecifiedNamespaceAsAdmin(e2eTestNamespace) + compat_otp.SetNamespacePrivileged(oc, e2eTestNamespace) + + compat_otp.By("2.0: Create a client pod, a deployment and the services") + createResourceFromFile(oc, e2eTestNamespace, clientPod) + ensurePodWithLabelReady(oc, e2eTestNamespace, clientPodLabel) + createResourceFromFile(oc, e2eTestNamespace, testPodSvc) + ensurePodWithLabelReady(oc, e2eTestNamespace, "name="+srvrcInfo) + + compat_otp.By("3.0: For the default WildcardsDisallowed wildcardPolicy of routeAdmission, check the ROUTER_ALLOW_WILDCARD_ROUTES env variable, which should be false") + routerpod := getOneRouterPodNameByIC(oc, "default") + namespaceOwnershipEnv := readRouterPodEnv(oc, routerpod, "ROUTER_ALLOW_WILDCARD_ROUTES") + o.Expect(namespaceOwnershipEnv).To(o.ContainSubstring("ROUTER_ALLOW_WILDCARD_ROUTES=false")) + + compat_otp.By("4.0: Create a route with wildcard-policy Subdomain, which should Not be Admitted") + routehost := "wildcard." + baseDomain + anyhost := "any." + baseDomain + createRoute(oc, e2eTestNamespace, "http", "unsecure80520", unsecSvcName, []string{"--wildcard-policy=Subdomain", "--hostname=" + routehost}) + ensureRouteIsNotAdmittedByIngressController(oc, e2eTestNamespace, "unsecure80520", "default") + + compat_otp.By("5.0: Debug node to backup the config.yaml, and restore it before the test finishes running") + nodeName := getByJsonPath(oc, "default", "nodes", "{.items[0].metadata.name}") + actualGenerationInt := getRouterDeploymentGeneration(oc, "router-default") + defer restoreConfigYaml(oc, e2eTestNamespace, caseID, nodeName) + backupConfigYaml(oc, e2eTestNamespace, caseID, nodeName) + + compat_otp.By("6.0: Debug node to set WildcardsAllowed wildcardPolicy in the config.yaml file") + ingressConfig := fmt.Sprintf(` +ingress: + routeAdmissionPolicy: + wildcardPolicy: "WildcardsAllowed"`) + + appendIngressToConfigYaml(oc, e2eTestNamespace, caseID, nodeName, ingressConfig) + ensureRouterDeployGenerationIs(oc, "default", strconv.Itoa(actualGenerationInt+1)) + + compat_otp.By("7.0. Check the ROUTER_ALLOW_WILDCARD_ROUTES env variable, which should be true") + routerpod = getOneNewRouterPodFromRollingUpdate(oc, "default") + namespaceOwnershipEnv = readRouterPodEnv(oc, routerpod, "ROUTER_ALLOW_WILDCARD_ROUTES") + o.Expect(namespaceOwnershipEnv).To(o.ContainSubstring("ROUTER_ALLOW_WILDCARD_ROUTES=true")) + + compat_otp.By("8.0: Curl the route with the two hostnames again, both should get 200 ok reponse") + podIP := getPodv4Address(oc, routerpod, "openshift-ingress") + toDst := routehost + ":80:" + podIP + curlCmd := []string{"-n", e2eTestNamespace, clientPodName, "--", "curl", "http://" + routehost, "-sI", "--resolve", toDst, "--connect-timeout", "10"} + ensureRouteIsAdmittedByIngressController(oc, e2eTestNamespace, "unsecure80520", "default") + repeatCmdOnClient(oc, curlCmd, "200 OK", 60, 1) + toDst = anyhost + ":80:" + podIP + curlCmd = []string{"-n", e2eTestNamespace, clientPodName, "--", "curl", "http://" + anyhost, "-sI", "--resolve", toDst, "--connect-timeout", "10"} + repeatCmdOnClient(oc, curlCmd, "200 OK", 60, 1) + + compat_otp.By("9.0: Debug node to set WildcardsDisabllowed wildcardPolicy in the config.yaml file") + ingressConfig = fmt.Sprintf(` +ingress: + routeAdmissionPolicy: + wildcardPolicy: "WildcardsDisallowed"`) + + appendIngressToConfigYaml(oc, e2eTestNamespace, caseID, nodeName, ingressConfig) + ensureRouterDeployGenerationIs(oc, "default", strconv.Itoa(actualGenerationInt+2)) + + compat_otp.By("10.0: For the configured WildcardsDisallowed wildcardPolicy of routeAdmission, check the ROUTER_ALLOW_WILDCARD_ROUTES env variable, which should be false") + routerpod = getOneNewRouterPodFromRollingUpdate(oc, "default") + namespaceOwnershipEnv = readRouterPodEnv(oc, routerpod, "ROUTER_ALLOW_WILDCARD_ROUTES") + o.Expect(namespaceOwnershipEnv).To(o.ContainSubstring("ROUTER_ALLOW_WILDCARD_ROUTES=false")) + + compat_otp.By(`11.0: Check the route's status, which should Not be Admitted`) + ensureRouteIsNotAdmittedByIngressController(oc, e2eTestNamespace, "unsecure80520", "default") + }) + + // includes OCP-81996 and OCP-84261 + // OCP-81996 - capture and log http cookies with specific prefixes via httpCaptureCookies option + // OCP-84261 - capture and log http cookies by the "httpCaptureCookies" option for Edge route and REEN route + // author: shudili@redhat.com + g.It("Author:shudili-MicroShiftOnly-NonPreRelease-High-81996-capture and log http cookies with specific prefixes via httpCaptureCookies option [Disruptive]", func() { + var ( + buildPruningBaseDir = testdata.FixturePath("router") + testPodSvc = filepath.Join(buildPruningBaseDir, "web-server-signed-deploy.yaml") + srvrcInfo = "web-server-deploy" + unsecSvcName = "service-unsecure" + secsvcName = "service-secure" + clientPod = filepath.Join(buildPruningBaseDir, "test-client-pod.yaml") + clientPodName = "hello-pod" + clientPodLabel = "app=hello-pod" + caseID = "81996" + e2eTestNamespace = "e2e-ne-" + caseID + "-" + getRandomString() + baseDomain = "apps.example.com" + routehost = "route-unsec" + caseID + "." + baseDomain + edgeRouteHost = "route-edge" + caseID + "." + baseDomain + reenRouteHost = "route-reen" + caseID + "." + baseDomain + ) + + compat_otp.By("1.0: Prepare a namespace for the testing") + defer oc.DeleteSpecifiedNamespaceAsAdmin(e2eTestNamespace) + oc.CreateSpecifiedNamespaceAsAdmin(e2eTestNamespace) + compat_otp.SetNamespacePrivileged(oc, e2eTestNamespace) + + compat_otp.By("2.0: Debug node to backup the config.yaml, and restore it before the test finishes running") + nodeName := getByJsonPath(oc, "default", "nodes", "{.items[0].metadata.name}") + defer restoreConfigYaml(oc, e2eTestNamespace, caseID, nodeName) + backupConfigYaml(oc, e2eTestNamespace, caseID, nodeName) + actualGenerationInt := getRouterDeploymentGeneration(oc, "router-default") + + compat_otp.By(`3.0: Debug node and configure "capture and log http cookies with specific prefixes via httpCaptureCookies option"`) + ingressConfig := fmt.Sprintf(` +ingress: + accessLogging: + httpCaptureCookies: + - matchType: Prefix + maxLength: 100 + namePrefix: foo + status: Enabled`) + + appendIngressToConfigYaml(oc, e2eTestNamespace, caseID, nodeName, ingressConfig) + ensureRouterDeployGenerationIs(oc, "default", strconv.Itoa(actualGenerationInt+1)) + + compat_otp.By("4.0: Create a client pod, a deployment and the services") + createResourceFromFile(oc, e2eTestNamespace, clientPod) + ensurePodWithLabelReady(oc, e2eTestNamespace, clientPodLabel) + createResourceFromFile(oc, e2eTestNamespace, testPodSvc) + ensurePodWithLabelReady(oc, e2eTestNamespace, "name="+srvrcInfo) + + compat_otp.By("5.0: Create a http/edge/reen route for the testing") + createRoute(oc, e2eTestNamespace, "http", "route-http", unsecSvcName, []string{"--hostname=" + routehost}) + ensureRouteIsAdmittedByIngressController(oc, e2eTestNamespace, "route-http", "default") + createRoute(oc, e2eTestNamespace, "edge", "route-edge", unsecSvcName, []string{"--hostname=" + edgeRouteHost}) + ensureRouteIsAdmittedByIngressController(oc, e2eTestNamespace, "route-edge", "default") + createRoute(oc, e2eTestNamespace, "reencrypt", "route-reen", secsvcName, []string{"--hostname=" + reenRouteHost}) + ensureRouteIsAdmittedByIngressController(oc, e2eTestNamespace, "route-reen", "default") + + compat_otp.By("6.0: Check httpCaptureCookies configuration in haproxy") + routerpod := getOneNewRouterPodFromRollingUpdate(oc, "default") + // ensureHaproxyBlockConfigContains(oc, routerpod, "defaults", []string{"capture cookie foo len 100"}) + + compat_otp.By("7.0: Curl the http route with cookie fo=nobar, expect to get 200 OK") + podIP := getPodv4Address(oc, routerpod, "openshift-ingress") + toDst := routehost + ":80:" + podIP + curlCmd := []string{"-n", e2eTestNamespace, clientPodName, "--", "curl", "http://" + routehost + "/index.html", "-sI", "-b", "fo=nobar", "--resolve", toDst, "--connect-timeout", "10"} + repeatCmdOnClient(oc, curlCmd, "200 OK", 90, 6) + + compat_otp.By("8.0: Curl the http route with cookie foo=bar, expect to get 200 OK") + curlCmd = []string{"-n", e2eTestNamespace, clientPodName, "--", "curl", "http://" + routehost + "/index.html", "-sI", "-b", "foo=bar", "--resolve", toDst, "--connect-timeout", "10"} + repeatCmdOnClient(oc, curlCmd, "200 OK", 90, 6) + + compat_otp.By("8.1: Curl the http route with cookie foo22=bar22, expect to get 200 OK") + curlCmd = []string{"-n", e2eTestNamespace, clientPodName, "--", "curl", "http://" + routehost + "/index.html", "-sI", "-b", "foo22=bar22", "--resolve", toDst, "--connect-timeout", "10"} + repeatCmdOnClient(oc, curlCmd, "200 OK", 90, 6) + + compat_otp.By("8.2: Curl the edge route with cookie foo=barforedge, expect to get 200 OK") + toDst = edgeRouteHost + ":443:" + podIP + curlCmd = []string{"-n", e2eTestNamespace, clientPodName, "--", "curl", "https://" + edgeRouteHost + "/index.html", "-skI", "-b", "foo=barforedge", "--resolve", toDst, "--connect-timeout", "10"} + repeatCmdOnClient(oc, curlCmd, "200 OK", 90, 6) + + compat_otp.By("8.3: Curl the reen route with cookie foo=barforreen, expect to get 200 OK") + toDst = reenRouteHost + ":443:" + podIP + curlCmd = []string{"-n", e2eTestNamespace, clientPodName, "--", "curl", "https://" + reenRouteHost + "/index.html", "-skI", "-b", "foo=barforreen", "--resolve", toDst, "--connect-timeout", "10"} + repeatCmdOnClient(oc, curlCmd, "200 OK", 90, 6) + + compat_otp.By("9.0: Check the router logs, which should contain both the cookie foo=bar and the url") + logs := waitRouterLogsAppear(oc, routerpod, "foo=bar") + o.Expect(logs).To(o.ContainSubstring("index.html")) + + compat_otp.By("10.0: Check the router logs, which should NOT contain both the cookie fo=nobar") + containerName := getByJsonPath(oc, "openshift-ingress", "pod/"+routerpod, "{.spec.containers[*].name}") + logs, err := oc.AsAdmin().WithoutNamespace().Run("logs").Args("-n", "openshift-ingress", "-c", strings.Split(containerName, " ")[1], routerpod).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(logs).NotTo(o.ContainSubstring("fo=nobar")) + + compat_otp.By("11.0: Check the router logs, which should contain both the cookie foo22=bar22 and the url") + logs = waitRouterLogsAppear(oc, routerpod, "foo22=bar22") + o.Expect(logs).To(o.ContainSubstring("index.html")) + + // check logs for edge route of OCP-84261 + compat_otp.By("12.0: Check the router logs, which should contain both the cookie foo=barforedge and the url for the edge route") + logs = waitRouterLogsAppear(oc, routerpod, "foo=barforedge") + o.Expect(logs).To(o.ContainSubstring("index.html")) + + // check logs for reen route of OCP-84261 + compat_otp.By("13.0: Check the router logs, which should contain both the cookie foo=barforreen and the url for the reen route") + logs = waitRouterLogsAppear(oc, routerpod, "foo=barforreen") + o.Expect(logs).To(o.ContainSubstring("index.html")) + }) + + // includes OCP-81997 and OCP-81998 + // author: shudili@redhat.com + g.It("Author:shudili-MicroShiftOnly-NonPreRelease-High-81997-capture and log http cookies with exact match via httpCaptureCookies option [Disruptive]", func() { + var ( + buildPruningBaseDir = testdata.FixturePath("router") + testPodSvc = filepath.Join(buildPruningBaseDir, "web-server-deploy.yaml") + srvrcInfo = "web-server-deploy" + unsecSvcName = "service-unsecure" + clientPod = filepath.Join(buildPruningBaseDir, "test-client-pod.yaml") + clientPodName = "hello-pod" + clientPodLabel = "app=hello-pod" + caseID = "81997" + e2eTestNamespace = "e2e-ne-" + caseID + "-" + getRandomString() + baseDomain = "apps.example.com" + routehost = "route-unsec" + caseID + "." + baseDomain + ) + + compat_otp.By("1.0: Prepare a namespace for the following testing") + defer oc.DeleteSpecifiedNamespaceAsAdmin(e2eTestNamespace) + oc.CreateSpecifiedNamespaceAsAdmin(e2eTestNamespace) + compat_otp.SetNamespacePrivileged(oc, e2eTestNamespace) + + compat_otp.By("2.0: Debug node to backup the config.yaml, and restore it before the test finishes running") + nodeName := getByJsonPath(oc, "default", "nodes", "{.items[0].metadata.name}") + defer restoreConfigYaml(oc, e2eTestNamespace, caseID, nodeName) + backupConfigYaml(oc, e2eTestNamespace, caseID, nodeName) + actualGenerationInt := getRouterDeploymentGeneration(oc, "router-default") + + compat_otp.By(`3.0: Debug node and configure "capture and log http cookies with specific prefixes via httpCaptureCookies option"`) + ingressConfig := fmt.Sprintf(`ingress: + accessLogging: + httpCaptureCookies: + - matchType: Exact + maxLength: 100 + name: foo + status: Enabled`) + + appendIngressToConfigYaml(oc, e2eTestNamespace, caseID, nodeName, ingressConfig) + ensureRouterDeployGenerationIs(oc, "default", strconv.Itoa(actualGenerationInt+1)) + + compat_otp.By("4.0: Create a client pod, a deployment and the services") + createResourceFromFile(oc, e2eTestNamespace, clientPod) + ensurePodWithLabelReady(oc, e2eTestNamespace, clientPodLabel) + createResourceFromFile(oc, e2eTestNamespace, testPodSvc) + ensurePodWithLabelReady(oc, e2eTestNamespace, "name="+srvrcInfo) + + compat_otp.By("5.0: Create a http route for the testing") + createRoute(oc, e2eTestNamespace, "http", "route-http", unsecSvcName, []string{"--hostname=" + routehost}) + ensureRouteIsAdmittedByIngressController(oc, e2eTestNamespace, "route-http", "default") + + compat_otp.By("6.0: Check httpCaptureCookies configuration in haproxy") + routerpod := getOneNewRouterPodFromRollingUpdate(oc, "default") + // ensureHaproxyBlockConfigContains(oc, routerpod, "defaults", []string{"capture cookie foo= len 100"}) + + compat_otp.By("7.0: Curl the http route with cookie fooor=nobar, expect to get 200 OK") + podIP := getPodv4Address(oc, routerpod, "openshift-ingress") + toDst := routehost + ":80:" + podIP + curlCmd := []string{"-n", e2eTestNamespace, clientPodName, "--", "curl", "http://" + routehost + "/index.html", "-sI", "-b", "fooor=nobar", "--resolve", toDst, "--connect-timeout", "10"} + repeatCmdOnClient(oc, curlCmd, "200 OK", 90, 6) + + compat_otp.By("8.0: Curl the http route with cookie foo=bar, expect to get 200 OK") + curlCmd = []string{"-n", e2eTestNamespace, clientPodName, "--", "curl", "http://" + routehost + "/index.html", "-sI", "-b", "foo=bar", "--resolve", toDst, "--connect-timeout", "10"} + repeatCmdOnClient(oc, curlCmd, "200 OK", 90, 6) + + compat_otp.By("9.0: Check the router logs, which should contain both the cookie foo=bar and the url") + logs := waitRouterLogsAppear(oc, routerpod, "foo=bar") + o.Expect(logs).To(o.ContainSubstring("index.html")) + + compat_otp.By("10.0: Check the router logs, which should NOT contain the cookie fooor=nobar") + containerName := getByJsonPath(oc, "openshift-ingress", "pod/"+routerpod, "{.spec.containers[*].name}") + logs, err := oc.AsAdmin().WithoutNamespace().Run("logs").Args("-n", "openshift-ingress", "-c", strings.Split(containerName, " ")[1], routerpod).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(logs).NotTo(o.ContainSubstring("fooor=nobar")) + + // OCP-81998(The httpCaptureCookies option strictly adheres to the maxlength parameter) + compat_otp.By("11.0: debug node and configure maxLength for the httpCaptureCookies") + ingressConfig = fmt.Sprintf(`ingress: + accessLogging: + httpCaptureCookies: + - matchType: Exact + maxLength: 10 + name: foo + status: Enabled`) + + appendIngressToConfigYaml(oc, e2eTestNamespace, caseID, nodeName, ingressConfig) + ensureRouterDeployGenerationIs(oc, "default", strconv.Itoa(actualGenerationInt+2)) + + compat_otp.By("12.0: Curl the http route with cookie foo=bar89abdef, expect to get 200 OK") + routerpod = getOneNewRouterPodFromRollingUpdate(oc, "default") + podIP = getPodv4Address(oc, routerpod, "openshift-ingress") + toDst = routehost + ":80:" + podIP + curlCmd = []string{"-n", e2eTestNamespace, clientPodName, "--", "curl", "http://" + routehost + "/index.html", "-sI", "-b", "foo=bar89abdef", "--resolve", toDst, "--connect-timeout", "10"} + repeatCmdOnClient(oc, curlCmd, "200 OK", 90, 6) + + compat_otp.By("13.0: Check the router logs, which should contain the cookie foo=bar89a, NOT foo=bar89abdef") + logs = waitRouterLogsAppear(oc, routerpod, "foo=bar89a") + o.Expect(logs).NotTo(o.ContainSubstring("foo=bar89ab")) + }) + + // includes OCP-82000, OCP-82002 and OCP-84259 + // OCP-82000 - capture and log specific http Request header via httpCaptureHeaders option + // OCP-82002 - capture and log specific http Response headers via httpCaptureHeaders option + // OCP-84259 - capture and log specific http header via "httpCaptureHeaders" option for Edge route and REEN route + // author: shudili@redhat.com + g.It("Author:shudili-MicroShiftOnly-NonPreRelease-High-82000-capture and log specific http Request header via httpCaptureHeaders option [Disruptive]", func() { + var ( + buildPruningBaseDir = testdata.FixturePath("router") + testPodSvc = filepath.Join(buildPruningBaseDir, "web-server-signed-deploy.yaml") + srvrcInfo = "web-server-deploy" + unsecSvcName = "service-unsecure" + secsvcName = "service-secure" + clientPod = filepath.Join(buildPruningBaseDir, "test-client-pod.yaml") + clientPodName = "hello-pod" + clientPodLabel = "app=hello-pod" + server = "nginx" + caseID = "82000" + e2eTestNamespace = "e2e-ne-" + caseID + "-" + getRandomString() + baseDomain = "apps.example.com" + routehost = "route-unsec" + caseID + "." + baseDomain + edgeRouteHost = "route-edge" + caseID + "." + baseDomain + reenRouteHost = "route-reen" + caseID + "." + baseDomain + ) + + compat_otp.By("1.0: Prepare a namespace for the following testing") + defer oc.DeleteSpecifiedNamespaceAsAdmin(e2eTestNamespace) + oc.CreateSpecifiedNamespaceAsAdmin(e2eTestNamespace) + compat_otp.SetNamespacePrivileged(oc, e2eTestNamespace) + + compat_otp.By("2.0: Debug node to backup the config.yaml, and restore it before the test finishes running") + nodeName := getByJsonPath(oc, "default", "nodes", "{.items[0].metadata.name}") + defer restoreConfigYaml(oc, e2eTestNamespace, caseID, nodeName) + backupConfigYaml(oc, e2eTestNamespace, caseID, nodeName) + actualGenerationInt := getRouterDeploymentGeneration(oc, "router-default") + + compat_otp.By("3.0: Debug node and configure capture and log http cookies with specific prefixes via httpCaptureCookies option") + ingressConfig := fmt.Sprintf(` +ingress: + accessLogging: + httpCaptureHeaders: + request: + - maxLength: 120 + name: Host + response: + - maxLength: 120 + name: "Server" + status: Enabled`) + + appendIngressToConfigYaml(oc, e2eTestNamespace, caseID, nodeName, ingressConfig) + ensureRouterDeployGenerationIs(oc, "default", strconv.Itoa(actualGenerationInt+1)) + + compat_otp.By("4.0: Create a client pod, a deployment and the services") + createResourceFromFile(oc, e2eTestNamespace, clientPod) + ensurePodWithLabelReady(oc, e2eTestNamespace, clientPodLabel) + createResourceFromFile(oc, e2eTestNamespace, testPodSvc) + ensurePodWithLabelReady(oc, e2eTestNamespace, "name="+srvrcInfo) + + compat_otp.By("5.0: Create a http/edge/reen route for the testing") + createRoute(oc, e2eTestNamespace, "http", "route-http", unsecSvcName, []string{"--hostname=" + routehost}) + ensureRouteIsAdmittedByIngressController(oc, e2eTestNamespace, "route-http", "default") + createRoute(oc, e2eTestNamespace, "edge", "route-edge", unsecSvcName, []string{"--hostname=" + edgeRouteHost}) + ensureRouteIsAdmittedByIngressController(oc, e2eTestNamespace, "route-edge", "default") + createRoute(oc, e2eTestNamespace, "reencrypt", "route-reen", secsvcName, []string{"--hostname=" + reenRouteHost}) + ensureRouteIsAdmittedByIngressController(oc, e2eTestNamespace, "route-reen", "default") + + compat_otp.By("6.0: Check httpCaptureHeaders configuration in haproxy") + routerpod := getOneNewRouterPodFromRollingUpdate(oc, "default") + frontendCfg := getBlockConfig(oc, routerpod, "frontend fe_sni") + o.Expect(frontendCfg).To(o.ContainSubstring("capture request header Host len 120")) + // check configuration for OCP-82002 + o.Expect(frontendCfg).To(o.ContainSubstring("capture response header Server len 120")) + + compat_otp.By("7.0: Curl the http route, expect to get 200 OK") + podIP := getPodv4Address(oc, routerpod, "openshift-ingress") + toDst := routehost + ":80:" + podIP + curlCmd := []string{"-n", e2eTestNamespace, clientPodName, "--", "curl", "http://" + routehost + "/index.html", "-sI", "--resolve", toDst, "--connect-timeout", "10"} + repeatCmdOnClient(oc, curlCmd, "200 OK", 90, 6) + + compat_otp.By("7.1: Curl the edge route, expect to get 200 OK") + toDst = edgeRouteHost + ":443:" + podIP + curlCmd = []string{"-n", e2eTestNamespace, clientPodName, "--", "curl", "https://" + edgeRouteHost + "/index.html", "-skI", "--resolve", toDst, "--connect-timeout", "10"} + repeatCmdOnClient(oc, curlCmd, "200 OK", 90, 6) + + compat_otp.By("7.2: Curl the reen route, expect to get 200 OK") + toDst = reenRouteHost + ":443:" + podIP + curlCmd = []string{"-n", e2eTestNamespace, clientPodName, "--", "curl", "https://" + reenRouteHost + "/index.html", "-skI", "--resolve", toDst, "--connect-timeout", "10"} + repeatCmdOnClient(oc, curlCmd, "200 OK", 90, 6) + + // check logs for OCP-82002 + compat_otp.By("8.0: Check the router logs, which should contain both the request host and the response server") + logs := waitRouterLogsAppear(oc, routerpod, routehost) + o.Expect(logs).To(o.MatchRegexp(server + "/" + "[0-9\\.]+")) + + // check logs for edge route of OCP-84259 + compat_otp.By("9.0: Check the router logs, which should contain both the request host of the edge route and the response server") + logs = waitRouterLogsAppear(oc, routerpod, edgeRouteHost) + o.Expect(logs).To(o.MatchRegexp(server + "/" + "[0-9\\.]+")) + + // check logs for reen route of OCP-84259 + compat_otp.By("10.0: Check the router logs, which should contain both the request host of the reen route and the response server") + logs = waitRouterLogsAppear(oc, routerpod, reenRouteHost) + o.Expect(logs).To(o.MatchRegexp(server + "/" + "[0-9\\.]+")) + }) + + // author: shudili@redhat.com + g.It("Author:shudili-MicroShiftOnly-High-82003-The httpCaptureHeaders option strictly adheres to the maxlength parameter [Disruptive]", func() { + var ( + buildPruningBaseDir = testdata.FixturePath("router") + testPodSvc = filepath.Join(buildPruningBaseDir, "web-server-deploy.yaml") + srvrcInfo = "web-server-deploy" + unsecSvcName = "service-unsecure" + clientPod = filepath.Join(buildPruningBaseDir, "test-client-pod.yaml") + clientPodName = "hello-pod" + clientPodLabel = "app=hello-pod" + server = "nginx" + caseID = "82003" + e2eTestNamespace = "e2e-ne-" + caseID + "-" + getRandomString() + baseDomain = "apps.example.com" + routehostPrefix = "route-unsec" + caseID + routehost = routehostPrefix + "." + baseDomain + ) + + compat_otp.By("1.0: Prepare a namespace for the following testing") + defer oc.DeleteSpecifiedNamespaceAsAdmin(e2eTestNamespace) + oc.CreateSpecifiedNamespaceAsAdmin(e2eTestNamespace) + compat_otp.SetNamespacePrivileged(oc, e2eTestNamespace) + + compat_otp.By("2.0: Debug node to backup the config.yaml, and restore it before the test finishes running") + nodeName := getByJsonPath(oc, "default", "nodes", "{.items[0].metadata.name}") + defer restoreConfigYaml(oc, e2eTestNamespace, caseID, nodeName) + backupConfigYaml(oc, e2eTestNamespace, caseID, nodeName) + actualGenerationInt := getRouterDeploymentGeneration(oc, "router-default") + + compat_otp.By("3.0: Debug node and configure a small maxLength for request/response httpCaptureHeaders") + ingressConfig := fmt.Sprintf(` +ingress: + accessLogging: + httpCaptureHeaders: + request: + - maxLength: 16 + name: Host + response: + - maxLength: 5 + name: "Server" + status: Enabled`) + + appendIngressToConfigYaml(oc, e2eTestNamespace, caseID, nodeName, ingressConfig) + ensureRouterDeployGenerationIs(oc, "default", strconv.Itoa(actualGenerationInt+1)) + + compat_otp.By("4.0: Create a client pod, a deployment and the services") + createResourceFromFile(oc, e2eTestNamespace, clientPod) + ensurePodWithLabelReady(oc, e2eTestNamespace, clientPodLabel) + createResourceFromFile(oc, e2eTestNamespace, testPodSvc) + ensurePodWithLabelReady(oc, e2eTestNamespace, "name="+srvrcInfo) + + compat_otp.By("5.0: Create a http route for the testing") + createRoute(oc, e2eTestNamespace, "http", "route-http", unsecSvcName, []string{"--hostname=" + routehost}) + ensureRouteIsAdmittedByIngressController(oc, e2eTestNamespace, "route-http", "default") + + compat_otp.By("6.0: Curl the http route, expect to get 200 OK") + routerpod := getOneNewRouterPodFromRollingUpdate(oc, "default") + podIP := getPodv4Address(oc, routerpod, "openshift-ingress") + toDst := routehost + ":80:" + podIP + curlCmd := []string{"-n", e2eTestNamespace, clientPodName, "--", "curl", "http://" + routehost + "/index.html", "-sI", "--resolve", toDst, "--connect-timeout", "10"} + repeatCmdOnClient(oc, curlCmd, "200 OK", 90, 6) + + compat_otp.By("7.0: Check the router logs, which should contain the request host header and response server header with values not exceeding the maxLength") + logs := waitRouterLogsAppear(oc, routerpod, routehostPrefix) + o.Expect(logs).NotTo(o.ContainSubstring(routehost)) + o.Expect(logs).To(o.ContainSubstring(server)) + o.Expect(logs).NotTo(o.ContainSubstring(server + "/")) + }) + + // author: shudili@redhat.com + g.It("Author:shudili-MicroShiftOnly-High-82004-custom http 503 and 404 error pages for ingress route [Disruptive]", func() { + var ( + buildPruningBaseDir = testdata.FixturePath("router") + configmapName = "custom-82004-error-code-pages" + cmFile1 = filepath.Join(buildPruningBaseDir, "error-page-503.http") + cmFile2 = filepath.Join(buildPruningBaseDir, "error-page-404.http") + testPodSvc = filepath.Join(buildPruningBaseDir, "web-server-deploy.yaml") + srvrcInfo = "web-server-deploy" + unsecSvcName = "service-unsecure" + clientPod = filepath.Join(buildPruningBaseDir, "test-client-pod.yaml") + clientPodName = "hello-pod" + clientPodLabel = "app=hello-pod" + caseID = "82004" + e2eTestNamespace = "e2e-ne-" + caseID + "-" + getRandomString() + baseDomain = "apps.example.com" + routehost = "route-unsec" + caseID + "." + baseDomain + notExistHost = "not-exist" + caseID + "." + baseDomain + ) + + compat_otp.By("1.0 Create a configmap with the customized error pages in openshift-ingress namespace") + _, err := oc.AsAdmin().WithoutNamespace().Run("create").Args("configmap", configmapName, "--from-file="+cmFile1, "--from-file="+cmFile2, "-n", "openshift-ingress").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + defer oc.AsAdmin().WithoutNamespace().Run("delete").Args("configmap", configmapName, "-n", "openshift-ingress").Output() + + compat_otp.By("2.0: Prepare a namespace for the following testing") + defer oc.DeleteSpecifiedNamespaceAsAdmin(e2eTestNamespace) + oc.CreateSpecifiedNamespaceAsAdmin(e2eTestNamespace) + compat_otp.SetNamespacePrivileged(oc, e2eTestNamespace) + + compat_otp.By("3.0: Debug node to backup the config.yaml, and restore it before the test finishes running") + nodeName := getByJsonPath(oc, "default", "nodes", "{.items[0].metadata.name}") + defer restoreConfigYaml(oc, e2eTestNamespace, caseID, nodeName) + backupConfigYaml(oc, e2eTestNamespace, caseID, nodeName) + actualGenerationInt := getRouterDeploymentGeneration(oc, "router-default") + + compat_otp.By("4.0: Debug node and configure custom ingress httpErrorCodePages") + ingressConfig := fmt.Sprintf(` +ingress: + httpErrorCodePages: + name: %s +`, configmapName) + + appendIngressToConfigYaml(oc, e2eTestNamespace, caseID, nodeName, ingressConfig) + ensureRouterDeployGenerationIs(oc, "default", strconv.Itoa(actualGenerationInt+1)) + + compat_otp.By("5.0: Create a client pod, a deployment and the services") + createResourceFromFile(oc, e2eTestNamespace, clientPod) + ensurePodWithLabelReady(oc, e2eTestNamespace, clientPodLabel) + createResourceFromFile(oc, e2eTestNamespace, testPodSvc) + ensurePodWithLabelReady(oc, e2eTestNamespace, "name="+srvrcInfo) + + compat_otp.By("6.0: Create a http route for the testing") + createRoute(oc, e2eTestNamespace, "http", "route-http", unsecSvcName, []string{"--hostname=" + routehost}) + ensureRouteIsAdmittedByIngressController(oc, e2eTestNamespace, "route-http", "default") + + compat_otp.By("7.0: Check customized error 503 and 404 pages in haproxy") + routerpod := getOneNewRouterPodFromRollingUpdate(oc, "default") + errorfileCfg := readRouterPodData(oc, routerpod, "cat /var/lib/haproxy/errorfiles/error-page-503.http", "Unavailable") + o.Expect(errorfileCfg).Should(o.And( + o.ContainSubstring(`HTTP/1.0 503 Service Unavailable`), + o.ContainSubstring(`Custom:Application Unavailable`))) + + errorfileCfg = readRouterPodData(oc, routerpod, "cat /var/lib/haproxy/errorfiles/error-page-404.http", "Not Found") + o.Expect(errorfileCfg).Should(o.And( + o.ContainSubstring(`HTTP/1.0 404 Not Found`), + o.ContainSubstring(`Custom:Not Found`))) + + compat_otp.By("8.0: Curl a not exist http route, expect to get the custom 404 error page") + podIP := getPodv4Address(oc, routerpod, "openshift-ingress") + toDst := notExistHost + ":80:" + podIP + curlCmd := []string{"-n", e2eTestNamespace, clientPodName, "--", "curl", "http://" + notExistHost, "-s", "--resolve", toDst, "--connect-timeout", "10"} + repeatCmdOnClient(oc, curlCmd, "Custom:Not Found", 60, 1) + + compat_otp.By("9.0: Scale down the deployment to 0, the curl a http route, expect to get the custom 503 error page") + scaleDeploy(oc, e2eTestNamespace, srvrcInfo, 0) + toDst = routehost + ":80:" + podIP + curlCmd = []string{"-n", e2eTestNamespace, clientPodName, "--", "curl", "http://" + routehost, "-s", "--resolve", toDst, "--connect-timeout", "10"} + repeatCmdOnClient(oc, curlCmd, "Custom:Application Unavailable", 60, 1) + }) + + // author: shudili@redhat.com + g.It("Author:shudili-MicroShiftOnly-NonPreRelease-High-82014-httpLogFormat for logging http requests [Disruptive]", func() { + var ( + buildPruningBaseDir = testdata.FixturePath("router") + testPodSvc = filepath.Join(buildPruningBaseDir, "web-server-deploy.yaml") + srvrcInfo = "web-server-deploy" + unsecSvcName = "service-unsecure" + clientPod = filepath.Join(buildPruningBaseDir, "test-client-pod.yaml") + clientPodName = "hello-pod" + clientPodLabel = "app=hello-pod" + caseID = "82014" + e2eTestNamespace = "e2e-ne-" + caseID + "-" + getRandomString() + baseDomain = "apps.example.com" + routehost = "route-unsec" + caseID + "." + baseDomain + jsonPath = "{.status.ingress[0].conditions[?(@.type==\"Admitted\")].status}" + ) + + compat_otp.By("1.0: Prepare a namespace for the following testing") + defer oc.DeleteSpecifiedNamespaceAsAdmin(e2eTestNamespace) + oc.CreateSpecifiedNamespaceAsAdmin(e2eTestNamespace) + compat_otp.SetNamespacePrivileged(oc, e2eTestNamespace) + + compat_otp.By("2.0: Debug node to backup the config.yaml, and restore it before the test finishes running") + nodeName := getByJsonPath(oc, "default", "nodes", "{.items[0].metadata.name}") + defer restoreConfigYaml(oc, e2eTestNamespace, caseID, nodeName) + backupConfigYaml(oc, e2eTestNamespace, caseID, nodeName) + actualGenerationInt := getRouterDeploymentGeneration(oc, "router-default") + + compat_otp.By(`3.0: Debug node and configure httpLogFormat with %{+Q}r`) + val := `%{+Q}r` + ingressConfig := fmt.Sprintf(` +ingress: + accessLogging: + httpLogFormat: "%s" + status: Enabled +`, val) + + appendIngressToConfigYaml(oc, e2eTestNamespace, caseID, nodeName, ingressConfig) + ensureRouterDeployGenerationIs(oc, "default", strconv.Itoa(actualGenerationInt+1)) + + compat_otp.By("4.0: Create a client pod, a deployment and the services") + createResourceFromFile(oc, e2eTestNamespace, clientPod) + ensurePodWithLabelReady(oc, e2eTestNamespace, clientPodLabel) + createResourceFromFile(oc, e2eTestNamespace, testPodSvc) + ensurePodWithLabelReady(oc, e2eTestNamespace, "name="+srvrcInfo) + + compat_otp.By("5.0: Create a http route for the testing") + createRoute(oc, e2eTestNamespace, "http", "route-http", unsecSvcName, []string{"--hostname=" + routehost}) + waitForOutputEquals(oc, e2eTestNamespace, "route/route-http", jsonPath, "True") + + compat_otp.By("6.0: Check httpLogFormat configuration in haproxy") + routerpod := getOneNewRouterPodFromRollingUpdate(oc, "default") + // OCPBUGS-60257(indentation issue of "frontend public" in haproxy) + // formatCfg := getBlockConfig(oc, routerpod, "defaults") + // expectedVal := fmt.Sprintf(`log-format "%s"`, val) + // o.Expect(formatCfg).To(o.ContainSubstring(expectedVal)) + + compat_otp.By("7.0: Curl the http route, expect to get 200 OK") + podIP := getPodv4Address(oc, routerpod, "openshift-ingress") + toDst := routehost + ":80:" + podIP + curlCmd := []string{"-n", e2eTestNamespace, clientPodName, "--", "curl", "http://" + routehost + "/path/second/index.html", "-sI", "--resolve", toDst, "--connect-timeout", "10"} + repeatCmdOnClient(oc, curlCmd, "200 OK", 90, 6) + + compat_otp.By("8.0: Check the router logs, which should contain the desired info including the http request url") + logs := waitRouterLogsAppear(oc, routerpod, "/path/second/index.html") + o.Expect(logs).To(o.MatchRegexp(`haproxy\[[0-9]+\]: "HEAD /path/second/index.html HTTP`)) + + compat_otp.By(`9.0: Debug node and configure httpLogFormat with %ci:%cp %si:%sp %HU %ST`) + val = `%ci:%cp %si:%sp %HU %ST` + ingressConfig = fmt.Sprintf(` +ingress: + accessLogging: + httpLogFormat: "%s" + status: Enabled +`, val) + + appendIngressToConfigYaml(oc, e2eTestNamespace, caseID, nodeName, ingressConfig) + ensureRouterDeployGenerationIs(oc, "default", strconv.Itoa(actualGenerationInt+2)) + + compat_otp.By("10.0: Check new httpLogFormat configuration in haproxy") + routerpod = getOneNewRouterPodFromRollingUpdate(oc, "default") + // OCPBUGS-60257(indentation issue of "frontend public" in haproxy) + // formatCfg = getBlockConfig(oc, routerpod, "defaults") + // expectedVal = fmt.Sprintf(`log-format "%s"`, val) + // o.Expect(formatCfg).To(o.ContainSubstring(expectedVal)) + + compat_otp.By("11.0: Curl the http route, expect to get 200 OK") + podIP = getPodv4Address(oc, routerpod, "openshift-ingress") + toDst = routehost + ":80:" + podIP + curlCmd = []string{"-n", e2eTestNamespace, clientPodName, "--", "curl", "http://" + routehost + "/path/second/index.html", "-sI", "--resolve", toDst, "--connect-timeout", "10"} + repeatCmdOnClient(oc, curlCmd, "200 OK", 90, 6) + + compat_otp.By("12.0: Check the router logs, which should contain the desired info including source & destination address and http status code") + logs = waitRouterLogsAppear(oc, routerpod, "/path/second/index.html") + o.Expect(logs).To(o.MatchRegexp(`haproxy\[[0-9]+\]: [0-9\\.a-fA-F:]+:[0-9]+ [0-9\\.a-fA-F:]+:8080 /path/second/index.html 200`)) + }) + + // author: shudili@redhat.com + g.It("Author:shudili-MicroShiftOnly-NonPreRelease-High-82015-logging destination to a syslog server [Disruptive]", func() { + var ( + buildPruningBaseDir = testdata.FixturePath("router") + testPodSvc = filepath.Join(buildPruningBaseDir, "web-server-deploy.yaml") + srvrcInfo = "web-server-deploy" + unsecSvcName = "service-unsecure" + syslogPod = filepath.Join(buildPruningBaseDir, "rsyslogd-pod.yaml") + syslogPodLable = "name=rsyslogd" + syslogPodName = "rsyslogd-pod" + clientPod = filepath.Join(buildPruningBaseDir, "test-client-pod.yaml") + clientPodName = "hello-pod" + clientPodLabel = "app=hello-pod" + caseID = "82014" + e2eTestNamespace = "e2e-ne-" + caseID + "-" + getRandomString() + baseDomain = "apps.example.com" + routehost = "route-unsec" + caseID + "." + baseDomain + ) + + compat_otp.By("1.0: Prepare a namespace for the following testing") + defer oc.DeleteSpecifiedNamespaceAsAdmin(e2eTestNamespace) + oc.CreateSpecifiedNamespaceAsAdmin(e2eTestNamespace) + compat_otp.SetNamespacePrivileged(oc, e2eTestNamespace) + + compat_otp.By("2.0: Create a client pod, a rsyslogd pod, a deployment and the services") + createResourceFromFile(oc, e2eTestNamespace, clientPod) + ensurePodWithLabelReady(oc, e2eTestNamespace, clientPodLabel) + createResourceFromFile(oc, e2eTestNamespace, syslogPod) + ensurePodWithLabelReady(oc, e2eTestNamespace, syslogPodLable) + createResourceFromFile(oc, e2eTestNamespace, testPodSvc) + ensurePodWithLabelReady(oc, e2eTestNamespace, "name="+srvrcInfo) + + compat_otp.By("3.0: Debug node to backup the config.yaml, and restore it before the test finishes running") + nodeName := getByJsonPath(oc, "default", "nodes", "{.items[0].metadata.name}") + defer restoreConfigYaml(oc, e2eTestNamespace, caseID, nodeName) + backupConfigYaml(oc, e2eTestNamespace, caseID, nodeName) + actualGenerationInt := getRouterDeploymentGeneration(oc, "router-default") + + compat_otp.By(`4.0: Debug node and configure the syslog destination`) + syslogIP := getByJsonPath(oc, e2eTestNamespace, "pods/"+syslogPodName, "{.status.podIP}") + ingressConfig := fmt.Sprintf(` +ingress: + accessLogging: + destination: + syslog: + address: %s + port: 514 + type: Syslog + status: Enabled +`, syslogIP) + + appendIngressToConfigYaml(oc, e2eTestNamespace, caseID, nodeName, ingressConfig) + ensureRouterDeployGenerationIs(oc, "default", strconv.Itoa(actualGenerationInt+1)) + + compat_otp.By("5.0: Create a http route for the testing") + createRoute(oc, e2eTestNamespace, "http", "route-http", unsecSvcName, []string{"--hostname=" + routehost}) + ensureRouteIsAdmittedByIngressController(oc, e2eTestNamespace, "route-http", "default") + + compat_otp.By("6.0: Check the syslog configuration in haproxy") + routerpod := getOneNewRouterPodFromRollingUpdate(oc, "default") + syslogCfg := getBlockConfig(oc, routerpod, "global") + expectedVal := fmt.Sprintf(`log %s:514 len 1024 local1 info`, syslogIP) + o.Expect(syslogCfg).To(o.ContainSubstring(expectedVal)) + + compat_otp.By("7.0: Curl the http route, expect to get 200 OK") + podIP := getPodv4Address(oc, routerpod, "openshift-ingress") + toDst := routehost + ":80:" + podIP + curlCmd := []string{"-n", e2eTestNamespace, clientPodName, "--", "curl", "http://" + routehost + "/path/second/index.html", "-sI", "--resolve", toDst, "--connect-timeout", "10"} + repeatCmdOnClient(oc, curlCmd, "200 OK", 90, 6) + + compat_otp.By("8.0: Check the logs of syslog pod, which should contain the haproxy log") + logs, err := oc.AsAdmin().WithoutNamespace().Run("logs").Args("-n", e2eTestNamespace, syslogPodName, "--tail=20").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(logs).To(o.MatchRegexp(`haproxy\[[0-9]+\]:.+/path/second/index.html`)) + + compat_otp.By(`9.0: Debug node and configure destination syslog with facility local2`) + ingressConfig = fmt.Sprintf(` +ingress: + accessLogging: + destination: + syslog: + address: %s + port: 514 + facility: local2 + type: Syslog + status: Enabled +`, syslogIP) + + appendIngressToConfigYaml(oc, e2eTestNamespace, caseID, nodeName, ingressConfig) + ensureRouterDeployGenerationIs(oc, "default", strconv.Itoa(actualGenerationInt+2)) + + compat_otp.By("10.0: Check the new syslog configuration in haproxy") + routerpod = getOneNewRouterPodFromRollingUpdate(oc, "default") + syslogCfg = getBlockConfig(oc, routerpod, "global") + expectedVal = fmt.Sprintf(`log %s:514 len 1024 local2 info`, syslogIP) + o.Expect(syslogCfg).To(o.ContainSubstring(expectedVal)) + }) + + // includes OCP-84260 and OCP-82042 + // OCP-84260 Negative test of the logging configuration + // OCP-82042 disable logging function + // author: shudili@redhat.com + g.It("Author:shudili-MicroShiftOnly-Medium-84260-Negative test of the logging configuration [Disruptive]", func() { + var ( + caseID = "84260" + e2eTestNamespace = "e2e-ne-" + caseID + "-" + getRandomString() + ) + + compat_otp.By("1.0: Prepare a namespace for the following testing") + defer oc.DeleteSpecifiedNamespaceAsAdmin(e2eTestNamespace) + oc.CreateSpecifiedNamespaceAsAdmin(e2eTestNamespace) + compat_otp.SetNamespacePrivileged(oc, e2eTestNamespace) + + compat_otp.By("2.0: Debug node to backup the config.yaml, and restore it before the test finishes running") + nodeName := getByJsonPath(oc, "default", "nodes", "{.items[0].metadata.name}") + defer restoreConfigYaml(oc, e2eTestNamespace, caseID, nodeName) + backupConfigYaml(oc, e2eTestNamespace, caseID, nodeName) + actualGenerationInt := getRouterDeploymentGeneration(oc, "router-default") + + compat_otp.By(`3.0: Debug node and try to configure invalid maxLength -1 of the httpCaptureCookies`) + ingressConfig := fmt.Sprintf(`ingress: + accessLogging: + httpCaptureCookies: + - matchType: Exact + maxLength: -1 + name: foo + status: Enabled`) + + output := appendInvalidIngressConfigToConfigYaml(oc, e2eTestNamespace, caseID, nodeName, ingressConfig) + o.Expect(output).To(o.ContainSubstring("Must be between 1 and 1024")) + + compat_otp.By(`4.0: Debug node and try to configure invalid maxLength 0 of the httpCaptureCookies`) + ingressConfig = fmt.Sprintf(`ingress: + accessLogging: + httpCaptureCookies: + - matchType: Exact + maxLength: 0 + name: foo + status: Enabled`) + + output = appendInvalidIngressConfigToConfigYaml(oc, e2eTestNamespace, caseID, nodeName, ingressConfig) + o.Expect(output).To(o.ContainSubstring("Must be between 1 and 1024")) + + compat_otp.By(`5.0: Debug node and try to configure the name of the httpCaptureCookies with specail characters including space, # and so on`) + ingressConfig = fmt.Sprintf(`ingress: + accessLogging: + httpCaptureCookies: + - matchType: Exact + maxLength: 100 + name: "foo 33#?-" + status: Enabled`) + + output = appendInvalidIngressConfigToConfigYaml(oc, e2eTestNamespace, caseID, nodeName, ingressConfig) + compat_otp.By("output is :" + output) + o.Expect(output).To(o.ContainSubstring("contains invalid characters")) + + compat_otp.By(`6.0: Debug node and try to configure invalid maxLength -1 of the httpCaptureHeaders`) + ingressConfig = fmt.Sprintf(`ingress: + accessLogging: + httpCaptureHeaders: + request: + - maxLength: -1 + name: Host + response: + - maxLength: 10 + name: "Server" + status: Enabled`) + + output = appendInvalidIngressConfigToConfigYaml(oc, e2eTestNamespace, caseID, nodeName, ingressConfig) + o.Expect(output).To(o.ContainSubstring("maxLength must be at least 1")) + + compat_otp.By(`7.0: Debug node and try to configure status with invalid Enable(not Enabled)`) + ingressConfig = fmt.Sprintf(`ingress: + accessLogging: + httpCaptureHeaders: + request: + - maxLength: 10 + name: Host + response: + - maxLength: 10 + name: "Server" + status: Enable`) + + output = appendInvalidIngressConfigToConfigYaml(oc, e2eTestNamespace, caseID, nodeName, ingressConfig) + o.Expect(output).To(o.ContainSubstring("invalid access logging status: Enable")) + + // OCP-82042 disable logging function + compat_otp.By("8.0: Debug node and configure capture and log http cookies with specific prefixes via httpCaptureCookies option, but without enable the status") + ingressConfig = fmt.Sprintf(` +ingress: + accessLogging: + httpCaptureCookies: + - matchType: Prefix + maxLength: 100 + namePrefix: foo +`) + + appendIngressToConfigYaml(oc, e2eTestNamespace, caseID, nodeName, ingressConfig) + ensureRouterDeployGenerationIs(oc, "default", strconv.Itoa(actualGenerationInt)) + + compat_otp.By("9.0: Check the configuration of haproxy which should not contain the logging httpCaptureCookies") + routerpod := getOneNewRouterPodFromRollingUpdate(oc, "default") + defaultsCfg := getBlockConfig(oc, routerpod, "defaults") + o.Expect(defaultsCfg).NotTo(o.ContainSubstring("capture cookie foo len 100")) + }) +}) diff --git a/tests-extension/test/e2e/route-admission.go b/tests-extension/test/e2e/route-admission.go new file mode 100644 index 000000000..09d5706cf --- /dev/null +++ b/tests-extension/test/e2e/route-admission.go @@ -0,0 +1,299 @@ +package router + +import ( + "fmt" + "os/exec" + "path/filepath" + + g "github.com/onsi/ginkgo/v2" + o "github.com/onsi/gomega" + compat_otp "github.com/openshift/origin/test/extended/util/compat_otp" + "github.com/openshift/router-tests-extension/test/testdata" +) + +var _ = g.Describe("[sig-network-edge] Network_Edge Component_Router", func() { + defer g.GinkgoRecover() + + var oc = compat_otp.NewCLI("router-admission", compat_otp.KubeConfigPath()) + + // Test case creater: hongli@redhat.com + g.It("Author:mjoseph-ROSA-OSD_CCS-ARO-Critical-27594-Set namespaceOwnership of routeAdmission to InterNamespaceAllowed", func() { + var ( + buildPruningBaseDir = testdata.FixturePath("router") + testPodSvc = filepath.Join(buildPruningBaseDir, "web-server-deploy.yaml") + customTemp = filepath.Join(buildPruningBaseDir, "ingresscontroller-np.yaml") + srvrcInfo = "web-server-deploy" + srvName = "service-unsecure" + e2eTestNamespace2 = "e2e-ne-ocp27594-" + getRandomString() + ingctrl = ingressControllerDescription{ + name: "ocp27594", + namespace: "openshift-ingress-operator", + domain: "", + template: customTemp, + } + ) + + compat_otp.By("1. Create an additional namespace for this scenario") + defer oc.DeleteSpecifiedNamespaceAsAdmin(e2eTestNamespace2) + oc.CreateSpecifiedNamespaceAsAdmin(e2eTestNamespace2) + e2eTestNamespace1 := oc.Namespace() + path1 := "/path/first" + path2 := "/path/second" + routehost := srvName + "-" + "ocp27594." + "apps.example.com" + + compat_otp.By("2. Create a custom ingresscontroller") + ingctrl.domain = ingctrl.name + "." + getBaseDomain(oc) + // Updating namespaceOwnership as 'InterNamespaceAllowed' in the yaml file + sedCmd := fmt.Sprintf(`sed -i'' -e 's|Strict|%s|g' %s`, "InterNamespaceAllowed", customTemp) + _, err := exec.Command("bash", "-c", sedCmd).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + defer ingctrl.delete(oc) + ingctrl.create(oc) + ensureRouterDeployGenerationIs(oc, ingctrl.name, "1") + custContPod := getOneNewRouterPodFromRollingUpdate(oc, ingctrl.name) + + compat_otp.By("3. Check the ROUTER_DISABLE_NAMESPACE_OWNERSHIP_CHECK env variable, which should be true") + namespaceOwnershipEnv := readRouterPodEnv(oc, custContPod, "ROUTER_DISABLE_NAMESPACE_OWNERSHIP_CHECK") + o.Expect(namespaceOwnershipEnv).To(o.ContainSubstring("ROUTER_DISABLE_NAMESPACE_OWNERSHIP_CHECK=true")) + + compat_otp.By("4. Create a server pod and an unsecure service in one ns") + createResourceFromFile(oc, e2eTestNamespace1, testPodSvc) + ensurePodWithLabelReady(oc, e2eTestNamespace1, "name="+srvrcInfo) + + compat_otp.By("5. Create a server pod and an unsecure service in the other ns") + operateResourceFromFile(oc, "create", e2eTestNamespace2, testPodSvc) + ensurePodWithLabelReady(oc, e2eTestNamespace2, "name="+srvrcInfo) + + compat_otp.By("6. Expose a http route with path " + path1 + " in the first ns") + err = oc.Run("expose").Args("service", srvName, "--hostname="+routehost, "--path="+path1, "-n", e2eTestNamespace1).Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + getRoutes(oc, e2eTestNamespace1) + waitForOutputEquals(oc, e2eTestNamespace1, "route", "{.items[0].metadata.name}", srvName) + + compat_otp.By("7. Create a edge route with the same hostname, but with different path " + path2 + " in the second ns") + err = oc.AsAdmin().Run("create").Args("route", "edge", "route-edge", "--service="+srvName, "--hostname="+routehost, "--path="+path2, "-n", e2eTestNamespace2).Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + getRoutes(oc, e2eTestNamespace2) + waitForOutputEquals(oc, e2eTestNamespace2, "route", "{.items[0].metadata.name}", "route-edge") + + compat_otp.By("8 Check the custom router pod and ensure " + e2eTestNamespace1 + " http route is loaded in haproxy.config") + ensureHaproxyBlockConfigContains(oc, custContPod, e2eTestNamespace1, []string{"backend be_http:" + e2eTestNamespace1 + ":service-unsecure"}) + + compat_otp.By("9. Check the custom router pod and ensure " + e2eTestNamespace2 + " edge route is loaded in haproxy.config") + ensureHaproxyBlockConfigContains(oc, custContPod, e2eTestNamespace2, []string{"backend be_edge_http:" + e2eTestNamespace2 + ":route-edge"}) + }) + + // Test case creater: hongli@redhat.com + g.It("Author:mjoseph-ROSA-OSD_CCS-ARO-Critical-27595-Set namespaceOwnership of routeAdmission to Strict", func() { + var ( + buildPruningBaseDir = testdata.FixturePath("router") + testPodSvc = filepath.Join(buildPruningBaseDir, "web-server-deploy.yaml") + srvrcInfo = "web-server-deploy" + e2eTestNamespace2 = "e2e-ne-ocp27595-" + getRandomString() + ) + + compat_otp.By("1. Create an additional namespace for this scenario") + defer oc.DeleteSpecifiedNamespaceAsAdmin(e2eTestNamespace2) + oc.CreateSpecifiedNamespaceAsAdmin(e2eTestNamespace2) + e2eTestNamespace1 := oc.Namespace() + path1 := "/path/first" + path2 := "/path/second" + routehost := "ocp27595.apps.example.com" + + // Strict is by default so just need to check in default router + compat_otp.By("2. Check the ROUTER_DISABLE_NAMESPACE_OWNERSHIP_CHECK env variable, which should be false") + routerpod := getOneRouterPodNameByIC(oc, "default") + namespaceOwnershipEnv := readRouterPodEnv(oc, routerpod, "ROUTER_DISABLE_NAMESPACE_OWNERSHIP_CHECK") + o.Expect(namespaceOwnershipEnv).To(o.ContainSubstring("ROUTER_DISABLE_NAMESPACE_OWNERSHIP_CHECK=false")) + + compat_otp.By("3. Create a server pod and an unsecure service in one ns") + createResourceFromFile(oc, e2eTestNamespace1, testPodSvc) + ensurePodWithLabelReady(oc, e2eTestNamespace1, "name="+srvrcInfo) + + compat_otp.By("4. Create a reen route with path " + path1 + " in the first ns") + err := oc.AsAdmin().WithoutNamespace().Run("create").Args("route", "reencrypt", "route-reen", "--service=service-secure", "--hostname="+routehost, "--path="+path1, "-n", e2eTestNamespace1).Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + getRoutes(oc, e2eTestNamespace1) + waitForOutputEquals(oc, e2eTestNamespace1, "route", "{.items[0].metadata.name}", "route-reen") + + compat_otp.By("5 Check the custom router pod and ensure " + e2eTestNamespace1 + " route is loaded in haproxy.config") + ensureHaproxyBlockConfigContains(oc, routerpod, e2eTestNamespace1, []string{"backend be_secure:" + e2eTestNamespace1 + ":route-reen"}) + compat_otp.By("6. Create a server pod and an unsecure service in the other ns") + operateResourceFromFile(oc, "create", e2eTestNamespace2, testPodSvc) + ensurePodWithLabelReady(oc, e2eTestNamespace2, "name="+srvrcInfo) + + compat_otp.By("7. Create a http route with the same hostname, but with different path " + path2 + " in the second ns") + err = oc.AsAdmin().WithoutNamespace().Run("expose").Args("service", "service-unsecure", "--hostname="+routehost, "--path="+path2, "-n", e2eTestNamespace2).Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + getRoutes(oc, e2eTestNamespace2) + waitForOutputEquals(oc, e2eTestNamespace2, "route", "{.items[0].metadata.name}", "service-unsecure") + + compat_otp.By("8. Confirm the route in the second ns is shown as HostAlreadyClaimed") + waitForOutputContains(oc, e2eTestNamespace2, "route", `{.items[*].status.ingress[?(@.routerName=="default")].conditions[*].reason}`, "HostAlreadyClaimed") + }) + + // Test case creater: hongli@redhat.com + // For OCP-27596 and OCP-27605 + g.It("Author:mjoseph-ROSA-OSD_CCS-ARO-Critical-27596-Update the namespaceOwnership of routeAdmission", func() { + var ( + buildPruningBaseDir = testdata.FixturePath("router") + customTemp = filepath.Join(buildPruningBaseDir, "ingresscontroller-np.yaml") + ingctrl = ingressControllerDescription{ + name: "ocp27596", + namespace: "openshift-ingress-operator", + domain: "", + template: customTemp, + } + ) + + compat_otp.By("1. Create a custom ingresscontroller") + ingctrl.domain = ingctrl.name + "." + getBaseDomain(oc) + defer ingctrl.delete(oc) + ingctrl.create(oc) + ensureRouterDeployGenerationIs(oc, ingctrl.name, "1") + + compat_otp.By("2. Check the ROUTER_DISABLE_NAMESPACE_OWNERSHIP_CHECK env variable, which should be false") + routerpod := getOneNewRouterPodFromRollingUpdate(oc, ingctrl.name) + namespaceOwnershipEnv := readRouterPodEnv(oc, routerpod, "ROUTER_DISABLE_NAMESPACE_OWNERSHIP_CHECK") + o.Expect(namespaceOwnershipEnv).To(o.ContainSubstring("ROUTER_DISABLE_NAMESPACE_OWNERSHIP_CHECK=false")) + + compat_otp.By("3. Patch the custom ingress controller and set namespaceOwnership to InterNamespaceAllowed") + patchResourceAsAdmin(oc, ingctrl.namespace, "ingresscontrollers/"+ingctrl.name, "{\"spec\":{\"routeAdmission\":{\"namespaceOwnership\":\"InterNamespaceAllowed\"}}}") + ensureRouterDeployGenerationIs(oc, ingctrl.name, "2") + + compat_otp.By("4. Check the ROUTER_DISABLE_NAMESPACE_OWNERSHIP_CHECK env variable, which should be true") + newRouterpod := getOneNewRouterPodFromRollingUpdate(oc, ingctrl.name) + namespaceOwnershipEnv = readRouterPodEnv(oc, newRouterpod, "ROUTER_DISABLE_NAMESPACE_OWNERSHIP_CHECK") + o.Expect(namespaceOwnershipEnv).To(o.ContainSubstring("ROUTER_DISABLE_NAMESPACE_OWNERSHIP_CHECK=true")) + + compat_otp.By("5. Patch the custom ingress controller and set namespaceOwnership to Strict") + patchResourceAsAdmin(oc, ingctrl.namespace, "ingresscontrollers/"+ingctrl.name, "{\"spec\":{\"routeAdmission\":{\"namespaceOwnership\":\"Strict\"}}}") + + compat_otp.By("6. Check the ROUTER_DISABLE_NAMESPACE_OWNERSHIP_CHECK env variable, which should be false") + namespaceOwnershipEnv = readRouterPodEnv(oc, routerpod, "ROUTER_DISABLE_NAMESPACE_OWNERSHIP_CHECK") + o.Expect(namespaceOwnershipEnv).To(o.ContainSubstring("ROUTER_DISABLE_NAMESPACE_OWNERSHIP_CHECK=false")) + + compat_otp.By("7. Patch the custom ingress controller and set namespaceOwnership to Null") + patchResourceAsAdmin(oc, ingctrl.namespace, "ingresscontrollers/"+ingctrl.name, "{\"spec\":{\"routeAdmission\":{\"namespaceOwnership\":null}}}") + ensureRouterDeployGenerationIs(oc, ingctrl.name, "3") + + compat_otp.By("8. Check the ROUTER_DISABLE_NAMESPACE_OWNERSHIP_CHECK env variable, which should be false") + newRouterpod = getOneNewRouterPodFromRollingUpdate(oc, ingctrl.name) + namespaceOwnershipEnv = readRouterPodEnv(oc, newRouterpod, "ROUTER_DISABLE_NAMESPACE_OWNERSHIP_CHECK") + o.Expect(namespaceOwnershipEnv).To(o.ContainSubstring("ROUTER_DISABLE_NAMESPACE_OWNERSHIP_CHECK=false")) + + // Incorporating the negative case OCP-27605 here + compat_otp.By("9. Patch the custom ingress controller and set namespaceOwnership to a invalid string like 'InvalidTest'") + output, _ := oc.AsAdmin().WithoutNamespace().Run("patch").Args("ingresscontroller/"+ingctrl.name, "-p", "{\"spec\":{\"routeAdmission\":{\"namespaceOwnership\":\"InvalidTest\"}}}", "--type=merge", "-n", ingctrl.namespace).Output() + o.Expect(output).To(o.ContainSubstring("spec.routeAdmission.namespaceOwnership: Unsupported value: \"InvalidTest\": supported values: \"InterNamespaceAllowed\", \"Strict\"")) + }) + + // Test case creater: hongli@redhat.com + g.It("Author:mjoseph-NonHyperShiftHOST-Critical-30190-Set wildcardPolicy of routeAdmission to WildcardsAllowed", func() { + var ( + buildPruningBaseDir = testdata.FixturePath("router") + testPodSvc = filepath.Join(buildPruningBaseDir, "web-server-deploy.yaml") + customTemp = filepath.Join(buildPruningBaseDir, "ingresscontroller-np.yaml") + srvName = "service-unsecure" + ingctrl = ingressControllerDescription{ + name: "ocp30190", + namespace: "openshift-ingress-operator", + domain: "", + template: customTemp, + } + ) + compat_otp.By("1. Create a custom ingresscontroller") + ns := oc.Namespace() + baseDomain := getBaseDomain(oc) + ingctrl.domain = ingctrl.name + "." + baseDomain + // Updating wildcardPolicy as 'WildcardsAllowed' in the yaml file + sedCmd := fmt.Sprintf(`sed -i'' -e 's|WildcardsDisallowed|%s|g' %s`, "WildcardsAllowed", customTemp) + _, err := exec.Command("bash", "-c", sedCmd).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + defer ingctrl.delete(oc) + ingctrl.create(oc) + ensureRouterDeployGenerationIs(oc, ingctrl.name, "1") + routehost := "wildcard." + ns + "." + ingctrl.domain + anyhost := "any." + ns + "." + ingctrl.domain + custContPod := getOneNewRouterPodFromRollingUpdate(oc, ingctrl.name) + + compat_otp.By("2. Check the ROUTER_ALLOW_WILDCARD_ROUTES env variable, which should be true") + namespaceOwnershipEnv := readRouterPodEnv(oc, custContPod, "ROUTER_ALLOW_WILDCARD_ROUTES") + o.Expect(namespaceOwnershipEnv).To(o.ContainSubstring("ROUTER_ALLOW_WILDCARD_ROUTES=true")) + + compat_otp.By("3. Create a server pod and an unsecure service") + createResourceFromFile(oc, ns, testPodSvc) + ensurePodWithLabelReady(oc, ns, "name=web-server-deploy") + + compat_otp.By("4. Expose a http wildcard route") + err = oc.WithoutNamespace().Run("expose").Args("service", srvName, "--hostname="+routehost, "-n", ns, "--wildcard-policy=Subdomain").Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + getRoutes(oc, ns) + + compat_otp.By("5. Check the reachability of the wildcard route") + ingressContPod := getPodListByLabel(oc, "openshift-ingress-operator", "name=ingress-operator") + iplist := getPodIP(oc, "openshift-ingress", custContPod) + toDst := routehost + ":80:" + iplist[0] + cmdOnPod := []string{"-n", "openshift-ingress-operator", ingressContPod[0], "--", "curl", "-I", "http://" + routehost, "--resolve", toDst, "--connect-timeout", "10"} + result, _ := repeatCmdOnClient(oc, cmdOnPod, "200", 30, 1) + o.Expect(result).To(o.ContainSubstring("200")) + + compat_otp.By("6. Check the reachability of the test route") + toDst = anyhost + ":80:" + iplist[0] + cmdOnPod = []string{"-n", "openshift-ingress-operator", ingressContPod[0], "--", "curl", "-I", "http://" + anyhost, "--resolve", toDst, "--connect-timeout", "10"} + result, _ = repeatCmdOnClient(oc, cmdOnPod, "200", 30, 1) + o.Expect(result).To(o.ContainSubstring("200")) + }) + + // Test case creater: hongli@redhat.com + // For OCP-30191 and OCP-30192 + g.It("Author:mjoseph-NonHyperShiftHOST-Medium-30191-Set wildcardPolicy of routeAdmission to WildcardsDisallowed", func() { + var ( + buildPruningBaseDir = testdata.FixturePath("router") + testPodSvc = filepath.Join(buildPruningBaseDir, "web-server-deploy.yaml") + customTemp = filepath.Join(buildPruningBaseDir, "ingresscontroller-np.yaml") + srvName = "service-unsecure" + ingctrl = ingressControllerDescription{ + name: "ocp30191", + namespace: "openshift-ingress-operator", + domain: "", + template: customTemp, + } + ) + compat_otp.By("1. Create a custom ingresscontroller") + ns := oc.Namespace() + ingctrl.domain = ingctrl.name + "." + getBaseDomain(oc) + defer ingctrl.delete(oc) + ingctrl.create(oc) + ensureRouterDeployGenerationIs(oc, ingctrl.name, "1") + routehost := "wildcard." + ns + "." + ingctrl.domain + custContPod := getOneNewRouterPodFromRollingUpdate(oc, ingctrl.name) + + compat_otp.By("2. Check the ROUTER_ALLOW_WILDCARD_ROUTES env variable, which should be false") + namespaceOwnershipEnv := readRouterPodEnv(oc, custContPod, "ROUTER_ALLOW_WILDCARD_ROUTES") + o.Expect(namespaceOwnershipEnv).To(o.ContainSubstring("ROUTER_ALLOW_WILDCARD_ROUTES=false")) + + compat_otp.By("3. Create a server pod and an unsecure service") + createResourceFromFile(oc, ns, testPodSvc) + ensurePodWithLabelReady(oc, ns, "name=web-server-deploy") + + compat_otp.By("4. Expose a http wildcard route") + err := oc.WithoutNamespace().Run("expose").Args("service", srvName, "--hostname="+routehost, "-n", ns, "--wildcard-policy=Subdomain").Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + getRoutes(oc, ns) + + compat_otp.By("5. Confirm the route status is 'RouteNotAdmitted' and confirm the route is not accessible") + ensureRouteIsNotAdmittedByIngressController(oc, ns, "service-unsecure", ingctrl.name) + ingressContPod := getPodListByLabel(oc, "openshift-ingress-operator", "name=ingress-operator") + iplist := getPodIP(oc, "openshift-ingress", custContPod) + curlCmd := fmt.Sprintf("curl --resolve %s:80:%s http://%s -I -k --connect-timeout 10", routehost, iplist[0], routehost) + statsOut, err := compat_otp.RemoteShPod(oc, "openshift-ingress-operator", ingressContPod[0], "sh", "-c", curlCmd) + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(statsOut).Should(o.ContainSubstring("HTTP/1.0 503 Service Unavailable")) + + // Incorporating the negative case OCP-30192 here + compat_otp.By("6. Patch the custom ingress controller and set wildcardPolicy to a invalid string like 'unknown'") + output, _ := oc.AsAdmin().WithoutNamespace().Run("patch").Args("ingresscontroller/"+ingctrl.name, "-p", "{\"spec\":{\"routeAdmission\":{\"wildcardPolicy\":\"unknown\"}}}", "--type=merge", "-n", ingctrl.namespace).Output() + o.Expect(output).To(o.ContainSubstring("spec.routeAdmission.wildcardPolicy: Unsupported value: \"unknown\": supported values: \"WildcardsAllowed\", \"WildcardsDisallowed\"")) + }) +}) diff --git a/tests-extension/test/e2e/route-hsts.go b/tests-extension/test/e2e/route-hsts.go new file mode 100644 index 000000000..c943fe7cd --- /dev/null +++ b/tests-extension/test/e2e/route-hsts.go @@ -0,0 +1,637 @@ +package router + +import ( + "path/filepath" + + g "github.com/onsi/ginkgo/v2" + o "github.com/onsi/gomega" + + //e2e "k8s.io/kubernetes/test/e2e/framework" + + compat_otp "github.com/openshift/origin/test/extended/util/compat_otp" + "github.com/openshift/router-tests-extension/test/testdata" +) + +var _ = g.Describe("[sig-network-edge] Network_Edge Component_Router", func() { + defer g.GinkgoRecover() + + var oc = compat_otp.NewCLI("router-hsts", compat_otp.KubeConfigPath()) + + // incorporate OCP-15976, OCP-16368 and OCP-16369 into one + // Test case creater: zzhao@redhat.com, modified by iamin@redhat.com - OCP-15976: The edge route should support HSTS + // Test case creater: zzhao@redhat.com - OCP-16368: The reencrypt route should support HSTS + // Test case creater: zzhao@redhat.com, modfiied by iamin@redhat.com - OCP-16369: The unsecure/passthrough route should NOT support HSTS + g.It("Author:iamin-ROSA-OSD_CCS-ARO-Critical-15976-The edge/reencrypt route supports HSTS but unsecure/passthrough route does not", func() { + buildPruningBaseDir := testdata.FixturePath("router") + testPodSvc := filepath.Join(buildPruningBaseDir, "web-server-signed-deploy.yaml") + + compat_otp.By("1. Create a server pod and its service") + routerPod := getOneRouterPodNameByIC(oc, "default") + ns := oc.Namespace() + createResourceFromWebServer(oc, ns, testPodSvc, "web-server-deploy") + + compat_otp.By("2. Create an edge route") + createRoute(oc, ns, "edge", "edge-route", "service-unsecure", []string{}) + ensureRouteIsAdmittedByIngressController(oc, ns, "edge-route", "default") + edgeHost := "edge-route-" + ns + ".apps." + getBaseDomain(oc) + + compat_otp.By("3. Set the Strict-Transport-Security max-age header as annotation") + setAnnotation(oc, ns, "route/edge-route", "haproxy.router.openshift.io/hsts_header=max-age=31536000") + ensureHaproxyBlockConfigContains(oc, routerPod, ns, []string{"set-header Strict-Transport-Security 'max-age=31536000'"}) + + compat_otp.By("4. Check the reachability of the host using the HSTS header") + result := waitForOutsideCurlContains("--head https://"+edgeHost, "-k", `200`) + o.Expect(result).To(o.ContainSubstring(`strict-transport-security: max-age=31536000`)) + + compat_otp.By("5. Set the Strict-Transport-Security max-age, subdomains header as annotation") + setAnnotation(oc, ns, "route/edge-route", "haproxy.router.openshift.io/hsts_header=max-age=31536000;includeSubDomains") + ensureHaproxyBlockConfigContains(oc, routerPod, ns, []string{"set-header Strict-Transport-Security 'max-age=31536000;includeSubDomains'"}) + + compat_otp.By("6. Check the reachability of the host using the HSTS header") + result = waitForOutsideCurlContains("--head https://"+edgeHost, "-k", `200`) + o.Expect(result).To(o.ContainSubstring(`strict-transport-security: max-age=31536000;includeSubDomains`)) + + compat_otp.By("7. Set the Strict-Transport-Security max-age, subdomains preload header as annotation") + setAnnotation(oc, ns, "route/edge-route", "haproxy.router.openshift.io/hsts_header=max-age=100;includeSubDomains;preload") + ensureHaproxyBlockConfigContains(oc, routerPod, ns, []string{"set-header Strict-Transport-Security 'max-age=100;includeSubDomains;preload'"}) + + compat_otp.By("8. Check the reachability of the host using the HSTS header") + result = waitForOutsideCurlContains("--head https://"+edgeHost, "-k", `200`) + o.Expect(result).To(o.ContainSubstring(`strict-transport-security: max-age=100;includeSubDomains;preload`)) + + // OCP-16368: The reencrypt route should support HSTS + compat_otp.By("9. Create a reencrypt route") + createRoute(oc, ns, "reencrypt", "reen-route", "service-secure", []string{}) + ensureRouteIsAdmittedByIngressController(oc, ns, "reen-route", "default") + reenHost := "reen-route-" + ns + ".apps." + getBaseDomain(oc) + + compat_otp.By("10. Set the Strict-Transport-Security header as annotation") + setAnnotation(oc, ns, "route/reen-route", "haproxy.router.openshift.io/hsts_header=max-age=100;includeSubDomains;preload") + ensureHaproxyBlockConfigContains(oc, routerPod, "backend be_secure:"+ns+":reen-route", []string{"set-header Strict-Transport-Security 'max-age=100;includeSubDomains;preload'"}) + + compat_otp.By("11. Check the reachability of the host using the HSTS header") + result = waitForOutsideCurlContains("--head https://"+reenHost, "-k", `200`) + o.Expect(result).To(o.ContainSubstring(`strict-transport-security: max-age=100;includeSubDomains;preload`)) + + //OCP-16369: The unsecure/passthrough route should NOT support HSTS + compat_otp.By("12. Create an unsecure route") + createRoute(oc, ns, "http", "unsec-route", "service-unsecure", []string{}) + ensureRouteIsAdmittedByIngressController(oc, ns, "unsec-route", "default") + unsecureHost := "unsec-route-" + ns + ".apps." + getBaseDomain(oc) + + compat_otp.By("13. Set the Strict-Transport-Security max-age, subdomains preload header and cookie-name header as annotation") + setAnnotation(oc, ns, "route/unsec-route", "haproxy.router.openshift.io/hsts_header=max-age=31536000;includeSubDomains;preload") + setAnnotation(oc, ns, "route/unsec-route", "router.openshift.io/cookie_name=unsecure-cookie_1") + searchOutput := ensureHaproxyBlockConfigContains(oc, routerPod, "backend be_http:"+ns+":unsec-route", []string{"cookie unsecure-cookie_1"}) + o.Expect(searchOutput).NotTo(o.ContainSubstring(`set-header Strict-Transport-Security`)) + + compat_otp.By("14. Check the reachability of the unsecure host using the HSTS header") + result = waitForOutsideCurlContains("--head http://"+unsecureHost, "", `200`) + o.Expect(result).NotTo(o.ContainSubstring(`strict-transport-security`)) + o.Expect(result).To(o.MatchRegexp("(S|s)et-(C|c)ookie: unsecure-cookie_1")) + + compat_otp.By("15. Create a passthrough route") + createRoute(oc, ns, "passthrough", "pass-route", "service-secure", []string{}) + ensureRouteIsAdmittedByIngressController(oc, ns, "pass-route", "default") + passthroughHost := "pass-route-" + ns + ".apps." + getBaseDomain(oc) + + compat_otp.By("16. Set the Strict-Transport-Security max-age header as annotation") + setAnnotation(oc, ns, "route/pass-route", "haproxy.router.openshift.io/hsts_header=max-age=31536000") + searchOutput = ensureHaproxyBlockConfigContains(oc, routerPod, "be_tcp:"+ns+":pass-route", []string{ns}) + o.Expect(searchOutput).NotTo(o.ContainSubstring(`set-header Strict-Transport-Security`)) + + compat_otp.By("17. Check the reachability of the passthrough host using the HSTS header") + result = waitForOutsideCurlContains("--head https://"+passthroughHost, "-k", `200`) + o.Expect(result).NotTo(o.ContainSubstring(`strict-transport-security`)) + }) + + // Test case creater: zzhao@redhat.com, modified by iamin@redhat.com - OCP-15977: Negative testing for route HSTS policy + g.It("Author:iamin-NonHyperShiftHOST-Low-15977-Negative testing for route HSTS policy", func() { + buildPruningBaseDir := testdata.FixturePath("router") + testPodSvc := filepath.Join(buildPruningBaseDir, "web-server-deploy.yaml") + + compat_otp.By("1. Create a server pod and its service") + routerPod := getOneRouterPodNameByIC(oc, "default") + ns := oc.Namespace() + createResourceFromWebServer(oc, ns, testPodSvc, "web-server-deploy") + + compat_otp.By("2. Create an edge route") + createRoute(oc, ns, "edge", "edge-route", "service-unsecure", []string{}) + ensureRouteIsAdmittedByIngressController(oc, ns, "edge-route", "default") + edgeHost := "edge-route-" + ns + ".apps." + getBaseDomain(oc) + + compat_otp.By("3. Set the Negative Strict-Transport-Security max-age header and cookie-name header as annotation") + setAnnotation(oc, ns, "route/edge-route", "haproxy.router.openshift.io/hsts_header=max-age=-20") + setAnnotation(oc, ns, "route/edge-route", "router.openshift.io/cookie_name=edge-with-invalid-hsts") + ensureHaproxyBlockConfigContains(oc, routerPod, ns, []string{"cookie edge-with-invalid-hsts"}) + ensureHaproxyBlockConfigNotContains(oc, routerPod, ns, []string{"set-header Strict-Transport-Security"}) + + compat_otp.By("4. Check the reachability of the host using a negative HSTS header") + result := waitForOutsideCurlContains("--head https://"+edgeHost, "-k", `200`) + o.Expect(result).NotTo(o.ContainSubstring(`strict-transport-security`)) + o.Expect(result).To(o.ContainSubstring(`set-cookie: edge-with-invalid-hsts`)) + + compat_otp.By("5. Set an invalid NOT 'includeSubDomains' Strict-Transport-Security header and cookie-name header as annotation") + setAnnotation(oc, ns, "route/edge-route", "haproxy.router.openshift.io/hsts_header=max-age=20;invalid") + setAnnotation(oc, ns, "route/edge-route", "router.openshift.io/cookie_name=edge-with-invalid-hsts-subdomain") + ensureHaproxyBlockConfigContains(oc, routerPod, ns, []string{"cookie edge-with-invalid-hsts-subdomain"}) + ensureHaproxyBlockConfigNotContains(oc, routerPod, ns, []string{"set-header Strict-Transport-Security"}) + + compat_otp.By("6. Check the reachability of the host using an invalid HSTS header") + result = waitForOutsideCurlContains("--head https://"+edgeHost, "-k", `200`) + o.Expect(result).NotTo(o.ContainSubstring(`strict-transport-security`)) + o.Expect(result).To(o.ContainSubstring(`set-cookie: edge-with-invalid-hsts-subdomain`)) + + compat_otp.By("7. Set an invalid NOT 'preload' Strict-Transport-Security header and cookie-name header as annotation") + setAnnotation(oc, ns, "route/edge-route", "haproxy.router.openshift.io/hsts_header=max-age=20;includeSubDomains;invalid") + setAnnotation(oc, ns, "route/edge-route", "router.openshift.io/cookie_name=edge-with-invalid-hsts-preload") + ensureHaproxyBlockConfigContains(oc, routerPod, ns, []string{"cookie edge-with-invalid-hsts-preload"}) + ensureHaproxyBlockConfigNotContains(oc, routerPod, ns, []string{"set-header Strict-Transport-Security"}) + + compat_otp.By("8. Check the reachability of the edge host using an invalid preload HSTS header") + result = waitForOutsideCurlContains("--head https://"+edgeHost, "-k", `200`) + o.Expect(result).NotTo(o.ContainSubstring(`strict-transport-security`)) + o.Expect(result).To(o.ContainSubstring(`set-cookie: edge-with-invalid-hsts-preload`)) + }) + + // author: aiyengar@redhat.com + // https://issues.redhat.com/browse/OCPBUGS-43431 + g.It("Author:aiyengar-NonHyperShiftHOST-Critical-43474-The includeSubDomainsPolicy parameter can configure subdomain policy to inherit the HSTS policy of parent domain [Serial]", func() { + buildPruningBaseDir := testdata.FixturePath("router") + customTemp := filepath.Join(buildPruningBaseDir, "ingresscontroller-np.yaml") + testPodSvc := filepath.Join(buildPruningBaseDir, "web-server-deploy.yaml") + var ( + ingctrl = ingressControllerDescription{ + name: "ocp43474", + namespace: "openshift-ingress-operator", + domain: "", + template: customTemp, + } + ) + compat_otp.By("Create one custom ingresscontroller") + baseDomain := getBaseDomain(oc) + ingctrl.domain = ingctrl.name + "." + baseDomain + defer ingctrl.delete(oc) + ingctrl.create(oc) + ensureRouterDeployGenerationIs(oc, ingctrl.name, "1") + + compat_otp.By("Deploy project with pods and service resources") + oc.SetupProject() + createResourceFromFile(oc, oc.Namespace(), testPodSvc) + ensurePodWithLabelReady(oc, oc.Namespace(), "name=web-server-deploy") + + compat_otp.By("Expose an edge route via the unsecure service inside project") + var output string + ingctldomain := getIngressctlDomain(oc, ingctrl.name) + routehost := "route-edge" + "-" + oc.Namespace() + "." + ingctrl.domain + createRoute(oc, oc.Namespace(), "edge", "route-edge", "service-unsecure", []string{"--hostname=" + routehost}) + output, err := oc.Run("get").Args("route").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(output).To(o.ContainSubstring("route-edge")) + + compat_otp.By("Annotate the edge route with preload HSTS header option") + setAnnotation(oc, oc.Namespace(), "route/route-edge", "haproxy.router.openshift.io/hsts_header=max-age=50000") + output, err = oc.Run("get").Args("route", "route-edge", "-o=jsonpath={.metadata.annotations}").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(output).To(o.ContainSubstring("haproxy.router.openshift.io/hsts_header")) + + compat_otp.By("Add the HSTS policy to global ingresses resource with IncludeSubdomain enforced to be absent") + defer patchGlobalResourceAsAdmin(oc, "ingresses.config.openshift.io/cluster", "[{\"op\":\"remove\" , \"path\" : \"/spec/requiredHSTSPolicies\" , \"value\" : [{\"domainPatterns\" :"+"['*"+"."+ingctldomain+"'"+"] , \"includeSubDomainsPolicy\" : \"RequireNoIncludeSubDomains\" , \"maxAge\":{}}]}]") + patchGlobalResourceAsAdmin(oc, "ingresses.config.openshift.io/cluster", "[{\"op\":\"add\" , \"path\" : \"/spec/requiredHSTSPolicies\" , \"value\" : [{\"domainPatterns\" :"+"['*"+"."+ingctldomain+"'"+"] , \"includeSubDomainsPolicy\" : \"RequireNoIncludeSubDomains\", \"maxAge\":{}}]}]") + output, err = oc.AsAdmin().WithoutNamespace().Run("get").Args("ingresses.config.openshift.io/cluster", "-o=jsonpath={.spec.requiredHSTSPolicies[0]}").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(output).To(o.ContainSubstring("RequireNoIncludeSubDomains")) + + compat_otp.By("Annotate the edge route with preload option to verify the effect") + output1, err2 := oc.Run("annotate").Args("-n", oc.Namespace(), "route/route-edge", "haproxy.router.openshift.io/hsts_header=max-age=50000;includeSubDomains", "--overwrite").Output() + o.Expect(err2).To(o.HaveOccurred()) + o.Expect(output1).To(o.ContainSubstring("HSTS includeSubDomains must not be specified")) + + compat_otp.By("Add the HSTS policy to global ingresses resource with IncludeSubdomain enforced to be present") + patchGlobalResourceAsAdmin(oc, "ingresses.config.openshift.io/cluster", "[{\"op\":\"add\" , \"path\" : \"/spec/requiredHSTSPolicies\" , \"value\" : [{\"domainPatterns\" :"+"['*"+"."+ingctldomain+"'"+"] , \"maxAge\":{}, \"includeSubDomainsPolicy\" : \"RequireIncludeSubDomains\"}]}]") + output, err = oc.AsAdmin().WithoutNamespace().Run("get").Args("ingresses.config.openshift.io/cluster", "-o=jsonpath={.spec.requiredHSTSPolicies[0]}").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(output).To(o.ContainSubstring("RequireIncludeSubDomains")) + + compat_otp.By("verify the enforced policy by overwriting the route annotation to disable Preload headers") + msg2, err := oc.Run("annotate").WithoutNamespace().Args("route/route-edge", "haproxy.router.openshift.io/hsts_header='max-age=50000'", "--overwrite").Output() + o.Expect(err).To(o.HaveOccurred()) + o.Expect(msg2).To(o.ContainSubstring("HSTS includeSubDomains must be specified")) + }) + + // author: aiyengar@redhat.com + // https://issues.redhat.com/browse/OCPBUGS-43431 + g.It("Author:aiyengar-NonHyperShiftHOST-High-43475-The includeSubDomainsPolicy option can be configured to be permissive with NoOpinion flag [Serial]", func() { + buildPruningBaseDir := testdata.FixturePath("router") + customTemp := filepath.Join(buildPruningBaseDir, "ingresscontroller-np.yaml") + testPodSvc := filepath.Join(buildPruningBaseDir, "web-server-deploy.yaml") + var ( + ingctrl = ingressControllerDescription{ + name: "ocp43475", + namespace: "openshift-ingress-operator", + domain: "", + template: customTemp, + } + ) + compat_otp.By("Create one custom ingresscontroller") + baseDomain := getBaseDomain(oc) + ingctrl.domain = ingctrl.name + "." + baseDomain + defer ingctrl.delete(oc) + ingctrl.create(oc) + ensureRouterDeployGenerationIs(oc, ingctrl.name, "1") + + compat_otp.By("Deploy project with pods and service resources") + oc.SetupProject() + createResourceFromFile(oc, oc.Namespace(), testPodSvc) + ensurePodWithLabelReady(oc, oc.Namespace(), "name=web-server-deploy") + + compat_otp.By("Expose an edge route via the unsecure service inside project") + var output string + ingctldomain := getIngressctlDomain(oc, ingctrl.name) + routehost := "route-edge" + "-" + oc.Namespace() + "." + ingctrl.domain + createRoute(oc, oc.Namespace(), "edge", "route-edge", "service-unsecure", []string{"--hostname=" + routehost}) + output, err := oc.Run("get").Args("route").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(output).To(o.ContainSubstring("route-edge")) + + compat_otp.By("Annotate the edge route with preload HSTS header option") + setAnnotation(oc, oc.Namespace(), "route/route-edge", "haproxy.router.openshift.io/hsts_header=max-age=50000") + output, err = oc.Run("get").Args("route", "route-edge", "-o=jsonpath={.metadata.annotations}").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(output).To(o.ContainSubstring("haproxy.router.openshift.io/hsts_header")) + + compat_otp.By("Add the HSTS policy to global ingresses resource with preload option set to NoOpinion") + defer patchGlobalResourceAsAdmin(oc, "ingresses.config.openshift.io/cluster", "[{\"op\":\"remove\" , \"path\" : \"/spec/requiredHSTSPolicies\" , \"value\" : [{\"domainPatterns\" :"+"['*"+"."+ingctldomain+"'"+"] , \"includeSubDomainsPolicy\" : \"RequireIncludeSubDomains\" , \"maxAge\":{}, \"preloadPolicy\" :\"NoOpinion\"}]}]") + patchGlobalResourceAsAdmin(oc, "ingresses.config.openshift.io/cluster", "[{\"op\":\"add\" , \"path\" : \"/spec/requiredHSTSPolicies\" , \"value\" : [{\"domainPatterns\" :"+"['*"+"."+ingctldomain+"'"+"] , \"maxAge\":{}, \"preloadPolicy\" :\"NoOpinion\"}]}]") + output, err = oc.AsAdmin().WithoutNamespace().Run("get").Args("ingresses.config.openshift.io/cluster", "-o=jsonpath={.spec.requiredHSTSPolicies[0]}").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(output).To(o.ContainSubstring("NoOpinion")) + + compat_otp.By("Annotate the edge route with preload option to verify") + _, err2 := oc.Run("annotate").WithoutNamespace().Args("route/route-edge", "haproxy.router.openshift.io/hsts_header=max-age=50000;preload", "--overwrite").Output() + o.Expect(err2).NotTo(o.HaveOccurred()) + + compat_otp.By("Annotate the edge route without preload option to verify") + _, err2 = oc.Run("annotate").WithoutNamespace().Args("route/route-edge", "haproxy.router.openshift.io/hsts_header=max-age=50000", "--overwrite").Output() + o.Expect(err2).NotTo(o.HaveOccurred()) + }) + + // author: aiyengar@redhat.com + // https://issues.redhat.com/browse/OCPBUGS-43431 + g.It("Author:aiyengar-NonHyperShiftHOST-Critical-43476-The PreloadPolicy option can be set to be enforced strictly to be present or absent in HSTS preload header checks [Serial]", func() { + buildPruningBaseDir := testdata.FixturePath("router") + customTemp := filepath.Join(buildPruningBaseDir, "ingresscontroller-np.yaml") + testPodSvc := filepath.Join(buildPruningBaseDir, "web-server-deploy.yaml") + var ( + ingctrl = ingressControllerDescription{ + name: "ocp43476", + namespace: "openshift-ingress-operator", + domain: "", + template: customTemp, + } + ) + compat_otp.By("Create one custom ingresscontroller") + baseDomain := getBaseDomain(oc) + ingctrl.domain = ingctrl.name + "." + baseDomain + defer ingctrl.delete(oc) + ingctrl.create(oc) + ensureRouterDeployGenerationIs(oc, ingctrl.name, "1") + + compat_otp.By("Deploy project with pods and service resources") + oc.SetupProject() + createResourceFromFile(oc, oc.Namespace(), testPodSvc) + ensurePodWithLabelReady(oc, oc.Namespace(), "name=web-server-deploy") + + compat_otp.By("Expose an edge route via the unsecure service inside project") + var output string + ingctldomain := getIngressctlDomain(oc, ingctrl.name) + routehost := "route-edge" + "-" + oc.Namespace() + "." + ingctrl.domain + createRoute(oc, oc.Namespace(), "edge", "route-edge", "service-unsecure", []string{"--hostname=" + routehost}) + output, err := oc.Run("get").Args("route").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(output).To(o.ContainSubstring("route-edge")) + + compat_otp.By("Annotate the edge route with preload HSTS header option") + setAnnotation(oc, oc.Namespace(), "route/route-edge", "haproxy.router.openshift.io/hsts_header=max-age=50000") + output, err = oc.Run("get").Args("route", "route-edge", "-o=jsonpath={.metadata.annotations}").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(output).To(o.ContainSubstring("haproxy.router.openshift.io/hsts_header")) + + compat_otp.By("Add the HSTS policy to global ingresses resource with preload enforced to be absent") + defer patchGlobalResourceAsAdmin(oc, "ingresses.config.openshift.io/cluster", "[{\"op\":\"remove\" , \"path\" : \"/spec/requiredHSTSPolicies\" , \"value\" : [{\"domainPatterns\" :"+"['*"+"."+ingctldomain+"'"+"] , \"includeSubDomainsPolicy\" : \"RequireIncludeSubDomains\" , \"maxAge\":{}, \"preloadPolicy\" :\"RequireNoPreload\"}]}]") + patchGlobalResourceAsAdmin(oc, "ingresses.config.openshift.io/cluster", "[{\"op\":\"add\" , \"path\" : \"/spec/requiredHSTSPolicies\" , \"value\" : [{\"domainPatterns\" :"+"['*"+"."+ingctldomain+"'"+"] , \"maxAge\":{}, \"preloadPolicy\" :\"RequireNoPreload\"}]}]") + output, err = oc.AsAdmin().WithoutNamespace().Run("get").Args("ingresses.config.openshift.io/cluster", "-o=jsonpath={.spec.requiredHSTSPolicies[0]}").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(output).To(o.ContainSubstring("RequireNoPreload")) + + compat_otp.By("Annotate the edge route with preload option to verify the effect") + output1, err2 := oc.Run("annotate").WithoutNamespace().Args("route/route-edge", "haproxy.router.openshift.io/hsts_header=max-age=50000;preload", "--overwrite").Output() + o.Expect(err2).To(o.HaveOccurred()) + o.Expect(output1).To(o.ContainSubstring("HSTS preload must not be specified")) + + compat_otp.By("Add the HSTS policy to global ingresses resource with preload enforced to be present") + patchGlobalResourceAsAdmin(oc, "ingresses.config.openshift.io/cluster", "[{\"op\":\"add\" , \"path\" : \"/spec/requiredHSTSPolicies\" , \"value\" : [{\"domainPatterns\" :"+"['*"+"."+ingctldomain+"'"+"] , \"maxAge\":{}, \"preloadPolicy\" :\"RequirePreload\"}]}]") + output, err = oc.AsAdmin().WithoutNamespace().Run("get").Args("ingresses.config.openshift.io/cluster", "-o=jsonpath={.spec.requiredHSTSPolicies[0]}").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(output).To(o.ContainSubstring("RequirePreload")) + + compat_otp.By("verify the enforced policy by overwriting the route annotation to disable Preload headers") + msg2, err := oc.Run("annotate").WithoutNamespace().Args("route/route-edge", "haproxy.router.openshift.io/hsts_header='max-age=50000'", "--overwrite").Output() + o.Expect(err).To(o.HaveOccurred()) + o.Expect(msg2).To(o.ContainSubstring("HSTS preload must be specified")) + }) + + // author: aiyengar@redhat.com + // https://issues.redhat.com/browse/OCPBUGS-43431 + g.It("Author:aiyengar-NonHyperShiftHOST-High-43478-The PreloadPolicy option can be configured to be permissive with NoOpinion flag [Serial]", func() { + buildPruningBaseDir := testdata.FixturePath("router") + customTemp := filepath.Join(buildPruningBaseDir, "ingresscontroller-np.yaml") + testPodSvc := filepath.Join(buildPruningBaseDir, "web-server-deploy.yaml") + var ( + ingctrl = ingressControllerDescription{ + name: "ocp43478", + namespace: "openshift-ingress-operator", + domain: "", + template: customTemp, + } + ) + compat_otp.By("Create one custom ingresscontroller") + baseDomain := getBaseDomain(oc) + ingctrl.domain = ingctrl.name + "." + baseDomain + defer ingctrl.delete(oc) + ingctrl.create(oc) + ensureRouterDeployGenerationIs(oc, ingctrl.name, "1") + + compat_otp.By("Deploy project with pods and service resources") + oc.SetupProject() + createResourceFromFile(oc, oc.Namespace(), testPodSvc) + ensurePodWithLabelReady(oc, oc.Namespace(), "name=web-server-deploy") + + compat_otp.By("Expose an edge route via the unsecure service inside project") + var output string + ingctldomain := getIngressctlDomain(oc, ingctrl.name) + routedomain := "route-edge" + "-" + oc.Namespace() + "." + ingctrl.domain + createRoute(oc, oc.Namespace(), "edge", "route-edge", "service-unsecure", []string{"--hostname=" + routedomain}) + output, err := oc.Run("get").Args("route").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(output).To(o.ContainSubstring("route-edge")) + + compat_otp.By("Annotate the edge route with preload HSTS header option") + setAnnotation(oc, oc.Namespace(), "route/route-edge", "haproxy.router.openshift.io/hsts_header=max-age=50000") + output, err = oc.Run("get").Args("route", "route-edge", "-o=jsonpath={.metadata.annotations}").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(output).To(o.ContainSubstring("haproxy.router.openshift.io/hsts_header")) + + compat_otp.By("Add the HSTS policy to global ingresses resource with preload option set to NoOpinion") + defer patchGlobalResourceAsAdmin(oc, "ingresses.config.openshift.io/cluster", "[{\"op\":\"remove\" , \"path\" : \"/spec/requiredHSTSPolicies\" , \"value\" : [{\"domainPatterns\" :"+"['*"+"."+ingctldomain+"'"+"] , \"includeSubDomainsPolicy\" : \"RequireIncludeSubDomains\" , \"maxAge\":{}, \"preloadPolicy\" :\"NoOpinion\"}]}]") + patchGlobalResourceAsAdmin(oc, "ingresses.config.openshift.io/cluster", "[{\"op\":\"add\" , \"path\" : \"/spec/requiredHSTSPolicies\" , \"value\" : [{\"domainPatterns\" :"+"['*"+"."+ingctldomain+"'"+"] , \"maxAge\":{}, \"preloadPolicy\" :\"NoOpinion\"}]}]") + output, err = oc.AsAdmin().WithoutNamespace().Run("get").Args("ingresses.config.openshift.io/cluster", "-o=jsonpath={.spec.requiredHSTSPolicies[0]}").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(output).To(o.ContainSubstring("NoOpinion")) + + compat_otp.By("Annotate the edge route with preload option to verify") + _, err2 := oc.Run("annotate").WithoutNamespace().Args("route/route-edge", "haproxy.router.openshift.io/hsts_header=max-age=50000;preload", "--overwrite").Output() + o.Expect(err2).NotTo(o.HaveOccurred()) + + compat_otp.By("Annotate the edge route without preload option to verify") + _, err2 = oc.Run("annotate").WithoutNamespace().Args("route/route-edge", "haproxy.router.openshift.io/hsts_header=max-age=50000", "--overwrite").Output() + o.Expect(err2).NotTo(o.HaveOccurred()) + }) + + // author: aiyengar@redhat.com + // https://issues.redhat.com/browse/OCPBUGS-43431 + g.It("Author:aiyengar-NonHyperShiftHOST-High-43479-The Maxage HSTS policy strictly adheres to validation of route based based on largestMaxAge and smallestMaxAge parameter [Serial]", func() { + buildPruningBaseDir := testdata.FixturePath("router") + customTemp := filepath.Join(buildPruningBaseDir, "ingresscontroller-np.yaml") + testPodSvc := filepath.Join(buildPruningBaseDir, "web-server-deploy.yaml") + var ( + ingctrl = ingressControllerDescription{ + name: "ocp43479", + namespace: "openshift-ingress-operator", + domain: "", + template: customTemp, + } + ) + compat_otp.By("Create one custom ingresscontroller") + baseDomain := getBaseDomain(oc) + ingctrl.domain = ingctrl.name + "." + baseDomain + defer ingctrl.delete(oc) + ingctrl.create(oc) + ensureRouterDeployGenerationIs(oc, ingctrl.name, "1") + + compat_otp.By("Deploy project with pods and service resources") + oc.SetupProject() + createResourceFromFile(oc, oc.Namespace(), testPodSvc) + ensurePodWithLabelReady(oc, oc.Namespace(), "name=web-server-deploy") + + compat_otp.By("Expose an edge route via the unsecure service inside project") + var output string + ingctldomain := getIngressctlDomain(oc, ingctrl.name) + routehost := "route-edge" + "-" + oc.Namespace() + "." + ingctrl.domain + createRoute(oc, oc.Namespace(), "edge", "route-edge", "service-unsecure", []string{"--hostname=" + routehost}) + output, err := oc.Run("get").Args("route").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(output).To(o.ContainSubstring("route-edge")) + + compat_otp.By("Annotate the edge route with preload HSTS header option") + setAnnotation(oc, oc.Namespace(), "route/route-edge", "haproxy.router.openshift.io/hsts_header=max-age=50000") + output, err = oc.Run("get").Args("route", "route-edge", "-o=jsonpath={.metadata.annotations}").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(output).To(o.ContainSubstring("haproxy.router.openshift.io/hsts_header")) + + compat_otp.By("Add the HSTS policy to global ingresses resource with preload option set to maxAge with lowest and highest timer option") + defer patchGlobalResourceAsAdmin(oc, "ingresses.config.openshift.io/cluster", "[{\"op\":\"remove\" , \"path\" : \"/spec/requiredHSTSPolicies\" , \"value\" : [{\"domainPatterns\" :"+"['*"+"."+ingctldomain+"'"+"] , \"maxAge\":{\"largestMaxAge\": 40000, \"smallestMaxAge\": 100 }}]}]") + patchGlobalResourceAsAdmin(oc, "ingresses.config.openshift.io/cluster", "[{\"op\":\"add\" , \"path\" : \"/spec/requiredHSTSPolicies\" , \"value\" : [{\"domainPatterns\" :"+"['*"+"."+ingctldomain+"'"+"] , \"maxAge\":{\"largestMaxAge\": 40000, \"smallestMaxAge\": 100 }}]}]") + output, err = oc.AsAdmin().WithoutNamespace().Run("get").Args("ingresses.config.openshift.io/cluster", "-o=jsonpath={.spec.requiredHSTSPolicies[0]}").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(output).To(o.ContainSubstring("largestMaxAge")) + + compat_otp.By("verify the enforced policy by overwriting the route annotation with largestMaxAge set higher than globally defined") + msg2, err := oc.Run("annotate").WithoutNamespace().Args("route/route-edge", "haproxy.router.openshift.io/hsts_header='max-age=50000'", "--overwrite").Output() + o.Expect(err).To(o.HaveOccurred()) + o.Expect(msg2).To(o.ContainSubstring("HSTS max-age is greater than maximum age 40000s")) + + compat_otp.By("verify the enforced policy by overwriting the route annotation with largestMaxAge set lower than globally defined") + msg2, err = oc.Run("annotate").WithoutNamespace().Args("route/route-edge", "haproxy.router.openshift.io/hsts_header='max-age=50'", "--overwrite").Output() + o.Expect(err).To(o.HaveOccurred()) + o.Expect(msg2).To(o.ContainSubstring("HSTS max-age is less than minimum age 100s")) + + }) + + // author: aiyengar@redhat.com + // https://issues.redhat.com/browse/OCPBUGS-43431 + g.It("Author:aiyengar-NonHyperShiftHOST-High-43480-The HSTS domain policy can be configure with multiple domainPatterns options [Disruptive]", func() { + buildPruningBaseDir := testdata.FixturePath("router") + customTemp := filepath.Join(buildPruningBaseDir, "ingresscontroller-np.yaml") + testPodSvc := filepath.Join(buildPruningBaseDir, "web-server-deploy.yaml") + var ( + ingctrl1 = ingressControllerDescription{ + name: "ocp43480-1", + namespace: "openshift-ingress-operator", + domain: "", + template: customTemp, + } + ingctrl2 = ingressControllerDescription{ + name: "ocp43480-2", + namespace: "openshift-ingress-operator", + domain: "", + template: customTemp, + } + ) + compat_otp.By("Create first custom ingresscontroller") + baseDomain := getBaseDomain(oc) + ingctrl1.domain = ingctrl1.name + "." + baseDomain + defer ingctrl1.delete(oc) + ingctrl1.create(oc) + ensureRouterDeployGenerationIs(oc, ingctrl1.name, "1") + + compat_otp.By("Create second custom ingresscontroller") + baseDomain = getBaseDomain(oc) + ingctrl2.domain = ingctrl2.name + "." + baseDomain + defer ingctrl2.delete(oc) + ingctrl2.create(oc) + ensureRouterDeployGenerationIs(oc, ingctrl2.name, "1") + + compat_otp.By("Deploy project with pods and service resources") + oc.SetupProject() + createResourceFromFile(oc, oc.Namespace(), testPodSvc) + ensurePodWithLabelReady(oc, oc.Namespace(), "name=web-server-deploy") + + compat_otp.By("Expose an edge route via the unsecure service through ingresscontroller 1 inside project") + var output1 string + ingctldomain1 := getIngressctlDomain(oc, ingctrl1.name) + routehost1 := "route-edge1" + "-" + oc.Namespace() + "." + ingctrl1.domain + createRoute(oc, oc.Namespace(), "edge", "route-edge1", "service-unsecure", []string{"--hostname=" + routehost1}) + output1, err := oc.Run("get").Args("route").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(output1).To(o.ContainSubstring("route-edge1")) + + compat_otp.By("Expose an edge route via the unsecure service through ingresscontroller 2 inside project") + var output2 string + ingctldomain2 := getIngressctlDomain(oc, ingctrl2.name) + routehost2 := "route-edge2" + "-" + oc.Namespace() + "." + ingctrl2.domain + createRoute(oc, oc.Namespace(), "edge", "route-edge2", "service-unsecure", []string{"--hostname=" + routehost2}) + output2, err = oc.Run("get").Args("route").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(output2).To(o.ContainSubstring("route-edge2")) + + compat_otp.By("Annotate the edge route 1 to enable HSTS header option") + setAnnotation(oc, oc.Namespace(), "route/route-edge1", "haproxy.router.openshift.io/hsts_header=max-age=4000") + output, err := oc.Run("get").Args("route", "route-edge1", "-o=jsonpath={.metadata.annotations}").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(output).To(o.ContainSubstring("haproxy.router.openshift.io/hsts_header")) + + compat_otp.By("Annotate the edge route 2 to enable HSTS header option") + setAnnotation(oc, oc.Namespace(), "route/route-edge2", "haproxy.router.openshift.io/hsts_header=max-age=2000") + output, err = oc.Run("get").Args("route", "route-edge2", "-o=jsonpath={.metadata.annotations}").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(output).To(o.ContainSubstring("haproxy.router.openshift.io/hsts_header")) + + compat_otp.By("Set a different HSTS maxage policy for each domain in the global ingresses configuration") + defer patchGlobalResourceAsAdmin(oc, "ingresses.config.openshift.io/cluster", "[{\"op\":\"remove\" , \"path\" : \"/spec/requiredHSTSPolicies\"}]") + patchGlobalResourceAsAdmin(oc, "ingresses.config.openshift.io/cluster", "[{\"op\":\"add\" , \"path\" : \"/spec/requiredHSTSPolicies\" , \"value\" : [{\"domainPatterns\" :"+"['*"+"."+ingctldomain1+"'"+"] , \"includeSubDomainsPolicy\":\"NoOpinion\",\"maxAge\":{\"largestMaxAge\":5000,\"smallestMaxAge\":1},\"preloadPolicy\":\"NoOpinion\"},{\"domainPatterns\":"+" ['*"+"."+ingctldomain2+"'"+"],\"includeSubDomainsPolicy\":\"NoOpinion\",\"maxAge\":{\"largestMaxAge\":3000,\"smallestMaxAge\":1},\"preloadPolicy\":\"NoOpinion\"}]}]") + output, err = oc.AsAdmin().WithoutNamespace().Run("get").Args("ingresses.config.openshift.io/cluster", "-o=jsonpath={.spec.requiredHSTSPolicies[0]}").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(output).To(o.ContainSubstring("largestMaxAge")) + + compat_otp.By("verify the enforced policy by overwriting the annotation for route 1 with max-age set higher than the largestMaxAge defined for the domain") + msg1, err := oc.Run("annotate").WithoutNamespace().Args("route/route-edge1", "haproxy.router.openshift.io/hsts_header='max-age=6000'", "--overwrite").Output() + o.Expect(err).To(o.HaveOccurred()) + o.Expect(msg1).To(o.ContainSubstring("HSTS max-age is greater than maximum age 5000s")) + + compat_otp.By("verify the enforced policy by overwriting the annotation for route 2 with max-age set higher than the largestMaxAge defined for the domain") + msg2, err := oc.Run("annotate").WithoutNamespace().Args("route/route-edge2", "haproxy.router.openshift.io/hsts_header='max-age=4000'", "--overwrite").Output() + o.Expect(err).To(o.HaveOccurred()) + o.Expect(msg2).To(o.ContainSubstring("HSTS max-age is greater than maximum age 3000s")) + + }) + + // author: aiyengar@redhat.com + // https://issues.redhat.com/browse/OCPBUGS-43431 + g.It("Author:aiyengar-NonHyperShiftHOST-High-43884-lobal HSTS policy can be enforced strictly on a specific namespace using namespaceSelector for given domain pattern filtering [Serial]", func() { + buildPruningBaseDir := testdata.FixturePath("router") + customTemp := filepath.Join(buildPruningBaseDir, "ingresscontroller-np.yaml") + testPodSvc := filepath.Join(buildPruningBaseDir, "web-server-deploy.yaml") + var ( + ingctrl = ingressControllerDescription{ + name: "ocp43884", + namespace: "openshift-ingress-operator", + domain: "", + template: customTemp, + } + ) + compat_otp.By("Create one custom ingresscontroller") + baseDomain := getBaseDomain(oc) + ingctrl.domain = ingctrl.name + "." + baseDomain + defer ingctrl.delete(oc) + ingctrl.create(oc) + ensureRouterDeployGenerationIs(oc, ingctrl.name, "1") + + compat_otp.By("Deploy project 1 with pods and service resources") + oc.SetupProject() + project1 := oc.Namespace() + createResourceFromFile(oc, project1, testPodSvc) + ensurePodWithLabelReady(oc, oc.Namespace(), "name=web-server-deploy") + + compat_otp.By("Deploy project 2 with pods and service resources") + oc.SetupProject() + project2 := oc.Namespace() + createResourceFromFile(oc, project2, testPodSvc) + ensurePodWithLabelReady(oc, oc.Namespace(), "name=web-server-deploy") + + compat_otp.By("set up HSTS policy for the custom domain with namespace selector set to label of project1 namespace") + ingctldomain := getIngressctlDomain(oc, ingctrl.name) + defer patchGlobalResourceAsAdmin(oc, "ingresses.config.openshift.io/cluster", "[{\"op\":\"remove\" , \"path\" : \"/spec/requiredHSTSPolicies\"}]") + patchGlobalResourceAsAdmin(oc, "ingresses.config.openshift.io/cluster", "[{\"op\":\"add\" , \"path\" : \"/spec/requiredHSTSPolicies\" , \"value\" : [{\"domainPatterns\" :"+"['*"+"."+ingctldomain+"'"+"],\"includeSubDomainsPolicy\":\"NoOpinion\",\"maxAge\":{\"largestMaxAge\":5000,\"smallestMaxAge\":1},\"namespaceSelector\":{\"matchLabels\":{\"kubernetes.io/metadata.name\":\""+project1+"\"}},\"preloadPolicy\":\"NoOpinion\"}]}]") + output, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("ingresses.config.openshift.io/cluster", "-o=jsonpath={.spec.requiredHSTSPolicies[0]}").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(output).To(o.ContainSubstring("largestMaxAge")) + + compat_otp.By("Test for outcome by creating an edge route via the HSTS implemented domain through the project1") + routehost1 := "route-edge" + "-" + project1 + "." + ingctrl.domain + output, err1 := oc.AsAdmin().WithoutNamespace().Run("create").Args("-n", project1, "route", "edge", "route-edge", "--service=service-unsecure", "--hostname="+routehost1).Output() + o.Expect(err1).To(o.HaveOccurred()) + o.Expect(output).To(o.ContainSubstring("HSTS max-age must be set correctly in HSTS annotation")) + + compat_otp.By("Test for outcome by creating an edge route via the default non-HSTS policy controlled domain through the project2") + routehost2 := "route-edge2" + "-" + project2 + "." + ingctrl.domain + createRoute(oc, project2, "edge", "route-edge", "service-unsecure", []string{"--hostname=" + routehost2}) + output2, err := oc.Run("get").Args("route").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(output2).To(o.ContainSubstring("route-edge2")) + + }) + + // author: aiyengar@redhat.com + g.It("Author:aiyengar-Low-43966-Negative values for largestMaxAge and smallestMaxAge option under Maxage HSTS policy are rejected", func() { + buildPruningBaseDir := testdata.FixturePath("router") + customTemp := filepath.Join(buildPruningBaseDir, "ingresscontroller-np.yaml") + var ( + ingctrl = ingressControllerDescription{ + name: "ocp43966", + namespace: "openshift-ingress-operator", + domain: "", + template: customTemp, + } + ) + compat_otp.By("Create one custom ingresscontroller") + baseDomain := getBaseDomain(oc) + ingctrl.domain = ingctrl.name + "." + baseDomain + defer ingctrl.delete(oc) + ingctrl.create(oc) + ensureRouterDeployGenerationIs(oc, ingctrl.name, "1") + + compat_otp.By("Add the HSTS policy with largestMaxAge set to negative value") + ingctldomain := getIngressctlDomain(oc, ingctrl.name) + patch := "[{\"op\":\"add\" , \"path\" : \"/spec/requiredHSTSPolicies\" , \"value\" : [{\"domainPatterns\" :" + "['*" + "." + ingctldomain + "'" + "] , \"maxAge\":{\"largestMaxAge\": -40000, \"smallestMaxAge\": 100 }}]}]" + output1, err := oc.AsAdmin().WithoutNamespace().Run("patch").Args("ingresses.config.openshift.io/cluster", "--patch="+patch, "--type=json").Output() + o.Expect(err).To(o.HaveOccurred()) + o.Expect(output1).To(o.ContainSubstring("largestMaxAge in body should be greater than or equal to 0")) + + compat_otp.By("Add the HSTS policy with smallestMaxAge set to negative value") + patch = "[{\"op\":\"add\" , \"path\" : \"/spec/requiredHSTSPolicies\" , \"value\" : [{\"domainPatterns\" :" + "['*" + "." + ingctldomain + "'" + "] , \"maxAge\":{\"largestMaxAge\": 40000, \"smallestMaxAge\": -100 }}]}]" + output2, err := oc.AsAdmin().WithoutNamespace().Run("patch").Args("ingresses.config.openshift.io/cluster", "--patch="+patch, "--type=json").Output() + o.Expect(err).To(o.HaveOccurred()) + o.Expect(output2).To(o.ContainSubstring("smallestMaxAge in body should be greater than or equal to 0")) + }) +}) diff --git a/tests-extension/test/e2e/route-weight.go b/tests-extension/test/e2e/route-weight.go new file mode 100644 index 000000000..dc031d5c0 --- /dev/null +++ b/tests-extension/test/e2e/route-weight.go @@ -0,0 +1,581 @@ +package router + +import ( + "fmt" + "os" + "path/filepath" + "strings" + + g "github.com/onsi/ginkgo/v2" + o "github.com/onsi/gomega" + compat_otp "github.com/openshift/origin/test/extended/util/compat_otp" + "github.com/openshift/router-tests-extension/test/testdata" +) + +var _ = g.Describe("[sig-network-edge] Network_Edge Component_Router", func() { + defer g.GinkgoRecover() + + var oc = compat_otp.NewCLI("route-weight", compat_otp.KubeConfigPath()) + + // author: hongli@redhat.com + g.It("Author:hongli-ROSA-OSD_CCS-ARO-Medium-10889-Sticky session could work normally after set weight for route", func() { + var ( + buildPruningBaseDir = testdata.FixturePath("router") + webServerTemplate = filepath.Join(buildPruningBaseDir, "template-web-server-deploy.yaml") + + webServerDeploy1 = webServerDeployDescription{ + deployName: "web-server-deploy1", + svcSecureName: "service-secure1", + svcUnsecureName: "service-unsecure1", + template: webServerTemplate, + namespace: "", + } + + webServerDeploy2 = webServerDeployDescription{ + deployName: "web-server-deploy2", + svcSecureName: "service-secure2", + svcUnsecureName: "service-unsecure2", + template: webServerTemplate, + namespace: "", + } + deploy1Label = "name=" + webServerDeploy1.deployName + deploy2Label = "name=" + webServerDeploy2.deployName + fileDir = "/tmp/OCP-10889" + cookie = fileDir + "/cookie" + routeName = "edge10889" + ) + + compat_otp.By("Deploy two sets of web-server and services") + ns := oc.Namespace() + webServerDeploy1.namespace = ns + webServerDeploy1.create(oc) + webServerDeploy2.namespace = ns + webServerDeploy2.create(oc) + ensurePodWithLabelReady(oc, ns, deploy1Label) + ensurePodWithLabelReady(oc, ns, deploy2Label) + + compat_otp.By("Create edge route and set route-backends with multi serivces") + createRoute(oc, ns, "edge", routeName, webServerDeploy1.svcUnsecureName, []string{}) + ensureRouteIsAdmittedByIngressController(oc, ns, routeName, "default") + // Note: the "balance roundrobin" is used for the route once set route-backends, no need to annotate the route" + err := oc.Run("set").Args("route-backends", routeName, "service-unsecure1=60", "service-unsecure2=40").Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + + compat_otp.By("Check haproxy.config and ensure deploy2 pod is added") + routerPod := getOneRouterPodNameByIC(oc, "default") + ensureHaproxyBlockConfigMatchRegexp(oc, routerPod, "be_edge_http:"+ns+":"+routeName, []string{"server pod:" + webServerDeploy2.deployName + ".+weight 170"}) + + compat_otp.By("Access the route, ensure web server 2 is in service and save the cookie") + defer os.RemoveAll(fileDir) + err = os.MkdirAll(fileDir, 0755) + o.Expect(err).NotTo(o.HaveOccurred()) + edgeRouteHost := getRouteHost(oc, ns, routeName) + curlCmd := fmt.Sprintf(`curl https://%s -sk -c %s --connect-timeout 10`, edgeRouteHost, cookie) + expectedOutput := []string{"Hello-OpenShift web-server-deploy2"} + repeatCmdOnClient(oc, curlCmd, expectedOutput, 60, 1) + + compat_otp.By("Access the route several times withoug cookie and ensure web server 1 is in service as well") + curlCmd = fmt.Sprintf(`curl https://%s -sk --connect-timeout 10`, edgeRouteHost) + expectedOutput = []string{"Hello-OpenShift web-server-deploy1"} + repeatCmdOnClient(oc, curlCmd, expectedOutput, 60, 1) + + compat_otp.By("Access the route with the saved cookie for 6 times, ensure only web server 2 provides the service") + curlCmd = fmt.Sprintf(`curl https://%s -sk -b %s --connect-timeout 10`, edgeRouteHost, cookie) + expectedOutput = []string{"Hello-OpenShift web-server-deploy1", "Hello-OpenShift web-server-deploy2"} + _, result := repeatCmdOnClient(oc, curlCmd, expectedOutput, 90, 6) + o.Expect(result[0]).To(o.Equal(0)) + o.Expect(result[1]).To(o.Equal(6)) + }) + + // author: hongli@redhat.com + // Includes OCP-11306: Set negative backends weight for ab routing + // OCP-15382: Set max backends weight for ab routing + g.It("Author:hongli-ROSA-OSD_CCS-ARO-Low-11351-Set backends weight to zero for ab routing", func() { + var ( + buildPruningBaseDir = testdata.FixturePath("router") + webServerTemplate = filepath.Join(buildPruningBaseDir, "template-web-server-deploy.yaml") + + webServerDeploy1 = webServerDeployDescription{ + deployName: "web-server-deploy1", + svcSecureName: "service-secure1", + svcUnsecureName: "service-unsecure1", + template: webServerTemplate, + namespace: "", + } + + webServerDeploy2 = webServerDeployDescription{ + deployName: "web-server-deploy2", + svcSecureName: "service-secure2", + svcUnsecureName: "service-unsecure2", + template: webServerTemplate, + namespace: "", + } + deploy1Label = "name=" + webServerDeploy1.deployName + deploy2Label = "name=" + webServerDeploy2.deployName + routeName = "edge11351" + ) + + compat_otp.By("Deploy two sets of web-server and services") + ns := oc.Namespace() + webServerDeploy1.namespace = ns + webServerDeploy1.create(oc) + webServerDeploy2.namespace = ns + webServerDeploy2.create(oc) + ensurePodWithLabelReady(oc, ns, deploy1Label) + ensurePodWithLabelReady(oc, ns, deploy2Label) + + compat_otp.By("Create edge route and set route-backends with multi serivces") + createRoute(oc, ns, "edge", routeName, webServerDeploy1.svcUnsecureName, []string{}) + ensureRouteIsAdmittedByIngressController(oc, ns, routeName, "default") + err := oc.Run("set").Args("route-backends", routeName, "service-unsecure1=0", "service-unsecure2=1").Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + + compat_otp.By("Check haproxy.config and ensure weight of deploy2 is 1") + routerPod := getOneRouterPodNameByIC(oc, "default") + ensureHaproxyBlockConfigMatchRegexp(oc, routerPod, "be_edge_http:"+ns+":"+routeName, []string{"server pod:" + webServerDeploy2.deployName + ".+weight 1"}) + + compat_otp.By("Access the route for 6 times, ensure only deploy2 is in service") + edgeRouteHost := getRouteHost(oc, ns, routeName) + curlCmd := fmt.Sprintf(`curl https://%s -sk --connect-timeout 10`, edgeRouteHost) + expectedOutput := []string{"Hello-OpenShift web-server-deploy1", "Hello-OpenShift web-server-deploy2"} + _, result := repeatCmdOnClient(oc, curlCmd, expectedOutput, 90, 6) + o.Expect(result[0]).To(o.Equal(0)) + o.Expect(result[1]).To(o.Equal(6)) + + compat_otp.By("Set route-backends to zero for all serivces/backends") + err = oc.Run("set").Args("route-backends", routeName, "--zero=true").Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + + compat_otp.By("Check haproxy.config and ensure weight of deploy2 is 0") + ensureHaproxyBlockConfigMatchRegexp(oc, routerPod, "be_edge_http:"+ns+":"+routeName, []string{"server pod:" + webServerDeploy2.deployName + ".+weight 0"}) + + compat_otp.By("Access the route for 6 times, ensure all request are failing") + curlCmd = fmt.Sprintf(`curl https://%s -skI --connect-timeout 10`, edgeRouteHost) + expectedOutput = []string{"503"} + _, result = repeatCmdOnClient(oc, curlCmd, expectedOutput, 90, 6) + o.Expect(result[0]).To(o.Equal(6)) + + compat_otp.By("Attempt to set route-backends to char") + output, err := oc.Run("set").Args("route-backends", routeName, "service-unsecure1=abc", "service-unsecure2=^*%").Output() + o.Expect(err).To(o.HaveOccurred()) + o.Expect(output).To(o.MatchRegexp("invalid argument.*WEIGHT must be a number")) + + compat_otp.By("Attempt to set route-backends to negative weight") + output, err = oc.Run("set").Args("route-backends", routeName, "service-unsecure1=-80", "service-unsecure2=-20").Output() + o.Expect(err).To(o.HaveOccurred()) + o.Expect(output).To(o.MatchRegexp("negative percentages are not allowed")) + + compat_otp.By("Attempt to set route-backends weight to 257") + output, err = oc.Run("set").Args("route-backends", routeName, "service-unsecure1=257", "service-unsecure2=0").Output() + o.Expect(err).To(o.HaveOccurred()) + o.Expect(output).To(o.MatchRegexp("weight must be an integer between 0 and 256")) + }) + + // author: hongli@redhat.com + // Includes OCP-11809: Set backends weight for passthough route + // OCP-11970: Set backends weight for reencrypt route + // OCP-12076: Set backends weight for unsecure route + g.It("Author:hongli-ROSA-OSD_CCS-ARO-High-11608-Set backends weight for edge/passthrough/reencrypt/unsecure route", func() { + var ( + buildPruningBaseDir = testdata.FixturePath("router") + webServerTemplate = filepath.Join(buildPruningBaseDir, "template-web-server-deploy.yaml") + destCA = filepath.Join(buildPruningBaseDir, "ca-bundle.pem") + + webServerDeploy1 = webServerDeployDescription{ + deployName: "web-server-deploy1", + svcSecureName: "service-secure1", + svcUnsecureName: "service-unsecure1", + template: webServerTemplate, + namespace: "", + } + + webServerDeploy2 = webServerDeployDescription{ + deployName: "web-server-deploy2", + svcSecureName: "service-secure2", + svcUnsecureName: "service-unsecure2", + template: webServerTemplate, + namespace: "", + } + deploy1Label = "name=" + webServerDeploy1.deployName + deploy2Label = "name=" + webServerDeploy2.deployName + edgeRouteName = "edge11608" + passRouteName = "pass11608" + reenRouteName = "reen11608" + unsecureRouteName = "unsecure11608" + ) + + compat_otp.By("Deploy two sets of web-server and services") + ns := oc.Namespace() + webServerDeploy1.namespace = ns + webServerDeploy1.create(oc) + webServerDeploy2.namespace = ns + webServerDeploy2.create(oc) + ensurePodWithLabelReady(oc, ns, deploy1Label) + ensurePodWithLabelReady(oc, ns, deploy2Label) + + compat_otp.By("Create edge route and set route-backends with multi serivces") + createRoute(oc, ns, "edge", edgeRouteName, webServerDeploy1.svcUnsecureName, []string{}) + ensureRouteIsAdmittedByIngressController(oc, ns, edgeRouteName, "default") + err := oc.Run("set").Args("route-backends", edgeRouteName, "service-unsecure1=10", "service-unsecure2=10").Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + + compat_otp.By("Create passthrough route and set route-backends with multi serivces") + createRoute(oc, ns, "passthrough", passRouteName, webServerDeploy1.svcSecureName, []string{}) + ensureRouteIsAdmittedByIngressController(oc, ns, passRouteName, "default") + err = oc.Run("set").Args("route-backends", passRouteName, "service-secure1=20%", "service-secure2=80%").Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + + compat_otp.By("Create reencrypt route and set route-backends with multi serivces") + createRoute(oc, ns, "reencrypt", reenRouteName, webServerDeploy1.svcSecureName, []string{"--dest-ca-cert=" + destCA}) + ensureRouteIsAdmittedByIngressController(oc, ns, reenRouteName, "default") + // Note: the "balance roundrobin" is used for the route once set route-backends, no need to annotate the route" + err = oc.Run("set").Args("route-backends", reenRouteName, "service-secure1=256", "service-secure2=256").Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + + compat_otp.By("Create unsecure route and set route-backends with multi serivces") + createRoute(oc, ns, "http", unsecureRouteName, webServerDeploy1.svcUnsecureName, []string{}) + ensureRouteIsAdmittedByIngressController(oc, ns, unsecureRouteName, "default") + // Note: the "balance roundrobin" is used for the route once set route-backends, no need to annotate the route" + err = oc.Run("set").Args("route-backends", unsecureRouteName, "service-unsecure1=50", "service-unsecure2=100").Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + + compat_otp.By("Check edge route weight in haproxy.config") + routerPod := getOneRouterPodNameByIC(oc, "default") + backendBegin := "be_edge_http:" + ns + ":" + edgeRouteName + ensureHaproxyBlockConfigMatchRegexp(oc, routerPod, backendBegin, []string{"server pod:" + webServerDeploy1.deployName + ".+weight 256", "server pod:" + webServerDeploy2.deployName + ".+weight 256"}) + + compat_otp.By("Check passthrough route weight in haproxy.config") + backendBegin = "be_tcp:" + ns + ":" + passRouteName + ensureHaproxyBlockConfigMatchRegexp(oc, routerPod, backendBegin, []string{"server pod:" + webServerDeploy1.deployName + ".+weight 64", "server pod:" + webServerDeploy2.deployName + ".+weight 256"}) + + compat_otp.By("Check reencryp route weight in haproxy.config") + backendBegin = "be_secure:" + ns + ":" + reenRouteName + ensureHaproxyBlockConfigMatchRegexp(oc, routerPod, backendBegin, []string{"server pod:" + webServerDeploy1.deployName + ".+weight 256", "server pod:" + webServerDeploy2.deployName + ".+weight 256"}) + + compat_otp.By("Check unsecure route weight in haproxy.config") + backendBegin = "be_http:" + ns + ":" + unsecureRouteName + ensureHaproxyBlockConfigMatchRegexp(oc, routerPod, backendBegin, []string{"server pod:" + webServerDeploy1.deployName + ".+weight 128", "server pod:" + webServerDeploy2.deployName + ".+weight 256"}) + }) + + // author: hongli@redhat.com + // Includes OCP-15259: Could not set more than 3 additional backends for route + // OCP-13521: The passthrough route with multiple service will set load balance policy to RoundRobin by default + g.It("Author:hongli-ROSA-OSD_CCS-ARO-Medium-12088-Set multiple backends weight for route", func() { + var ( + buildPruningBaseDir = testdata.FixturePath("router") + webServerTemplate = filepath.Join(buildPruningBaseDir, "template-web-server-deploy.yaml") + + webServerDeploy1 = webServerDeployDescription{ + deployName: "web-server-deploy1", + svcSecureName: "service-secure1", + svcUnsecureName: "service-unsecure1", + template: webServerTemplate, + namespace: "", + } + + webServerDeploy2 = webServerDeployDescription{ + deployName: "web-server-deploy2", + svcSecureName: "service-secure2", + svcUnsecureName: "service-unsecure2", + template: webServerTemplate, + namespace: "", + } + + webServerDeploy3 = webServerDeployDescription{ + deployName: "web-server-deploy3", + svcSecureName: "service-secure3", + svcUnsecureName: "service-unsecure3", + template: webServerTemplate, + namespace: "", + } + + webServerDeploy4 = webServerDeployDescription{ + deployName: "web-server-deploy4", + svcSecureName: "service-secure4", + svcUnsecureName: "service-unsecure4", + template: webServerTemplate, + namespace: "", + } + + deploy1Label = "name=" + webServerDeploy1.deployName + deploy2Label = "name=" + webServerDeploy2.deployName + deploy3Label = "name=" + webServerDeploy3.deployName + deploy4Label = "name=" + webServerDeploy4.deployName + routeName = "pass12088" + ) + + compat_otp.By("Deploy four sets of web-server and services, three of them will be set as alternate backends") + ns := oc.Namespace() + webServerDeploy1.namespace = ns + webServerDeploy1.create(oc) + webServerDeploy2.namespace = ns + webServerDeploy2.create(oc) + webServerDeploy3.namespace = ns + webServerDeploy3.create(oc) + webServerDeploy4.namespace = ns + webServerDeploy4.create(oc) + ensurePodWithLabelReady(oc, ns, deploy1Label) + ensurePodWithLabelReady(oc, ns, deploy2Label) + ensurePodWithLabelReady(oc, ns, deploy3Label) + ensurePodWithLabelReady(oc, ns, deploy4Label) + + compat_otp.By("Create edge route and set route-backends with multi serivces") + createRoute(oc, ns, "passthrough", routeName, webServerDeploy1.svcSecureName, []string{}) + ensureRouteIsAdmittedByIngressController(oc, ns, routeName, "default") + // Note: the "balance roundrobin" is used for the route once set route-backends, no need to annotate the route" + err := oc.Run("set").Args("route-backends", routeName, "service-secure1=10", "service-secure2=20", "service-secure3=30", "service-secure4=40").Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + + compat_otp.By("Check haproxy.config and ensure weight of deploy2/3/4 is added and balance is roundrobin") + routerPod := getOneRouterPodNameByIC(oc, "default") + backendBegin := "be_tcp:" + ns + ":" + routeName + ensureHaproxyBlockConfigMatchRegexp(oc, routerPod, backendBegin, []string{"server pod:" + webServerDeploy1.deployName + ".+weight 64", "server pod:" + webServerDeploy2.deployName + ".+weight 128", "server pod:" + webServerDeploy3.deployName + ".+weight 192", "server pod:" + webServerDeploy4.deployName + ".+weight 256"}) + + compat_otp.By("Attempt to set route-backends more than 3 alternate backends") + output, err := oc.Run("set").Args("route-backends", routeName, "service-secure1=1", "service-secure2=1", "service-secure3=1", "service-secure4=1", "service-secure5=1").Output() + o.Expect(err).To(o.HaveOccurred()) + o.Expect(output).To(o.MatchRegexp("cannot specify more than 3 .*backends")) + }) + + // author: hongli@redhat.com + g.It("Author:hongli-ROSA-OSD_CCS-ARO-Medium-15902-Endpoint will end up weight 1 when scaled weight per endpoint is less than 1", func() { + var ( + buildPruningBaseDir = testdata.FixturePath("router") + webServerTemplate = filepath.Join(buildPruningBaseDir, "template-web-server-deploy.yaml") + + webServerDeploy1 = webServerDeployDescription{ + deployName: "web-server-deploy1", + svcSecureName: "service-secure1", + svcUnsecureName: "service-unsecure1", + template: webServerTemplate, + namespace: "", + } + + webServerDeploy2 = webServerDeployDescription{ + deployName: "web-server-deploy2", + svcSecureName: "service-secure2", + svcUnsecureName: "service-unsecure2", + template: webServerTemplate, + namespace: "", + } + deploy1Label = "name=" + webServerDeploy1.deployName + deploy2Label = "name=" + webServerDeploy2.deployName + routeName = "edge15902" + ) + + compat_otp.By("Deploy two sets of web-server and services") + ns := oc.Namespace() + webServerDeploy1.namespace = ns + webServerDeploy1.create(oc) + webServerDeploy2.namespace = ns + webServerDeploy2.create(oc) + ensurePodWithLabelReady(oc, ns, deploy1Label) + ensurePodWithLabelReady(oc, ns, deploy2Label) + + compat_otp.By("Scale deploy1 to replicas 2") + scaleDeploy(oc, ns, webServerDeploy1.deployName, 2) + waitForOutputEquals(oc, ns, "deployment/"+webServerDeploy1.deployName, "{.status.readyReplicas}", "2") + + compat_otp.By("Create edge route and set route-backends with multi serivces") + createRoute(oc, ns, "edge", routeName, webServerDeploy1.svcUnsecureName, []string{}) + ensureRouteIsAdmittedByIngressController(oc, ns, routeName, "default") + err := oc.Run("set").Args("route-backends", routeName, "service-unsecure1=1", "service-unsecure2=256").Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + + compat_otp.By("Check haproxy.config and ensure weight of each deploy1 pod is 1") + routerPod := getOneRouterPodNameByIC(oc, "default") + backendBegin := "be_edge_http:" + ns + ":" + routeName + backendConfig := ensureHaproxyBlockConfigMatchRegexp(oc, routerPod, backendBegin, []string{"server pod:" + webServerDeploy1.deployName + ".+weight 1", "server pod:" + webServerDeploy2.deployName + ".+weight 256"}) + o.Expect(strings.Count(backendConfig, "weight 1")).To(o.Equal(2)) + }) + + // author: hongli@redhat.com + // Includes OCP-15994: Each endpoint gets weight/numberOfEndpoints portion of the requests - passthrough route + // OCP-15993: Each endpoint gets weight/numberOfEndpoints portion of the requests - edge route + // OCP-15995: Each endpoint gets weight/numberOfEndpoints portion of the requests - reencrypt route + g.It("Author:hongli-ROSA-OSD_CCS-ARO-Medium-15910-Each endpoint gets weight/numberOfEndpoints portion of the requests", func() { + var ( + buildPruningBaseDir = testdata.FixturePath("router") + webServerTemplate = filepath.Join(buildPruningBaseDir, "template-web-server-deploy.yaml") + destCA = filepath.Join(buildPruningBaseDir, "ca-bundle.pem") + + webServerDeploy1 = webServerDeployDescription{ + deployName: "web-server-deploy1", + svcSecureName: "service-secure1", + svcUnsecureName: "service-unsecure1", + template: webServerTemplate, + namespace: "", + } + + webServerDeploy2 = webServerDeployDescription{ + deployName: "web-server-deploy2", + svcSecureName: "service-secure2", + svcUnsecureName: "service-unsecure2", + template: webServerTemplate, + namespace: "", + } + deploy1Label = "name=" + webServerDeploy1.deployName + deploy2Label = "name=" + webServerDeploy2.deployName + edgeRouteName = "edge15910" + passRouteName = "pass15910" + reenRouteName = "reen15910" + unsecureRouteName = "unsecure15910" + ) + + compat_otp.By("Deploy two sets of web-server and services") + ns := oc.Namespace() + webServerDeploy1.namespace = ns + webServerDeploy1.create(oc) + webServerDeploy2.namespace = ns + webServerDeploy2.create(oc) + ensurePodWithLabelReady(oc, ns, deploy1Label) + ensurePodWithLabelReady(oc, ns, deploy2Label) + + compat_otp.By("Scale deploy1 to replicas 2 and scale deploy2 to replicas 3") + scaleDeploy(oc, ns, webServerDeploy1.deployName, 2) + waitForOutputEquals(oc, ns, "deployment/"+webServerDeploy1.deployName, "{.status.readyReplicas}", "2") + scaleDeploy(oc, ns, webServerDeploy2.deployName, 3) + waitForOutputEquals(oc, ns, "deployment/"+webServerDeploy2.deployName, "{.status.readyReplicas}", "3") + + compat_otp.By("Create edge route and set route-backends with multi serivces") + createRoute(oc, ns, "edge", edgeRouteName, webServerDeploy1.svcUnsecureName, []string{}) + ensureRouteIsAdmittedByIngressController(oc, ns, edgeRouteName, "default") + err := oc.Run("set").Args("route-backends", edgeRouteName, "service-unsecure1=10", "service-unsecure2=10").Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + + compat_otp.By("Create passthrough route and set route-backends with multi serivces") + createRoute(oc, ns, "passthrough", passRouteName, webServerDeploy1.svcSecureName, []string{}) + ensureRouteIsAdmittedByIngressController(oc, ns, passRouteName, "default") + err = oc.Run("set").Args("route-backends", passRouteName, "service-secure1=20%", "service-secure2=80%").Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + + compat_otp.By("Create reencrypt route and set route-backends with multi serivces") + createRoute(oc, ns, "reencrypt", reenRouteName, webServerDeploy1.svcSecureName, []string{"--dest-ca-cert=" + destCA}) + ensureRouteIsAdmittedByIngressController(oc, ns, reenRouteName, "default") + err = oc.Run("set").Args("route-backends", reenRouteName, "service-secure1=256", "service-secure2=256").Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + + compat_otp.By("Create unsecure route and set route-backends with multi serivces") + createRoute(oc, ns, "http", unsecureRouteName, webServerDeploy1.svcUnsecureName, []string{}) + ensureRouteIsAdmittedByIngressController(oc, ns, unsecureRouteName, "default") + err = oc.Run("set").Args("route-backends", unsecureRouteName, "service-unsecure1=50", "service-unsecure2=100").Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + + compat_otp.By("Check edge route weight in haproxy.config") + routerPod := getOneRouterPodNameByIC(oc, "default") + backendBegin := "be_edge_http:" + ns + ":" + edgeRouteName + backendConfig := ensureHaproxyBlockConfigMatchRegexp(oc, routerPod, backendBegin, []string{"server pod:" + webServerDeploy1.deployName + ".+weight 256", "server pod:" + webServerDeploy2.deployName + ".+weight 170"}) + o.Expect(strings.Count(backendConfig, "weight 256")).To(o.Equal(2)) + o.Expect(strings.Count(backendConfig, "weight 170")).To(o.Equal(3)) + + compat_otp.By("Check passthrough route weight in haproxy.config") + backendBegin = "be_tcp:" + ns + ":" + passRouteName + backendConfig = ensureHaproxyBlockConfigMatchRegexp(oc, routerPod, backendBegin, []string{"server pod:" + webServerDeploy1.deployName + ".+weight 96", "server pod:" + webServerDeploy2.deployName + ".+weight 256"}) + o.Expect(strings.Count(backendConfig, "weight 96")).To(o.Equal(2)) + o.Expect(strings.Count(backendConfig, "weight 256")).To(o.Equal(3)) + + compat_otp.By("Check reencryp route weight in haproxy.config") + backendBegin = "be_secure:" + ns + ":" + reenRouteName + backendConfig = ensureHaproxyBlockConfigMatchRegexp(oc, routerPod, backendBegin, []string{"server pod:" + webServerDeploy1.deployName + ".+weight 256", "server pod:" + webServerDeploy2.deployName + ".+weight 170"}) + o.Expect(strings.Count(backendConfig, "weight 256")).To(o.Equal(2)) + o.Expect(strings.Count(backendConfig, "weight 170")).To(o.Equal(3)) + + compat_otp.By("Check unsecure route weight in haproxy.config") + backendBegin = "be_http:" + ns + ":" + unsecureRouteName + backendConfig = ensureHaproxyBlockConfigMatchRegexp(oc, routerPod, backendBegin, []string{"server pod:" + webServerDeploy1.deployName + ".+weight 192", "server pod:" + webServerDeploy2.deployName + ".+weight 256"}) + o.Expect(strings.Count(backendConfig, "weight 192")).To(o.Equal(2)) + o.Expect(strings.Count(backendConfig, "weight 256")).To(o.Equal(3)) + }) + + // author: shudili@redhat.com + g.It("Author:shudili-ROSA-OSD_CCS-ARO-Critical-67093-Alternate Backends and Weights for a route work well", func() { + var ( + buildPruningBaseDir = testdata.FixturePath("router") + testPodSvcTP = filepath.Join(buildPruningBaseDir, "template-web-server-deploy.yaml") + + webServerDeploy1 = webServerDeployDescription{ + deployName: "web-server-deploy01", + svcSecureName: "service-secure01", + svcUnsecureName: "service-unsecure01", + template: testPodSvcTP, + namespace: "", + } + + webServerDeploy2 = webServerDeployDescription{ + deployName: "web-server-deploy02", + svcSecureName: "service-secure02", + svcUnsecureName: "service-unsecure02", + template: testPodSvcTP, + namespace: "", + } + + webServerDeploy3 = webServerDeployDescription{ + deployName: "web-server-deploy03", + svcSecureName: "service-secure03", + svcUnsecureName: "service-unsecure03", + template: testPodSvcTP, + namespace: "", + } + srv1Label = "name=" + webServerDeploy1.deployName + srv2Label = "name=" + webServerDeploy2.deployName + srv3Label = "name=" + webServerDeploy3.deployName + service1Name = webServerDeploy1.svcUnsecureName + service2Name = webServerDeploy2.svcUnsecureName + service3Name = webServerDeploy3.svcUnsecureName + ) + + compat_otp.By("Create 3 server pods and 3 unsecure services") + ns := oc.Namespace() + webServerDeploy1.namespace = ns + webServerDeploy1.create(oc) + webServerDeploy2.namespace = ns + webServerDeploy2.create(oc) + webServerDeploy3.namespace = ns + webServerDeploy3.create(oc) + ensurePodWithLabelReady(oc, ns, srv1Label) + ensurePodWithLabelReady(oc, ns, srv2Label) + ensurePodWithLabelReady(oc, ns, srv3Label) + + compat_otp.By("Expose a route with the unsecure service inside the project") + output, SrvErr := oc.Run("expose").Args("service", service1Name).Output() + o.Expect(SrvErr).NotTo(o.HaveOccurred()) + o.Expect(output).To(o.ContainSubstring(service1Name)) + + // the below test step was for [OCPBUGS-29690] haproxy shouldn't be oom + compat_otp.By("check the default weights for the selected routes are 1") + routerpod := getOneRouterPodNameByIC(oc, "default") + srvPod1Name, err := oc.Run("get").Args("pods", "-l", srv1Label, "-o=jsonpath=\"{.items[0].metadata.name}\"").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + srvPod2Name, err := oc.Run("get").Args("pods", "-l", srv2Label, "-o=jsonpath=\"{.items[0].metadata.name}\"").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + srvPod3Name, err := oc.Run("get").Args("pods", "-l", srv3Label, "-o=jsonpath=\"{.items[0].metadata.name}\"").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + srvPod1Name = strings.Trim(srvPod1Name, "\"") + srvPod2Name = strings.Trim(srvPod2Name, "\"") + srvPod3Name = strings.Trim(srvPod3Name, "\"") + // make sure all ingress-canary pods are ready + ensurePodWithLabelReady(oc, "openshift-ingress-canary", `ingresscanary.operator.openshift.io/daemonset-ingresscanary=canary_controller`) + selectedSrvNum := fmt.Sprintf("cat haproxy.config | grep -E \"server pod:ingress-canary|server pod:%s|server pod:%s|server pod:%s\"| wc -l", srvPod1Name, srvPod3Name, srvPod3Name) + selectedWeight1Num := fmt.Sprintf("cat haproxy.config | grep -E \"server pod:ingress-canary|server pod:%s|server pod:%s|server pod:%s\"| grep \"weight 1\" |wc -l", srvPod1Name, srvPod3Name, srvPod3Name) + srvPodNum, err := oc.AsAdmin().WithoutNamespace().Run("exec").Args("-n", "openshift-ingress", routerpod, "--", "bash", "-c", selectedSrvNum).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + weight1Num, err := oc.AsAdmin().WithoutNamespace().Run("exec").Args("-n", "openshift-ingress", routerpod, "--", "bash", "-c", selectedWeight1Num).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(srvPodNum).To(o.Equal(weight1Num)) + + compat_otp.By("patch the route with alternate backends and weights") + patchRrAlBackend := "{\"metadata\":{\"annotations\":{\"haproxy.router.openshift.io/balance\": \"roundrobin\"}}, " + + "\"spec\": {\"to\": {\"kind\": \"Service\", \"name\": \"" + service1Name + "\", \"weight\": 20}, \"alternateBackends\": [{\"kind\": \"Service\", \"name\": \"" + service2Name + "\", \"weight\": 10}, {\"kind\": \"Service\", \"name\": \"" + service3Name + "\", \"weight\": 10}]}}" + err = oc.AsAdmin().WithoutNamespace().Run("patch").Args("-n", ns, "route/"+service1Name, "--type=merge", "-p", patchRrAlBackend).Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + + compat_otp.By("check the route's backend config") + backend := "be_http:" + ns + ":" + service1Name + bk1 := ensureHaproxyBlockConfigMatchRegexp(oc, routerpod, backend, []string{"server pod:" + srvPod1Name + ".+weight 256"}) + o.Expect(strings.Count(bk1, "weight 256") >= 1).To(o.BeTrue()) + bk2 := ensureHaproxyBlockConfigMatchRegexp(oc, routerpod, backend, []string{"server pod:" + srvPod2Name + ".+weight 128"}) + o.Expect(strings.Count(bk2, "weight 128") >= 1).To(o.BeTrue()) + bk3 := ensureHaproxyBlockConfigMatchRegexp(oc, routerpod, backend, []string{"server pod:" + srvPod3Name + ".+weight 128"}) + o.Expect(strings.Count(bk3, "weight 128") >= 1).To(o.BeTrue()) + }) +}) diff --git a/tests-extension/test/e2e/route.go b/tests-extension/test/e2e/route.go new file mode 100644 index 000000000..31bca0f54 --- /dev/null +++ b/tests-extension/test/e2e/route.go @@ -0,0 +1,2451 @@ +package router + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "strconv" + "strings" + "time" + + g "github.com/onsi/ginkgo/v2" + o "github.com/onsi/gomega" + compat_otp "github.com/openshift/origin/test/extended/util/compat_otp" + "github.com/openshift/router-tests-extension/test/testdata" + clusterinfra "github.com/openshift/origin/test/extended/util/compat_otp/clusterinfra" + "k8s.io/apimachinery/pkg/util/wait" + e2e "k8s.io/kubernetes/test/e2e/framework" +) + +var _ = g.Describe("[sig-network-edge] Network_Edge Component_Router", func() { + defer g.GinkgoRecover() + + var oc = compat_otp.NewCLI("routes", compat_otp.KubeConfigPath()) + + // incorporate OCP-10024, OCP-11883 and OCP-12122 into one + // Test case creater: zzhao@redhat.com - OCP-10024 Route could NOT be updated after created + // Test case creater: zzhao@redhat.com - OCP-11883 Be able to add more alias for service + // Test case creater: zzhao@redhat.com - OCP-12122 Alias will be invalid after removing it + g.It("Author:mjoseph-Critical-10024-Route could NOT be updated after created", func() { + var ( + buildPruningBaseDir = testdata.FixturePath("router") + testPodSvc = filepath.Join(buildPruningBaseDir, "web-server-deploy.yaml") + customTemp2 = filepath.Join(buildPruningBaseDir, "subdomain-routes/route.yaml") + unSecSvcName = "service-unsecure" + aliasRoute = "service-unsecure2" + edgeRoute = "ocp10024-unsecure" + rut = routeDescription{ + namespace: "", + domain: "", + subDomain: "ocp10024", + template: customTemp2, + } + ) + + compat_otp.By("1: Create an edge route using the route yaml file") + ns := oc.Namespace() + baseDomain := getBaseDomain(oc) + rut.domain = "apps" + "." + baseDomain + rut.namespace = ns + rut.create(oc) + getRoutes(oc, ns) + ensureRouteIsAdmittedByIngressController(oc, ns, edgeRoute, "default") + + compat_otp.By("2: Try to update the hostname for route using a test user and confirm it is not possible") + patchOutput, _ := oc.WithoutNamespace().Run("patch").Args("route/"+edgeRoute, "-p", "{\"spec\":{\"host\":\"www.changeroute.com\"}}", "--type=merge", "-n", ns).Output() + o.Expect(patchOutput).To(o.ContainSubstring(`spec.host: Invalid value: "www.changeroute.com"`)) + + // OCP-11883: Be able to add more alias for service + compat_otp.By("3: Create a server and its service") + createResourceFromFile(oc, ns, testPodSvc) + ensurePodWithLabelReady(oc, ns, "name=web-server-deploy") + + compat_otp.By("4: Create a http route using the service-unsecure service") + routerpod := getOneRouterPodNameByIC(oc, "default") + createRoute(oc, ns, "http", unSecSvcName, unSecSvcName, []string{}) + ensureRouteIsAdmittedByIngressController(oc, ns, unSecSvcName, "default") + backendName1 := "be_http:" + ns + ":service-unsecure" + ensureHaproxyBlockConfigContains(oc, routerpod, backendName1, []string{"service-unsecure:http"}) + + compat_otp.By("5: Create another http route (alias) using the same service") + createRoute(oc, ns, "http", aliasRoute, unSecSvcName, []string{}) + ensureRouteIsAdmittedByIngressController(oc, ns, aliasRoute, "default") + getRoutes(oc, ns) + backendName2 := "be_http:" + ns + ":service-unsecure2" + ensureHaproxyBlockConfigContains(oc, routerpod, backendName2, []string{"service-unsecure:http"}) + + // OCP-12122 Alias will be invalid after removing it + compat_otp.By("6: Delete the alias route and verify that route is not accessible") + err := oc.AsAdmin().Run("delete").Args("-n", ns, "route", aliasRoute).Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + routeOutput, _ := oc.AsAdmin().WithoutNamespace().Run("get").Args("route", "-n", ns, aliasRoute, "-ojsonpath={.status.ingress[?(@.routerName==\"default\")].conditions[*].status}").Output() + o.Expect(routeOutput).To(o.ContainSubstring(`routes.route.openshift.io "service-unsecure2" not found`)) + + compat_otp.By("7: Confirming the alias route got removed from haproxy") + waitErr := wait.PollImmediate(3*time.Second, 60*time.Second, func() (bool, error) { + noBackendConfig := readRouterPodData(oc, routerpod, "cat haproxy.config", "be_http:"+ns) + if !strings.Contains(noBackendConfig, aliasRoute) { + return true, nil + } + e2e.Logf("Still waiting for the alias route to get removed from the haproxy") + return false, nil + }) + o.Expect(waitErr).NotTo(o.HaveOccurred(), "The alias route'service-unsecure2' never get removed") + }) + + // author: shudili@redhat.com + g.It("Author:shudili-ROSA-OSD_CCS-ARO-Critical-10043-Set balance leastconn for passthrough routes", func() { + var ( + buildPruningBaseDir = testdata.FixturePath("router") + testPodSvc = filepath.Join(buildPruningBaseDir, "web-server-deploy.yaml") + svcName = "service-secure" + ) + + compat_otp.By("1.0 Create a server pod and its services") + ns := oc.Namespace() + createResourceFromFile(oc, ns, testPodSvc) + ensurePodWithLabelReady(oc, ns, "name=web-server-deploy") + + compat_otp.By("2.0 Create a passthrough route") + createRoute(oc, ns, "passthrough", "route-pass", svcName, []string{"--hostname=" + "passth10043" + ".apps." + getBaseDomain(oc)}) + ensureRouteIsAdmittedByIngressController(oc, ns, "route-pass", "default") + + compat_otp.By(`3.0 Add the balance=leastconn annotation to the routes`) + setAnnotation(oc, ns, "route/route-pass", "haproxy.router.openshift.io/balance=leastconn") + + compat_otp.By(`4.0 Check the balance leastconn configuration in haproxy`) + routerpod := getOneRouterPodNameByIC(oc, "default") + backendStart := fmt.Sprintf("backend be_tcp:%s:%s", ns, "route-pass") + ensureHaproxyBlockConfigContains(oc, routerpod, backendStart, []string{"balance leastconn"}) + }) + + // bugzilla: 1368525 + g.It("Author:shudili-ROSA-OSD_CCS-ARO-Medium-10207-NetworkEdge Should use the same cookies for secure and insecure access when insecureEdgeTerminationPolicy set to allow for edge/reencrypt route", func() { + var ( + buildPruningBaseDir = testdata.FixturePath("router") + baseTemp = filepath.Join(buildPruningBaseDir, "ingresscontroller-np.yaml") + testPodSvc = filepath.Join(buildPruningBaseDir, "web-server-signed-deploy.yaml") + clientPod = filepath.Join(buildPruningBaseDir, "test-client-pod.yaml") + clientPodName = "hello-pod" + clientPodLabel = "app=hello-pod" + srvrcInfo = "web-server-deploy" + unSecSvcName = "service-unsecure" + secSvcName = "service-secure" + podFileDir = "/data/OCP-10207-cookie" + fileDir = "/tmp/OCP-10207-cookie" + ingctrl = ingressControllerDescription{ + name: "ocp10207", + namespace: "openshift-ingress-operator", + domain: "", + template: baseTemp, + } + ) + + compat_otp.By("1.0: Prepare file folder and file for testing") + defer os.RemoveAll(fileDir) + err := os.MkdirAll(fileDir, 0755) + o.Expect(err).NotTo(o.HaveOccurred()) + updateFilebySedCmd(testPodSvc, "replicas: 1", "replicas: 2") + + compat_otp.By("2.0: Create a client pod, two server pods and the service") + ns := oc.Namespace() + err = oc.AsAdmin().WithoutNamespace().Run("create").Args("-n", ns, "-f", clientPod).Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + ensurePodWithLabelReady(oc, ns, clientPodLabel) + // create the cookie folder in the client pod + err = oc.AsAdmin().WithoutNamespace().Run("exec").Args("-n", ns, clientPodName, "--", "mkdir", podFileDir).Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + srvPodList := createResourceFromWebServer(oc, ns, testPodSvc, srvrcInfo) + + compat_otp.By("3.0: Create a custom ingresscontroller and an edge route with insecure_policy Allow") + ingctrl.domain = ingctrl.name + "." + getBaseDomain(oc) + routehost := "edge10207" + "." + ingctrl.domain + defer ingctrl.delete(oc) + ingctrl.create(oc) + ensureCustomIngressControllerAvailable(oc, ingctrl.name) + createRoute(oc, ns, "edge", "route-edge10207", unSecSvcName, []string{"--hostname=" + routehost, "--insecure-policy=Allow"}) + ensureRouteIsAdmittedByIngressController(oc, ns, "route-edge10207", "default") + + compat_otp.By("4.0: Curl the edge route for two times, one with saving the cookie for the second server") + routerpod := getOneRouterPodNameByIC(oc, ingctrl.name) + podIP := getPodv4Address(oc, routerpod, "openshift-ingress") + toDst := routehost + ":443:" + podIP + curlCmd := []string{"-n", ns, clientPodName, "--", "curl", "https://" + routehost, "-ks", "--resolve", toDst, "--connect-timeout", "10"} + expectOutput := []string{"Hello-OpenShift " + srvPodList[0] + " http-8080"} + repeatCmdOnClient(oc, curlCmd, expectOutput, 60, 1) + curlCmd = []string{"-n", ns, clientPodName, "--", "curl", "https://" + routehost, "-ks", "-c", podFileDir + "/cookie-10207", "--resolve", toDst, "--connect-timeout", "10"} + expectOutput = []string{"Hello-OpenShift " + srvPodList[1] + " http-8080"} + repeatCmdOnClient(oc, curlCmd, expectOutput, 120, 1) + + compat_otp.By("5.0: Open the cookie file and check the contents") + // access the cookie file and confirm that the output contains false and false + err = oc.AsAdmin().WithoutNamespace().Run("cp").Args("-n", ns, clientPodName+":"+podFileDir+"/cookie-10207", fileDir+"/cookie-10207").Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + checkCookieFile(fileDir+"/cookie-10207", "FALSE\t/\tFALSE") + + compat_otp.By("6.0: Curl the edge route with the cookie, expect forwarding to the second server") + curlCmdWithCookie := []string{"-n", ns, clientPodName, "--", "curl", "https://" + routehost, "-ks", "-b", podFileDir + "/cookie-10207", "--resolve", toDst, "--connect-timeout", "10"} + expectOutput = []string{"Hello-OpenShift " + srvPodList[0] + " http-8080", "Hello-OpenShift " + srvPodList[1] + " http-8080"} + _, result := repeatCmdOnClient(oc, curlCmdWithCookie, expectOutput, 120, 6) + o.Expect(result[1]).To(o.Equal(6)) + + compat_otp.By("7.0: Patch the edge route with Redirect tls insecureEdgeTerminationPolicy, then curl the edge route with the cookie, expect forwarding to the second server") + patchResourceAsAdmin(oc, ns, "route/route-edge10207", `{"spec":{"tls": {"insecureEdgeTerminationPolicy":"Redirect"}}}`) + toDst2 := routehost + ":80:" + podIP + curlCmdWithCookie = []string{"-n", ns, clientPodName, "--", "curl", "http://" + routehost, "-ksSL", "-b", podFileDir + "/cookie-10207", "--resolve", toDst, "--resolve", toDst2, "--connect-timeout", "10"} + _, result = repeatCmdOnClient(oc, curlCmdWithCookie, expectOutput, 120, 6) + o.Expect(result[1]).To(o.Equal(6)) + + compat_otp.By("8.0: Create a reencrypt route with Allow policy") + reenhost := "reen10207" + "." + ingctrl.domain + toDst = reenhost + ":443:" + podIP + toDst2 = reenhost + ":80:" + podIP + createRoute(oc, ns, "reencrypt", "route-reen10207", secSvcName, []string{"--hostname=" + reenhost, "--insecure-policy=Allow"}) + ensureRouteIsAdmittedByIngressController(oc, ns, "route-reen10207", "default") + + compat_otp.By("9.0: Curl the route and generate a cookie file") + curlCmdWithCookie = []string{"-n", ns, clientPodName, "--", "curl", "http://" + reenhost, "-ks", "-c", podFileDir + "/reen-cookie", "--resolve", toDst, "--resolve", toDst2, "--connect-timeout", "10"} + expectOutput = []string{"Hello-OpenShift " + srvPodList[0] + " https-8443"} + repeatCmdOnClient(oc, curlCmdWithCookie, expectOutput, 60, 1) + + compat_otp.By("10.0: Open the cookie file and check the contents") + // access the cookie file and confirm that the output contains false and false + err = oc.AsAdmin().WithoutNamespace().Run("cp").Args("-n", ns, clientPodName+":"+podFileDir+"/reen-cookie", fileDir+"/reen-cookie").Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + checkCookieFile(fileDir+"/reen-cookie", "FALSE\t/\tFALSE") + }) + + g.It("Author:mjoseph-ROSA-OSD_CCS-ARO-Critical-10660-Service endpoint can be work well if the mapping pod ip is updated", func() { + var ( + buildPruningBaseDir = testdata.FixturePath("router") + testPodSvc = filepath.Join(buildPruningBaseDir, "web-server-deploy.yaml") + unSecSvcName = "service-unsecure" + serverName = "web-server-deploy" + ) + + compat_otp.By("1. Create a server pod and its service") + ns := oc.Namespace() + createResourceFromWebServer(oc, ns, testPodSvc, "web-server-deploy") + + compat_otp.By("2. Check the service endpoints") + epJsonPath := "{.subsets[0].addresses[0].ip}:{.subsets[0].ports[0].port}" + epIPregExp := "([0-9]+.[0-9]+.[0-9]+.[0-9]+|[0-9a-zA-Z]+:[0-9a-zA-Z:]+)" + epSearchOutput := waitForOutputMatchRegexp(oc, ns, "endpoints/"+unSecSvcName, epJsonPath, epIPregExp) + o.Expect(epSearchOutput).NotTo(o.ContainSubstring("NotMatch")) + ep := getByJsonPath(oc, ns, "endpoints/"+unSecSvcName, epJsonPath) + + compat_otp.By("3. Delete the server pod and check the endpoint") + scaleDeploy(oc, ns, serverName, 0) + // there will not an ip assigned for the EP after the pod is removed + noneEP := getByJsonPath(oc, ns, "endpoints/"+unSecSvcName, epJsonPath) + o.Expect(noneEP).To(o.ContainSubstring(":")) + + compat_otp.By("4. Create the pod again and recheck the service endpoints") + scaleDeploy(oc, ns, serverName, 1) + epSearchOutput = waitForOutputMatchRegexp(oc, ns, "endpoints/"+unSecSvcName, epJsonPath, epIPregExp) + o.Expect(epSearchOutput).NotTo(o.ContainSubstring("NotMatch")) + newEP := getByJsonPath(oc, ns, "endpoints/"+unSecSvcName, epJsonPath) + // the new IP assigned will be different from the old one + o.Expect(newEP).NotTo(o.Equal(ep)) + }) + + // author: iamin@redhat.com + g.It("Author:iamin-ROSA-OSD_CCS-ARO-Low-10943-NetworkEdge Set invalid timeout server for route", func() { + var ( + buildPruningBaseDir = testdata.FixturePath("router") + testPodSvc = filepath.Join(buildPruningBaseDir, "web-server-deploy.yaml") + unSecSvcName = "service-unsecure" + ) + + compat_otp.By("1.0: Create single pod and the service") + ns := oc.Namespace() + createResourceFromFile(oc, ns, testPodSvc) + ensurePodWithLabelReady(oc, ns, "name=web-server-deploy") + output, err := oc.Run("get").Args("service").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(output).To(o.ContainSubstring(unSecSvcName)) + + compat_otp.By("2.0: Create an unsecure route") + + createRoute(oc, ns, "http", unSecSvcName, unSecSvcName, []string{}) + output, err = oc.Run("get").Args("route").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(output).To(o.ContainSubstring(unSecSvcName)) + + compat_otp.By("3.0: Annotate unsecure route") + setAnnotation(oc, ns, "route/"+unSecSvcName, "haproxy.router.openshift.io/timeout=-2s") + findAnnotation := getAnnotation(oc, ns, "route", unSecSvcName) + o.Expect(findAnnotation).To(o.ContainSubstring(`haproxy.router.openshift.io/timeout":"-2s`)) + + compat_otp.By("4.0: Check HAProxy file for timeout tunnel") + routerpod := getOneRouterPodNameByIC(oc, "default") + ensureHaproxyBlockConfigNotContains(oc, routerpod, ns, []string{"timeout server -2s"}) + }) + + // author: iamin@redhat.com + // combine OCP-9651, OCP-9717 + g.It("Author:iamin-ROSA-OSD_CCS-ARO-NonHyperShiftHOST-Critical-11036-NetworkEdge Set insecureEdgeTerminationPolicy to Redirect for passthrough/edge/reencrypt route", func() { + var ( + buildPruningBaseDir = testdata.FixturePath("router") + testPodSvc = filepath.Join(buildPruningBaseDir, "web-server-signed-deploy.yaml") + SvcName = "service-secure" + unSecSvc = "service-unsecure" + ) + + compat_otp.By("1.0: Create single pod, service and a passthrough/edge/reencrypt route") + ns := oc.Namespace() + srvPodList := createResourceFromWebServer(oc, ns, testPodSvc, "web-server-deploy") + output, err := oc.Run("get").Args("service").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(output).To(o.And(o.ContainSubstring(unSecSvc), o.ContainSubstring(SvcName))) + createRoute(oc, ns, "passthrough", "passthrough-route", SvcName, []string{}) + createRoute(oc, ns, "reencrypt", "reen-route", SvcName, []string{}) + createRoute(oc, ns, "edge", "edge-route", unSecSvc, []string{}) + output, err = oc.Run("get").Args("route").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(output).To(o.And(o.ContainSubstring("passthrough-route"), o.ContainSubstring("reen-route"), o.ContainSubstring("edge-route"))) + + compat_otp.By("2.0: Add Redirect in tls") + patchResourceAsAdmin(oc, ns, "route/passthrough-route", `{"spec":{"tls": {"insecureEdgeTerminationPolicy":"Redirect"}}}`) + output, err = oc.Run("get").Args("route/passthrough-route", "-n", ns, "-o=jsonpath={.spec.tls}").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(output).To(o.ContainSubstring(`"insecureEdgeTerminationPolicy":"Redirect"`)) + + compat_otp.By("3.0: Test Route Http request is redirected to https") + routehost := "passthrough-route-" + ns + ".apps." + getBaseDomain(oc) + waitForOutsideCurlContains("http://"+routehost, "-I -k", "ocation: https://"+routehost) + waitForOutsideCurlContains("http://"+routehost, "-L -k", "Hello-OpenShift "+srvPodList[0]+" https-8443") + + compat_otp.By("4.0: Attempt to update route policy to Allow") + result, _ := oc.AsAdmin().WithoutNamespace().Run("patch").Args("route/passthrough-route", "-p", `{"spec":{"tls": {"insecureEdgeTerminationPolicy":"Allow"}}}`, "-n", ns).Output() + o.Expect(result).To(o.ContainSubstring("invalid value for InsecureEdgeTerminationPolicy option, acceptable values are None, Redirect, or empty")) + + compat_otp.By("5.0: Add Redirect in reencrypt tls") + patchResourceAsAdmin(oc, ns, "route/reen-route", `{"spec":{"tls": {"insecureEdgeTerminationPolicy":"Redirect"}}}`) + output, err = oc.Run("get").Args("route/reen-route", "-n", ns, "-o=jsonpath={.spec.tls}").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(output).To(o.ContainSubstring(`"insecureEdgeTerminationPolicy":"Redirect"`)) + + compat_otp.By("6.0: Test Route Http request is redirected to https") + reenhost := "reen-route-" + ns + ".apps." + getBaseDomain(oc) + waitForOutsideCurlContains("http://"+reenhost, "-I -k", "ocation: https://"+reenhost) + waitForOutsideCurlContains("http://"+reenhost, "-L -k", "Hello-OpenShift "+srvPodList[0]+" https-8443") + + compat_otp.By("7.0: Add Redirect in edge tls") + patchResourceAsAdmin(oc, ns, "route/edge-route", `{"spec":{"tls": {"insecureEdgeTerminationPolicy":"Redirect"}}}`) + output, err = oc.Run("get").Args("route/edge-route", "-n", ns, "-o=jsonpath={.spec.tls}").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(output).To(o.ContainSubstring(`"insecureEdgeTerminationPolicy":"Redirect"`)) + + compat_otp.By("8.0: Test Route Http request is redirected to https") + edgehost := "edge-route-" + ns + ".apps." + getBaseDomain(oc) + waitForOutsideCurlContains("http://"+edgehost, "-I -k", "ocation: https://"+edgehost) + waitForOutsideCurlContains("http://"+edgehost, "-L -k", "Hello-OpenShift "+srvPodList[0]+" http-8080") + + compat_otp.By("9.0: Attempt to update route policy to invalid value") + result, _ = oc.AsAdmin().WithoutNamespace().Run("patch").Args("route/edge-route", "-p", `{"spec":{"tls": {"insecureEdgeTerminationPolicy":"Abc"}}}`, "-n", ns).Output() + o.Expect(result).To(o.ContainSubstring("invalid value for InsecureEdgeTerminationPolicy option, acceptable values are None, Allow, Redirect, or empty")) + + }) + + // author: iamin@redhat.com + g.It("Author:iamin-ROSA-OSD_CCS-ARO-Medium-11067-NetworkEdge oc help information should contain option wildcard-policy", func() { + var ( + buildPruningBaseDir = testdata.FixturePath("router") + testPodSvc = filepath.Join(buildPruningBaseDir, "web-server-deploy.yaml") + svcName = "service-secure" + ) + + compat_otp.By("1.0: Create single pod, service") + ns := oc.Namespace() + createResourceFromWebServer(oc, ns, testPodSvc, "web-server-deploy") + + compat_otp.By("2.0: Check help section for expose service") + output, err := oc.Run("expose").Args("service", svcName, "--help").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(output).To(o.ContainSubstring("--wildcard-policy=")) + + compat_otp.By("3.0: Check help section for edge route creation") + output, err = oc.Run("create").Args("route", "edge", "route-edge", "--help").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(output).To(o.ContainSubstring("--wildcard-policy=")) + + compat_otp.By("4.0: Check help section for passthrough route creation") + output, err = oc.Run("create").Args("route", "passthrough", "route-pass", "--help").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(output).To(o.ContainSubstring("--wildcard-policy=")) + + compat_otp.By("5.0: Check help section for reencrypt route creation") + output, err = oc.Run("create").Args("route", "reencrypt", "route-reen", "--help").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(output).To(o.ContainSubstring("--wildcard-policy=")) + }) + + // merge OCP-11042(NetworkEdge NetworkEdge Disable haproxy hash based sticky session for edge termination routes) to OCP-11130 + g.It("Author:shudili-ROSA-OSD_CCS-ARO-Critical-11130-NetworkEdge Enable/Disable haproxy cookies based sticky session for edge termination routes", func() { + var ( + buildPruningBaseDir = testdata.FixturePath("router") + baseTemp = filepath.Join(buildPruningBaseDir, "ingresscontroller-np.yaml") + clientPod = filepath.Join(buildPruningBaseDir, "test-client-pod.yaml") + clientPodName = "hello-pod" + clientPodLabel = "app=hello-pod" + testPodSvc = filepath.Join(buildPruningBaseDir, "web-server-deploy.yaml") + srvrcInfo = "web-server-deploy" + unSecSvcName = "service-unsecure" + fileDir = "/data/OCP-11130-cookie" + ingctrl = ingressControllerDescription{ + name: "ocp11130", + namespace: "openshift-ingress-operator", + domain: "", + template: baseTemp, + } + ) + + compat_otp.By("1.0: Updated replicas in the web-server-deploy file for testing") + updateFilebySedCmd(testPodSvc, "replicas: 1", "replicas: 2") + + compat_otp.By("2.0: Create a client pod, two server pods and the service") + ns := oc.Namespace() + err := oc.AsAdmin().WithoutNamespace().Run("create").Args("-n", ns, "-f", clientPod).Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + ensurePodWithLabelReady(oc, ns, clientPodLabel) + // create the cookie folder in the client pod + err = oc.AsAdmin().WithoutNamespace().Run("exec").Args("-n", ns, clientPodName, "--", "mkdir", fileDir).Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + srvPodList := createResourceFromWebServer(oc, ns, testPodSvc, srvrcInfo) + + compat_otp.By("3.0: Create an edge route") + ingctrl.domain = ingctrl.name + "." + getBaseDomain(oc) + routehost := "edge11130" + "." + ingctrl.domain + defer ingctrl.delete(oc) + ingctrl.create(oc) + ensureCustomIngressControllerAvailable(oc, ingctrl.name) + createRoute(oc, ns, "edge", "route-edge11130", unSecSvcName, []string{"--hostname=" + routehost}) + ensureRouteIsAdmittedByIngressController(oc, ns, "route-edge11130", "default") + + compat_otp.By("4.0: Curl the edge route, make sure saving the cookie for server 1") + routerpod := getOneRouterPodNameByIC(oc, ingctrl.name) + podIP := getPodv4Address(oc, routerpod, "openshift-ingress") + toDst := routehost + ":443:" + podIP + curlCmd := []string{"-n", ns, clientPodName, "--", "curl", "https://" + routehost, "-ks", "-c", fileDir + "/cookie-11130", "--resolve", toDst, "--connect-timeout", "10"} + + expectOutput := []string{"Hello-OpenShift " + srvPodList[0] + " http-8080"} + repeatCmdOnClient(oc, curlCmd, expectOutput, 120, 1) + + compat_otp.By("5.0: Curl the edge route, make sure could get response from server 2") + curlCmd = []string{"-n", ns, clientPodName, "--", "curl", "https://" + routehost, "-ks", "--resolve", toDst, "--connect-timeout", "10"} + expectOutput = []string{"Hello-OpenShift " + srvPodList[1] + " http-8080"} + repeatCmdOnClient(oc, curlCmd, expectOutput, 120, 1) + + compat_otp.By("6.0: Curl the edge route with the cookie, expect all are forwarded to the server 1") + curlCmdWithCookie := []string{"-n", ns, clientPodName, "--", "curl", "https://" + routehost, "-ks", "-b", fileDir + "/cookie-11130", "--resolve", toDst, "--connect-timeout", "10"} + expectOutput = []string{"Hello-OpenShift " + srvPodList[0] + " http-8080", "Hello-OpenShift " + srvPodList[1] + " http-8080"} + _, result := repeatCmdOnClient(oc, curlCmdWithCookie, expectOutput, 120, 6) + o.Expect(result[0]).To(o.Equal(6)) + + // Disable haproxy hash based sticky session for edge termination routes + compat_otp.By("7.0: Annotate the edge route with haproxy.router.openshift.io/disable_cookies=true") + _, err = oc.Run("annotate").WithoutNamespace().Args("-n", ns, "route/route-edge11130", "haproxy.router.openshift.io/disable_cookies=true", "--overwrite").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + + compat_otp.By("8.0: Curl the edge route, and save the cookie for the backend server") + curlCmd = []string{"-n", ns, clientPodName, "--", "curl", "https://" + routehost, "-ks", "-c", fileDir + "/cookie-11130", "--resolve", toDst, "--connect-timeout", "10"} + expectOutput = []string{"Hello-OpenShift"} + repeatCmdOnClient(oc, curlCmd, expectOutput, 120, 1) + + compat_otp.By("9.0: Curl the edge route with the cookie, expect forwarding to the two server") + expectOutput = []string{"Hello-OpenShift " + srvPodList[0] + " http-8080", "Hello-OpenShift " + srvPodList[1] + " http-8080"} + _, result = repeatCmdOnClient(oc, curlCmdWithCookie, expectOutput, 150, 15) + o.Expect(result[0] > 0).To(o.BeTrue()) + o.Expect(result[1] > 0).To(o.BeTrue()) + o.Expect(result[0] + result[1]).To(o.Equal(15)) + }) + + // incorporate OCP-11619, OCP-10914 and OCP-11325 into one + // Test case creater: bmeng@redhat.com - OCP-11619-Limit the number of TCP connection per IP in specified time period + // Test case creater: yadu@redhat.com - OCP-10914: Protect from ddos by limiting TCP concurrent connection for route + // Test case creater: hongli@redhat.com - OCP-11325: Limit the number of http request per ip + g.It("Author:mjoseph-ROSA-OSD_CCS-ARO-Critical-11619-Limit the number of TCP connection per IP in specified time period", func() { + var ( + buildPruningBaseDir = testdata.FixturePath("router") + testPodSvc = filepath.Join(buildPruningBaseDir, "web-server-deploy.yaml") + ) + + compat_otp.By("1. Create a server") + ns := oc.Namespace() + createResourceFromFile(oc, ns, testPodSvc) + ensurePodWithLabelReady(oc, ns, "name=web-server-deploy") + + compat_otp.By("2. Create a passthrough route in the namespace") + createRoute(oc, ns, "passthrough", "mypass", "service-secure", []string{}) + output := getRoutes(oc, ns) + o.Expect(output).To(o.ContainSubstring("mypass")) + + compat_otp.By("3. Check the reachability of the passthrough route") + routehost := getRouteHost(oc, ns, "mypass") + curlCmd := fmt.Sprintf(`curl -k https://%s --connect-timeout 10`, routehost) + repeatCmdOnClient(oc, curlCmd, "Hello-OpenShift", 60, 1) + + compat_otp.By("4. Annotate the route to limit the TCP nums per ip and verify") + setAnnotation(oc, ns, "route/mypass", "haproxy.router.openshift.io/rate-limit-connections=true") + setAnnotation(oc, ns, "route/mypass", "haproxy.router.openshift.io/rate-limit-connections.rate-tcp=2") + findAnnotation := getAnnotation(oc, ns, "route", "mypass") + o.Expect(findAnnotation).NotTo(o.ContainSubstring(`haproxy.router.openshift.io/rate-limit-connections: "true"`)) + o.Expect(findAnnotation).NotTo(o.ContainSubstring(`haproxy.router.openshift.io/rate-limit-connections.rate-tcp: "2"`)) + + compat_otp.By("5. Verify the haproxy configuration to ensure the tcp rate limit is configured") + podName := getOneRouterPodNameByIC(oc, "default") + backendName := "be_tcp:" + ns + ":mypass" + ensureHaproxyBlockConfigContains(oc, podName, backendName, []string{"src_conn_rate", "tcp-request content reject if { src_conn_rate ge 2 }"}) + + // OCP-10914: Protect from ddos by limiting TCP concurrent connection for route + compat_otp.By("6. Expose a service in the namespace") + createRoute(oc, ns, "http", "service-unsecure", "service-unsecure", []string{}) + output = getRoutes(oc, ns) + o.Expect(output).To(o.ContainSubstring("service-unsecure")) + + compat_otp.By("7. Check the reachability of the http route") + routehost = getRouteHost(oc, ns, "service-unsecure") + curlCmd = fmt.Sprintf(`curl http://%s --connect-timeout 10`, routehost) + repeatCmdOnClient(oc, curlCmd, "Hello-OpenShift", 30, 1) + + compat_otp.By("8. Annotate the route to limit the concurrent TCP connections rate and verify") + setAnnotation(oc, ns, "route/service-unsecure", "haproxy.router.openshift.io/rate-limit-connections=true") + setAnnotation(oc, ns, "route/service-unsecure", "haproxy.router.openshift.io/rate-limit-connections.concurrent-tcp=2") + findAnnotation = getAnnotation(oc, ns, "route", "service-unsecure") + o.Expect(findAnnotation).NotTo(o.ContainSubstring(`haproxy.router.openshift.io/rate-limit-connections: "true"`)) + o.Expect(findAnnotation).NotTo(o.ContainSubstring(`haproxy.router.openshift.io/rate-limit-connections.concurrent-tcp: "2"`)) + + compat_otp.By("9. Verify the haproxy configuration to ensure the tcp rate limit is configured") + backendName1 := "be_http:" + ns + ":service-unsecure" + ensureHaproxyBlockConfigContains(oc, podName, backendName1, []string{"src_conn_cur", "tcp-request content reject if { src_conn_cur ge 2 }"}) + + // OCP-11325: Limit the number of http request per ip + compat_otp.By("10. Annotate the route to limit the http request nums per ip and verify") + setAnnotation(oc, ns, "route/service-unsecure", "haproxy.router.openshift.io/rate-limit-connections.concurrent-tcp-") + setAnnotation(oc, ns, "route/service-unsecure", "haproxy.router.openshift.io/rate-limit-connections.rate-http=3") + findAnnotation = getAnnotation(oc, ns, "route", "service-unsecure") + o.Expect(findAnnotation).NotTo(o.ContainSubstring(`haproxy.router.openshift.io/rate-limit-connections: "true"`)) + o.Expect(findAnnotation).NotTo(o.ContainSubstring(`haproxy.router.openshift.io/rate-limit-connections.rate-http: "3"`)) + + compat_otp.By("11. Verify the haproxy configuration to ensure the http rate limit is configured") + ensureHaproxyBlockConfigContains(oc, podName, backendName1, []string{"src_http_req_rate", "tcp-request content reject if { src_http_req_rate ge 3 }"}) + }) + + // author: iamin@redhat.com + g.It("Author:iamin-ROSA-OSD_CCS-ARO-Critical-11635-NetworkEdge Set timeout server for passthough route", func() { + var ( + buildPruningBaseDir = testdata.FixturePath("router") + testPodSvc = filepath.Join(buildPruningBaseDir, "httpbin-deploy.yaml") + secureSvcName = "httpbin-svc-secure" + ) + + compat_otp.By("1.0: Create single pod and the service") + ns := oc.Namespace() + createResourceFromFile(oc, ns, testPodSvc) + ensurePodWithLabelReady(oc, ns, "name=httpbin-pod") + + compat_otp.By("2.0: Create a passthrough route") + routeName := "route-passthrough11635" + routehost := routeName + "-" + ns + ".apps." + getBaseDomain(oc) + + createRoute(oc, ns, "passthrough", routeName, secureSvcName, []string{}) + ensureRouteIsAdmittedByIngressController(oc, ns, routeName, "default") + + compat_otp.By("3.0: Annotate passthrough route") + setAnnotation(oc, ns, "route/"+routeName, "haproxy.router.openshift.io/timeout=3s") + findAnnotation := getAnnotation(oc, ns, "route", routeName) + o.Expect(findAnnotation).To(o.ContainSubstring(`haproxy.router.openshift.io/timeout":"3s`)) + + compat_otp.By("4.0: Curl the edge route for two times, one with normal delay and other above timeout delay") + waitForOutsideCurlContains("https://"+routehost+"/delay/2", "-kI", `200 OK`) + waitForOutsideCurlContains("https://"+routehost+"/delay/5", "-kI", `exit status`) + + compat_otp.By("5.0: Check HAProxy file for timeout tunnel") + routerpod := getOneRouterPodNameByIC(oc, "default") + ensureHaproxyBlockConfigContains(oc, routerpod, ns, []string{routeName, "timeout tunnel 3s"}) + }) + + // author: shudili@redhat.com + g.It("Author:shudili-ROSA-OSD_CCS-ARO-Medium-11728-haproxy hash based sticky session for tcp mode passthrough routes", func() { + // skip this case on IBM and PowerVS platforms due to multiple private IPs in the test environment(please refer to OCPQE-28604 and OCPQE-31210) + compat_otp.SkipIfPlatformType(oc, "IBMCloud, PowerVS") + + // if the ingress canary route isn't accessable from outside, skip it + if !isCanaryRouteAvailable(oc) { + g.Skip("Skip for the ingress canary route could not be available to the outside") + } + + var ( + buildPruningBaseDir = testdata.FixturePath("router") + testPodSvc = filepath.Join(buildPruningBaseDir, "web-server-deploy.yaml") + srvrcInfo = "web-server-deploy" + secSvcName = "service-secure" + routeName = "route-pass11728" + ) + + compat_otp.By("1.0: Updated replicas in the web-server-deploy file for testing") + updateFilebySedCmd(testPodSvc, "replicas: 1", "replicas: 2") + + compat_otp.By("2.0: Create two server pods and the service") + ns := oc.Namespace() + srvPodList := createResourceFromWebServer(oc, ns, testPodSvc, srvrcInfo) + + compat_otp.By("3.0: Create a passthrough route") + routehost := routeName + ".apps." + getBaseDomain(oc) + createRoute(oc, ns, "passthrough", routeName, secSvcName, []string{"--hostname=" + routehost}) + ensureRouteIsAdmittedByIngressController(oc, ns, routeName, "default") + + compat_otp.By("4.0: Check the passthrough route configuration in haproxy") + routerpod := getOneRouterPodNameByIC(oc, "default") + backendStart := fmt.Sprintf(`backend be_tcp:%s:%s`, ns, routeName) + ensureHaproxyBlockConfigContains(oc, routerpod, backendStart, []string{routeName, "balance source", "hash-type consistent"}) + + compat_otp.By("5.0: Curl the passthrough route, and save the output") + curlCmd := fmt.Sprintf(`curl https://%s -sk --connect-timeout 10`, routehost) + outputWithOneServer, _ := repeatCmdOnClient(oc, curlCmd, "Hello-OpenShift", 60, 1) + + compat_otp.By("6.0: Curl the passthrough route for 6 times, all are forwarded to the expected server") + expectOutput := []string{"Hello-OpenShift " + srvPodList[0], "Hello-OpenShift " + srvPodList[1]} + output, matchedList := repeatCmdOnClient(oc, curlCmd, expectOutput, 90, 6) + o.Expect(output).To(o.ContainSubstring(outputWithOneServer)) + o.Expect(matchedList[0] + matchedList[1]).To(o.Equal(6)) + o.Expect(matchedList[0] * matchedList[1]).To(o.Equal(0)) + }) + + // author: iamin@redhat.com + g.It("Author:iamin-ROSA-OSD_CCS-ARO-High-11982-NetworkEdge Set timeout server for http route", func() { + var ( + buildPruningBaseDir = testdata.FixturePath("router") + testPodSvc = filepath.Join(buildPruningBaseDir, "httpbin-deploy.yaml") + insecureSvcName = "httpbin-svc-insecure" + ) + + compat_otp.By("1.0: Create single pod and the service") + ns := oc.Namespace() + createResourceFromFile(oc, ns, testPodSvc) + ensurePodWithLabelReady(oc, ns, "name=httpbin-pod") + output, err := oc.Run("get").Args("service").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(output).To(o.ContainSubstring(insecureSvcName)) + + compat_otp.By("2.0: Create an http route") + routeName := "route-http11982" + routehost := routeName + "-" + ns + ".apps." + getBaseDomain(oc) + + createRoute(oc, ns, "http", routeName, insecureSvcName, []string{}) + output, err = oc.Run("get").Args("route").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(output).To(o.ContainSubstring(routeName)) + + compat_otp.By("3.0: Annotate http route") + setAnnotation(oc, ns, "route/"+routeName, "haproxy.router.openshift.io/timeout=2s") + findAnnotation := getAnnotation(oc, ns, "route", routeName) + o.Expect(findAnnotation).To(o.ContainSubstring(`haproxy.router.openshift.io/timeout":"2s`)) + + compat_otp.By("4.0: Curl the http route for two times, one with normal delay and other above timeout delay") + waitForOutsideCurlContains("http://"+routehost+"/delay/1", "-I", `200 OK`) + // some proxies return "Gateway Timeout" but some return "Gateway Time-out" + waitForOutsideCurlContains("http://"+routehost+"/delay/5", "-I", `504 Gateway Time`) + + compat_otp.By("5.0: Check HAProxy file for timeout tunnel") + routerpod := getOneRouterPodNameByIC(oc, "default") + ensureHaproxyBlockConfigContains(oc, routerpod, ns, []string{routeName, "timeout server 2s"}) + }) + + // bug: 1374772 + g.It("Author:shudili-ROSA-OSD_CCS-ARO-Critical-12091-haproxy config information should be clean when changing the service to another route", func() { + var ( + buildPruningBaseDir = testdata.FixturePath("router") + customTemp = filepath.Join(buildPruningBaseDir, "ingresscontroller-np.yaml") + clientPod = filepath.Join(buildPruningBaseDir, "test-client-pod.yaml") + clientPodName = "hello-pod" + clientPodLabel = "app=hello-pod" + webServerTemplate = filepath.Join(buildPruningBaseDir, "template-web-server-deploy.yaml") + webServerDeploy1 = webServerDeployDescription{ + deployName: "web-server-deploy1", + svcSecureName: "service-secure1", + svcUnsecureName: "service-unsecure1", + template: webServerTemplate, + namespace: "", + } + + webServerDeploy2 = webServerDeployDescription{ + deployName: "web-server-deploy2", + svcSecureName: "service-secure2", + svcUnsecureName: "service-unsecure2", + template: webServerTemplate, + namespace: "", + } + deploy1Label = "name=" + webServerDeploy1.deployName + deploy2Label = "name=" + webServerDeploy2.deployName + unsecureRouteName = "unsecure12091" + + ingctrl = ingressControllerDescription{ + name: "12091", + namespace: "openshift-ingress-operator", + domain: "", + template: customTemp, + } + ) + + compat_otp.By("1.0 Create a custom ingresscontroller") + baseDomain := getBaseDomain(oc) + ingctrl.domain = ingctrl.name + "." + baseDomain + defer ingctrl.delete(oc) + ingctrl.create(oc) + ensureRouterDeployGenerationIs(oc, ingctrl.name, "1") + + compat_otp.By("2.0: Create a client pod and deploy two sets of web-server and services") + ns := oc.Namespace() + err := oc.AsAdmin().WithoutNamespace().Run("create").Args("-n", ns, "-f", clientPod).Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + ensurePodWithLabelReady(oc, ns, clientPodLabel) + webServerDeploy1.namespace = ns + webServerDeploy2.namespace = ns + webServerDeploy1.create(oc) + webServerDeploy2.create(oc) + ensurePodWithLabelReady(oc, ns, deploy1Label) + ensurePodWithLabelReady(oc, ns, deploy2Label) + pod1Name := getPodListByLabel(oc, ns, deploy1Label)[0] + pod2Name := getPodListByLabel(oc, ns, deploy2Label)[0] + + compat_otp.By("3.0: Create a unsecure route") + routehost := unsecureRouteName + "." + ingctrl.domain + createRoute(oc, ns, "http", unsecureRouteName, webServerDeploy1.svcUnsecureName, []string{"--hostname=" + routehost}) + ensureRouteIsAdmittedByIngressController(oc, ns, unsecureRouteName, ingctrl.name) + + compat_otp.By("4.0: Add the balance=roundrobin annotation to the route, then check it in haproxy") + routerpod := getOneNewRouterPodFromRollingUpdate(oc, ingctrl.name) + backendStart := fmt.Sprintf(`backend be_http:%s:%s`, ns, unsecureRouteName) + setAnnotation(oc, ns, "route/"+unsecureRouteName, "haproxy.router.openshift.io/balance=roundrobin") + ensureHaproxyBlockConfigContains(oc, routerpod, backendStart, []string{"balance roundrobin"}) + + compat_otp.By("5.0: Curl the http route, make sure the first server is hit") + podIP := getPodv4Address(oc, routerpod, "openshift-ingress") + toDst := routehost + ":80:" + podIP + curlCmd := []string{"-n", ns, clientPodName, "--", "curl", "http://" + routehost, "-s", "--resolve", toDst, "--connect-timeout", "10"} + expectOutput := []string{"Hello-OpenShift " + pod1Name + " http-8080"} + repeatCmdOnClient(oc, curlCmd, expectOutput, 60, 1) + + compat_otp.By("6.0: Patch the http route with spec to another service") + toAnotherService := fmt.Sprintf(`{"spec":{"to":{"name": "%s"}}}`, webServerDeploy2.svcUnsecureName) + patchResourceAsAdmin(oc, ns, "route/"+unsecureRouteName, toAnotherService) + + compat_otp.By("7.0: Check the route configuration in haproxy, make sure the first service disappeared and the second service present") + pod1IP := getByJsonPath(oc, ns, "pod/"+pod1Name, `{.status.podIP}`) + ensureHaproxyBlockConfigNotContains(oc, routerpod, backendStart, []string{webServerDeploy1.svcUnsecureName, pod1IP}) + ensureHaproxyBlockConfigContains(oc, routerpod, backendStart, []string{webServerDeploy2.svcUnsecureName}) + + compat_otp.By("8.0: Curl the route for 10 times, all are forwarded to the second server") + expectOutput = []string{"Hello-OpenShift " + pod1Name + " http-8080", "Hello-OpenShift " + pod2Name + " http-8080"} + _, result := repeatCmdOnClient(oc, curlCmd, expectOutput, 180, 10) + o.Expect(result[1]).To(o.Equal(10)) + }) + + // incorporate OCP-12506 OCP-15115 OCP-16368 into one + // Test case creater: hongli@redhat.com - OCP-12506: Hostname of componentRoutes should be RFC compliant + // Test case creater: zzhao@redhat.com - OCP-15115: Harden haproxy to prevent the PROXY header from being passed for reencrypt route + g.It("Author:mjoseph-High-12506-reencrypt route with no cert if a router is configured with a default wildcard cert", func() { + buildPruningBaseDir := testdata.FixturePath("router") + testPodSvc := filepath.Join(buildPruningBaseDir, "web-server-deploy.yaml") + caCert := filepath.Join(buildPruningBaseDir, "ca-bundle.pem") + + compat_otp.By("1. Create a server pod and its service") + ns := oc.Namespace() + defaultContPod := getOneNewRouterPodFromRollingUpdate(oc, "default") + createResourceFromWebServer(oc, ns, testPodSvc, "web-server-deploy") + + compat_otp.By("2. Create a reen route") + createRoute(oc, ns, "reencrypt", "12506-no-cert", "service-secure", []string{"--dest-ca-cert=" + caCert}) + getRoutes(oc, ns) + + compat_otp.By("3. Confirm whether the destination certificate is present") + waitForOutputContains(oc, ns, "route/12506-no-cert", "{.spec.tls}", "destinationCACertificate") + + compat_otp.By("4. Check the router pod and ensure the routes are loaded in haproxy.config of default controller") + ensureHaproxyBlockConfigContains(oc, defaultContPod, ns, []string{"backend be_secure:" + ns + ":12506-no-cert"}) + + compat_otp.By("5. Check the reachability of the host in the default controller") + reenHost := "12506-no-cert-" + ns + ".apps." + getBaseDomain(oc) + waitForOutsideCurlContains("https://"+reenHost, "-k", `Hello-OpenShift web-server-deploy`) + + // OCP-15115: Harden haproxy to prevent the PROXY header from being passed for reencrypt route + compat_otp.By("6. Access the route with 'proxy' header and confirm the proxy is carried with it") + result := waitForOutsideCurlContains("--head -H proxy:10.10.10.10 https://"+reenHost, "-k", `200`) + o.Expect(result).NotTo(o.ContainSubstring(`proxy:10.10.10.10`)) + }) + + // incorporate OCP-12562 and OCP-12575 into one + // Test case creater: hongli@redhat.com - OCP-12562 The path specified in route can work well for edge terminated + // Test case creater: hongli@redhat.com - OCP-12575 The path specified in route can work well for unsecure + g.It("Author:mjoseph-ROSA-OSD_CCS-ARO-Critical-12562-The path specified in route can work well for edge/unsecure termination", func() { + buildPruningBaseDir := testdata.FixturePath("router") + testPodSvc := filepath.Join(buildPruningBaseDir, "web-server-deploy.yaml") + unSecSvc := "service-unsecure" + edgeRoute := "12562-edge" + httpRoute := "12562-http" + + compat_otp.By("1. Create a server pod and its service") + ns := oc.Namespace() + baseDomain := getBaseDomain(oc) + createResourceFromWebServer(oc, ns, testPodSvc, "web-server-deploy") + + compat_otp.By("2. Create a edge route with path") + edgeHost := edgeRoute + "-" + ns + ".apps." + baseDomain + createRoute(oc, ns, "edge", edgeRoute, unSecSvc, []string{"--path=/test"}) + ensureRouteIsAdmittedByIngressController(oc, ns, edgeRoute, "default") + + compat_otp.By("3. Curl the edge routes with and without path") + waitForOutsideCurlContains("https://"+edgeHost+"/test/", "-k", "Hello-OpenShift-Path-Test") + waitForOutsideCurlContains("https://"+edgeHost, "-k", "Application is not available") + + compat_otp.By("4. Remove path and check the reachability of the edge route without path") + patchResourceAsAdmin(oc, ns, "route/"+edgeRoute, `{"spec":{"path": ""}}`) + ensureRouteIsAdmittedByIngressController(oc, ns, edgeRoute, "default") + waitForOutsideCurlContains("https://"+edgeHost, "-k", "Hello-OpenShift") + + compat_otp.By("5. Re-add the path and again check the reachability of the edge route with path") + patchResourceAsAdmin(oc, ns, "route/"+edgeRoute, `{"spec":{"path": "/test"}}`) + ensureRouteIsAdmittedByIngressController(oc, ns, edgeRoute, "default") + output := getByJsonPath(oc, ns, "route/"+edgeRoute, `{.items[*].status.ingress[?(@.routerName=="default")].conditions[*].reason}`) + o.Expect(output).NotTo(o.ContainSubstring("HostAlreadyClaimed")) + waitForOutsideCurlContains("https://"+edgeHost+"/test/", "-k", "Hello-OpenShift-Path-Test") + + // OCP-12575: The path specified in route can work well for unsecure + compat_otp.By("6. Create a http route") + httpHost := httpRoute + "-" + ns + ".apps." + baseDomain + createRoute(oc, ns, "http", httpRoute, unSecSvc, []string{"--path=/test"}) + ensureRouteIsAdmittedByIngressController(oc, ns, httpRoute, "default") + + compat_otp.By("7. Curl the edge route and with and without path") + waitForOutsideCurlContains("http://"+httpHost+"/test/", "-k", "Hello-OpenShift-Path-Test") + waitForOutsideCurlContains("http://"+httpHost, "-k", "Application is not available") + + compat_otp.By("8. Remove path and check the reachability of the http route without path") + patchResourceAsAdmin(oc, ns, "route/"+httpRoute, `{"spec":{"path": ""}}`) + ensureRouteIsAdmittedByIngressController(oc, ns, httpRoute, "default") + waitForOutsideCurlContains("http://"+httpHost, "-k", "Hello-OpenShift") + + compat_otp.By("9. Re-add the path and again check the reachability of the http route with path") + patchResourceAsAdmin(oc, ns, "route/"+httpRoute, `{"spec":{"path": "/test"}}`) + ensureRouteIsAdmittedByIngressController(oc, ns, httpRoute, "default") + output1 := getByJsonPath(oc, ns, "route/"+httpRoute, `{.items[*].status.ingress[?(@.routerName=="default")].conditions[*].reason}`) + o.Expect(output1).NotTo(o.ContainSubstring("HostAlreadyClaimed")) + waitForOutsideCurlContains("http://"+httpHost+"/test/", "-k", "Hello-OpenShift-Path-Test") + }) + + // Test case creater: hongli@redhat.com + g.It("Author:mjoseph-Critical-12564-The path specified in route can work well for reencrypt terminated", func() { + buildPruningBaseDir := testdata.FixturePath("router") + testPodSvc := filepath.Join(buildPruningBaseDir, "web-server-deploy.yaml") + caCert := filepath.Join(buildPruningBaseDir, "ca-bundle.pem") + + compat_otp.By("1. Create a server pod and its service") + ns := oc.Namespace() + defaultContPod := getOneNewRouterPodFromRollingUpdate(oc, "default") + createResourceFromWebServer(oc, ns, testPodSvc, "web-server-deploy") + + compat_otp.By("2. Create a reen route") + createRoute(oc, ns, "reencrypt", "12564-reencrypt", "service-secure", []string{"--dest-ca-cert=" + caCert, "--path=/test"}) + getRoutes(oc, ns) + + compat_otp.By("3. Confirm whether the destination certificate is present") + waitForOutputContains(oc, ns, "route/12564-reencrypt", "{.spec.tls}", "destinationCACertificate") + + compat_otp.By("4. Check the router pod and ensure the routes are loaded in haproxy.config of default controller") + ensureHaproxyBlockConfigContains(oc, defaultContPod, ns, []string{"backend be_secure:" + ns + ":12564-reencrypt"}) + + compat_otp.By("5. Check the reachability of the in the specified path") + reenHostWithPath := "12564-reencrypt-" + ns + ".apps." + getBaseDomain(oc) + "/test/" + waitForOutsideCurlContains("https://"+reenHostWithPath, "-k", `Hello-OpenShift-Path-Test web-server-deploy`) + + compat_otp.By("6. Check the reachability of the host in the default controller") + reenHostWithOutPath := "12564-reencrypt-" + ns + ".apps." + getBaseDomain(oc) + waitForOutsideCurlContains("https://"+reenHostWithOutPath, "-kI", "503 Service Unavailable") + }) + + // incorporate OCP-12652, OCP-12556 and OCP-13248 into one + // Test case creater: zzhao@redhat.com - OCP-12652 The later route should be HostAlreadyClaimed when there is a same host exist + // Test case creater: zzhao@redhat.com - OCP-12556 Create a route without host named + // Test case creater: zzhao@redhat.com - OCP-13248 The hostname should be converted to available route when met special character + g.It("Author:mjoseph-ROSA-OSD_CCS-ARO-Critical-12652-The later route should be HostAlreadyClaimed when there is a same host exist", func() { + var ( + buildPruningBaseDir = testdata.FixturePath("router") + testPodSvc = filepath.Join(buildPruningBaseDir, "web-server-signed-deploy.yaml") + srvrcInfo = "web-server-deploy" + unSecSvcName = "service-unsecure" + secSvcName = "service-secure" + unsecureRoute = "route.12652" + httpRoute = "route.edge" + reenRoute = "route.reen" + passthroughRoute = "route.pass" + e2eTestNamespace2 = "e2e-ne-ocp22652-" + getRandomString() + ) + + compat_otp.By("1. Create an additional namespace for this scenario") + defer oc.DeleteSpecifiedNamespaceAsAdmin(e2eTestNamespace2) + oc.CreateSpecifiedNamespaceAsAdmin(e2eTestNamespace2) + e2eTestNamespace1 := oc.Namespace() + baseDomain := getBaseDomain(oc) + httpRoutehost1 := "route-12652-" + e2eTestNamespace1 + ".apps." + baseDomain + httpRoutehost2 := "route-12652-" + e2eTestNamespace2 + ".apps." + baseDomain + + compat_otp.By("2. Create a server pod and an unsecure service in one ns") + createResourceFromWebServer(oc, e2eTestNamespace1, testPodSvc, srvrcInfo) + + compat_otp.By("3: Create a http and edge route") + // Http route is created without hostname + createRoute(oc, e2eTestNamespace1, "http", unsecureRoute, unSecSvcName, []string{}) + ensureRouteIsAdmittedByIngressController(oc, e2eTestNamespace1, unsecureRoute, "default") + createRoute(oc, e2eTestNamespace1, "edge", httpRoute, unSecSvcName, []string{"--hostname=www.route-edge.com"}) + ensureRouteIsAdmittedByIngressController(oc, e2eTestNamespace1, httpRoute, "default") + + compat_otp.By("4. Create a server pod and an unsecure service in the other ns") + operateResourceFromFile(oc, "create", e2eTestNamespace2, testPodSvc) + ensurePodWithLabelReady(oc, e2eTestNamespace2, "name="+srvrcInfo) + + compat_otp.By("5: Create a http and edge route in the other ns with same host name") + // Http route is created without hostname + _, err := oc.AsAdmin().WithoutNamespace().Run("expose").Args("-n", e2eTestNamespace2, "service", unSecSvcName, "--name="+unsecureRoute).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + ensureRouteIsAdmittedByIngressController(oc, e2eTestNamespace2, unsecureRoute, "default") + _, err = oc.AsAdmin().WithoutNamespace().Run("create").Args("-n", e2eTestNamespace2, "route", "edge", httpRoute, "--service="+unSecSvcName, "--hostname=www.route-edge.com").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + + compat_otp.By("6. Confirm the route in the second ns is shown as HostAlreadyClaimed") + waitForOutputContains(oc, e2eTestNamespace2, "route", `{.items[*].status.ingress[?(@.routerName=="default")].conditions[*].reason}`, "HostAlreadyClaimed") + + // OCP-12556 NetworkEdge Create a route without host named + compat_otp.By("7: Check the http routes in both namespace are reachable without explicity configuring hostname") + waitForOutsideCurlContains("http://"+httpRoutehost1, "", `Hello-OpenShift web-server-deploy`) + waitForOutsideCurlContains("http://"+httpRoutehost2, "", `Hello-OpenShift web-server-deploy`) + + // OCP-13248 The hostname should be converted to available route when met special character + compat_otp.By("8: Create passthrough and reen route in first namespace") + createRoute(oc, e2eTestNamespace1, "passthrough", passthroughRoute, secSvcName, []string{}) + ensureRouteIsAdmittedByIngressController(oc, e2eTestNamespace1, passthroughRoute, "default") + createRoute(oc, e2eTestNamespace1, "reencrypt", reenRoute, secSvcName, []string{}) + ensureRouteIsAdmittedByIngressController(oc, e2eTestNamespace1, reenRoute, "default") + + compat_otp.By("9: Check these routes whose names have '.' decoded to '-'") + output := getRoutes(oc, e2eTestNamespace1) + o.Expect(output).To(o.And(o.ContainSubstring("route-reen"), o.ContainSubstring("route-edge"), o.ContainSubstring("route-pass"), o.ContainSubstring("route-12652"))) + }) + + // author: iamin@redhat.com + g.It("Author:iamin-ROSA-OSD_CCS-ARO-NonHyperShiftHOST-Critical-13753-NetworkEdge Check the cookie if using secure mode when insecureEdgeTerminationPolicy to Redirect for edge/reencrypt route", func() { + var ( + buildPruningBaseDir = testdata.FixturePath("router") + testPodSvc = filepath.Join(buildPruningBaseDir, "web-server-signed-deploy.yaml") + srvrcInfo = "web-server-deploy" + unSecSvcName = "service-unsecure" + SvcName = "service-secure" + fileDir = "/tmp/OCP-13753-cookie" + ) + + compat_otp.By("1.0: Prepare file folder and file for testing") + defer os.RemoveAll(fileDir) + err := os.MkdirAll(fileDir, 0755) + o.Expect(err).NotTo(o.HaveOccurred()) + + compat_otp.By("2.0: Create two server pods and the service") + ns := oc.Namespace() + srvPodList := createResourceFromWebServer(oc, ns, testPodSvc, srvrcInfo) + + compat_otp.By("3.0: Create an edge and reencrypt route with insecure_policy Redirect") + edgehost := "edge-route-" + ns + ".apps." + getBaseDomain(oc) + reenhost := "reen-route-" + ns + ".apps." + getBaseDomain(oc) + createRoute(oc, ns, "edge", "edge-route", unSecSvcName, []string{"--insecure-policy=Redirect"}) + ensureRouteIsAdmittedByIngressController(oc, ns, "edge-route", "default") + output, err := oc.Run("get").Args("route/edge-route", "-n", ns, "-o=jsonpath={.spec.tls}").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(output).To(o.ContainSubstring(`"insecureEdgeTerminationPolicy":"Redirect"`)) + + createRoute(oc, ns, "reencrypt", "reen-route", SvcName, []string{"--insecure-policy=Redirect"}) + ensureRouteIsAdmittedByIngressController(oc, ns, "reen-route", "default") + output, err = oc.Run("get").Args("route/reen-route", "-n", ns, "-o=jsonpath={.spec.tls}").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(output).To(o.ContainSubstring(`"insecureEdgeTerminationPolicy":"Redirect"`)) + + compat_otp.By("4.0: Curl the edge route and generate a cookie file") + waitForOutsideCurlContains("http://"+edgehost, "-v -L -k -c "+fileDir+"/edge-cookie", "Hello-OpenShift "+srvPodList[0]+" http-8080") + + compat_otp.By("5.0: Open the cookie file and check the contents") + // access the cookie file and confirm that the output contains false and true + checkCookieFile(fileDir+"/edge-cookie", "FALSE\t/\tTRUE") + + compat_otp.By("6.0: Curl the reencrypt route and generate a cookie file") + waitForOutsideCurlContains("http://"+reenhost, "-v -L -k -c "+fileDir+"/reen-cookie", "Hello-OpenShift "+srvPodList[0]+" https-8443") + + compat_otp.By("7.0: Open the cookie file and check the contents") + // access the cookie file and confirm that the output contains false and true + checkCookieFile(fileDir+"/reen-cookie", "FALSE\t/\tTRUE") + + }) + + // author: iamin@redhat.com + //combine OCP-9650 + g.It("Author:iamin-ROSA-OSD_CCS-ARO-NonHyperShiftHOST-Critical-13839-NetworkEdge Set insecureEdgeTerminationPolicy to Allow for reencrypt/edge route", func() { + var ( + buildPruningBaseDir = testdata.FixturePath("router") + testPodSvc = filepath.Join(buildPruningBaseDir, "web-server-signed-deploy.yaml") + SvcName = "service-secure" + unSecSvc = "service-unsecure" + ) + + compat_otp.By("1.0: Create single pod, service and reencrypt and edge route") + ns := oc.Namespace() + srvPodList := createResourceFromWebServer(oc, ns, testPodSvc, "web-server-deploy") + output, err := oc.Run("get").Args("service").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(output).To(o.And(o.ContainSubstring(unSecSvc), o.ContainSubstring(SvcName))) + createRoute(oc, ns, "reencrypt", "reen-route", SvcName, []string{}) + createRoute(oc, ns, "edge", "edge-route", unSecSvc, []string{}) + output, err = oc.Run("get").Args("route").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(output).To(o.And(o.ContainSubstring("reen-route"), o.ContainSubstring("edge-route"))) + + compat_otp.By("2.0: Add Allow policy in tls") + patchResourceAsAdmin(oc, ns, "route/reen-route", `{"spec":{"tls": {"insecureEdgeTerminationPolicy":"Allow"}}}`) + output, err = oc.Run("get").Args("route/reen-route", "-n", ns, "-o=jsonpath={.spec.tls}").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(output).To(o.ContainSubstring(`"insecureEdgeTerminationPolicy":"Allow"`)) + + compat_otp.By("3.0: Test Route is accessible using http and https") + routehost := "reen-route-" + ns + ".apps." + getBaseDomain(oc) + waitForOutsideCurlContains("http://"+routehost, "-k", "Hello-OpenShift "+srvPodList[0]+" https-8443 default") + waitForOutsideCurlContains("https://"+routehost, "-k", "Hello-OpenShift "+srvPodList[0]+" https-8443 default") + + compat_otp.By("4.0: Add Allow in edge tls") + patchResourceAsAdmin(oc, ns, "route/edge-route", `{"spec":{"tls": {"insecureEdgeTerminationPolicy":"Allow"}}}`) + output, err = oc.Run("get").Args("route/edge-route", "-n", ns, "-o=jsonpath={.spec.tls}").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(output).To(o.ContainSubstring(`"insecureEdgeTerminationPolicy":"Allow"`)) + + compat_otp.By("5.0: Test Route is accessible using http and https") + edgehost := "edge-route-" + ns + ".apps." + getBaseDomain(oc) + waitForOutsideCurlContains("http://"+edgehost, "-k", "Hello-OpenShift "+srvPodList[0]+" http-8080") + waitForOutsideCurlContains("https://"+edgehost, "-k", "Hello-OpenShift "+srvPodList[0]+" http-8080") + + }) + + // author: iamin@redhat.com + g.It("Author:iamin-ROSA-OSD_CCS-ARO-NonHyperShiftHOST-Critical-14678-NetworkEdge Only the host in whitelist could access unsecure/edge/reencrypt/passthrough routes", func() { + var ( + buildPruningBaseDir = testdata.FixturePath("router") + unSecSvcName = "service-unsecure" + signedPod = filepath.Join(buildPruningBaseDir, "web-server-signed-deploy.yaml") + ) + + compat_otp.By("1.0: Create Pod and Services") + ns := oc.Namespace() + routerpod := getOneRouterPodNameByIC(oc, "default") + createResourceFromFile(oc, ns, signedPod) + ensurePodWithLabelReady(oc, ns, "name=web-server-deploy") + + compat_otp.By("2.0: Create an unsecure, edge, reencrypt and passthrough route") + domain := getIngressctlDomain(oc, "default") + unsecureRoute := "route-unsecure" + unsecureHost := unsecureRoute + "-" + ns + "." + domain + edgeRoute := "route-edge" + edgeHost := edgeRoute + "-" + ns + "." + domain + passthroughRoute := "route-passthrough" + passthroughHost := passthroughRoute + "-" + ns + "." + domain + reenRoute := "route-reen" + reenHost := reenRoute + "-" + ns + "." + domain + + createRoute(oc, ns, "http", unsecureRoute, unSecSvcName, []string{}) + ensureRouteIsAdmittedByIngressController(oc, ns, "route-unsecure", "default") + createRoute(oc, ns, "edge", edgeRoute, unSecSvcName, []string{}) + ensureRouteIsAdmittedByIngressController(oc, ns, "route-edge", "default") + createRoute(oc, ns, "passthrough", passthroughRoute, "service-secure", []string{}) + ensureRouteIsAdmittedByIngressController(oc, ns, "route-passthrough", "default") + createRoute(oc, ns, "reencrypt", reenRoute, "service-secure", []string{}) + ensureRouteIsAdmittedByIngressController(oc, ns, "route-reen", "default") + + compat_otp.By("3.0: Annotate unsecure, edge, reencrypt and passthrough route") + setAnnotation(oc, ns, "route/"+unsecureRoute, `haproxy.router.openshift.io/ip_whitelist=0.0.0.0/0 ::/0`) + findAnnotation := getAnnotation(oc, ns, "route", unsecureRoute) + o.Expect(findAnnotation).To(o.ContainSubstring(`haproxy.router.openshift.io/ip_whitelist":"0.0.0.0/0 ::/0`)) + setAnnotation(oc, ns, "route/"+edgeRoute, `haproxy.router.openshift.io/ip_whitelist=0.0.0.0/0 ::/0`) + findAnnotation = getAnnotation(oc, ns, "route", edgeRoute) + o.Expect(findAnnotation).To(o.ContainSubstring(`haproxy.router.openshift.io/ip_whitelist":"0.0.0.0/0 ::/0`)) + setAnnotation(oc, ns, "route/"+passthroughRoute, `haproxy.router.openshift.io/ip_whitelist=0.0.0.0/0 ::/0`) + findAnnotation = getAnnotation(oc, ns, "route", passthroughRoute) + o.Expect(findAnnotation).To(o.ContainSubstring(`haproxy.router.openshift.io/ip_whitelist":"0.0.0.0/0 ::/0`)) + setAnnotation(oc, ns, "route/"+reenRoute, `haproxy.router.openshift.io/ip_whitelist=0.0.0.0/0 ::/0`) + findAnnotation = getAnnotation(oc, ns, "route", reenRoute) + o.Expect(findAnnotation).To(o.ContainSubstring(`haproxy.router.openshift.io/ip_whitelist":"0.0.0.0/0 ::/0`)) + + compat_otp.By("4.0: access the routes using the IP from the whitelist") + waitForOutsideCurlContains("http://"+unsecureHost, "", `Hello-OpenShift web-server-deploy`) + waitForOutsideCurlContains("https://"+edgeHost, "-k", `Hello-OpenShift web-server-deploy`) + waitForOutsideCurlContains("https://"+passthroughHost, "-k", `Hello-OpenShift web-server-deploy`) + waitForOutsideCurlContains("https://"+reenHost, "-k", `Hello-OpenShift web-server-deploy`) + + compat_otp.By("5.0: re-annotate routes with a random IP") + setAnnotation(oc, ns, "route/"+unsecureRoute, `haproxy.router.openshift.io/ip_whitelist=5.6.7.8`) + findAnnotation = getAnnotation(oc, ns, "route", unsecureRoute) + o.Expect(findAnnotation).To(o.ContainSubstring(`haproxy.router.openshift.io/ip_whitelist":"5.6.7.8`)) + setAnnotation(oc, ns, "route/"+edgeRoute, `haproxy.router.openshift.io/ip_whitelist=5.6.7.8`) + findAnnotation = getAnnotation(oc, ns, "route", edgeRoute) + o.Expect(findAnnotation).To(o.ContainSubstring(`haproxy.router.openshift.io/ip_whitelist":"5.6.7.8`)) + setAnnotation(oc, ns, "route/"+passthroughRoute, `haproxy.router.openshift.io/ip_whitelist=5.6.7.8`) + findAnnotation = getAnnotation(oc, ns, "route", passthroughRoute) + o.Expect(findAnnotation).To(o.ContainSubstring(`haproxy.router.openshift.io/ip_whitelist":"5.6.7.8`)) + setAnnotation(oc, ns, "route/"+reenRoute, `haproxy.router.openshift.io/ip_whitelist=5.6.7.8`) + findAnnotation = getAnnotation(oc, ns, "route", reenRoute) + o.Expect(findAnnotation).To(o.ContainSubstring(`haproxy.router.openshift.io/ip_whitelist":"5.6.7.8`)) + + compat_otp.By("6.0: attempt to access the routes without an IP in the whitelist") + cmd := fmt.Sprintf(`curl --connect-timeout 10 -s %s %s 2>&1`, "-I", "http://"+unsecureHost) + result, _ := exec.Command("bash", "-c", cmd).Output() + // use -I for 2 different scenarios, squid result has failure bad gateway, otherwise uses exit status + if strings.Contains(string(result), `squid`) { + waitForOutsideCurlContains("http://"+unsecureHost, "-I", `Bad Gateway`) + } else { + waitForOutsideCurlContains("http://"+unsecureHost, "", `exit status`) + } + waitForOutsideCurlContains("https://"+edgeHost, "-k", `exit status`) + waitForOutsideCurlContains("https://"+passthroughHost, "-k", `exit status`) + waitForOutsideCurlContains("https://"+reenHost, "-k", `exit status`) + + compat_otp.By("7.0: Check HaProxy if the IP in the whitelist annotation exists") + ensureHaproxyBlockConfigContains(oc, routerpod, ns+":"+unsecureRoute, []string{"acl allowlist src 5.6.7.8"}) + ensureHaproxyBlockConfigContains(oc, routerpod, ns+":"+edgeRoute, []string{"acl allowlist src 5.6.7.8"}) + ensureHaproxyBlockConfigContains(oc, routerpod, ns+":"+passthroughRoute, []string{"acl allowlist src 5.6.7.8"}) + ensureHaproxyBlockConfigContains(oc, routerpod, ns+":"+reenRoute, []string{"acl allowlist src 5.6.7.8"}) + }) + + // author: iamin@redhat.com + g.It("Author:iamin-ROSA-OSD_CCS-ARO-Low-14680-NetworkEdge Add invalid value in annotation whitelist to route", func() { + var ( + buildPruningBaseDir = testdata.FixturePath("router") + testPodSvc = filepath.Join(buildPruningBaseDir, "web-server-deploy.yaml") + unSecSvcName = "service-unsecure" + ) + + compat_otp.By("1.0: Create Pod and Services") + ns := oc.Namespace() + routerpod := getOneRouterPodNameByIC(oc, "default") + createResourceFromFile(oc, ns, testPodSvc) + ensurePodWithLabelReady(oc, ns, "name=web-server-deploy") + + compat_otp.By("2.0: Create an unsecure, route") + unsecureRoute := "route-unsecure" + unsecureHost := unsecureRoute + "-" + ns + ".apps." + getBaseDomain(oc) + + createRoute(oc, ns, "http", unsecureRoute, unSecSvcName, []string{}) + ensureRouteIsAdmittedByIngressController(oc, ns, "route-unsecure", "default") + + compat_otp.By("3.0: Annotate route with invalid whitelist value") + setAnnotation(oc, ns, "route/"+unsecureRoute, `haproxy.router.openshift.io/ip_whitelist='192.abc.123.0'`) + findAnnotation := getAnnotation(oc, ns, "route", unsecureRoute) + o.Expect(findAnnotation).To(o.ContainSubstring(`haproxy.router.openshift.io/ip_whitelist":"'192.abc.123.0'"`)) + + compat_otp.By("4.0: access the route using any host since whitelist is not in effect") + waitForOutsideCurlContains("http://"+unsecureHost, "", `Hello-OpenShift web-server-deploy`) + + compat_otp.By("5.0: re-annotate route with IP that all Hosts can access") + setAnnotation(oc, ns, "route/"+unsecureRoute, `haproxy.router.openshift.io/ip_whitelist=0.0.0.0/0`) + findAnnotation = getAnnotation(oc, ns, "route", unsecureRoute) + o.Expect(findAnnotation).To(o.ContainSubstring(`haproxy.router.openshift.io/ip_whitelist":"0.0.0.0/0`)) + + compat_otp.By("6.0: all hosts can access the route") + waitForOutsideCurlContains("http://"+unsecureHost, "", `Hello-OpenShift web-server-deploy`) + + compat_otp.By("7.0: Check HaProxy if the IP in the whitelist annotation exists") + ensureHaproxyBlockConfigContains(oc, routerpod, ns+":"+unsecureRoute, []string{"acl allowlist src 0.0.0.0/0"}) + }) + + // incorporate OCP-15028 OCP-15071 OCP-15072 OCP-15073 into one + // Test case creater: zzhao@redhat.com - OCP-15028: The router can do a case-insensitive match of a hostname for unsecure route + // Test case creater: zzhao@redhat.com - OCP-15071: The router can do a case-insensitive match of a hostname for edge route + // Test case creater: zzhao@redhat.com - OCP-15072: The router can do a case-insensitive match of a hostname for passthrough route + // Test case creater: zzhao@redhat.com - OCP-15073: The router can do a case-insensitive match of a hostname for reencrypt route + g.It("Author:mjoseph-ROSA-OSD_CCS-ARO-High-15028-router can do a case-insensitive match of a hostname for unsecure/edge/passthrough/reencrypt route", func() { + buildPruningBaseDir := testdata.FixturePath("router") + testPodSvc := filepath.Join(buildPruningBaseDir, "web-server-deploy.yaml") + caCert := filepath.Join(buildPruningBaseDir, "ca-bundle.pem") + UnsecureSvcName := "service-unsecure" + SecureSvcName := "service-secure" + + compat_otp.By("1. Create a server pod and its service") + ns := oc.Namespace() + baseDomain := getBaseDomain(oc) + createResourceFromWebServer(oc, ns, testPodSvc, "web-server-deploy") + + compat_otp.By("2. Create a unsecure route and ensure the routes are loaded in haproxy.config of default controlle") + createRoute(oc, ns, "http", "15028-http", UnsecureSvcName, []string{}) + ensureRouteIsAdmittedByIngressController(oc, ns, "15028-http", "default") + httpHostCapital := "15028-HTTP-" + ns + ".apps." + baseDomain + + compat_otp.By("3. Create a edge route and ensure the routes are loaded in haproxy.config of default controlle") + createRoute(oc, ns, "edge", "15028-edge", UnsecureSvcName, []string{}) + ensureRouteIsAdmittedByIngressController(oc, ns, "15028-edge", "default") + edgeHostCapital := "15028-EDGE-" + ns + ".apps." + baseDomain + + compat_otp.By("4. Create a passthrough route and ensure the routes are loaded in haproxy.config of default controlle") + createRoute(oc, ns, "passthrough", "15028-pass", SecureSvcName, []string{}) + ensureRouteIsAdmittedByIngressController(oc, ns, "15028-pass", "default") + passHostCapital := "15028-PASS-" + ns + ".apps." + baseDomain + + compat_otp.By("5. Create a reen route and ensure the routes are loaded in haproxy.config of default controlle") + createRoute(oc, ns, "reencrypt", "15028-reen", SecureSvcName, []string{"--dest-ca-cert=" + caCert}) + ensureRouteIsAdmittedByIngressController(oc, ns, "15028-reen", "default") + reenHostCapital := "15028-REEN-" + ns + ".apps." + getBaseDomain(oc) + getRoutes(oc, ns) + + // OCP-15028: The router can do a case-insensitive match of a hostname for unsecure route + compat_otp.By("6. Check the reachability of case-insensitive match of the hostname for the unsecure route") + waitForOutsideCurlContains("http://"+httpHostCapital, "-k", `Hello-OpenShift web-server-deploy`) + + // OCP-15071: The router can do a case-insensitive match of a hostname for edge route + compat_otp.By("7. Check the reachability of case-insensitive match of the hostname for the edge route") + waitForOutsideCurlContains("https://"+edgeHostCapital, "-k", `Hello-OpenShift web-server-deploy`) + + // OCP-15072: The router can do a case-insensitive match of a hostname for passthrough route + compat_otp.By("8. Check the reachability of case-insensitive match of the hostname for the passthrough route") + waitForOutsideCurlContains("https://"+passHostCapital, "-k", `Hello-OpenShift web-server-deploy`) + + // OCP-15073: The router can do a case-insensitive match of a hostname for reencrypt route + compat_otp.By("9. Check the reachability of case-insensitive match of the hostname for the reencrypt route") + waitForOutsideCurlContains("https://"+reenHostCapital, "-k", `Hello-OpenShift web-server-deploy`) + }) + + // incorporate OCP-10762 OCP-15113 OCP-15114 into one + // Test case creater: zzhao@redhat.com - OCP-10762: Check the header forward format + // Test case creater: zzhao@redhat.com - OCP-15113: Harden haproxy to prevent the PROXY header from being passed for unsecure route + // Test case creater: zzhao@redhat.com - OCP-15114: Harden haproxy to prevent the PROXY header from being passed for edge route + g.It("Author:mjoseph-ROSA-OSD_CCS-ARO-High-15113-Harden haproxy to prevent the PROXY header from being passed for unsecure/edge route", func() { + buildPruningBaseDir := testdata.FixturePath("router") + testPodSvc := filepath.Join(buildPruningBaseDir, "httpbin-deploy.yaml") + UnsecureSvcName := "httpbin-svc-insecure" + + compat_otp.By("1. Create a server pod and its service") + ns := oc.Namespace() + baseDomain := getBaseDomain(oc) + createResourceFromFile(oc, ns, testPodSvc) + ensurePodWithLabelReady(oc, ns, "name=httpbin-pod") + + compat_otp.By("2. Create a unsecure route") + createRoute(oc, ns, "http", "15113-http", UnsecureSvcName, []string{}) + ensureRouteIsAdmittedByIngressController(oc, ns, "15113-http", "default") + getRoutes(oc, ns) + httpHost := "15113-http-" + ns + ".apps." + baseDomain + + compat_otp.By("3. Create a edge route") + createRoute(oc, ns, "edge", "15113-edge", UnsecureSvcName, []string{}) + ensureRouteIsAdmittedByIngressController(oc, ns, "15113-edge", "default") + getRoutes(oc, ns) + edgeHost := "15113-edge-" + ns + ".apps." + baseDomain + + // OCP-10762: Check the header forward format + compat_otp.By("4. Access the route and check the header forward format") + result := waitForOutsideCurlContains("http://"+httpHost+"/headers", "", `proto=http`) + o.Expect(result).To(o.ContainSubstring(`"Forwarded": "for=`)) + result = waitForOutsideCurlContains("https://"+edgeHost+"/headers", "-k", `proto=https`) + o.Expect(result).To(o.ContainSubstring(`"Forwarded": "for=`)) + + // OCP-15113: Harden haproxy to prevent the PROXY header from being passed for unsecure route + compat_otp.By("5. Access the route with 'proxy' header and confirm the proxy is not carried with it") + result = waitForOutsideCurlContains(httpHost+"/headers", "-H proxy:10.10.10.10", `"Host": "`+httpHost) + o.Expect(result).NotTo(o.ContainSubstring(`proxy:10.10.10.10`)) + + // OCP-15114: Harden haproxy to prevent the PROXY header from being passed for edge route + compat_otp.By("6. Access the route with 'proxy' header and confirm the proxy is not carried with it") + result = waitForOutsideCurlContains("https://"+edgeHost+"/headers", "-k -H proxy:10.10.10.10", `"Host": "`+edgeHost) + o.Expect(result).NotTo(o.ContainSubstring(`proxy:10.10.10.10`)) + }) + + // merge OCP-15874(NetworkEdge can set cookie name for reencrypt routes by annotation) to OCP-15873 + g.It("Author:shudili-ROSA-OSD_CCS-ARO-Critical-15873-NetworkEdge can set cookie name for edge/reen routes by annotation", func() { + var ( + buildPruningBaseDir = testdata.FixturePath("router") + baseTemp = filepath.Join(buildPruningBaseDir, "ingresscontroller-np.yaml") + clientPod = filepath.Join(buildPruningBaseDir, "test-client-pod.yaml") + clientPodName = "hello-pod" + clientPodLabel = "app=hello-pod" + testPodSvc = filepath.Join(buildPruningBaseDir, "web-server-signed-deploy.yaml") + srvrcInfo = "web-server-deploy" + unSecSvcName = "service-unsecure" + secSvcName = "service-secure" + fileDir = "/data/OCP-15873-cookie" + ingctrl = ingressControllerDescription{ + name: "15873", + namespace: "openshift-ingress-operator", + domain: "", + template: baseTemp, + } + ) + + compat_otp.By("1.0: Updated replicas in the web-server-signed-deploy.yaml for testing") + updateFilebySedCmd(testPodSvc, "replicas: 1", "replicas: 2") + + compat_otp.By("2.0: Create a client pod, two server pods and the service") + ns := oc.Namespace() + err := oc.AsAdmin().WithoutNamespace().Run("create").Args("-n", ns, "-f", clientPod).Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + ensurePodWithLabelReady(oc, ns, clientPodLabel) + // create the cookie folder in the client pod + err = oc.AsAdmin().WithoutNamespace().Run("exec").Args("-n", ns, clientPodName, "--", "mkdir", fileDir).Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + srvPodList := createResourceFromWebServer(oc, ns, testPodSvc, srvrcInfo) + + compat_otp.By("3.0: Create an edge route") + ingctrl.domain = ingctrl.name + "." + getBaseDomain(oc) + routehost := "edge15873" + "." + ingctrl.domain + defer ingctrl.delete(oc) + ingctrl.create(oc) + ensureCustomIngressControllerAvailable(oc, ingctrl.name) + createRoute(oc, ns, "edge", "route-edge15873", unSecSvcName, []string{"--hostname=" + routehost}) + ensureRouteIsAdmittedByIngressController(oc, ns, "route-edge15873", "default") + + compat_otp.By("4.0: Set the cookie name by route annotation with router.openshift.io/cookie_name=2-edge_cookie") + _, err = oc.Run("annotate").WithoutNamespace().Args("-n", ns, "route/route-edge15873", "router.openshift.io/cookie_name=2-edge_cookie", "--overwrite").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + + compat_otp.By("5.0: Curl the edge route, and check the Set-Cookie header is set") + routerpod := getOneRouterPodNameByIC(oc, ingctrl.name) + podIP := getPodv4Address(oc, routerpod, "openshift-ingress") + toDst := routehost + ":443:" + podIP + curlCmd := []string{"-n", ns, clientPodName, "--", "curl", "https://" + routehost, "-kvs", "--resolve", toDst, "--connect-timeout", "10"} + expectOutput := []string{"set-cookie: 2-edge_cookie=[0-9a-z]+"} + repeatCmdOnClient(oc, curlCmd, expectOutput, 60, 1) + + compat_otp.By("6.0: Curl the edge route, saving the cookie for one server") + curlCmd = []string{"-n", ns, clientPodName, "--", "curl", "https://" + routehost, "-ks", "-c" + fileDir + "/cookie-15873", "--resolve", toDst, "--connect-timeout", "10"} + expectOutput = []string{"Hello-OpenShift " + srvPodList[1] + " http-8080"} + repeatCmdOnClient(oc, curlCmd, expectOutput, 120, 1) + + compat_otp.By("7.0: Curl the edge route with the cookie, expect all are forwarded to the desired server") + curlCmdWithCookie := []string{"-n", ns, clientPodName, "--", "curl", "https://" + routehost, "-ks", "-b", fileDir + "/cookie-15873", "--resolve", toDst, "--connect-timeout", "10"} + expectOutput = []string{"Hello-OpenShift " + srvPodList[0] + " http-8080", "Hello-OpenShift " + srvPodList[1] + " http-8080"} + _, result := repeatCmdOnClient(oc, curlCmdWithCookie, expectOutput, 120, 6) + o.Expect(result[1]).To(o.Equal(6)) + + // test for NetworkEdge can set cookie name for reencrypt routes by annotation + compat_otp.By("8.0: Create a reencrypt route") + routehost = "reen15873" + "." + ingctrl.domain + toDst = routehost + ":443:" + podIP + createRoute(oc, ns, "reencrypt", "route-reen15873", secSvcName, []string{"--hostname=" + routehost}) + ensureRouteIsAdmittedByIngressController(oc, ns, "route-reen15873", "default") + + compat_otp.By("9.0: Set the cookie name by route annotation with router.openshift.io/cookie_name=_reen-cookie3") + _, err = oc.Run("annotate").WithoutNamespace().Args("-n", ns, "route/route-reen15873", "router.openshift.io/cookie_name=_reen-cookie3", "--overwrite").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + + compat_otp.By("10.0: Curl the reencrypt route, and check the Set-Cookie header is set") + curlCmd = []string{"-n", ns, clientPodName, "--", "curl", "https://" + routehost, "-kv", "--resolve", toDst, "--connect-timeout", "10"} + expectOutput = []string{"set-cookie: _reen-cookie3=[0-9a-z]+"} + repeatCmdOnClient(oc, curlCmd, expectOutput, 60, 1) + + compat_otp.By("11.0: Curl the reen route, saving the cookie for one server") + curlCmd = []string{"-n", ns, clientPodName, "--", "curl", "https://" + routehost, "-k", "-c", fileDir + "/cookie-15873", "--resolve", toDst, "--connect-timeout", "10"} + expectOutput = []string{"Hello-OpenShift " + srvPodList[1] + " https-8443"} + repeatCmdOnClient(oc, curlCmd, expectOutput, 120, 1) + + compat_otp.By("12.0: Curl the reen route with the cookie, expect all are forwarded to the desired server") + curlCmdWithCookie = []string{"-n", ns, clientPodName, "--", "curl", "https://" + routehost, "-ks", "-b", fileDir + "/cookie-15873", "--resolve", toDst, "--connect-timeout", "10"} + expectOutput = []string{"Hello-OpenShift +" + srvPodList[0] + " +https-8443", "Hello-OpenShift +" + srvPodList[1] + " +https-8443"} + _, result = repeatCmdOnClient(oc, curlCmdWithCookie, expectOutput, 120, 6) + o.Expect(result[1]).To(o.Equal(6)) + }) + + // author: iamin@redhat.com + g.It("Author:iamin-ROSA-OSD_CCS-ARO-Medium-16732-NetworkEdge Check haproxy.config when overwriting 'timeout server' which was already specified", func() { + var ( + buildPruningBaseDir = testdata.FixturePath("router") + testPodSvc = filepath.Join(buildPruningBaseDir, "web-server-deploy.yaml") + srvrcInfo = "web-server-deploy" + unSecSvcName = "service-unsecure" + ) + + compat_otp.By("1.0: Create single pod and the service") + ns := oc.Namespace() + createResourceFromFile(oc, ns, testPodSvc) + ensurePodWithLabelReady(oc, ns, "name="+srvrcInfo) + output, err := oc.Run("get").Args("service").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(output).To(o.ContainSubstring(unSecSvcName)) + + compat_otp.By("2.0: Create an unsecure route") + routeName := unSecSvcName + + createRoute(oc, ns, "http", unSecSvcName, unSecSvcName, []string{}) + output, err = oc.Run("get").Args("route").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(output).To(o.ContainSubstring(unSecSvcName)) + + compat_otp.By("3.0: Annotate unsecure route") + setAnnotation(oc, ns, "route/"+routeName, "haproxy.router.openshift.io/timeout=5s") + findAnnotation := getAnnotation(oc, ns, "route", routeName) + o.Expect(findAnnotation).To(o.ContainSubstring(`haproxy.router.openshift.io/timeout":"5s`)) + + compat_otp.By("4.0: Check HAProxy file for timeout server") + routerpod := getOneRouterPodNameByIC(oc, "default") + ensureHaproxyBlockConfigContains(oc, routerpod, ns+":"+routeName, []string{"timeout server 5s"}) + + // overwrite annotation with same parameter to check whether haProxy shows the same annotation twice + compat_otp.By("5.0: Overwrite route annotation") + setAnnotation(oc, ns, "route/"+routeName, "haproxy.router.openshift.io/timeout=5s") + + compat_otp.By("6.0: Check HAProxy file again for timeout server") + ensureHaproxyBlockConfigContains(oc, routerpod, ns, []string{ns + ":" + routeName}) + }) + + g.It("Author:mjoseph-NonHyperShiftHOST-ROSA-OSD_CCS-ARO-Critical-17145-haproxy router support websocket via unsecure route", func() { + // On AWS (non private) proxy cluster, the url `*.apps.` is unreachable from a test client pod + // See also https://issues.redhat.com/browse/OCPQE-30244 + if checkProxy(oc) { + g.Skip("Skipping on proxy cluster since no proxy ENV in the test client pod") + } + + var ( + buildPruningBaseDir = testdata.FixturePath("router") + testPodSvc = filepath.Join(buildPruningBaseDir, "websocket-deploy.yaml") + unsecSvcName = "ws-unsecure" + clientPod = filepath.Join(buildPruningBaseDir, "test-client-pod.yaml") + clientPodLabel = "app=hello-pod" + ) + + compat_otp.By("1: Create a client pod, a server and its services in a namespace") + ns := oc.Namespace() + baseDomain := getBaseDomain(oc) + // Client pod with websocket client tool + updateFilebySedCmd(clientPod, + "quay.io/openshifttest/nginx-alpine@sha256:cee6930776b92dc1e93b73f9e5965925d49cff3d2e91e1d071c2f0ff72cbca29", + "quay.io/openshifttest/hello-sdn@sha256:c89445416459e7adea9a5a416b3365ed3d74f2491beb904d61dc8d1eb89a72a4") + createResourceFromFile(oc, ns, clientPod) + ensurePodWithLabelReady(oc, ns, clientPodLabel) + + // Server pod with websocket testing capablity + createResourceFromFile(oc, ns, testPodSvc) + ensurePodWithLabelReady(oc, ns, "name=hello-websocket") + + compat_otp.By("2: Create a http route for the testing") + routehost := "unsecure17145-" + ns + ".apps." + baseDomain + createRoute(oc, ns, "http", "unsecure17145", unsecSvcName, []string{"--hostname=" + routehost}) + ensureRouteIsAdmittedByIngressController(oc, ns, "unsecure17145", "default") + + compat_otp.By("3: Curl the http route to ensure server is ready") + curlCmd := []string{"-n", ns, "hello-pod", "--", "curl", "http://" + routehost + "/echo", "--connect-timeout", "10"} + repeatCmdOnClient(oc, curlCmd, "not websocket protocol", 60, 1) + + compat_otp.By("4: Use the unsecure route for confirming the websocket is working") + cmd := fmt.Sprintf("(echo WebsocketTesting ; sleep 20) | ws ws://%s/echo", routehost) + output, err := oc.AsAdmin().WithoutNamespace().Run("exec").Args("-n", ns, "hello-pod", "--", "bash", "-c", cmd).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + // The websocket will echo what ever input we give + o.Expect(output).To(o.ContainSubstring(`< WebsocketTesting`)) + }) + + // author: iamin@redhat.com + //combining OCP-18482 and OCP-18489 into one test + g.It("Author:iamin-ROSA-OSD_CCS-ARO-Critical-18482-NetworkEdge limits backend pod max concurrent connections for unsecure, edge, reen, passthrough route", func() { + var ( + buildPruningBaseDir = testdata.FixturePath("router") + testPodSvc = filepath.Join(buildPruningBaseDir, "web-server-deploy.yaml") + unSecSvcName = "service-unsecure" + secSvcName = "service-secure" + ) + + compat_otp.By("1.0: Create single pod and the services") + ns := oc.Namespace() + createResourceFromFile(oc, ns, testPodSvc) + ensurePodWithLabelReady(oc, ns, "name=web-server-deploy") + output, err := oc.Run("get").Args("service").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(output).To(o.And(o.ContainSubstring(unSecSvcName), o.ContainSubstring(secSvcName))) + + compat_otp.By("2.0: Create an unsecure, edge and reencrypt route") + unsecureRoute := "route-unsecure" + edgeRoute := "route-edge" + reenRoute := "route-reen" + passthroughRoute := "route-passthrough" + + createRoute(oc, ns, "http", unsecureRoute, unSecSvcName, []string{}) + ensureRouteIsAdmittedByIngressController(oc, ns, "route-unsecure", "default") + createRoute(oc, ns, "edge", edgeRoute, unSecSvcName, []string{}) + ensureRouteIsAdmittedByIngressController(oc, ns, "route-edge", "default") + createRoute(oc, ns, "reencrypt", reenRoute, secSvcName, []string{}) + ensureRouteIsAdmittedByIngressController(oc, ns, "route-reen", "default") + createRoute(oc, ns, "passthrough", passthroughRoute, secSvcName, []string{}) + ensureRouteIsAdmittedByIngressController(oc, ns, "route-passthrough", "default") + + compat_otp.By("3.0: Annotate the routes with rate-limit annotations") + setAnnotation(oc, ns, "route/"+unsecureRoute, "haproxy.router.openshift.io/pod-concurrent-connections=1") + findAnnotation := getAnnotation(oc, ns, "route", unsecureRoute) + o.Expect(findAnnotation).To(o.ContainSubstring(`haproxy.router.openshift.io/pod-concurrent-connections":"1`)) + setAnnotation(oc, ns, "route/"+edgeRoute, "haproxy.router.openshift.io/pod-concurrent-connections=2") + findAnnotation = getAnnotation(oc, ns, "route", edgeRoute) + o.Expect(findAnnotation).To(o.ContainSubstring(`haproxy.router.openshift.io/pod-concurrent-connections":"2`)) + setAnnotation(oc, ns, "route/"+reenRoute, "haproxy.router.openshift.io/pod-concurrent-connections=3") + findAnnotation = getAnnotation(oc, ns, "route", reenRoute) + o.Expect(findAnnotation).To(o.ContainSubstring(`haproxy.router.openshift.io/pod-concurrent-connections":"3`)) + setAnnotation(oc, ns, "route/"+passthroughRoute, "haproxy.router.openshift.io/pod-concurrent-connections=2") + findAnnotation = getAnnotation(oc, ns, "route", passthroughRoute) + o.Expect(findAnnotation).To(o.ContainSubstring(`haproxy.router.openshift.io/pod-concurrent-connections":"2`)) + + compat_otp.By("4.0: Check HAProxy file for route rate-limit annotation") + routerpod := getOneRouterPodNameByIC(oc, "default") + ensureHaproxyBlockConfigContains(oc, routerpod, unsecureRoute, []string{"maxconn 1"}) + ensureHaproxyBlockConfigContains(oc, routerpod, edgeRoute, []string{"maxconn 2"}) + ensureHaproxyBlockConfigContains(oc, routerpod, reenRoute, []string{"maxconn 3"}) + ensureHaproxyBlockConfigContains(oc, routerpod, passthroughRoute, []string{"maxconn 2"}) + }) + + // author: iamin@redhat.com + g.It("Author:iamin-ROSA-OSD_CCS-ARO-Medium-18490-NetworkEdge limits multiple backend pods max concurrent connections", func() { + var ( + buildPruningBaseDir = testdata.FixturePath("router") + testPodSvc = filepath.Join(buildPruningBaseDir, "web-server-deploy.yaml") + unSecSvcName = "service-unsecure" + ) + + compat_otp.By("1.0: Create single pod and its services") + ns := oc.Namespace() + createResourceFromFile(oc, ns, testPodSvc) + ensurePodWithLabelReady(oc, ns, "name=web-server-deploy") + output, err := oc.Run("get").Args("service").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(output).To(o.ContainSubstring(unSecSvcName)) + + compat_otp.By("2.0: Scale deployment to have 2 pods") + output, err = oc.AsAdmin().WithoutNamespace().Run("scale").Args("-n", ns, "deployment/web-server-deploy", "--replicas=2").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(output).To(o.ContainSubstring("web-server-deploy scaled")) + waitForOutputEquals(oc, ns, "deployment/web-server-deploy", "{.status.availableReplicas}", "2") + + compat_otp.By("3.0: Create an edge route") + edgeRoute := "route-edge" + createRoute(oc, ns, "edge", edgeRoute, unSecSvcName, []string{}) + ensureRouteIsAdmittedByIngressController(oc, ns, "route-edge", "default") + + compat_otp.By("4.0: Annotate the edge route with rate-limit annotation") + setAnnotation(oc, ns, "route/"+edgeRoute, "haproxy.router.openshift.io/pod-concurrent-connections=1") + findAnnotation := getAnnotation(oc, ns, "route", edgeRoute) + o.Expect(findAnnotation).To(o.ContainSubstring(`haproxy.router.openshift.io/pod-concurrent-connections":"1`)) + + compat_otp.By("5.0: Check HAProxy file for route rate-limit annotation") + routerpod := getOneRouterPodNameByIC(oc, "default") + searchOutput := ensureHaproxyBlockConfigContains(oc, routerpod, edgeRoute, []string{"maxconn 1"}) + count := strings.Count(searchOutput, "maxconn 1") + o.Expect(count).To(o.Equal(2), "Expected the substring to appear exactly twice") + }) + + // Test case creater: zzhao@redhat.com + g.It("Author:mjoseph-ROSA-OSD_CCS-ARO-Medium-19804-Unsecure route with path and another tls route with same hostname can work at the same time", func() { + buildPruningBaseDir := testdata.FixturePath("router") + testPodSvc := filepath.Join(buildPruningBaseDir, "web-server-deploy.yaml") + unSecSvc := "service-unsecure" + edgeRoute := "19804-edge" + httpRoute := "19804-http" + + compat_otp.By("1. Create a server pod and its service") + ns := oc.Namespace() + baseDomain := getBaseDomain(oc) + createResourceFromWebServer(oc, ns, testPodSvc, "web-server-deploy") + + compat_otp.By("2. Create a http route") + httpHost := edgeRoute + "-" + ns + ".apps." + baseDomain + createRoute(oc, ns, "http", httpRoute, unSecSvc, []string{"--hostname=" + httpHost, "--path=/test"}) + ensureRouteIsAdmittedByIngressController(oc, ns, httpRoute, "default") + + compat_otp.By("3. Create a edge route") + edgeHost := edgeRoute + "-" + ns + ".apps." + baseDomain + createRoute(oc, ns, "edge", edgeRoute, unSecSvc, []string{"--insecure-policy=Allow"}) + ensureRouteIsAdmittedByIngressController(oc, ns, edgeRoute, "default") + + compat_otp.By("4. Access the route without path") + getRoutes(oc, ns) + waitForOutsideCurlContains("https://"+httpHost+"/test/", "-k", "Hello-OpenShift-Path-Test") + + compat_otp.By("5. Access the route with path") + waitForOutsideCurlContains("https://"+edgeHost, "-k", "Hello-OpenShift") + }) + + // author: iamin@redhat.com + //combining OCP-34106 and OCP-34168 into one + g.It("Author:iamin-ROSA-OSD_CCS-ARO-High-34106-NetworkEdge Routes annotated with 'haproxy.router.openshift.io/rewrite-target=/path' will replace and rewrite http request with specified '/path'", func() { + var ( + buildPruningBaseDir = testdata.FixturePath("router") + testPodSvc = filepath.Join(buildPruningBaseDir, "web-server-deploy.yaml") + unSecSvcName = "service-unsecure" + ) + + compat_otp.By("1.0: Create single pod, service") + ns := oc.Namespace() + createResourceFromWebServer(oc, ns, testPodSvc, "web-server-deploy") + + compat_otp.By("2.0: Expose the service to create http unsecure route") + createRoute(oc, ns, "http", unSecSvcName, unSecSvcName, []string{}) + ensureRouteIsAdmittedByIngressController(oc, ns, "service-unsecure", "default") + + compat_otp.By("3.0: Annotate unsecure route with path rewrite target") + setAnnotation(oc, ns, "route/"+unSecSvcName, `haproxy.router.openshift.io/rewrite-target=/path/second/`) + findAnnotation := getAnnotation(oc, ns, "route", unSecSvcName) + o.Expect(findAnnotation).To(o.ContainSubstring(`haproxy.router.openshift.io/rewrite-target":"/path/second/`)) + + compat_otp.By("4.0: Curl route to see if the route will rewrite to second path") + domain := getIngressctlDomain(oc, "default") + unsecureHost := unSecSvcName + "-" + ns + "." + domain + waitForOutsideCurlContains("http://"+unsecureHost, "", `second-test web-server-deploy`) + + compat_otp.By("5.0: Annotate unsecure route with rewrite target") + setAnnotation(oc, ns, "route/"+unSecSvcName, `haproxy.router.openshift.io/rewrite-target=/`) + findAnnotation = getAnnotation(oc, ns, "route", unSecSvcName) + o.Expect(findAnnotation).To(o.ContainSubstring(`haproxy.router.openshift.io/rewrite-target":"/`)) + + compat_otp.By("6.0: Curl route with different post-fixes in the web-server app") + waitForOutsideCurlContains("http://"+unsecureHost+"/", "", `Hello-OpenShift web-server-deploy`) + waitForOutsideCurlContains("http://"+unsecureHost+"/test/", "", `Hello-OpenShift-Path-Test web-server-deploy`) + waitForOutsideCurlContains("http://"+unsecureHost+"/path/", "", `ocp-test web-server-deploy`) + waitForOutsideCurlContains("http://"+unsecureHost+"/path/second/", "", `second-test web-server-deploy`) + + }) + + // author: iamin@redhat.com + g.It("Author:iamin-ROSA-OSD_CCS-ARO-Critical-38671-NetworkEdge 'haproxy.router.openshift.io/timeout-tunnel' annotation gets applied alongside 'haproxy.router.openshift.io/timeout' for clear/edge/reencrypt routes", func() { + var ( + buildPruningBaseDir = testdata.FixturePath("router") + testPodSvc = filepath.Join(buildPruningBaseDir, "web-server-deploy.yaml") + srvrcInfo = "web-server-deploy" + unSecSvcName = "service-unsecure" + ) + + compat_otp.By("1.0: Create single pod and 3 services") + ns := oc.Namespace() + createResourceFromFile(oc, ns, testPodSvc) + ensurePodWithLabelReady(oc, ns, "name="+srvrcInfo) + output, err := oc.Run("get").Args("service").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(output).To(o.And(o.ContainSubstring(unSecSvcName), o.ContainSubstring("service-secure"))) + + compat_otp.By("2.0: Create a clear HTTP, edge and reen route") + routeName := unSecSvcName + + createRoute(oc, ns, "http", unSecSvcName, unSecSvcName, []string{}) + createRoute(oc, ns, "edge", "edge-route", unSecSvcName, []string{}) + createRoute(oc, ns, "reencrypt", "reen-route", "service-secure", []string{}) + output, err = oc.Run("get").Args("route").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(output).To(o.And(o.ContainSubstring(unSecSvcName), o.ContainSubstring("edge-route"), o.ContainSubstring("reen-route"))) + + compat_otp.By("3.0: Annotate all 3 routes") + setAnnotation(oc, ns, "route/"+routeName, "haproxy.router.openshift.io/timeout=15s") + findAnnotation := getAnnotation(oc, ns, "route", routeName) + o.Expect(findAnnotation).To(o.ContainSubstring(`haproxy.router.openshift.io/timeout":"15s`)) + + setAnnotation(oc, ns, "route/edge-route", "haproxy.router.openshift.io/timeout=15s") + findAnnotation = getAnnotation(oc, ns, "route", "edge-route") + o.Expect(findAnnotation).To(o.ContainSubstring(`haproxy.router.openshift.io/timeout":"15s`)) + + setAnnotation(oc, ns, "route/reen-route", "haproxy.router.openshift.io/timeout=15s") + findAnnotation = getAnnotation(oc, ns, "route", "reen-route") + o.Expect(findAnnotation).To(o.ContainSubstring(`haproxy.router.openshift.io/timeout":"15s`)) + + compat_otp.By("4.0: Check HAProxy file for timeout server on the routes") + routerpod := getOneRouterPodNameByIC(oc, "default") + ensureHaproxyBlockConfigContains(oc, routerpod, ns+":"+routeName, []string{"timeout server 15s"}) + ensureHaproxyBlockConfigContains(oc, routerpod, ns+":edge-route", []string{"timeout server 15s"}) + ensureHaproxyBlockConfigContains(oc, routerpod, ns+":reen-route", []string{"timeout server 15s"}) + + compat_otp.By("5.0: Annotate all routes with timeout tunnel") + setAnnotation(oc, ns, "route/"+routeName, "haproxy.router.openshift.io/timeout-tunnel=5s") + findAnnotation = getAnnotation(oc, ns, "route", routeName) + o.Expect(findAnnotation).To(o.ContainSubstring(`haproxy.router.openshift.io/timeout-tunnel":"5s`)) + + setAnnotation(oc, ns, "route/edge-route", "haproxy.router.openshift.io/timeout-tunnel=5s") + findAnnotation = getAnnotation(oc, ns, "route", "edge-route") + o.Expect(findAnnotation).To(o.ContainSubstring(`haproxy.router.openshift.io/timeout-tunnel":"5s`)) + + setAnnotation(oc, ns, "route/reen-route", "haproxy.router.openshift.io/timeout-tunnel=5s") + findAnnotation = getAnnotation(oc, ns, "route", "reen-route") + o.Expect(findAnnotation).To(o.ContainSubstring(`haproxy.router.openshift.io/timeout-tunnel":"5s`)) + + compat_otp.By("6.0: Check HAProxy file for timeout tunnel on the routes") + ensureHaproxyBlockConfigContains(oc, routerpod, ns+":"+routeName, []string{"timeout tunnel 5s"}) + ensureHaproxyBlockConfigContains(oc, routerpod, ns+":"+routeName, []string{"timeout tunnel 5s"}) + ensureHaproxyBlockConfigContains(oc, routerpod, ns+":"+routeName, []string{"timeout tunnel 5s"}) + }) + + // author: iamin@redhat.com + g.It("Author:iamin-ROSA-OSD_CCS-ARO-High-38672-NetworkEdge 'haproxy.router.openshift.io/timeout-tunnel' annotation takes precedence over 'haproxy.router.openshift.io/timeout' values for passthrough routes", func() { + var ( + buildPruningBaseDir = testdata.FixturePath("router") + testPodSvc = filepath.Join(buildPruningBaseDir, "web-server-deploy.yaml") + secSvcName = "service-secure" + ) + + compat_otp.By("1.0: Create single pod, service") + ns := oc.Namespace() + createResourceFromWebServer(oc, ns, testPodSvc, "web-server-deploy") + ensurePodWithLabelReady(oc, ns, "name=web-server-deploy") + + compat_otp.By("2.0: Create a passthrough route") + routeName := "38672-route-passth" + createRoute(oc, ns, "passthrough", routeName, secSvcName, []string{}) + ensureRouteIsAdmittedByIngressController(oc, ns, routeName, "default") + + compat_otp.By("3.0: Annotate passthrough route with two timeout annotations") + setAnnotation(oc, ns, "route/"+routeName, `haproxy.router.openshift.io/timeout=15s`) + setAnnotation(oc, ns, "route/"+routeName, `haproxy.router.openshift.io/timeout-tunnel=5s`) + findAnnotation := getAnnotation(oc, ns, "route", routeName) + o.Expect(findAnnotation).To(o.ContainSubstring(`haproxy.router.openshift.io/timeout":"15s`)) + o.Expect(findAnnotation).To(o.ContainSubstring(`haproxy.router.openshift.io/timeout-tunnel":"5s`)) + + compat_otp.By("4.0: Check HaProxy to see if timeout tunnel overrides timeout") + routerpod := getOneRouterPodNameByIC(oc, "default") + ensureHaproxyBlockConfigContains(oc, routerpod, routeName, []string{"timeout tunnel 5s"}) + + compat_otp.By("5.0: Remove the timeout tunnel annotation") + setAnnotation(oc, ns, "route/"+routeName, `haproxy.router.openshift.io/timeout-tunnel-`) + + compat_otp.By("6.0: Check Haproxy to see if timeout annotation is present") + ensureHaproxyBlockConfigContains(oc, routerpod, routeName, []string{"timeout tunnel 15s"}) + }) + + // author: aiyengar@redhat.com + g.It("Author:aiyengar-ROSA-OSD_CCS-ARO-Medium-42230-route can be configured to whitelist more than 61 ips/CIDRs", func() { + var ( + buildPruningBaseDir = testdata.FixturePath("router") + output string + testPodSvc = filepath.Join(buildPruningBaseDir, "web-server-deploy.yaml") + ) + compat_otp.By("Create pod, svc resources") + ns := oc.Namespace() + createResourceFromFile(oc, ns, testPodSvc) + ensurePodWithLabelReady(oc, ns, "name=web-server-deploy") + + compat_otp.By("expose a service in the namespace") + createRoute(oc, ns, "http", "service-unsecure", "service-unsecure", []string{}) + output, err := oc.Run("get").Args("-n", ns, "route").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(output).To(o.ContainSubstring("service-unsecure")) + + compat_otp.By("annotate the route with haproxy.router.openshift.io/ip_whitelist with 61 CIDR values and verify") + setAnnotation(oc, ns, "route/service-unsecure", "haproxy.router.openshift.io/ip_whitelist=192.168.0.0/24 192.168.1.0/24 192.168.2.0/24 192.168.3.0/24 192.168.4.0/24 192.168.5.0/24 192.168.6.0/24 192.168.7.0/24 192.168.8.0/24 192.168.9.0/24 192.168.10.0/24 192.168.11.0/24 192.168.12.0/24 192.168.13.0/24 192.168.14.0/24 192.168.15.0/24 192.168.16.0/24 192.168.17.0/24 192.168.18.0/24 192.168.19.0/24 192.168.20.0/24 192.168.21.0/24 192.168.22.0/24 192.168.23.0/24 192.168.24.0/24 192.168.25.0/24 192.168.26.0/24 192.168.27.0/24 192.168.28.0/24 192.168.29.0/24 192.168.30.0/24 192.168.31.0/24 192.168.32.0/24 192.168.33.0/24 192.168.34.0/24 192.168.35.0/24 192.168.36.0/24 192.168.37.0/24 192.168.38.0/24 192.168.39.0/24 192.168.40.0/24 192.168.41.0/24 192.168.42.0/24 192.168.43.0/24 192.168.44.0/24 192.168.45.0/24 192.168.46.0/24 192.168.47.0/24 192.168.48.0/24 192.168.49.0/24 192.168.50.0/24 192.168.51.0/24 192.168.52.0/24 192.168.53.0/24 192.168.54.0/24 192.168.55.0/24 192.168.56.0/24 192.168.57.0/24 192.168.58.0/24 192.168.59.0/24 192.168.60.0/24") + output, err = oc.Run("get").Args("-n", ns, "route", "service-unsecure", "-o=jsonpath={.metadata.annotations}").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(output).To(o.ContainSubstring("haproxy.router.openshift.io/ip_whitelist")) + + compat_otp.By("verify the acl whitelist parameter inside router pod for whitelist with 61 CIDR values") + podName := getOneRouterPodNameByIC(oc, "default") + //backendName is the leading context of the route + backendName := "be_http:" + ns + ":service-unsecure" + ensureHaproxyBlockConfigContains(oc, podName, backendName, []string{"acl allowlist src 192.168.0.0/24", "tcp-request content reject if !allowlist"}) + ensureHaproxyBlockConfigNotContains(oc, podName, backendName, []string{"acl allowlist src -f /var/lib/haproxy/router/allowlists/"}) + + compat_otp.By("annotate the route with haproxy.router.openshift.io/ip_whitelist with more than 61 CIDR values and verify") + setAnnotation(oc, ns, "route/service-unsecure", "haproxy.router.openshift.io/ip_whitelist=192.168.0.0/24 192.168.1.0/24 192.168.2.0/24 192.168.3.0/24 192.168.4.0/24 192.168.5.0/24 192.168.6.0/24 192.168.7.0/24 192.168.8.0/24 192.168.9.0/24 192.168.10.0/24 192.168.11.0/24 192.168.12.0/24 192.168.13.0/24 192.168.14.0/24 192.168.15.0/24 192.168.16.0/24 192.168.17.0/24 192.168.18.0/24 192.168.19.0/24 192.168.20.0/24 192.168.21.0/24 192.168.22.0/24 192.168.23.0/24 192.168.24.0/24 192.168.25.0/24 192.168.26.0/24 192.168.27.0/24 192.168.28.0/24 192.168.29.0/24 192.168.30.0/24 192.168.31.0/24 192.168.32.0/24 192.168.33.0/24 192.168.34.0/24 192.168.35.0/24 192.168.36.0/24 192.168.37.0/24 192.168.38.0/24 192.168.39.0/24 192.168.40.0/24 192.168.41.0/24 192.168.42.0/24 192.168.43.0/24 192.168.44.0/24 192.168.45.0/24 192.168.46.0/24 192.168.47.0/24 192.168.48.0/24 192.168.49.0/24 192.168.50.0/24 192.168.51.0/24 192.168.52.0/24 192.168.53.0/24 192.168.54.0/24 192.168.55.0/24 192.168.56.0/24 192.168.57.0/24 192.168.58.0/24 192.168.59.0/24 192.168.60.0/24 192.168.61.0/24") + output1, err1 := oc.Run("get").Args("-n", ns, "route", "service-unsecure", "-o=jsonpath={.metadata.annotations}").Output() + o.Expect(err1).NotTo(o.HaveOccurred()) + o.Expect(output1).To(o.ContainSubstring("haproxy.router.openshift.io/ip_whitelist")) + + compat_otp.By("verify the acl whitelist parameter inside router pod for whitelist with 62 CIDR values") + //backendName is the leading context of the route + ensureHaproxyBlockConfigContains(oc, podName, backendName, []string{`acl allowlist src -f /var/lib/haproxy/router/allowlists/` + ns + `:service-unsecure.txt`, `tcp-request content reject if !allowlist`}) + }) + + // author: mjoseph@redhat.com + g.It("Author:mjoseph-ROSA-OSD_CCS-ARO-High-45399-ingress controller continue to function normally with unexpected high timeout value", func() { + var ( + buildPruningBaseDir = testdata.FixturePath("router") + output string + testPodSvc = filepath.Join(buildPruningBaseDir, "web-server-deploy.yaml") + ) + compat_otp.By("Create pod, svc resources") + ns := oc.Namespace() + createResourceFromFile(oc, ns, testPodSvc) + ensurePodWithLabelReady(oc, ns, "name=web-server-deploy") + + compat_otp.By("expose a service in the namespace") + createRoute(oc, ns, "http", "service-secure", "service-secure", []string{}) + output, err := oc.Run("get").Args("-n", ns, "route").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(output).To(o.ContainSubstring("service-secure")) + + compat_otp.By("annotate the route with haproxy.router.openshift.io/timeout annotation to high value and verify") + setAnnotation(oc, ns, "route/service-secure", "haproxy.router.openshift.io/timeout=9999d") + output, err = oc.Run("get").Args("-n", ns, "route", "service-secure", "-o=jsonpath={.metadata.annotations}").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(output).To(o.ContainSubstring(`haproxy.router.openshift.io/timeout":"9999d`)) + + compat_otp.By("Verify the haproxy configuration for the set timeout value") + podName := getOneRouterPodNameByIC(oc, "default") + ensureHaproxyBlockConfigContains(oc, podName, ns, []string{"timeout server 2147483647ms"}) + + compat_otp.By("Verify the pod logs to see any timer overflow error messages") + log, err := oc.AsAdmin().WithoutNamespace().Run("logs").Args("-n", "openshift-ingress", podName, "-c", "router").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(log).NotTo(o.ContainSubstring(`timer overflow`)) + }) + + // author: hongli@redhat.com + g.It("Author:hongli-ROSA-OSD_CCS-ARO-High-45741-ingress canary route redirects http to https", func() { + var ns = "openshift-ingress-canary" + compat_otp.By("get the ingress route host") + canaryRouteHost := getByJsonPath(oc, ns, "route/canary", "{.status.ingress[0].host}") + o.Expect(canaryRouteHost).Should(o.ContainSubstring(`canary-openshift-ingress-canary.apps`)) + + compat_otp.By("curl canary route via http and redirects to https") + waitForOutsideCurlContains("http://"+canaryRouteHost, "-I", "302 Found") + waitForOutsideCurlContains("http://"+canaryRouteHost, "-kL", "Healthcheck requested") + waitForOutsideCurlContains("https://"+canaryRouteHost, "-k", "Healthcheck requested") + }) + + // author: iamin@redhat.com + g.It("Author:iamin-ROSA-OSD_CCS-ARO-NonPreRelease-PreChkUpgrade-High-45955-Unidling a route work without user intervention", func() { + var ( + buildPruningBaseDir = testdata.FixturePath("router") + testPodSvc = filepath.Join(buildPruningBaseDir, "web-server-deploy.yaml") + ns = "ocp45955" + ) + + networkType := compat_otp.CheckNetworkType(oc) + if !(strings.Contains(networkType, "openshiftsdn") || strings.Contains(networkType, "ovn")) { + g.Skip("Skipping because idling is not supported on the network type") + } + + compat_otp.By("1.0: Create a new project and a web-server application") + oc.CreateSpecifiedNamespaceAsAdmin(ns) + err := oc.AsAdmin().WithoutNamespace().Run("create").Args("-f", testPodSvc, "-n", ns).Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + ensurePodWithLabelReady(oc, ns, "name=web-server-deploy") + podName := getOnePodNameByLabel(oc, ns, "name=web-server-deploy") + + compat_otp.By("2.0: Confirm that the service exists and expose it") + _, err = oc.AsAdmin().WithoutNamespace().Run("get").Args("-n", ns, "service", "service-unsecure").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + _, err = oc.AsAdmin().WithoutNamespace().Run("expose").Args("service", "service-unsecure", "-n", ns).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + ensureRouteIsAdmittedByIngressController(oc, ns, "service-unsecure", "default") + + compat_otp.By("3.0: Access the exposed route") + routehost := getRouteHost(oc, ns, "service-unsecure") + curlCmd := fmt.Sprintf(`curl http://%s --connect-timeout 10`, routehost) + repeatCmdOnClient(oc, curlCmd, "Hello-OpenShift", 120, 1) + + compat_otp.By("4.0: Idle the service and wait for the pod resource to dissapear") + err = oc.AsAdmin().WithoutNamespace().Run("idle").Args("service-unsecure", "-n", ns).Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + + errResource := waitForResourceToDisappear(oc, ns, "pod/"+podName) + o.Expect(errResource).NotTo(o.HaveOccurred()) + + compat_otp.By("5.0: Confirm the service annotation is present") + findAnnotation := getAnnotation(oc, ns, "service", "service-unsecure") + o.Expect(findAnnotation).To(o.ContainSubstring(`idling.alpha.openshift.io/unidle-targets":"[{\"kind\":\"Deployment\",\"name\":\"web-server-deploy\",\"group\":\"apps\",\"replicas\":1}]`)) + }) + + // author: iamin@redhat.com + g.It("Author:iamin-ROSA-OSD_CCS-ARO-NonPreRelease-PstChkUpgrade-High-45955-Unidling a route work without user intervention", func() { + var ( + ns = "ocp45955" + ) + + networkType := compat_otp.CheckNetworkType(oc) + if !(strings.Contains(networkType, "openshiftsdn") || strings.Contains(networkType, "ovn")) { + g.Skip("Skipping because idling is not supported on the network type") + } + + compat_otp.By("1.0: Confirm that the service is still available with the annotation") + _, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("-n", ns, "service", "service-unsecure").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + findAnnotation := getAnnotation(oc, ns, "service", "service-unsecure") + o.Expect(findAnnotation).To(o.ContainSubstring(`idling.alpha.openshift.io/unidle-targets":"[{\"kind\":\"Deployment\",\"name\":\"web-server-deploy\",\"group\":\"apps\",\"replicas\":1}]`)) + + compat_otp.By("2.0: Access the route to unidle the service") + routehost := getRouteHost(oc, ns, "service-unsecure") + curlCmd := fmt.Sprintf(`curl http://%s --connect-timeout 10`, routehost) + repeatCmdOnClient(oc, curlCmd, "Hello-OpenShift", 120, 1) + + compat_otp.By("3.0: Check the service to see if the idle annotation is removed") + findAnnotation = getAnnotation(oc, ns, "service", "service-unsecure") + o.Expect(findAnnotation).NotTo(o.ContainSubstring(`idling.alpha.openshift.io/unidle-targets`)) + }) + + // author: mjoseph@redhat.com + g.It("Author:mjoseph-ROSA-OSD_CCS-ARO-High-49802-HTTPS redirect happens even if there is a more specific http-only", func() { + // curling through default controller will not work for proxy cluster. + if checkProxy(oc) { + g.Skip("This is proxy cluster, skip the test.") + } + var ( + buildPruningBaseDir = testdata.FixturePath("router") + testPodSvc = filepath.Join(buildPruningBaseDir, "web-server-deploy.yaml") + customTemp = filepath.Join(buildPruningBaseDir, "49802-route.yaml") + rut = routeDescription{ + namespace: "", + template: customTemp, + } + ) + + compat_otp.By("Create a pod") + baseDomain := getBaseDomain(oc) + ns := oc.Namespace() + createResourceFromFile(oc, ns, testPodSvc) + ensurePodWithLabelReady(oc, ns, "name=web-server-deploy") + podName := getPodListByLabel(oc, ns, "name=web-server-deploy") + defaultContPod := getOneRouterPodNameByIC(oc, "default") + + compat_otp.By("create routes and get the details") + rut.namespace = ns + rut.create(oc) + getRoutes(oc, ns) + + compat_otp.By("check the reachability of the secure route with redirection") + waitForCurl(oc, podName[0], baseDomain, "hello-pod-"+ns+".apps.", "HTTP/1.1 302 Found", "") + waitForCurl(oc, podName[0], baseDomain, "hello-pod-"+ns+".apps.", `location: https://hello-pod-`, "") + + compat_otp.By("check the reachability of the insecure routes") + waitForCurl(oc, podName[0], baseDomain+"/test/", "hello-pod-http-"+ns+".apps.", "HTTP/1.1 200 OK", "") + + compat_otp.By("check the reachability of the secure route") + curlCmd := fmt.Sprintf("curl -I -k https://hello-pod-%s.apps.%s --connect-timeout 10", ns, baseDomain) + statsOut, err := compat_otp.RemoteShPod(oc, ns, podName[0], "sh", "-c", curlCmd) + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(statsOut).Should(o.ContainSubstring("HTTP/1.1 200 OK")) + + compat_otp.By("check the router pod and ensure the routes are loaded in haproxy.config") + searchOutput := readRouterPodData(oc, defaultContPod, "cat haproxy.config", "hello-pod") + o.Expect(searchOutput).To(o.ContainSubstring("backend be_edge_http:" + ns + ":hello-pod")) + searchOutput1 := readRouterPodData(oc, defaultContPod, "cat haproxy.config", "hello-pod-http") + o.Expect(searchOutput1).To(o.ContainSubstring("backend be_http:" + ns + ":hello-pod-http")) + }) + + // author: mjoseph@redhat.com + g.It("Author:mjoseph-Critical-53696-Route status should updates accordingly when ingress routes cleaned up [Disruptive]", func() { + var ( + buildPruningBaseDir = testdata.FixturePath("router") + customTemp = filepath.Join(buildPruningBaseDir, "ingresscontroller-np.yaml") + ingctrl = ingressControllerDescription{ + name: "ocp53696", + namespace: "openshift-ingress-operator", + domain: "", + template: customTemp, + } + ) + + compat_otp.By("check the intial canary route status") + ensureRouteIsAdmittedByIngressController(oc, "openshift-ingress-canary", "canary", "default") + + compat_otp.By("shard the default ingress controller") + actualGen, _ := oc.AsAdmin().WithoutNamespace().Run("get").Args("deployment/router-default", "-n", "openshift-ingress", "-o=jsonpath={.metadata.generation}").Output() + defer patchResourceAsAdmin(oc, "openshift-ingress-operator", "ingresscontrollers/default", "{\"spec\":{\"routeSelector\":{\"matchLabels\":{\"type\":null}}}}") + patchResourceAsAdmin(oc, "openshift-ingress-operator", "ingresscontrollers/default", "{\"spec\":{\"routeSelector\":{\"matchLabels\":{\"type\":\"shard\"}}}}") + // After patching the default congtroller generation should be +1 + actualGenerationInt, _ := strconv.Atoi(actualGen) + ensureRouterDeployGenerationIs(oc, "default", strconv.Itoa(actualGenerationInt+1)) + + compat_otp.By("check whether canary route status is cleared") + checkRouteDetailsRemoved(oc, "openshift-ingress-canary", "canary", "default") + + compat_otp.By("patch the controller back to default check the canary route status") + patchResourceAsAdmin(oc, "openshift-ingress-operator", "ingresscontrollers/default", "{\"spec\":{\"routeSelector\":{\"matchLabels\":{\"type\":null}}}}") + ensureRouterDeployGenerationIs(oc, "default", strconv.Itoa(actualGenerationInt+2)) + ensureRouteIsAdmittedByIngressController(oc, "openshift-ingress-canary", "canary", "default") + + compat_otp.By("Create a shard ingresscontroller") + baseDomain := getBaseDomain(oc) + ingctrl.domain = "shard." + baseDomain + ingctrlResource := "ingresscontrollers/" + ingctrl.name + defer ingctrl.delete(oc) + ingctrl.create(oc) + ensureRouterDeployGenerationIs(oc, ingctrl.name, "1") + + compat_otp.By("patch the shard controller and check the canary route status") + patchResourceAsAdmin(oc, ingctrl.namespace, ingctrlResource, "{\"spec\":{\"nodePlacement\":{\"nodeSelector\":{\"matchLabels\":{\"node-role.kubernetes.io/worker\":\"\"}}}}}") + ensureRouterDeployGenerationIs(oc, ingctrl.name, "2") + ensureRouteIsAdmittedByIngressController(oc, "openshift-ingress-canary", "canary", "default") + ensureRouteIsAdmittedByIngressController(oc, "openshift-ingress-canary", "canary", ingctrl.name) + + compat_otp.By("delete the shard and check the status") + custContPod := getOneNewRouterPodFromRollingUpdate(oc, ingctrl.name) + ingctrl.delete(oc) + err3 := waitForResourceToDisappear(oc, "openshift-ingress", "pod/"+custContPod) + compat_otp.AssertWaitPollNoErr(err3, fmt.Sprintf("Router %v failed to fully terminate", "pod/"+custContPod)) + ensureRouteIsAdmittedByIngressController(oc, "openshift-ingress-canary", "canary", "default") + checkRouteDetailsRemoved(oc, "openshift-ingress-canary", "canary", ingctrl.name) + }) + + // bugzilla: 2021446 + // no ingress-operator pod on HyperShift guest cluster so this case is not available + g.It("Author:mjoseph-NonHyperShiftHOST-High-55895-Ingress should be in degraded status when canary route is not available [Disruptive]", func() { + compat_otp.By("Check the intial co/ingress and canary route status") + ensureClusterOperatorNormal(oc, "ingress", 1, 10) + ensureRouteIsAdmittedByIngressController(oc, "openshift-ingress-canary", "canary", "default") + + compat_otp.By("Check the reachability of the canary route") + baseDomain := getBaseDomain(oc) + operatorPod := getPodListByLabel(oc, "openshift-ingress-operator", "name=ingress-operator") + routehost := "canary-openshift-ingress-canary.apps." + baseDomain + cmdOnPod := []string{operatorPod[0], "-n", "openshift-ingress-operator", "--", "curl", "-k", "https://" + routehost, "--connect-timeout", "10"} + repeatCmdOnClient(oc, cmdOnPod, "Healthcheck requested", 30, 1) + + compat_otp.By("Patch the ingress controller and deleting the canary route") + actualGen, _ := oc.AsAdmin().WithoutNamespace().Run("get").Args("deployment/router-default", "-n", "openshift-ingress", "-o=jsonpath={.metadata.generation}").Output() + defer ensureClusterOperatorNormal(oc, "ingress", 3, 300) + defer patchResourceAsAdmin(oc, "openshift-ingress-operator", "ingresscontrollers/default", "{\"spec\":{\"routeSelector\":null}}") + patchResourceAsAdmin(oc, "openshift-ingress-operator", "ingresscontrollers/default", "{\"spec\":{\"routeSelector\":{\"matchLabels\":{\"type\":\"default\"}}}}") + // Deleting canary route + err := oc.AsAdmin().Run("delete").Args("-n", "openshift-ingress-canary", "route", "canary").Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + // After patching the default congtroller generation should be +1 + actualGenerationInt, _ := strconv.Atoi(actualGen) + ensureRouterDeployGenerationIs(oc, "default", strconv.Itoa(actualGenerationInt+1)) + + compat_otp.By("Check whether the canary route status cleared and confirm the route is not accessible") + checkRouteDetailsRemoved(oc, "openshift-ingress-canary", "canary", "default") + cmdOnPod = []string{operatorPod[0], "-n", "openshift-ingress-operator", "--", "curl", "-Ik", "https://" + routehost, "--connect-timeout", "10"} + repeatCmdOnClient(oc, cmdOnPod, "503", 120, 1) + + // Wait may be about 300 seconds + compat_otp.By("Check the ingress operator status to confirm it is in degraded state cause by canary route") + jpath := "{.status.conditions[*].message}" + waitForOutputContains(oc, "default", "co/ingress", jpath, "The \"default\" ingress controller reports Degraded=True") + waitForOutputContains(oc, "default", "co/ingress", jpath, "Canary route is not admitted by the default ingress controller") + }) + + // bugzilla: 1934904 + // Jira: OCPBUGS-9274 + // no openshift-machine-api namespace on HyperShift guest cluster so this case is not available + g.It("Author:mjoseph-NonHyperShiftHOST-NonPreRelease-High-56240-Canary daemonset can schedule pods to both worker and infra nodes [Disruptive]", func() { + var ( + infrastructureName = clusterinfra.GetInfrastructureName(oc) + machineSetName = infrastructureName + "-56240" + ) + + compat_otp.By("Check the intial machines and canary pod details") + getResourceName(oc, "openshift-machine-api", "machine") + getResourceName(oc, "openshift-ingress-canary", "pods") + + compat_otp.By("Create a new machineset") + clusterinfra.SkipConditionally(oc) + ms := clusterinfra.MachineSetDescription{Name: machineSetName, Replicas: 1} + defer ms.DeleteMachineSet(oc) + ms.CreateMachineSet(oc) + + compat_otp.By("Update machineset to schedule infra nodes") + out, _ := oc.AsAdmin().WithoutNamespace().Run("patch").Args("machinesets.machine.openshift.io", machineSetName, "-n", "openshift-machine-api", "-p", `{"spec":{"template":{"spec":{"taints":null}}}}`, "--type=merge").Output() + o.Expect(out).To(o.ContainSubstring("machineset.machine.openshift.io/" + machineSetName + " patched")) + out, _ = oc.AsAdmin().WithoutNamespace().Run("patch").Args("machinesets.machine.openshift.io", machineSetName, "-n", "openshift-machine-api", "-p", `{"spec":{"template":{"spec":{"metadata":{"labels":{"ingress": "true", "node-role.kubernetes.io/infra": ""}}}}}}`, "--type=merge").Output() + o.Expect(out).To(o.ContainSubstring("machineset.machine.openshift.io/" + machineSetName + " patched")) + updatedMachineName := clusterinfra.WaitForMachinesRunningByLabel(oc, 1, "machine.openshift.io/cluster-api-machineset="+machineSetName) + + compat_otp.By("Reschedule the running machineset with infra details") + clusterinfra.DeleteMachine(oc, updatedMachineName[0]) + updatedMachineName1 := clusterinfra.WaitForMachinesRunningByLabel(oc, 1, "machine.openshift.io/cluster-api-machineset="+machineSetName) + + compat_otp.By("Check the canary deamonset is scheduled on infra node which is newly created") + // confirm the new machineset is already created + updatedMachineSetName := clusterinfra.ListWorkerMachineSetNames(oc) + checkGivenStringPresentOrNot(true, updatedMachineSetName, machineSetName) + // confirm infra node presence among the nodes + infraNode := getByLabelAndJsonPath(oc, "default", "node", "node-role.kubernetes.io/infra", "{.items[*].metadata.name}") + // confirm a canary pod got scheduled on to the infra node + searchInDescribeResource(oc, "node", infraNode, "canary") + + compat_otp.By("Confirming the canary namespace is over-rided with the default node selector") + annotations, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("ns", "openshift-ingress-canary", "-ojsonpath={.metadata.annotations}").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(annotations).To(o.ContainSubstring(`openshift.io/node-selector":""`)) + + compat_otp.By("Confirming the canary daemonset has the default tolerations included for infra role") + tolerations := getByJsonPath(oc, "openshift-ingress-canary", "daemonset/ingress-canary", "{.spec.template.spec.tolerations}") + o.Expect(tolerations).To(o.ContainSubstring(`key":"node-role.kubernetes.io/infra`)) + + compat_otp.By("Tainting the infra nodes with 'NoSchedule' and confirm canary pods continues to remain up and functional on those nodes") + nodeNameOfMachine := clusterinfra.GetNodeNameFromMachine(oc, updatedMachineName1[0]) + output, err := oc.AsAdmin().WithoutNamespace().Run("adm").Args("taint", "nodes", nodeNameOfMachine, "node-role.kubernetes.io/infra:NoSchedule").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(output).To(o.ContainSubstring("node/" + nodeNameOfMachine + " tainted")) + // confirm the canary pod is still present in the infra node + searchInDescribeResource(oc, "node", infraNode, "canary") + + compat_otp.By("Tainting the infra nodes with 'NoExecute' and confirm canary pods continues to remain up and functional on those nodes") + output1, err1 := oc.AsAdmin().WithoutNamespace().Run("adm").Args("taint", "nodes", nodeNameOfMachine, "node-role.kubernetes.io/infra:NoExecute").Output() + o.Expect(err1).NotTo(o.HaveOccurred()) + o.Expect(output1).To(o.ContainSubstring("node/" + nodeNameOfMachine + " tainted")) + // confirm the canary pod is still present in the infra node + searchInDescribeResource(oc, "node", infraNode, "canary") + }) + + g.It("Author:mjoseph-ROSA-OSD_CCS-ARO-Medium-63004-Ipv6 addresses are also acceptable for whitelisting", func() { + var ( + buildPruningBaseDir = testdata.FixturePath("router") + output string + testPodSvc = filepath.Join(buildPruningBaseDir, "web-server-deploy.yaml") + ) + + compat_otp.By("Create a server pod") + ns := oc.Namespace() + createResourceFromFile(oc, ns, testPodSvc) + ensurePodWithLabelReady(oc, ns, "name=web-server-deploy") + + compat_otp.By("expose a service in the namespace") + createRoute(oc, ns, "http", "service-unsecure", "service-unsecure", []string{}) + output, err := oc.Run("get").Args("route").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(output).To(o.ContainSubstring("service-unsecure")) + + compat_otp.By("Annotate the route with Ipv6 subnet and verify it") + setAnnotation(oc, ns, "route/service-unsecure", "haproxy.router.openshift.io/ip_whitelist=2600:14a0::/40") + output, err = oc.Run("get").Args("route", "service-unsecure", "-o=jsonpath={.metadata.annotations}").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(output).To(o.ContainSubstring(`"haproxy.router.openshift.io/ip_whitelist":"2600:14a0::/40"`)) + + compat_otp.By("Verify the acl whitelist parameter inside router pod with Ipv6 address") + defaultPod := getOneRouterPodNameByIC(oc, "default") + backendName := "be_http:" + ns + ":service-unsecure" + ensureHaproxyBlockConfigContains(oc, defaultPod, backendName, []string{"acl allowlist src 2600:14a0::/40"}) + }) + + // author: hongli@redhat.com + g.It("Author:hongli-ROSA-OSD_CCS-ARO-High-73771-router can load secret", func() { + var ( + buildPruningBaseDir = testdata.FixturePath("router") + testPodSvc = filepath.Join(buildPruningBaseDir, "web-server-signed-deploy.yaml") + requiredRole = filepath.Join(buildPruningBaseDir, "ocp73771-role.yaml") + unsecsvcName = "service-unsecure" + secsvcName = "service-secure" + tmpdir = "/tmp/OCP-73771-CA/" + caKey = tmpdir + "ca.key" + caCrt = tmpdir + "ca.crt" + serverKey = tmpdir + "server.key" + serverCsr = tmpdir + "server.csr" + serverCrt = tmpdir + "server.crt" + multiServerCrt = tmpdir + "multiserver.crt" + ) + compat_otp.By("Create pod, svc resources") + ns := oc.Namespace() + createResourceFromFile(oc, ns, testPodSvc) + ensurePodWithLabelReady(oc, ns, "name=web-server-deploy") + + compat_otp.By("Create edge/passthrough/reencrypt routes and all should be reachable") + extraParas := []string{} + createRoute(oc, ns, "edge", "myedge", unsecsvcName, extraParas) + createRoute(oc, ns, "passthrough", "mypass", secsvcName, extraParas) + createRoute(oc, ns, "reencrypt", "myreen", secsvcName, extraParas) + edgeRouteHost := getRouteHost(oc, ns, "myedge") + passRouteHost := getRouteHost(oc, ns, "mypass") + reenRouteHost := getRouteHost(oc, ns, "myreen") + waitForOutsideCurlContains("https://"+edgeRouteHost, "-k", "Hello-OpenShift") + waitForOutsideCurlContains("https://"+passRouteHost, "-k", "Hello-OpenShift") + waitForOutsideCurlContains("https://"+reenRouteHost, "-k", "Hello-OpenShift") + + compat_otp.By("should be failed if patch the edge route without required role and secret") + err1 := "Forbidden: router serviceaccount does not have permission to get this secret" + err2 := "Forbidden: router serviceaccount does not have permission to watch this secret" + err3 := "Forbidden: router serviceaccount does not have permission to list this secret" + err4 := `Not found: "secrets \"mytls\" not found` + output, err := oc.WithoutNamespace().Run("patch").Args("-n", ns, "route/myedge", "-p", `{"spec":{"tls":{"externalCertificate":{"name":"mytls"}}}}`, "--type=merge").Output() + o.Expect(err).To(o.HaveOccurred()) + o.Expect(output).Should(o.And( + o.ContainSubstring(err1), + o.ContainSubstring(err2), + o.ContainSubstring(err3), + o.ContainSubstring(err4))) + + compat_otp.By("create required role/rolebinding and secret") + // create required role and rolebinding + createResourceFromFile(oc, ns, requiredRole) + // prepare the tmp folder and create self-signed cerfitcate + defer os.RemoveAll(tmpdir) + err = os.MkdirAll(tmpdir, 0755) + o.Expect(err).NotTo(o.HaveOccurred()) + opensslNewCa(caKey, caCrt, "/CN=ne-root-ca") + opensslNewCsr(serverKey, serverCsr, "/CN=ne-server-cert") + // san just contains edge route host but not reen route host + san := "subjectAltName=DNS:" + edgeRouteHost + opensslSignCsr(san, serverCsr, caCrt, caKey, serverCrt) + err = oc.AsAdmin().WithoutNamespace().Run("create").Args("-n", ns, "secret", "tls", "mytls", "--cert="+serverCrt, "--key="+serverKey).Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + + compat_otp.By("patch the edge and reen route, but only edge route should be reachable") + patchResourceAsAdmin(oc, ns, "route/myedge", `{"spec":{"tls":{"externalCertificate":{"name":"mytls"}}}}`) + patchResourceAsAdmin(oc, ns, "route/myreen", `{"spec":{"tls":{"externalCertificate":{"name":"mytls"}}}}`) + curlOptions := fmt.Sprintf("--cacert %v", caCrt) + waitForOutsideCurlContains("https://"+edgeRouteHost, curlOptions, "Hello-OpenShift") + repeatCmdOnClient(oc, fmt.Sprintf("curl https://%s %s --connect-timeout 10", reenRouteHost, curlOptions), `exit status (51|60)`, 60, 1) + + compat_otp.By("renew the server certificate with multi SAN and refresh the secret") + // multiSan contains both edge and reen route host + multiSan := san + ", DNS:" + reenRouteHost + opensslSignCsr(multiSan, serverCsr, caCrt, caKey, multiServerCrt) + newSecretYaml, err := oc.Run("create").Args("-n", ns, "secret", "tls", "mytls", "--cert="+multiServerCrt, "--key="+serverKey, "--dry-run=client", "-o=yaml").OutputToFile("ocp73771-newsecret.yaml") + o.Expect(err).NotTo(o.HaveOccurred()) + err = oc.WithoutNamespace().Run("apply").Args("-f", newSecretYaml).Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + + compat_otp.By("with the updated secret, both edge and reen route should be reachable") + waitForOutsideCurlContains("https://"+edgeRouteHost, curlOptions, "Hello-OpenShift") + waitForOutsideCurlContains("https://"+reenRouteHost, curlOptions, "Hello-OpenShift") + + compat_otp.By("should failed to patch passthrough route with externalCertificate") + output, err = oc.WithoutNamespace().Run("patch").Args("-n", ns, "route/mypass", "-p", `{"spec":{"tls":{"externalCertificate":{"name":"mytls"}}}}`, "--type=merge").Output() + o.Expect(err).To(o.HaveOccurred()) + o.Expect(output).To(o.ContainSubstring("passthrough termination does not support certificate")) + + compat_otp.By("edge route reports error after deleting the referenced secret") + err = oc.Run("delete").Args("-n", ns, "secret", "mytls").Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + waitForOutputContains(oc, ns, "route/myedge", `{.status.ingress[?(@.routerName=="default")].conditions[*]}`, "ExternalCertificateValidationFailed") + + // https://issues.redhat.com/browse/OCPBUGS-33958 (4.19+) + compat_otp.By("edge and reen route should be recovered after recreating the referenced secret") + err = oc.WithoutNamespace().Run("apply").Args("-f", newSecretYaml).Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + ensureRouteIsAdmittedByIngressController(oc, ns, "myedge", "default") + waitForOutsideCurlContains("https://"+edgeRouteHost, curlOptions, "Hello-OpenShift") + waitForOutsideCurlContains("https://"+reenRouteHost, curlOptions, "Hello-OpenShift") + }) + + // author: iamin@redhat.com + g.It("Author:iamin-ROSA-OSD_CCS-ARO-Critical-77080-NetworkEdge Only host in allowlist can access unsecure/edge/reencrypt/passthrough routes", func() { + var ( + buildPruningBaseDir = testdata.FixturePath("router") + unSecSvcName = "service-unsecure" + secSvcName = "service-secure" + signedPod = filepath.Join(buildPruningBaseDir, "web-server-signed-deploy.yaml") + ) + + compat_otp.By("1.0: Create Pod and Services") + ns := oc.Namespace() + routerpod := getOneRouterPodNameByIC(oc, "default") + srvPodList := createResourceFromWebServer(oc, ns, signedPod, "web-server-deploy") + + compat_otp.By("2.0: Create an unsecure, edge, reencrypt and passthrough route") + domain := getIngressctlDomain(oc, "default") + unsecureRoute := "route-unsecure" + unsecureHost := unsecureRoute + "-" + ns + "." + domain + edgeRoute := "route-edge" + edgeHost := edgeRoute + "-" + ns + "." + domain + passthroughRoute := "route-passthrough" + passthroughHost := passthroughRoute + "-" + ns + "." + domain + reenRoute := "route-reen" + reenHost := reenRoute + "-" + ns + "." + domain + + createRoute(oc, ns, "http", unsecureRoute, unSecSvcName, []string{}) + ensureRouteIsAdmittedByIngressController(oc, ns, "route-unsecure", "default") + createRoute(oc, ns, "edge", edgeRoute, unSecSvcName, []string{}) + ensureRouteIsAdmittedByIngressController(oc, ns, "route-edge", "default") + createRoute(oc, ns, "passthrough", passthroughRoute, secSvcName, []string{}) + ensureRouteIsAdmittedByIngressController(oc, ns, "route-passthrough", "default") + createRoute(oc, ns, "reencrypt", reenRoute, secSvcName, []string{}) + ensureRouteIsAdmittedByIngressController(oc, ns, "route-reen", "default") + + compat_otp.By("3.0: Annotate unsecure, edge, reencrypt and passthrough route") + setAnnotation(oc, ns, "route/"+unsecureRoute, `haproxy.router.openshift.io/ip_allowlist=0.0.0.0/0 ::/0`) + findAnnotation := getAnnotation(oc, ns, "route", unsecureRoute) + o.Expect(findAnnotation).To(o.ContainSubstring(`haproxy.router.openshift.io/ip_allowlist":"0.0.0.0/0 ::/0`)) + setAnnotation(oc, ns, "route/"+edgeRoute, `haproxy.router.openshift.io/ip_allowlist=0.0.0.0/0 ::/0`) + findAnnotation = getAnnotation(oc, ns, "route", edgeRoute) + o.Expect(findAnnotation).To(o.ContainSubstring(`haproxy.router.openshift.io/ip_allowlist":"0.0.0.0/0 ::/0`)) + setAnnotation(oc, ns, "route/"+passthroughRoute, `haproxy.router.openshift.io/ip_allowlist=0.0.0.0/0 ::/0`) + findAnnotation = getAnnotation(oc, ns, "route", passthroughRoute) + o.Expect(findAnnotation).To(o.ContainSubstring(`haproxy.router.openshift.io/ip_allowlist":"0.0.0.0/0 ::/0`)) + setAnnotation(oc, ns, "route/"+reenRoute, `haproxy.router.openshift.io/ip_allowlist=0.0.0.0/0 ::/0`) + findAnnotation = getAnnotation(oc, ns, "route", reenRoute) + o.Expect(findAnnotation).To(o.ContainSubstring(`haproxy.router.openshift.io/ip_allowlist":"0.0.0.0/0 ::/0`)) + + compat_otp.By("4.0: access the routes using the IP from the allowlist") + waitForOutsideCurlContains("http://"+unsecureHost, "", `Hello-OpenShift `+srvPodList[0]+` http-8080`) + waitForOutsideCurlContains("https://"+edgeHost, "-k", `Hello-OpenShift `+srvPodList[0]+` http-8080`) + waitForOutsideCurlContains("https://"+passthroughHost, "-k", `Hello-OpenShift `+srvPodList[0]+` https-8443 default`) + waitForOutsideCurlContains("https://"+reenHost, "-k", `Hello-OpenShift `+srvPodList[0]+` https-8443 default`) + + compat_otp.By("5.0: re-annotate routes with a random IP") + setAnnotation(oc, ns, "route/"+unsecureRoute, `haproxy.router.openshift.io/ip_allowlist=1050::5:600:300c:326b`) + findAnnotation = getAnnotation(oc, ns, "route", unsecureRoute) + o.Expect(findAnnotation).To(o.ContainSubstring(`haproxy.router.openshift.io/ip_allowlist":"1050::5:600:300c:326b`)) + setAnnotation(oc, ns, "route/"+edgeRoute, `haproxy.router.openshift.io/ip_allowlist=8.8.8.8`) + findAnnotation = getAnnotation(oc, ns, "route", edgeRoute) + o.Expect(findAnnotation).To(o.ContainSubstring(`haproxy.router.openshift.io/ip_allowlist":"8.8.8.8`)) + setAnnotation(oc, ns, "route/"+passthroughRoute, `haproxy.router.openshift.io/ip_allowlist=1050::5:600:300c:326b`) + findAnnotation = getAnnotation(oc, ns, "route", passthroughRoute) + o.Expect(findAnnotation).To(o.ContainSubstring(`haproxy.router.openshift.io/ip_allowlist":"1050::5:600:300c:326b`)) + setAnnotation(oc, ns, "route/"+reenRoute, `haproxy.router.openshift.io/ip_allowlist=8.8.4.4`) + findAnnotation = getAnnotation(oc, ns, "route", reenRoute) + o.Expect(findAnnotation).To(o.ContainSubstring(`haproxy.router.openshift.io/ip_allowlist":"8.8.4.4`)) + + compat_otp.By("6.0: attempt to access the routes without an IP in the allowlist") + cmd := fmt.Sprintf(`curl --connect-timeout 10 -s %s %s 2>&1`, "-I", "http://"+unsecureHost) + result, _ := exec.Command("bash", "-c", cmd).Output() + // use -I for 2 different scenarios, squid result has failure bad gateway, otherwise uses exit status + if strings.Contains(string(result), `squid`) { + waitForOutsideCurlContains("http://"+unsecureHost, "-I", `Bad Gateway`) + } else { + waitForOutsideCurlContains("http://"+unsecureHost, "", `exit status`) + } + waitForOutsideCurlContains("https://"+edgeHost, "-k", `exit status`) + waitForOutsideCurlContains("https://"+passthroughHost, "-k", `exit status`) + waitForOutsideCurlContains("https://"+reenHost, "-k", `exit status`) + + compat_otp.By("7.0: Check HaProxy if the IP in the allowlist annotation exists") + ensureHaproxyBlockConfigContains(oc, routerpod, ns+":"+unsecureRoute, []string{"acl allowlist src 1050::5:600:300c:326b", "tcp-request content reject if !allowlist"}) + ensureHaproxyBlockConfigContains(oc, routerpod, ns+":"+edgeRoute, []string{"acl allowlist src 8.8.8.8", "tcp-request content reject if !allowlist"}) + ensureHaproxyBlockConfigContains(oc, routerpod, ns+":"+passthroughRoute, []string{"acl allowlist src 1050::5:600:300c:326b", "tcp-request content reject if !allowlist"}) + ensureHaproxyBlockConfigContains(oc, routerpod, ns+":"+reenRoute, []string{"acl allowlist src 8.8.4.4", "tcp-request content reject if !allowlist"}) + }) + + // author: iamin@redhat.com + g.It("Author:iamin-ROSA-OSD_CCS-ARO-Critical-77082-NetworkEdge Route gives allowlist precedence when whitelist and allowlist annotations are both present", func() { + var ( + buildPruningBaseDir = testdata.FixturePath("router") + testPod = filepath.Join(buildPruningBaseDir, "web-server-deploy.yaml") + unSecSvcName = "service-unsecure" + ) + + compat_otp.By("1.0: Create Pod and Services") + ns := oc.Namespace() + routerpod := getOneRouterPodNameByIC(oc, "default") + srvPodList := createResourceFromWebServer(oc, ns, testPod, "web-server-deploy") + ensurePodWithLabelReady(oc, ns, "name=web-server-deploy") + + compat_otp.By("2.0: Create an unsecure route") + unsecureRoute := "route-unsecure" + unsecureHost := unsecureRoute + "-" + ns + ".apps." + getBaseDomain(oc) + createRoute(oc, ns, "http", unsecureRoute, unSecSvcName, []string{}) + ensureRouteIsAdmittedByIngressController(oc, ns, "route-unsecure", "default") + + compat_otp.By("3.0: Annotate unsecure route") + setAnnotation(oc, ns, "route/"+unsecureRoute, `haproxy.router.openshift.io/ip_whitelist=0.0.0.0/0 ::/0`) + findAnnotation := getAnnotation(oc, ns, "route", unsecureRoute) + o.Expect(findAnnotation).To(o.ContainSubstring(`haproxy.router.openshift.io/ip_whitelist":"0.0.0.0/0 ::/0`)) + + compat_otp.By("4.0: access the route using the IP from the whitelist") + waitForOutsideCurlContains("http://"+unsecureHost, "", `Hello-OpenShift `+srvPodList[0]+` http-8080`) + + compat_otp.By("5.0: add allowlist annotation with non valid host IP") + setAnnotation(oc, ns, "route/"+unsecureRoute, `haproxy.router.openshift.io/ip_allowlist=1.2.3.4`) + findAnnotation = getAnnotation(oc, ns, "route", unsecureRoute) + o.Expect(findAnnotation).To(o.ContainSubstring(`haproxy.router.openshift.io/ip_allowlist":"1.2.3.4`)) + + compat_otp.By("6.0: attempt to access the routes without an IP in the allowlist") + cmd := fmt.Sprintf(`curl --connect-timeout 10 -s %s %s 2>&1`, "-I", "http://"+unsecureHost) + result, _ := exec.Command("bash", "-c", cmd).Output() + // use -I for 2 different scenarios, squid result has failure bad gateway, otherwise uses exit status + if strings.Contains(string(result), `squid`) { + waitForOutsideCurlContains("http://"+unsecureHost, "-I", `Bad Gateway`) + } else { + waitForOutsideCurlContains("http://"+unsecureHost, "", `exit status`) + } + + compat_otp.By("7.0: annotate route with a valid public client IP in the allowlist and an invalid host IP in the whitelist") + setAnnotation(oc, ns, "route/"+unsecureRoute, `haproxy.router.openshift.io/ip_allowlist=0.0.0.0/0 ::/0`) + findAnnotation = getAnnotation(oc, ns, "route", unsecureRoute) + o.Expect(findAnnotation).To(o.ContainSubstring(`haproxy.router.openshift.io/ip_allowlist":"0.0.0.0/0 ::/0`)) + + setAnnotation(oc, ns, "route/"+unsecureRoute, `haproxy.router.openshift.io/ip_whitelist=1.2.3.4`) + findAnnotation1 := getAnnotation(oc, ns, "route", unsecureRoute) + o.Expect(findAnnotation1).To(o.ContainSubstring(`haproxy.router.openshift.io/ip_whitelist":"1.2.3.4`)) + + waitForOutsideCurlContains("http://"+unsecureHost, "", `Hello-OpenShift `+srvPodList[0]+` http-8080`) + + compat_otp.By("8.0: Check HaProxy if the allowlist annotation exists and tcp request exist") + ensureHaproxyBlockConfigContains(oc, routerpod, ns+":"+unsecureRoute, []string{"acl allowlist src", "tcp-request content reject if !allowlist"}) + }) + + // author: iamin@redhat.com + // Combines OCP-77091 and OCP 77086 tests for allowlist epic NE:1100 + g.It("Author:iamin-ROSA-OSD_CCS-ARO-High-77091-NetworkEdge Route does not enable allowlist with than 61 CIDRs and if invalid IP annotation is given", func() { + var ( + buildPruningBaseDir = testdata.FixturePath("router") + testPod = filepath.Join(buildPruningBaseDir, "web-server-deploy.yaml") + unSecSvcName = "service-unsecure" + ) + + compat_otp.By("1.0: Create Pod and Services") + ns := oc.Namespace() + routerpod := getOneRouterPodNameByIC(oc, "default") + srvPodList := createResourceFromWebServer(oc, ns, testPod, "web-server-deploy") + ensurePodWithLabelReady(oc, ns, "name=web-server-deploy") + + compat_otp.By("2.0: Create an edge route") + edgeRoute := "route-edge" + edgeHost := edgeRoute + "-" + ns + ".apps." + getBaseDomain(oc) + createRoute(oc, ns, "edge", edgeRoute, unSecSvcName, []string{}) + ensureRouteIsAdmittedByIngressController(oc, ns, "route-edge", "default") + + compat_otp.By("3.0: annotate route with an invalid IP and try to access route") + setAnnotation(oc, ns, "route/"+edgeRoute, `haproxy.router.openshift.io/ip_allowlist=192.abc.123.0`) + findAnnotation := getAnnotation(oc, ns, "route", edgeRoute) + o.Expect(findAnnotation).To(o.ContainSubstring(`haproxy.router.openshift.io/ip_allowlist":"192.abc.123.0`)) + + waitForOutsideCurlContains("https://"+edgeHost, "-k", `Hello-OpenShift `+srvPodList[0]+` http-8080`) + + compat_otp.By("4.0: Check HaProxy to confirm the allowlist annotation does not occur") + ensureHaproxyBlockConfigNotContains(oc, routerpod, ns+":"+edgeRoute, []string{"acl allowlist src", "tcp-request content reject if !allowlist"}) + + //OCP-77091 route does not enable whitelist with more than 61 CIDRs + compat_otp.By("5.0: Create an unsecure route") + unsecureRoute := "route-unsecure" + createRoute(oc, ns, "http", unsecureRoute, unSecSvcName, []string{}) + ensureRouteIsAdmittedByIngressController(oc, ns, "route-unsecure", "default") + + compat_otp.By("6.0: Annotate unsecure route with 61 CIDRs") + setAnnotation(oc, ns, "route/"+unsecureRoute, `haproxy.router.openshift.io/ip_allowlist=192.168.0.0/24 192.168.1.0/24 192.168.2.0/24 192.168.3.0/24 192.168.4.0/24 192.168.5.0/24 192.168.6.0/24 192.168.7.0/24 192.168.8.0/24 192.168.9.0/24 192.168.10.0/24 192.168.11.0/24 192.168.12.0/24 192.168.13.0/24 192.168.14.0/24 192.168.15.0/24 192.168.16.0/24 192.168.17.0/24 192.168.18.0/24 192.168.19.0/24 192.168.20.0/24 192.168.21.0/24 192.168.22.0/24 192.168.23.0/24 192.168.24.0/24 192.168.25.0/24 192.168.26.0/24 192.168.27.0/24 192.168.28.0/24 192.168.29.0/24 192.168.30.0/24 192.168.31.0/24 192.168.32.0/24 192.168.33.0/24 192.168.34.0/24 192.168.35.0/24 192.168.36.0/24 192.168.37.0/24 192.168.38.0/24 192.168.39.0/24 192.168.40.0/24 192.168.41.0/24 192.168.42.0/24 192.168.43.0/24 192.168.44.0/24 192.168.45.0/24 192.168.46.0/24 192.168.47.0/24 192.168.48.0/24 192.168.49.0/24 192.168.50.0/24 192.168.51.0/24 192.168.52.0/24 192.168.53.0/24 192.168.54.0/24 192.168.55.0/24 192.168.56.0/24 192.168.57.0/24 192.168.58.0/24 192.168.59.0/24 192.168.60.0/24`) + findAnnotation = getAnnotation(oc, ns, "route", unsecureRoute) + o.Expect(findAnnotation).To(o.ContainSubstring(`haproxy.router.openshift.io/ip_allowlist":"`)) + + compat_otp.By("7.0: Check HaProxy if the allowlist annotation exists and tcp request exist") + ensureHaproxyBlockConfigContains(oc, routerpod, ns+":"+unsecureRoute, []string{"acl allowlist src 192.168.0.0/24", "tcp-request content reject if !allowlist"}) + + compat_otp.By("8.0: add allowlist annotation with more than 61 CIDRs") + setAnnotation(oc, ns, "route/"+unsecureRoute, `haproxy.router.openshift.io/ip_allowlist=192.168.0.0/24 192.168.1.0/24 192.168.2.0/24 192.168.3.0/24 192.168.4.0/24 192.168.5.0/24 192.168.6.0/24 192.168.7.0/24 192.168.8.0/24 192.168.9.0/24 192.168.10.0/24 192.168.11.0/24 192.168.12.0/24 192.168.13.0/24 192.168.14.0/24 192.168.15.0/24 192.168.16.0/24 192.168.17.0/24 192.168.18.0/24 192.168.19.0/24 192.168.20.0/24 192.168.21.0/24 192.168.22.0/24 192.168.23.0/24 192.168.24.0/24 192.168.25.0/24 192.168.26.0/24 192.168.27.0/24 192.168.28.0/24 192.168.29.0/24 192.168.30.0/24 192.168.31.0/24 192.168.32.0/24 192.168.33.0/24 192.168.34.0/24 192.168.35.0/24 192.168.36.0/24 192.168.37.0/24 192.168.38.0/24 192.168.39.0/24 192.168.40.0/24 192.168.41.0/24 192.168.42.0/24 192.168.43.0/24 192.168.44.0/24 192.168.45.0/24 192.168.46.0/24 192.168.47.0/24 192.168.48.0/24 192.168.49.0/24 192.168.50.0/24 192.168.51.0/24 192.168.52.0/24 192.168.53.0/24 192.168.54.0/24 192.168.55.0/24 192.168.56.0/24 192.168.57.0/24 192.168.58.0/24 192.168.59.0/24 192.168.60.0/24 192.168.61.0/24`) + findAnnotation = getAnnotation(oc, ns, "route", unsecureRoute) + o.Expect(findAnnotation).To(o.ContainSubstring(`haproxy.router.openshift.io/ip_allowlist":"`)) + + compat_otp.By("9.0: Check HaProxy if the allowlist annotation exists and tcp request exist") + ensureHaproxyBlockConfigContains(oc, routerpod, "backend be_http:"+ns+":"+unsecureRoute, []string{`acl allowlist src -f /var/lib/haproxy/router/allowlists/` + ns + ":" + unsecureRoute + ".txt", "tcp-request content reject if !allowlist"}) + ensureHaproxyBlockConfigNotContains(oc, routerpod, "backend be_http:"+ns+":"+unsecureRoute, []string{"acl allowlist src 192.168.0.0/24"}) + }) + + // OCPBUGS-47773 + g.It("Author:shudili-ROSA-OSD_CCS-ARO-Critical-85274-Route spec path that have specail characters should not cause HaProxy error and ingress degraded", func() { + var ( + buildPruningBaseDir = testdata.FixturePath("router") + testPodSvc = filepath.Join(buildPruningBaseDir, "web-server-deploy.yaml") + unSecSvcName = "service-unsecure" + ingressJsonPath = `{.status.conditions[?(@.type=="Available")].status}{.status.conditions[?(@.type=="Progressing")].status}{.status.conditions[?(@.type=="Degraded")].status}` + ) + + // skip the test if ingress co is abnormal + status := getByJsonPath(oc, "default", "co/ingress", ingressJsonPath) + if status != "TrueFalseFalse" { + g.Skip("ingress co is abnormal") + } + + compat_otp.By("1.0: Create a single pod and the service") + ns := oc.Namespace() + createResourceFromFile(oc, ns, testPodSvc) + ensurePodWithLabelReady(oc, ns, "name=web-server-deploy") + output, err := oc.Run("get").Args("service").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(output).To(o.ContainSubstring(unSecSvcName)) + + compat_otp.By("2.0: Create an unsecure route") + createRoute(oc, ns, "http", unSecSvcName, unSecSvcName, []string{}) + ensureRouteIsAdmittedByIngressController(oc, ns, unSecSvcName, "default") + + compat_otp.By(`3.0: Try to patch the route spec.path with "/route-admission-test#2", which includes the # character`) + specPath := `{"spec": {"path": "/route-admission-test#2"}}` + output, err = oc.AsAdmin().WithoutNamespace().Run("patch").Args("route/"+unSecSvcName, "-p", specPath, "--type=merge", "-n", ns).Output() + o.Expect(err).To(o.HaveOccurred()) + o.Expect(output).To(o.ContainSubstring(`cannot contain # or spaces`)) + + compat_otp.By(`4.0: Try to patch the route spec.path with "/route-admission-test 22", which includes the space character`) + specPath = `{"spec": {"path": "/route-admission-test 22"}}` + output, err = oc.AsAdmin().WithoutNamespace().Run("patch").Args("route/"+unSecSvcName, "-p", specPath, "--type=merge", "-n", ns).Output() + o.Expect(err).To(o.HaveOccurred()) + o.Expect(output).To(o.ContainSubstring(`cannot contain # or spaces`)) + + compat_otp.By("5.0: Check the ingress co, make sure it is normal") + status = getByJsonPath(oc, "default", "co/ingress", ingressJsonPath) + o.Expect(status).To(o.ContainSubstring("TrueFalseFalse")) + }) +}) diff --git a/tests-extension/test/e2e/tls.go b/tests-extension/test/e2e/tls.go new file mode 100644 index 000000000..34e63decd --- /dev/null +++ b/tests-extension/test/e2e/tls.go @@ -0,0 +1,601 @@ +package router + +import ( + "fmt" + "os" + "path/filepath" + "regexp" + "strings" + + g "github.com/onsi/ginkgo/v2" + o "github.com/onsi/gomega" + e2e "k8s.io/kubernetes/test/e2e/framework" + + "github.com/openshift/origin/test/extended/util/compat_otp" + "github.com/openshift/router-tests-extension/test/testdata" +) + +var _ = g.Describe("[sig-network-edge] Network_Edge Component_Router", func() { + defer g.GinkgoRecover() + + var oc = compat_otp.NewCLI("router-tls", compat_otp.KubeConfigPath()) + + // incorporate OCP-12557, OCP-12563 into one + // Test case creater: bmeng@redhat.com - OCP-12557: Only the certs file of the certain route will be updated when the route is updated + // Test case creater: hongli@redhat.com - OCP-12563: The certs for the edge/reencrypt termination routes should be removed when the routes removed + g.It("Author:shudili-ROSA-OSD_CCS-ARO-High-12563-The certs for the edge/reencrypt termination routes should be removed when the routes removed", func() { + // skip the test if featureSet is set there + if compat_otp.IsTechPreviewNoUpgrade(oc) { + g.Skip("Skip for the haproxy was't the realtime for the backend configuration after enabled DynamicConfigurationManager") + } + + var ( + buildPruningBaseDir = testdata.FixturePath("router") + testPodSvc = filepath.Join(buildPruningBaseDir, "web-server-deploy.yaml") + unSecSvcName = "service-unsecure" + secSvcName = "service-secure" + dirname = "/tmp/OCP-12563" + caSubj = "/CN=NE-Test-Root-CA" + caCrt = dirname + "/12563-ca.crt" + caKey = dirname + "/12563-ca.key" + edgeRouteSubj = "/CN=example-edge.com" + edgeRouteCrt = dirname + "/12563-edgeroute.crt" + edgeRouteKey = dirname + "/12563-edgeroute.key" + edgeRouteCsr = dirname + "/12563-edgeroute.csr" + reenRouteSubj = "/CN=example-reen.com" + reenRouteCrt = dirname + "/12563-reenroute.crt" + reenRouteKey = dirname + "/12563-reenroute.key" + reenRouteCsr = dirname + "/12563-reenroute.csr" + reenRouteDstSubj = "/CN=example-reen-dst.com" + reenRouteDstCrt = dirname + "/12563-reenroutedst.crt" + reenRouteDstKey = dirname + "/12563-reenroutedst.key" + ) + + compat_otp.By("1.0 Create a file folder and prepair for testing") + defer os.RemoveAll(dirname) + err := os.MkdirAll(dirname, 0755) + o.Expect(err).NotTo(o.HaveOccurred()) + baseDomain := getBaseDomain(oc) + edgeRoute := "edge12563.apps." + baseDomain + reenRoute := "reen12563.apps." + baseDomain + + compat_otp.By("2.0: Use openssl to create ca certification and key") + opensslNewCa(caKey, caCrt, caSubj) + + compat_otp.By("3.0: Create a user CSR and the user key for the edge route") + opensslNewCsr(edgeRouteKey, edgeRouteCsr, edgeRouteSubj) + + compat_otp.By("3.1: Sign the user CSR and generate the certificate for the edge route") + san := "subjectAltName = DNS:" + edgeRoute + opensslSignCsr(san, edgeRouteCsr, caCrt, caKey, edgeRouteCrt) + + compat_otp.By("4.0: Create a user CSR and the user key for the reen route") + opensslNewCsr(reenRouteKey, reenRouteCsr, reenRouteSubj) + + compat_otp.By("4.1: Sign the user CSR and generate the certificate for the reen route") + san = "subjectAltName = DNS:" + reenRoute + opensslSignCsr(san, reenRouteCsr, caCrt, caKey, reenRouteCrt) + + compat_otp.By("5.0: Use openssl to create certification and key for the destination certification of the reen route") + opensslNewCa(reenRouteDstKey, reenRouteDstCrt, reenRouteDstSubj) + + compat_otp.By("6.0 Create a deployment") + ns := oc.Namespace() + createResourceFromFile(oc, ns, testPodSvc) + ensurePodWithLabelReady(oc, ns, "name=web-server-deploy") + + compat_otp.By("7.0: Create the edge route and the reen route") + createRoute(oc, ns, "edge", "route-edge", unSecSvcName, []string{"--hostname=" + edgeRoute, "--ca-cert=" + caCrt, "--cert=" + edgeRouteCrt, "--key=" + edgeRouteKey}) + createRoute(oc, ns, "reencrypt", "route-reen", secSvcName, []string{"--hostname=" + reenRoute, "--ca-cert=" + caCrt, "--cert=" + reenRouteCrt, "--key=" + reenRouteKey, "--dest-ca-cert=" + reenRouteDstCrt}) + ensureRouteIsAdmittedByIngressController(oc, ns, "route-edge", "default") + ensureRouteIsAdmittedByIngressController(oc, ns, "route-reen", "default") + + compat_otp.By("8.0: Check the certs for the edge/reencrypt termination routes") + routerpod := getOneRouterPodNameByIC(oc, "default") + edgeCertIntialTime := checkRouteCertificationInRouterPod(oc, ns, "route-edge", routerpod, "certs", "--hasCert") + reenCertIntialTime := checkRouteCertificationInRouterPod(oc, ns, "route-reen", routerpod, "certs", "--hasCert") + + compat_otp.By("9.0: Check the cacert for the reencrypt termination route") + reencaCertIntialTime := checkRouteCertificationInRouterPod(oc, ns, "route-reen", routerpod, "cacerts", "--hasCert") + + // OCP-12557: Only the certs file of the certain route will be updated when that route is updated + compat_otp.By("10.0: Show the cert files creation time") + e2e.Logf("The intial edge certificate creation details is %s", edgeCertIntialTime) + e2e.Logf("The intial reen certificate creation details is %s", reenCertIntialTime) + e2e.Logf("The intial reen CA certificate creation details is %s", reencaCertIntialTime) + + compat_otp.By("11.0: Patch the reen route with path varibale") + patchResourceAsAdmin(oc, ns, "route/route-reen", `{"spec": {"path": "/test"}}`) + output, err := oc.Run("get").Args("route/route-reen", "-n", ns, "-o=jsonpath={.spec.path}").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(output).To(o.ContainSubstring(`/test`)) + + compat_otp.By("12.0: Recheck the creation time of the certs") + // the cert details of reen route will be updated and the edge route will be same + edgeUpdatedCertTime := checkRouteCertificationInRouterPod(oc, ns, "route-edge", routerpod, "certs", "--hasCert") + reenUpdatedCertTime := checkRouteCertificationInRouterPod(oc, ns, "route-reen", routerpod, "certs", "--hasCert") + reencaCertUpdatedTime := checkRouteCertificationInRouterPod(oc, ns, "route-reen", routerpod, "cacerts", "--hasCert") + e2e.Logf("The Updated edge certificate creation details is %s", edgeUpdatedCertTime) + e2e.Logf("The Updated reen certificate creation details is %s", reenUpdatedCertTime) + e2e.Logf("The Updated reen CA certificate creation details is %s", reencaCertUpdatedTime) + o.Expect(edgeCertIntialTime).To(o.ContainSubstring(edgeUpdatedCertTime)) + o.Expect(reenCertIntialTime).NotTo(o.ContainSubstring(reenUpdatedCertTime)) + o.Expect(reencaCertIntialTime).NotTo(o.ContainSubstring(reencaCertUpdatedTime)) + + compat_otp.By("13.0: Delete the two routes") + err = oc.AsAdmin().WithoutNamespace().Run("delete").Args("-n", ns, "route", "route-edge").Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + err = oc.AsAdmin().WithoutNamespace().Run("delete").Args("-n", ns, "route", "route-reen").Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + + compat_otp.By("14.0: Check the certs for the edge/reencrypt termination routes again after deleted the routes") + checkRouteCertificationInRouterPod(oc, ns, "route-edge", routerpod, "certs", "--noCert") + checkRouteCertificationInRouterPod(oc, ns, "route-reen", routerpod, "certs", "--noCert") + + compat_otp.By("15.0: Check the cacert for the reencrypt termination route again after deleted the route") + checkRouteCertificationInRouterPod(oc, ns, "route-reen", routerpod, "cacerts", "--noCert") + }) + + // author: iamin@redhat.com + //Combine tls cases OCP-12573 and OCP-19799 + g.It("Author:iamin-ROSA-OSD_CCS-ARO-High-12573-Default haproxy router should be able to skip invalid cert route", func() { + var ( + buildPruningBaseDir = testdata.FixturePath("router") + testPodSvc = filepath.Join(buildPruningBaseDir, "web-server-deploy.yaml") + unSecSvcName = "service-unsecure" + secSvcName = "service-secure" + dirname = "/tmp/OCP-12573" + caSubj = "/CN=NE-Test-Root-CA" + caCrt1 = dirname + "/12573-ca1.crt" + caKey1 = dirname + "/12573-ca1.key" + caCrt2 = dirname + "/12573-ca2.crt" + caKey2 = dirname + "/12573-ca2.key" + edgeRouteSubj = "/CN=example-edge.com" + edgeRouteCrt1 = dirname + "/12573-edgeroute1.crt" + edgeRouteKey1 = dirname + "/12573-edgeroute1.key" + edgeRouteCsr1 = dirname + "/12573-edgeroute1.csr" + edgeRouteCrt2 = dirname + "/12573-edgeroute2.crt" + edgeRouteKey2 = dirname + "/12573-edgeroute2.key" + edgeRouteCsr2 = dirname + "/12573-edgeroute2.csr" + reenRouteSubj = "/CN=example-reen.com" + reenRouteCrt1 = dirname + "/12573-reenroute1.crt" + reenRouteKey1 = dirname + "/12573-reenroute1.key" + reenRouteCsr1 = dirname + "/12573-reenroute1.csr" + reenRouteCrt2 = dirname + "/12573-reenroute2.crt" + reenRouteKey2 = dirname + "/12573-reenroute2.key" + reenRouteCsr2 = dirname + "/12573-reenroute2.csr" + reenRouteDstSubj = "/CN=example-reen-dst.com" + reenRouteDstCrt1 = dirname + "/12573-reenroutedst1.crt" + reenRouteDstKey1 = dirname + "/12573-reenroutedst1.key" + reenRouteDstCrt2 = dirname + "/12573-reenroutedst2.crt" + reenRouteDstKey2 = dirname + "/12573-reenroutedst2.key" + ) + + compat_otp.By("1.0 Create a file folder and prepare for testing") + defer os.RemoveAll(dirname) + err := os.MkdirAll(dirname, 0755) + o.Expect(err).NotTo(o.HaveOccurred()) + baseDomain := getBaseDomain(oc) + edgeRoute1 := "ocp12573-edge1.apps." + baseDomain + reenRoute1 := "ocp12573-reen1.apps." + baseDomain + edgeRoute2 := "ocp12573-edge2.apps." + baseDomain + reenRoute2 := "ocp12573-reen2.apps." + baseDomain + + compat_otp.By("2.0: Use openssl to create ca certification and key") + opensslNewCa(caKey1, caCrt1, caSubj) + opensslNewCa(caKey2, caCrt2, caSubj) + + compat_otp.By("3.0: Create a user CSR and the user key for the edge route") + opensslNewCsr(edgeRouteKey1, edgeRouteCsr1, edgeRouteSubj) + opensslNewCsr(edgeRouteKey2, edgeRouteCsr2, edgeRouteSubj) + + compat_otp.By("3.1: Sign the user CSR and generate the certificate for the edge route") + san1 := "subjectAltName = DNS:" + edgeRoute1 + san2 := "subjectAltName = DNS:" + edgeRoute2 + opensslSignCsr(san1, edgeRouteCsr1, caCrt1, caKey1, edgeRouteCrt1) + opensslSignCsr(san2, edgeRouteCsr2, caCrt2, caKey2, edgeRouteCrt2) + + compat_otp.By("4.0: Create a user CSR and the user key for the reen route") + opensslNewCsr(reenRouteKey1, reenRouteCsr1, reenRouteSubj) + opensslNewCsr(reenRouteKey2, reenRouteCsr2, reenRouteSubj) + + compat_otp.By("4.1: Sign the user CSR and generate the certificate for the reen route") + san1 = "subjectAltName = DNS:" + reenRoute1 + san2 = "subjectAltName = DNS:" + reenRoute2 + opensslSignCsr(san1, reenRouteCsr1, caCrt1, caKey1, reenRouteCrt1) + opensslSignCsr(san2, reenRouteCsr2, caCrt2, caKey2, reenRouteCrt2) + + compat_otp.By("5.0: Use openssl to create certification and key for the destination certification of the reen route") + opensslNewCa(reenRouteDstKey1, reenRouteDstCrt1, reenRouteDstSubj) + opensslNewCa(reenRouteDstKey2, reenRouteDstCrt2, reenRouteDstSubj) + + compat_otp.By("6.0 Create a deployment") + ns := oc.Namespace() + createResourceFromFile(oc, ns, testPodSvc) + ensurePodWithLabelReady(oc, ns, "name=web-server-deploy") + + compat_otp.By("7.0: Create the edge route with invalid caCert and key") + createRoute(oc, ns, "edge", "route-edge", unSecSvcName, []string{"--hostname=" + edgeRoute1, "--ca-cert=" + caCrt1, "--cert=" + edgeRouteCrt2, "--key=" + edgeRouteKey2}) + createRoute(oc, ns, "edge", "route-edge2", unSecSvcName, []string{"--hostname=" + edgeRoute2, "--ca-cert=" + caCrt2, "--cert=" + edgeRouteCrt2, "--key=" + edgeRouteKey1}) + + compat_otp.By("8.0: Create the reencrypt route with invalid cert and destCA") + createRoute(oc, ns, "reencrypt", "route-reen", secSvcName, []string{"--hostname=" + reenRoute1, "--ca-cert=" + caCrt1, "--cert=" + reenRouteCrt2, "--key=" + reenRouteKey1, "--dest-ca-cert=" + reenRouteDstCrt1}) + createRoute(oc, ns, "reencrypt", "route-reen2", secSvcName, []string{"--hostname=" + reenRoute2, "--ca-cert=" + caCrt1, "--cert=" + reenRouteCrt2, "--key=" + reenRouteKey1, "--dest-ca-cert=" + reenRouteDstCrt1}) + + compat_otp.By("9.0: Check the routes Host section") + routeOutput := getRoutes(oc, ns) + o.Expect(strings.Count(routeOutput, `ExtendedValidationFailed`) == 4).To(o.BeTrue()) + + compat_otp.By("10.0: Edit the tls spec section of the edge route") + patchResourceAsAdmin(oc, ns, "route/route-edge", `{"spec":{"tls" :{"key": "qe","certificate": "ocp","caCertificate": "redhat"}}}`) + + compat_otp.By("11.0: Check the tls spec to see updated info") + patch, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("route", "route-edge", "-n", ns, "-ojsonpath={.spec.tls}").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(patch).To(o.ContainSubstring(`"caCertificate":"redhat","certificate":"ocp","key":"qe"`)) + }) + + // author: iamin@redhat.com + g.It("Author:iamin-ROSA-OSD_CCS-ARO-High-14089-Route cannot be accessed if the backend cannot be matched by the default destination CA of router", func() { + var ( + buildPruningBaseDir = testdata.FixturePath("router") + testPodSvc = filepath.Join(buildPruningBaseDir, "web-server-deploy.yaml") + secSvcName = "service-secure" + ) + + compat_otp.By("1.0 Create a deployment") + ns := oc.Namespace() + createResourceFromFile(oc, ns, testPodSvc) + ensurePodWithLabelReady(oc, ns, "name=web-server-deploy") + + compat_otp.By("2.0 Create the reencrypt route with the backend not matched the the default destination CA of router") + createRoute(oc, ns, "reencrypt", "route-reen", secSvcName, []string{}) + ensureRouteIsAdmittedByIngressController(oc, ns, "route-reen", "default") + + compat_otp.By("3.0: Curl the route while it uses the defaulted destCA") + reenHost := "route-reen-" + ns + ".apps." + getBaseDomain(oc) + waitForOutsideCurlContains("https://"+reenHost, "-I -k", `HTTP/1.0 503`) + + compat_otp.By("4.0: Check the route help section") + output, err := oc.Run("create").Args("route", "reencrypt", "-h").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(output).To(o.And(o.ContainSubstring("--dest-ca-cert"), o.ContainSubstring("Defaults to the Service CA"))) + }) + + // also includes OCP-25665/25666/25668/25703 + // author: hongli@redhat.com + g.It("Author:hongli-WRS-ROSA-OSD_CCS-ARO-Critical-25702-V-BR.12-the tlsSecurityProfile in ingresscontroller can be updated", func() { + buildPruningBaseDir := testdata.FixturePath("router") + customTemp := filepath.Join(buildPruningBaseDir, "ingresscontroller-np.yaml") + var ( + ingctrl = ingressControllerDescription{ + name: "ocp25702", + namespace: "openshift-ingress-operator", + domain: "", + template: customTemp, + } + ) + + compat_otp.By("create custom IC without tls profile config (Intermediate is default)") + baseDomain := getBaseDomain(oc) + ingctrl.domain = ingctrl.name + "." + baseDomain + defer ingctrl.delete(oc) + ingctrl.create(oc) + ensureCustomIngressControllerAvailable(oc, ingctrl.name) + + // OCP-25703 + compat_otp.By("check default TLS config and it should be same to Intermediate profile") + newrouterpod := getOneRouterPodNameByIC(oc, ingctrl.name) + env := readRouterPodEnv(oc, newrouterpod, "SSL_MIN_VERSION") + o.Expect(env).To(o.ContainSubstring(`SSL_MIN_VERSION=TLSv1.2`)) + env = readRouterPodEnv(oc, newrouterpod, "ROUTER_CIPHER") + o.Expect(env).To(o.ContainSubstring(`ROUTER_CIPHERSUITES=TLS_AES_128_GCM_SHA256:TLS_AES_256_GCM_SHA384:TLS_CHACHA20_POLY1305_SHA256`)) + o.Expect(env).To(o.ContainSubstring(`ROUTER_CIPHERS=ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384`)) + + // OCP-25665 + compat_otp.By("patch custom IC with tls profile Old and check the config") + patchResourceAsAdmin(oc, ingctrl.namespace, "ingresscontroller/"+ingctrl.name, `{"spec":{"tlsSecurityProfile":{"type":"Old"}}}`) + ensureRouterDeployGenerationIs(oc, ingctrl.name, "2") + newrouterpod = getOneNewRouterPodFromRollingUpdate(oc, ingctrl.name) + env = readRouterPodEnv(oc, newrouterpod, "SSL_MIN_VERSION") + o.Expect(env).To(o.ContainSubstring(`SSL_MIN_VERSION=TLSv1.1`)) + env = readRouterPodEnv(oc, newrouterpod, "ROUTER_CIPHER") + o.Expect(env).To(o.ContainSubstring(`ROUTER_CIPHERSUITES=TLS_AES_128_GCM_SHA256:TLS_AES_256_GCM_SHA384:TLS_CHACHA20_POLY1305_SHA256`)) + o.Expect(env).To(o.ContainSubstring(`ROUTER_CIPHERS=ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384:DHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA:ECDHE-RSA-AES256-SHA:DHE-RSA-AES128-SHA256:DHE-RSA-AES256-SHA256:AES128-GCM-SHA256:AES256-GCM-SHA384:AES128-SHA256:AES256-SHA256:AES128-SHA:AES256-SHA:DES-CBC3-SHA`)) + + // OCP-25666 + compat_otp.By("patch custom IC with tls profile Intermidiate and check the config") + patchResourceAsAdmin(oc, ingctrl.namespace, "ingresscontroller/"+ingctrl.name, `{"spec":{"tlsSecurityProfile":{"type":"Intermediate"}}}`) + ensureRouterDeployGenerationIs(oc, ingctrl.name, "3") + newrouterpod = getOneNewRouterPodFromRollingUpdate(oc, ingctrl.name) + env = readRouterPodEnv(oc, newrouterpod, "SSL_MIN_VERSION") + o.Expect(env).To(o.ContainSubstring(`SSL_MIN_VERSION=TLSv1.2`)) + env = readRouterPodEnv(oc, newrouterpod, "ROUTER_CIPHER") + o.Expect(env).To(o.ContainSubstring(`ROUTER_CIPHERSUITES=TLS_AES_128_GCM_SHA256:TLS_AES_256_GCM_SHA384:TLS_CHACHA20_POLY1305_SHA256`)) + o.Expect(env).To(o.ContainSubstring(`ROUTER_CIPHERS=ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384`)) + + // OCP-25668 + compat_otp.By("patch custom IC with tls profile Custom and check the config") + patchResourceAsAdmin(oc, ingctrl.namespace, "ingresscontroller/"+ingctrl.name, `{"spec":{"tlsSecurityProfile":{"type":"Custom","custom":{"ciphers":["DHE-RSA-AES256-GCM-SHA384","ECDHE-ECDSA-AES256-GCM-SHA384"],"minTLSVersion":"VersionTLS12"}}}}`) + ensureRouterDeployGenerationIs(oc, ingctrl.name, "4") + newrouterpod = getOneNewRouterPodFromRollingUpdate(oc, ingctrl.name) + env = readRouterPodEnv(oc, newrouterpod, "SSL_MIN_VERSION") + o.Expect(env).To(o.ContainSubstring(`SSL_MIN_VERSION=TLSv1.2`)) + env = readRouterPodEnv(oc, newrouterpod, "ROUTER_CIPHER") + o.Expect(env).To(o.ContainSubstring(`ROUTER_CIPHERS=DHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES256-GCM-SHA384`)) + }) + + // author: mjoseph@redhat.com + g.It("Author:mjoseph-WRS-Critical-43284-V-CM.01-setting tlssecurityprofile to TLSv1.3", func() { + buildPruningBaseDir := testdata.FixturePath("router") + customTemp := filepath.Join(buildPruningBaseDir, "ingresscontroller-np.yaml") + var ( + ingctrl = ingressControllerDescription{ + name: "ocp43284", + namespace: "openshift-ingress-operator", + domain: "", + template: customTemp, + } + ) + + compat_otp.By("create and patch the ingresscontroller to enable tls security profile to modern type TLSv1.3") + baseDomain := getBaseDomain(oc) + ingctrl.domain = ingctrl.name + "." + baseDomain + defer ingctrl.delete(oc) + ingctrl.create(oc) + ensureRouterDeployGenerationIs(oc, ingctrl.name, "1") + patchResourceAsAdmin(oc, ingctrl.namespace, "ingresscontroller/ocp43284", "{\"spec\":{\"tlsSecurityProfile\":{\"type\":\"Modern\"}}}") + ensureRouterDeployGenerationIs(oc, ingctrl.name, "2") + + compat_otp.By("check the env variable of the router pod") + newrouterpod := getOneNewRouterPodFromRollingUpdate(oc, "ocp43284") + tlsProfile := readRouterPodEnv(oc, newrouterpod, "TLS") + o.Expect(tlsProfile).To(o.ContainSubstring(`SSL_MIN_VERSION=TLSv1.3`)) + o.Expect(tlsProfile).To(o.ContainSubstring(`ROUTER_CIPHERSUITES=TLS_AES_128_GCM_SHA256:TLS_AES_256_GCM_SHA384:TLS_CHACHA20_POLY1305_SHA256`)) + + compat_otp.By("check the haproxy config on the router pod to ensure the ssl version TLSv1.3 is reflected") + tlsVersion := readRouterPodData(oc, newrouterpod, "cat haproxy.config", "ssl-min-ver") + o.Expect(tlsVersion).To(o.ContainSubstring(`ssl-default-bind-options ssl-min-ver TLSv1.3`)) + + compat_otp.By("check the haproxy config on the router pod to ensure the tls1.3 ciphers are enabled") + tlsCliper := readRouterPodData(oc, newrouterpod, "cat haproxy.config", "sl-default-bind-ciphersuites") + o.Expect(tlsCliper).To(o.ContainSubstring(`ssl-default-bind-ciphersuites TLS_AES_128_GCM_SHA256:TLS_AES_256_GCM_SHA384:TLS_CHACHA20_POLY1305_SHA256`)) + }) + + // author: hongli@redhat.com + g.It("Author:hongli-WRS-LEVEL0-Critical-43300-V-ACS.05-enable client certificate with optional policy", func() { + buildPruningBaseDir := testdata.FixturePath("router") + customTemp := filepath.Join(buildPruningBaseDir, "ingresscontroller-np.yaml") + cmFile := filepath.Join(buildPruningBaseDir, "ca-bundle.pem") + var ( + ingctrl = ingressControllerDescription{ + name: "ocp43300", + namespace: "openshift-ingress-operator", + domain: "", + template: customTemp, + } + ) + + compat_otp.By("create configmap client-ca-xxxxx in namespace openshift-config") + defer deleteConfigMap(oc, "openshift-config", "client-ca-43300") + createConfigMapFromFile(oc, "openshift-config", "client-ca-43300", cmFile) + + compat_otp.By("create and patch custom IC to enable client certificate with Optional policy") + baseDomain := getBaseDomain(oc) + ingctrl.domain = ingctrl.name + "." + baseDomain + defer ingctrl.delete(oc) + ingctrl.create(oc) + ensureRouterDeployGenerationIs(oc, ingctrl.name, "1") + patchResourceAsAdmin(oc, ingctrl.namespace, "ingresscontroller/ocp43300", "{\"spec\":{\"clientTLS\":{\"clientCA\":{\"name\":\"client-ca-43300\"},\"clientCertificatePolicy\":\"Optional\"}}}") + ensureRouterDeployGenerationIs(oc, ingctrl.name, "2") + + compat_otp.By("check client certification config after custom router rolled out") + newrouterpod := getOneNewRouterPodFromRollingUpdate(oc, ingctrl.name) + env := readRouterPodEnv(oc, newrouterpod, "ROUTER_MUTUAL_TLS_AUTH") + o.Expect(env).To(o.ContainSubstring(`ROUTER_MUTUAL_TLS_AUTH=optional`)) + o.Expect(env).To(o.ContainSubstring(`ROUTER_MUTUAL_TLS_AUTH_CA=/etc/pki/tls/client-ca/ca-bundle.pem`)) + }) + + // author: hongli@redhat.com + g.It("Author:hongli-WRS-Medium-43301-V-ACS.05-enable client certificate with required policy", func() { + buildPruningBaseDir := testdata.FixturePath("router") + customTemp := filepath.Join(buildPruningBaseDir, "ingresscontroller-np.yaml") + cmFile := filepath.Join(buildPruningBaseDir, "ca-bundle.pem") + var ( + ingctrl = ingressControllerDescription{ + name: "ocp43301", + namespace: "openshift-ingress-operator", + domain: "", + template: customTemp, + } + ) + + compat_otp.By("create configmap client-ca-xxxxx in namespace openshift-config") + defer deleteConfigMap(oc, "openshift-config", "client-ca-43301") + createConfigMapFromFile(oc, "openshift-config", "client-ca-43301", cmFile) + + compat_otp.By("create and patch custom IC to enable client certificate with required policy") + baseDomain := getBaseDomain(oc) + ingctrl.domain = ingctrl.name + "." + baseDomain + defer ingctrl.delete(oc) + ingctrl.create(oc) + ensureRouterDeployGenerationIs(oc, ingctrl.name, "1") + patchResourceAsAdmin(oc, ingctrl.namespace, "ingresscontroller/ocp43301", "{\"spec\":{\"clientTLS\":{\"clientCA\":{\"name\":\"client-ca-43301\"},\"clientCertificatePolicy\":\"Required\",\"allowedSubjectPatterns\":[\"www.test2.com\"]}}}") + ensureRouterDeployGenerationIs(oc, ingctrl.name, "2") + + compat_otp.By("check client certification config after custom router rolled out") + newrouterpod := getOneNewRouterPodFromRollingUpdate(oc, ingctrl.name) + env := readRouterPodEnv(oc, newrouterpod, "ROUTER_MUTUAL_TLS_AUTH") + o.Expect(env).To(o.ContainSubstring(`ROUTER_MUTUAL_TLS_AUTH=required`)) + o.Expect(env).To(o.ContainSubstring(`ROUTER_MUTUAL_TLS_AUTH_CA=/etc/pki/tls/client-ca/ca-bundle.pem`)) + o.Expect(env).To(o.ContainSubstring(`ROUTER_MUTUAL_TLS_AUTH_FILTER=(?:www.test2.com)`)) + }) + + // bugzilla: 2025624 + g.It("Author:mjoseph-Longduration-NonPreRelease-High-49750-After certificate rotation, ingress router's metrics endpoint will auto update certificates [Disruptive]", func() { + // Check whether the authentication operator is present or not + output, err := oc.WithoutNamespace().AsAdmin().Run("get").Args("route", "oauth-openshift", "-n", "openshift-authentication").Output() + if strings.Contains(output, "namespaces \"openshift-authentication\" not found") || err != nil { + g.Skip("This cluster dont have authentication operator, so skipping the test.") + } + var ( + ingressLabel = "ingresscontroller.operator.openshift.io/deployment-ingresscontroller=default" + ) + + compat_otp.By("Check the metrics endpoint to get the intial certificate details") + routerpod := getOneRouterPodNameByIC(oc, "default") + curlCmd := fmt.Sprintf("curl -k -v https://localhost:1936/metrics --connect-timeout 10") + statsOut, err := compat_otp.RemoteShPod(oc, "openshift-ingress", routerpod, "sh", "-c", curlCmd) + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(strings.Contains(statsOut, "CAfile: /etc/pki/tls/certs/ca-bundle.crt")).Should(o.BeTrue()) + dateRe := regexp.MustCompile("(start date.*)") + certStartDate := dateRe.FindAllString(string(statsOut), -1) + + compat_otp.By("Delete the default CA certificate in openshift-service-ca namespace") + defer ensureAllClusterOperatorsNormal(oc, 920) + err1 := oc.AsAdmin().WithoutNamespace().Run("delete").Args("secret", "signing-key", "-n", "openshift-service-ca").Execute() + o.Expect(err1).NotTo(o.HaveOccurred()) + + compat_otp.By("Waiting for some time till the cluster operators stabilize") + ensureClusterOperatorNormal(oc, "authentication", 5, 720) + + compat_otp.By("Check the router logs to see the certificate in the metrics reloaded") + ensureLogsContainString(oc, "openshift-ingress", ingressLabel, "reloaded metrics certificate") + + compat_otp.By("Check the metrics endpoint to get the certificate details after reload") + curlCmd1 := fmt.Sprintf("curl -k -vvv https://localhost:1936/metrics --connect-timeout 10") + statsOut1, err3 := compat_otp.RemoteShPod(oc, "openshift-ingress", routerpod, "sh", "-c", curlCmd1) + o.Expect(err3).NotTo(o.HaveOccurred()) + o.Expect(strings.Contains(statsOut1, "CAfile: /etc/pki/tls/certs/ca-bundle.crt")).Should(o.BeTrue()) + certStartDate1 := dateRe.FindAllString(string(statsOut1), -1) + // Cross check the start date of the ceritificate is not same after reloading + o.Expect(certStartDate1[0]).NotTo(o.Equal(certStartDate[0])) + }) + + // author: mjoseph@redhat.com + g.It("Author:mjoseph-Critical-50842-destination-ca-certificate-secret annotation for destination CA Opaque certifcate", func() { + buildPruningBaseDir := testdata.FixturePath("router") + testPodSvc := filepath.Join(buildPruningBaseDir, "web-server-deploy.yaml") + ingressTemp := filepath.Join(buildPruningBaseDir, "ingress-destCA.yaml") + customTemp := filepath.Join(buildPruningBaseDir, "ingresscontroller-np.yaml") + caCert := filepath.Join(buildPruningBaseDir, "ca-bundle.pem") + var ( + ingctrl = ingressControllerDescription{ + name: "ocp50842", + namespace: "openshift-ingress-operator", + domain: "", + template: customTemp, + } + ing = ingressDescription{ + name: "ingress-dca-opq", + namespace: "", + domain: "", + serviceName: "service-secure", + template: ingressTemp, + } + ) + + compat_otp.By("Create a pod") + ns := oc.Namespace() + baseDomain := getBaseDomain(oc) + createResourceFromFile(oc, ns, testPodSvc) + ensurePodWithLabelReady(oc, ns, "name=web-server-deploy") + podName := getPodListByLabel(oc, ns, "name=web-server-deploy") + + compat_otp.By("create custom ingresscontroller") + ingctrl.domain = ingctrl.name + "." + baseDomain + defer ingctrl.delete(oc) + ingctrl.create(oc) + ensureRouterDeployGenerationIs(oc, ingctrl.name, "1") + custContPod := getOneNewRouterPodFromRollingUpdate(oc, "ocp50842") + + compat_otp.By("create a secret with destination CA Opaque certificate") + createGenericSecret(oc, ns, "service-secret", "tls.crt", caCert) + + compat_otp.By("create ingress and get the details") + ing.domain = ingctrl.name + "." + baseDomain + ing.namespace = ns + ing.create(oc) + getIngress(oc, ns) + getRoutes(oc, ns) + routeNames := getResourceName(oc, ns, "route") + + compat_otp.By("check whether route details are present in custom controller domain") + waitForOutputContains(oc, ns, "route/"+routeNames[0], "{.metadata.annotations}", `"route.openshift.io/destination-ca-certificate-secret":"service-secret"`) + host := fmt.Sprintf(`service-secure-%s.ocp50842.%s`, ns, baseDomain) + waitForOutputEquals(oc, ns, "route/"+routeNames[0], "{.spec.host}", host) + + compat_otp.By("check the reachability of the host in custom controller") + controlerIP := getPodv4Address(oc, custContPod, "openshift-ingress") + curlCmd := []string{"-n", ns, podName[0], "--", "curl", "https://service-secure-" + ns + + ".ocp50842." + baseDomain + ":443", "-k", "-I", "--resolve", "service-secure-" + ns + ".ocp50842." + + baseDomain + ":443:" + controlerIP, "--connect-timeout", "10"} + repeatCmdOnClient(oc, curlCmd, "200", 30, 1) + + compat_otp.By("check the router pod and ensure the routes are loaded in haproxy.config of custom controller") + searchOutput := readRouterPodData(oc, custContPod, "cat haproxy.config", "ingress-dca-opq") + o.Expect(searchOutput).To(o.ContainSubstring("backend be_secure:" + ns + ":" + routeNames[0])) + }) + + // author: mjoseph@redhat.com + g.It("Author:mjoseph-Critical-51980-destination-ca-certificate-secret annotation for destination CA TLS certifcate", func() { + buildPruningBaseDir := testdata.FixturePath("router") + testPodSvc := filepath.Join(buildPruningBaseDir, "web-server-signed-deploy.yaml") + ingressTemp := filepath.Join(buildPruningBaseDir, "ingress-destCA.yaml") + customTemp := filepath.Join(buildPruningBaseDir, "ingresscontroller-np.yaml") + var ( + ingctrl = ingressControllerDescription{ + name: "ocp51980", + namespace: "openshift-ingress-operator", + domain: "", + template: customTemp, + } + ing = ingressDescription{ + name: "ingress-dca-tls", + namespace: "", + domain: "", + serviceName: "service-secure", + template: ingressTemp, + } + ) + + compat_otp.By("Create a pod") + baseDomain := getBaseDomain(oc) + ns := oc.Namespace() + createResourceFromFile(oc, ns, testPodSvc) + ensurePodWithLabelReady(oc, ns, "name=web-server-deploy") + podName := getPodListByLabel(oc, ns, "name=web-server-deploy") + + compat_otp.By("create custom ingresscontroller") + ingctrl.domain = ingctrl.name + "." + baseDomain + defer ingctrl.delete(oc) + ingctrl.create(oc) + ensureRouterDeployGenerationIs(oc, ingctrl.name, "1") + custContPod := getOneNewRouterPodFromRollingUpdate(oc, "ocp51980") + + compat_otp.By("create ingress and get the details") + ing.domain = ingctrl.name + "." + baseDomain + ing.namespace = ns + ing.create(oc) + getIngress(oc, ns) + getRoutes(oc, ns) + routeNames := getResourceName(oc, ns, "route") + + compat_otp.By("check whether route details are present in custom controller domain") + output := getByJsonPath(oc, ns, "route/"+routeNames[0], "{.metadata.annotations}") + o.Expect(output).Should(o.ContainSubstring(`"route.openshift.io/destination-ca-certificate-secret":"service-secret"`)) + output = getByJsonPath(oc, ns, "route/"+routeNames[0], "{.spec.host}") + o.Expect(output).Should(o.ContainSubstring(`service-secure-%s.ocp51980.%s`, ns, baseDomain)) + + compat_otp.By("check the router pod and ensure the routes are loaded in haproxy.config of custom controller") + searchOutput := pollReadPodData(oc, "openshift-ingress", custContPod, "cat haproxy.config", "ingress-dca-tls") + o.Expect(searchOutput).To(o.ContainSubstring("backend be_secure:" + ns + ":" + routeNames[0])) + + compat_otp.By("check the reachability of the host in custom controller") + controlerIP := getPodv4Address(oc, custContPod, "openshift-ingress") + curlCmd := []string{"-n", ns, podName[0], "--", "curl", "https://service-secure-" + ns + + ".ocp51980." + baseDomain + ":443", "-k", "-I", "--resolve", "service-secure-" + ns + ".ocp51980." + + baseDomain + ":443:" + controlerIP, "--connect-timeout", "10"} + repeatCmdOnClient(oc, curlCmd, "200", 30, 1) + }) +}) diff --git a/tests-extension/test/e2e/tuning.go b/tests-extension/test/e2e/tuning.go new file mode 100644 index 000000000..fa9c5e146 --- /dev/null +++ b/tests-extension/test/e2e/tuning.go @@ -0,0 +1,680 @@ +package router + +import ( + "fmt" + "os" + "path/filepath" + + g "github.com/onsi/ginkgo/v2" + o "github.com/onsi/gomega" + compat_otp "github.com/openshift/origin/test/extended/util/compat_otp" + "github.com/openshift/router-tests-extension/test/testdata" +) + +var _ = g.Describe("[sig-network-edge] Network_Edge Component_Router", func() { + defer g.GinkgoRecover() + + var oc = compat_otp.NewCLI("router-tunning", compat_otp.KubeConfigPath()) + + // incorporate OCP-40747, OCP-40748, OCP-40821 and OCP-40822 + // Test case creater: mjoseph@redhat.com - OCP-40747 The 'tune.maxrewrite' value can be modified with 'headerBufferMaxRewriteBytes' parameter + // Test case creater: mjoseph@redhat.com - OCP-40748 The 'tune.bufsize' value can be modified with 'headerBufferBytes' parameter + // Test case creater: mjoseph@redhat.com - OCP-40821 The 'tune.bufsize' and 'tune.maxwrite' values can be defined per haproxy router basis + // Test case creater: shudili@redhat.com - OCP-40822 The 'headerBufferBytes' and 'headerBufferMaxRewriteBytes' strictly honours the default minimum values + g.It("Author:mjoseph-Critical-40747-The 'tune.bufsize' and 'tune.maxwrite' values can be modified by ingresscontroller tuningOptions", func() { + buildPruningBaseDir := testdata.FixturePath("router") + customTemp := filepath.Join(buildPruningBaseDir, "ingresscontroller-tuning.yaml") + var ( + ingctrl = ingressControllerDescription{ + name: "ocp40747a", + namespace: "openshift-ingress-operator", + domain: "", + template: customTemp, + } + ingctrl2 = ingressControllerDescription{ + name: "ocp40747b", + namespace: "openshift-ingress-operator", + domain: "", + template: customTemp, + } + ) + + compat_otp.By("1: Create a custom ingresscontroller, and get its router name") + baseDomain := getBaseDomain(oc) + ingctrl.domain = ingctrl.name + "." + baseDomain + defer ingctrl.delete(oc) + ingctrl.create(oc) + ensureRouterDeployGenerationIs(oc, ingctrl.name, "1") + routerpod := getOneNewRouterPodFromRollingUpdate(oc, ingctrl.name) + + // OCP-40821 The 'tune.bufsize' and 'tune.maxwrite' values can be defined per haproxy router basis + compat_otp.By("2: Check the haproxy config on the router pod for existing maxrewrite and bufsize value") + ensureHaproxyBlockConfigContains(oc, routerpod, "global", []string{"tune.bufsize 16385", "tune.maxrewrite 4097"}) + + compat_otp.By("3: Create a second custom ingresscontroller, and get its router name") + baseDomain = getBaseDomain(oc) + ingctrl2.domain = ingctrl2.name + "." + baseDomain + defer ingctrl2.delete(oc) + ingctrl2.create(oc) + ensureRouterDeployGenerationIs(oc, ingctrl2.name, "1") + + compat_otp.By("4: Patch the second ingresscontroller with maxrewrite and bufsize value") + ingctrlResource2 := "ingresscontrollers/" + ingctrl2.name + patchResourceAsAdmin(oc, ingctrl.namespace, ingctrlResource2, "{\"spec\":{\"tuningOptions\" :{\"headerBufferBytes\": 18000, \"headerBufferMaxRewriteBytes\":10000}}}") + ensureRouterDeployGenerationIs(oc, ingctrl2.name, "2") + newSecondRouterpod := getOneNewRouterPodFromRollingUpdate(oc, ingctrl2.name) + + compat_otp.By("5: Check the haproxy config on the router pod of second ingresscontroller for the tune.bufsize buffer value") + ensureHaproxyBlockConfigContains(oc, newSecondRouterpod, "global", []string{"tune.bufsize 18000", "tune.maxrewrite 10000"}) + + // OCP-40822 The 'headerBufferBytes' and 'headerBufferMaxRewriteBytes' strictly honours the default minimum values + compat_otp.By("6: Patch ingresscontroller with minimum values and check whether it is configurable") + output1, _ := oc.AsAdmin().WithoutNamespace().Run("patch").Args("ingresscontroller/ocp40747b", "-p", "{\"spec\":{\"tuningOptions\" :{\"headerBufferBytes\": 8192, \"headerBufferMaxRewriteBytes\":2048}}}", "--type=merge", "-n", ingctrl2.namespace).Output() + o.Expect(output1).To(o.ContainSubstring(`The IngressController "ocp40747b" is invalid`)) + o.Expect(output1).To(o.ContainSubstring("spec.tuningOptions.headerBufferMaxRewriteBytes: Invalid value: 2048: spec.tuningOptions.headerBufferMaxRewriteBytes in body should be greater than or equal to 4096")) + o.Expect(output1).To(o.ContainSubstring("spec.tuningOptions.headerBufferBytes: Invalid value: 8192: spec.tuningOptions.headerBufferBytes in body should be greater than or equal to 16384")) + + // OCP-40747 The 'tune.maxrewrite' value can be modified with 'headerBufferMaxRewriteBytes' parameter + compat_otp.By("7: Patch ingresscontroller with tune.maxrewrite buffer value") + ingctrlResource := "ingresscontrollers/" + ingctrl.name + patchResourceAsAdmin(oc, ingctrl.namespace, ingctrlResource, "{\"spec\":{\"tuningOptions\" :{\"headerBufferMaxRewriteBytes\": 8192}}}") + ensureRouterDeployGenerationIs(oc, ingctrl.name, "2") + + compat_otp.By("8: Check the haproxy config on the router pod for the tune.maxrewrite buffer value") + newrouterpod := getOneNewRouterPodFromRollingUpdate(oc, ingctrl.name) + ensureHaproxyBlockConfigContains(oc, newrouterpod, "tune.maxrewrite", []string{"tune.maxrewrite 8192"}) + + // OCP-40748 The 'tune.bufsize' value can be modified with 'headerBufferBytes' parameter + compat_otp.By("9: Patch ingresscontroller with tune.bufsize buffer value") + patchResourceAsAdmin(oc, ingctrl.namespace, ingctrlResource, "{\"spec\":{\"tuningOptions\" :{\"headerBufferBytes\": 18000}}}") + ensureRouterDeployGenerationIs(oc, ingctrl.name, "3") + + compat_otp.By("10: check the haproxy config on the router pod for the tune.bufsize buffer value") + newrouterpod = getOneNewRouterPodFromRollingUpdate(oc, ingctrl.name) + ensureHaproxyBlockConfigContains(oc, newrouterpod, "tune.bufsize", []string{"tune.bufsize 18000"}) + }) + + // incorporate OCP-41110 and OCP-41128 + // Test case creater: shudili@redhat.com - OCP-41110 The threadCount ingresscontroller parameter controls the nbthread option for the haproxy router + // Test case creater: mjoseph@redhat.com - OCP-41128 Ingresscontroller should not accept invalid nbthread setting + g.It("Author:shudili-LEVEL0-Critical-41110-The threadCount ingresscontroller parameter controls the nbthread option for the haproxy router", func() { + buildPruningBaseDir := testdata.FixturePath("router") + customTemp := filepath.Join(buildPruningBaseDir, "ingresscontroller-np.yaml") + var ( + ingctrl = ingressControllerDescription{ + name: "ocp41110", + namespace: "openshift-ingress-operator", + domain: "", + template: customTemp, + } + threadcount = "6" + threadcountDefault = "4" + threadcount1 = "-1" + threadcount2 = "512" + threadcount3 = `"abc"` + ) + + compat_otp.By("1: Create a ingresscontroller with threadCount set") + baseDomain := getBaseDomain(oc) + ingctrl.domain = ingctrl.name + "." + baseDomain + defer ingctrl.delete(oc) + ingctrl.create(oc) + ensureRouterDeployGenerationIs(oc, ingctrl.name, "1") + + compat_otp.By("2: Check the router env to verify the default value of ROUTER_THREADS is applied") + podname := getOneNewRouterPodFromRollingUpdate(oc, ingctrl.name) + threadValue := readRouterPodEnv(oc, podname, "ROUTER_THREADS") + o.Expect(threadValue).To(o.ContainSubstring("ROUTER_THREADS=" + threadcountDefault)) + + compat_otp.By("3: Patch the new ingresscontroller with tuningOptions/threadCount " + threadcount) + ingctrlResource := "ingresscontrollers/" + ingctrl.name + patchResourceAsAdmin(oc, ingctrl.namespace, ingctrlResource, "{\"spec\": {\"tuningOptions\": {\"threadCount\": "+threadcount+"}}}") + ensureRouterDeployGenerationIs(oc, ingctrl.name, "2") + + compat_otp.By("4: Check the router env to verify the PROXY variable ROUTER_THREADS with " + threadcount + " is applied") + newpodname := getOneNewRouterPodFromRollingUpdate(oc, ingctrl.name) + dssearch := readRouterPodEnv(oc, newpodname, "ROUTER_THREADS") + o.Expect(dssearch).To(o.ContainSubstring("ROUTER_THREADS=" + threadcount)) + + compat_otp.By("5: Check the haproxy config on the router pod to ensure the nbthread is updated") + ensureHaproxyBlockConfigContains(oc, newpodname, "nbthread", []string{"nbthread " + threadcount}) + + // OCP-41128 Ingresscontroller should not accept invalid nbthread setting + compat_otp.By("6: Patch the new ingresscontroller with negative(" + threadcount1 + ") value as threadCount") + output1, _ := oc.AsAdmin().WithoutNamespace().Run("patch").Args( + "ingresscontroller/"+ingctrl.name, "-p", "{\"spec\": {\"tuningOptions\": {\"threadCount\": "+threadcount1+"}}}", + "--type=merge", "-n", ingctrl.namespace).Output() + o.Expect(output1).To(o.ContainSubstring("Invalid value: -1: spec.tuningOptions.threadCount in body should be greater than or equal to 1")) + + compat_otp.By("7: Patch the new ingresscontroller with high(" + threadcount2 + ") value for threadCount") + output2, _ := oc.AsAdmin().WithoutNamespace().Run("patch").Args( + "ingresscontroller/"+ingctrl.name, "-p", "{\"spec\": {\"tuningOptions\": {\"threadCount\": "+threadcount2+"}}}", + "--type=merge", "-n", ingctrl.namespace).Output() + o.Expect(output2).To(o.ContainSubstring("Invalid value: 512: spec.tuningOptions.threadCount in body should be less than or equal to 64")) + + compat_otp.By("8: Patch the new ingresscontroller with string(" + threadcount3 + ") value for threadCount") + output3, _ := oc.AsAdmin().WithoutNamespace().Run("patch").Args( + "ingresscontroller/"+ingctrl.name, "-p", "{\"spec\": {\"tuningOptions\": {\"threadCount\": "+threadcount3+"}}}", + "--type=merge", "-n", ingctrl.namespace).Output() + o.Expect(output3).To(o.ContainSubstring(`Invalid value: "string": spec.tuningOptions.threadCount in body must be of type integer: "string"`)) + }) + + // author: aiyengar@redhat.com + g.It("Author:aiyengar-Critical-43105-The tcp client/server fin and default timeout for the ingresscontroller can be modified via tuningOptions parameterss", func() { + buildPruningBaseDir := testdata.FixturePath("router") + customTemp := filepath.Join(buildPruningBaseDir, "ingresscontroller-np.yaml") + var ( + ingctrl = ingressControllerDescription{ + name: "43105", + namespace: "openshift-ingress-operator", + domain: "", + template: customTemp, + } + ) + + compat_otp.By("Create a custom ingresscontroller, and get its router name") + baseDomain := getBaseDomain(oc) + ingctrl.domain = ingctrl.name + "." + baseDomain + defer ingctrl.delete(oc) + ingctrl.create(oc) + ensureRouterDeployGenerationIs(oc, ingctrl.name, "1") + routerpod := getOneNewRouterPodFromRollingUpdate(oc, ingctrl.name) + + compat_otp.By("Verify the default server/client fin and default timeout values") + ensureHaproxyBlockConfigContains(oc, routerpod, "defaults", []string{"timeout client 30s", "timeout client-fin 1s", "timeout server 30s", "timeout server-fin 1s"}) + + compat_otp.By("Patch ingresscontroller with new timeout options") + ingctrlResource := "ingresscontrollers/" + ingctrl.name + patchResourceAsAdmin(oc, ingctrl.namespace, ingctrlResource, "{\"spec\":{\"tuningOptions\" :{\"clientFinTimeout\": \"3s\",\"clientTimeout\":\"33s\",\"serverFinTimeout\":\"3s\",\"serverTimeout\":\"33s\"}}}") + ensureRouterDeployGenerationIs(oc, ingctrl.name, "2") + newrouterpod := getOneNewRouterPodFromRollingUpdate(oc, ingctrl.name) + + compat_otp.By("verify the timeout variables from the new router pods") + checkenv := readRouterPodEnv(oc, newrouterpod, "TIMEOUT") + o.Expect(checkenv).To(o.ContainSubstring(`ROUTER_CLIENT_FIN_TIMEOUT=3s`)) + o.Expect(checkenv).To(o.ContainSubstring(`ROUTER_DEFAULT_CLIENT_TIMEOUT=33s`)) + o.Expect(checkenv).To(o.ContainSubstring(`ROUTER_DEFAULT_SERVER_TIMEOUT=33`)) + o.Expect(checkenv).To(o.ContainSubstring(`ROUTER_DEFAULT_SERVER_FIN_TIMEOUT=3s`)) + }) + + // incorporate OCP-43111 and OCP-43112 + // Test case creater: shudili@redhat.com - OCP-43111 The tcp client/server and tunnel timeouts for ingresscontroller will remain unchanged for negative values + // Test case creater: shudili@redhat.com - OCP-43112 timeout tunnel parameter for the haproxy pods an be modified with TuningOptions option in the ingresscontroller + g.It("Author:aiyengar-LEVEL0-Critical-43112-Timeout tunnel parameter for the haproxy pods an be modified with TuningOptions option in the ingresscontroller", func() { + buildPruningBaseDir := testdata.FixturePath("router") + customTemp := filepath.Join(buildPruningBaseDir, "ingresscontroller-np.yaml") + var ( + ingctrl = ingressControllerDescription{ + name: "43112", + namespace: "openshift-ingress-operator", + domain: "", + template: customTemp, + } + ) + + compat_otp.By("1: Create a custom ingresscontroller, and get its router name") + baseDomain := getBaseDomain(oc) + ingctrl.domain = ingctrl.name + "." + baseDomain + defer ingctrl.delete(oc) + ingctrl.create(oc) + ensureRouterDeployGenerationIs(oc, ingctrl.name, "1") + routerpod := getOneNewRouterPodFromRollingUpdate(oc, ingctrl.name) + compat_otp.By("Verify the default tls values") + ensureHaproxyBlockConfigContains(oc, routerpod, "timeout tunnel", []string{"timeout tunnel 1h"}) + + compat_otp.By("2: Patch ingresscontroller with a tunnel timeout option") + ingctrlResource := "ingresscontrollers/" + ingctrl.name + patchResourceAsAdmin(oc, ingctrl.namespace, ingctrlResource, "{\"spec\":{\"tuningOptions\" :{\"tunnelTimeout\": \"2h\"}}}") + ensureRouterDeployGenerationIs(oc, ingctrl.name, "2") + newrouterpod := getOneNewRouterPodFromRollingUpdate(oc, ingctrl.name) + + compat_otp.By("3: Verify the new tls inspect timeout value in the router pod") + checkenv := readRouterPodEnv(oc, newrouterpod, "ROUTER_DEFAULT_TUNNEL_TIMEOUT") + o.Expect(checkenv).To(o.ContainSubstring(`ROUTER_DEFAULT_TUNNEL_TIMEOUT=2h`)) + + // OCP-43111 The tcp client/server and tunnel timeouts for ingresscontroller will remain unchanged for negative values + compat_otp.By("4: Patch ingresscontroller with negative values for the tuningOptions settings and check the ingress operator config post the change") + patchResourceAsAdmin(oc, ingctrl.namespace, ingctrlResource, `{"spec":{"tuningOptions" :{"clientFinTimeout": "-7s","clientTimeout": "-33s","serverFinTimeout": "-3s","serverTimeout": "-27s","tlsInspectDelay": "-11s","tunnelTimeout": "-1h"}}}`) + output := getByJsonPath(oc, "openshift-ingress-operator", "ingresscontroller/"+ingctrl.name, "{.spec.tuningOptions}") + o.Expect(output).To(o.ContainSubstring("{\"clientFinTimeout\":\"-7s\",\"clientTimeout\":\"-33s\",\"reloadInterval\":\"0s\",\"serverFinTimeout\":\"-3s\",\"serverTimeout\":\"-27s\",\"tlsInspectDelay\":\"-11s\",\"tunnelTimeout\":\"-1h\"}")) + + compat_otp.By("5: Check the timeout option set in the haproxy pods post the changes applied") + ensureHaproxyBlockConfigContains(oc, routerpod, "defaults", []string{"timeout connect 5s", "timeout client 30s", "timeout client-fin 1s", "timeout server 30s", "timeout server-fin 1s", "timeout tunnel 1h"}) + }) + + // author: aiyengar@redhat.com + g.It("Author:aiyengar-Critical-43113-Tcp inspect-delay for the haproxy pod can be modified via the TuningOptions parameters in the ingresscontroller", func() { + buildPruningBaseDir := testdata.FixturePath("router") + customTemp := filepath.Join(buildPruningBaseDir, "ingresscontroller-np.yaml") + var ( + ingctrl = ingressControllerDescription{ + name: "43113", + namespace: "openshift-ingress-operator", + domain: "", + template: customTemp, + } + ) + + compat_otp.By("Create a custom ingresscontroller, and get its router name") + baseDomain := getBaseDomain(oc) + ingctrl.domain = ingctrl.name + "." + baseDomain + defer ingctrl.delete(oc) + ingctrl.create(oc) + ensureRouterDeployGenerationIs(oc, ingctrl.name, "1") + routerpod := getOneNewRouterPodFromRollingUpdate(oc, ingctrl.name) + + compat_otp.By("Verify the default tls values") + ensureHaproxyBlockConfigContains(oc, routerpod, "tcp-request inspect-delay", []string{"tcp-request inspect-delay 5s"}) + + compat_otp.By("Patch ingresscontroller with a tls inspect timeout option") + ingctrlResource := "ingresscontrollers/" + ingctrl.name + patchResourceAsAdmin(oc, ingctrl.namespace, ingctrlResource, "{\"spec\":{\"tuningOptions\" :{\"tlsInspectDelay\": \"15s\"}}}") + ensureRouterDeployGenerationIs(oc, ingctrl.name, "2") + newrouterpod := getOneNewRouterPodFromRollingUpdate(oc, ingctrl.name) + + compat_otp.By("verify the new tls inspect timeout value in the router pod") + checkenv := readRouterPodEnv(oc, newrouterpod, "ROUTER_INSPECT_DELAY") + o.Expect(checkenv).To(o.ContainSubstring(`ROUTER_INSPECT_DELAY=15s`)) + }) + + // incorporate OCP-50662 and OCP-50663 + // Test case creater: shudili@redhat.com - OCP-50662 Make ROUTER_BACKEND_CHECK_INTERVAL Configurable + // Test case creater: shudili@redhat.com - OCP-50663 Negative Test of Make ROUTER_BACKEND_CHECK_INTERVAL Configurable + g.It("Author:shudili-High-50662-Make ROUTER_BACKEND_CHECK_INTERVAL Configurable", func() { + buildPruningBaseDir := testdata.FixturePath("router") + baseTemp := filepath.Join(buildPruningBaseDir, "ingresscontroller-np.yaml") + extraParas := " tuningOptions:\n healthCheckInterval: 20s\n" + customTemp1 := addExtraParametersToYamlFile(baseTemp, "spec:", extraParas) + defer os.Remove(customTemp1) + extraParas = " tuningOptions:\n healthCheckInterval: 100m\n" + customTemp2 := addExtraParametersToYamlFile(baseTemp, "spec:", extraParas) + defer os.Remove(customTemp2) + ingctrl1 := ingressControllerDescription{ + name: "ocp50662one", + namespace: "openshift-ingress-operator", + domain: "", + template: customTemp1, + } + ingctrl2 := ingressControllerDescription{ + name: "ocp50662two", + namespace: "openshift-ingress-operator", + domain: "", + template: customTemp2, + } + ingctrlResource1 := "ingresscontrollers/" + ingctrl1.name + ingctrlResource2 := "ingresscontrollers/" + ingctrl2.name + + compat_otp.By("1: Create two custom ICs for testing ROUTER_BACKEND_CHECK_INTERVAL") + baseDomain := getBaseDomain(oc) + ingctrl1.domain = ingctrl1.name + "." + baseDomain + ingctrl2.domain = ingctrl2.name + "." + baseDomain + defer ingctrl1.delete(oc) + ingctrl1.create(oc) + defer ingctrl2.delete(oc) + ingctrl2.create(oc) + ensureRouterDeployGenerationIs(oc, ingctrl1.name, "1") + ensureRouterDeployGenerationIs(oc, ingctrl2.name, "1") + + compat_otp.By("2: Check ROUTER_BACKEND_CHECK_INTERVAL env in a route pod of IC ocp50662one, which should be 20s") + podname1 := getOneNewRouterPodFromRollingUpdate(oc, ingctrl1.name) + hciSearch := readRouterPodEnv(oc, podname1, "ROUTER_BACKEND_CHECK_INTERVAL") + o.Expect(hciSearch).To(o.ContainSubstring("ROUTER_BACKEND_CHECK_INTERVAL=20s")) + + compat_otp.By("3: Check ROUTER_BACKEND_CHECK_INTERVAL env in a route pod of IC ocp50662two, which should be 100m") + podname2 := getOneNewRouterPodFromRollingUpdate(oc, ingctrl2.name) + hciSearch = readRouterPodEnv(oc, podname2, "ROUTER_BACKEND_CHECK_INTERVAL") + o.Expect(hciSearch).To(o.ContainSubstring("ROUTER_BACKEND_CHECK_INTERVAL=100m")) + + compat_otp.By("4: Patch tuningOptions/healthCheckInterval with max 2147483647ms to IC ocp50662one, while tuningOptions/healthCheckInterval 0s to the IC ocp50662two") + healthCheckInterval := "2147483647ms" + patchResourceAsAdmin(oc, ingctrl1.namespace, ingctrlResource1, "{\"spec\": {\"tuningOptions\": {\"healthCheckInterval\": \""+healthCheckInterval+"\"}}}") + healthCheckInterval = "0s" + patchResourceAsAdmin(oc, ingctrl2.namespace, ingctrlResource2, "{\"spec\": {\"tuningOptions\": {\"healthCheckInterval\": \""+healthCheckInterval+"\"}}}") + ensureRouterDeployGenerationIs(oc, ingctrl1.name, "2") + ensureRouterDeployGenerationIs(oc, ingctrl2.name, "2") + + compat_otp.By("5: Check ROUTER_BACKEND_CHECK_INTERVAL env in a route pod of IC ocp50662one, which should be 2147483647ms") + podname1 = getOneNewRouterPodFromRollingUpdate(oc, ingctrl1.name) + hciSearch = readRouterPodEnv(oc, podname1, "ROUTER_BACKEND_CHECK_INTERVAL") + o.Expect(hciSearch).To(o.ContainSubstring("ROUTER_BACKEND_CHECK_INTERVAL=2147483647ms")) + + compat_otp.By("6: Try to find the ROUTER_BACKEND_CHECK_INTERVAL env in a route pod which shouldn't be seen by default") + podname2 = getOneNewRouterPodFromRollingUpdate(oc, ingctrl2.name) + cmd := fmt.Sprintf("/usr/bin/env | grep %s", "ROUTER_BACKEND_CHECK_INTERVAL") + _, err := oc.AsAdmin().WithoutNamespace().Run("exec").Args("-n", "openshift-ingress", podname2, "--", "bash", "-c", cmd).Output() + o.Expect(err).To(o.HaveOccurred()) + + // OCP-50663 Negative Test of Make ROUTER_BACKEND_CHECK_INTERVAL Configurable + compat_otp.By("7: Try to patch tuningOptions/healthCheckInterval 2147483900ms which is larger than the max healthCheckInterval, to the ingress-controller") + NegHealthCheckInterval := "2147483900ms" + patchResourceAsAdmin(oc, ingctrl1.namespace, ingctrlResource1, "{\"spec\": {\"tuningOptions\": {\"healthCheckInterval\": \""+NegHealthCheckInterval+"\"}}}") + ensureRouterDeployGenerationIs(oc, ingctrl1.name, "2") + + compat_otp.By("8: Check ROUTER_BACKEND_CHECK_INTERVAL env in a route pod which should be the max: 2147483647ms") + podname := getOneNewRouterPodFromRollingUpdate(oc, ingctrl1.name) + hciSearch = readRouterPodEnv(oc, podname, "ROUTER_BACKEND_CHECK_INTERVAL") + o.Expect(hciSearch).To(o.ContainSubstring("ROUTER_BACKEND_CHECK_INTERVAL=" + "2147483647ms")) + + compat_otp.By("9: Try to patch tuningOptions/healthCheckInterval -1s which is a minus value, to the ingress-controller") + NegHealthCheckInterval = "-1s" + output, err1 := oc.AsAdmin().WithoutNamespace().Run("patch").Args(ingctrlResource1, "-p", "{\"spec\": {\"tuningOptions\": {\"healthCheckInterval\": \""+NegHealthCheckInterval+"\"}}}", "--type=merge", "-n", ingctrl1.namespace).Output() + o.Expect(err1).To(o.HaveOccurred()) + o.Expect(output).To(o.ContainSubstring("Invalid value: \"-1s\"")) + + compat_otp.By("10: Try to patch tuningOptions/healthCheckInterval abc which is a string, to the ingress-controller") + NegHealthCheckInterval = "0abc" + output, err2 := oc.AsAdmin().WithoutNamespace().Run("patch").Args(ingctrlResource1, "-p", "{\"spec\": {\"tuningOptions\": {\"healthCheckInterval\": \""+NegHealthCheckInterval+"\"}}}", "--type=merge", "-n", ingctrl1.namespace).Output() + o.Expect(err2).To(o.HaveOccurred()) + o.Expect(output).To(o.ContainSubstring("Invalid value: \"0abc\":")) + }) + + // incorporate OCP-50926 and OCP-50928 + // Test case creater: shudili@redhat.com - OCP-50926 Support a Configurable ROUTER_MAX_CONNECTIONS in HAproxy + // Test case creater: shudili@redhat.com - OCP-50928 Negative test of Support a Configurable ROUTER_MAX_CONNECTIONS in HAproxy + g.It("Author:shudili-High-50926-Support a Configurable ROUTER_MAX_CONNECTIONS in HAproxy", func() { + var ( + buildPruningBaseDir = testdata.FixturePath("router") + customTemp = filepath.Join(buildPruningBaseDir, "ingresscontroller-tuning.yaml") + ingctrl = ingressControllerDescription{ + name: "ocp50926", + namespace: "openshift-ingress-operator", + domain: "", + template: customTemp, + } + ingctrlResource = "ingresscontrollers/" + ingctrl.name + ) + compat_otp.By("1: Create a custom IC with tuningOptions/maxConnections -1 specified by ingresscontroller-tuning.yaml") + baseDomain := getBaseDomain(oc) + ingctrl.domain = ingctrl.name + "." + baseDomain + defer ingctrl.delete(oc) + ingctrl.create(oc) + ensureRouterDeployGenerationIs(oc, ingctrl.name, "1") + + compat_otp.By("2: Check ROUTER_MAX_CONNECTIONS env under a route pod for the configured maxConnections -1, which should be auto") + podname := getOneNewRouterPodFromRollingUpdate(oc, ingctrl.name) + maxConnSearch := readRouterPodEnv(oc, podname, "ROUTER_MAX_CONNECTIONS") + o.Expect(maxConnSearch).To(o.ContainSubstring("ROUTER_MAX_CONNECTIONS=auto")) + + compat_otp.By("3: Check maxconn in haproxy.config which won't appear after configured tuningOptions/maxConnections with -1") + cmd := fmt.Sprintf("%s | grep \"%s\"", "cat haproxy.config", "maxconn") + _, err := oc.AsAdmin().WithoutNamespace().Run("exec").Args("-n", "openshift-ingress", podname, "--", "bash", "-c", cmd).Output() + o.Expect(err).To(o.HaveOccurred()) + + compat_otp.By("4: Patch tuningOptions/maxConnections with 2000 to IC ocp50926") + patchResourceAsAdmin(oc, ingctrl.namespace, ingctrlResource, "{\"spec\": {\"tuningOptions\": {\"maxConnections\": 2000}}}") + ensureRouterDeployGenerationIs(oc, ingctrl.name, "2") + + compat_otp.By("5: Check ROUTER_MAX_CONNECTIONS env under a router pod of IC ocp50926, which should be 2000") + podname = getOneNewRouterPodFromRollingUpdate(oc, ingctrl.name) + maxConnSearch = readRouterPodEnv(oc, podname, "ROUTER_MAX_CONNECTIONS") + o.Expect(maxConnSearch).To(o.ContainSubstring("ROUTER_MAX_CONNECTIONS=2000")) + + compat_otp.By("6: Check maxconn in haproxy.config under a router pod of IC ocp50926, which should be 2000") + ensureHaproxyBlockConfigContains(oc, podname, "maxconn", []string{"maxconn 2000"}) + + compat_otp.By("7: Patch tuningOptions/maxConnections with max 2000000 to IC ocp50926") + patchResourceAsAdmin(oc, ingctrl.namespace, ingctrlResource, "{\"spec\": {\"tuningOptions\": {\"maxConnections\": 2000000}}}") + ensureRouterDeployGenerationIs(oc, ingctrl.name, "3") + + compat_otp.By("8: Check ROUTER_MAX_CONNECTIONS env under a router pod of IC ocp50926, which should be 2000000") + podname = getOneNewRouterPodFromRollingUpdate(oc, ingctrl.name) + maxConnSearch = readRouterPodEnv(oc, podname, "ROUTER_MAX_CONNECTIONS") + o.Expect(maxConnSearch).To(o.ContainSubstring("ROUTER_MAX_CONNECTIONS=2000000")) + + compat_otp.By("9: Check maxconn in haproxy.config under a router pod of IC ocp50926, which should be 2000000") + ensureHaproxyBlockConfigContains(oc, podname, "maxconn", []string{"maxconn 2000000"}) + + // OCP-50928 Negative test of Support a Configurable ROUTER_MAX_CONNECTIONS in HAproxy + compat_otp.By("10: Patch tuningOptions/maxConnections 0 to the IC") + patchResourceAsAdmin(oc, ingctrl.namespace, ingctrlResource, "{\"spec\": {\"tuningOptions\": {\"maxConnections\": 0}}}") + ensureRouterDeployGenerationIs(oc, ingctrl.name, "4") + + compat_otp.By("11: Try to Check ROUTER_MAX_CONNECTIONS env in a route pod set to the default maxConnections by 0") + podname = getOneNewRouterPodFromRollingUpdate(oc, ingctrl.name) + cmd = fmt.Sprintf("/usr/bin/env | grep %s", "ROUTER_MAX_CONNECTIONS") + _, err = oc.AsAdmin().WithoutNamespace().Run("exec").Args("-n", "openshift-ingress", podname, "--", "bash", "-c", cmd).Output() + o.Expect(err).To(o.HaveOccurred()) + + compat_otp.By("12: Check maxconn in haproxy.config under a router pod of the IC , which should be 50000") + ensureHaproxyBlockConfigContains(oc, podname, "maxconn", []string{"maxconn 50000"}) + + compat_otp.By("13: Try to patch the ingress-controller with tuningOptions/maxConnections 1999, which is less than the min 2000") + NegMaxConnections := "1999" + output, err2 := oc.AsAdmin().WithoutNamespace().Run("patch").Args(ingctrlResource, "-p", "{\"spec\": {\"tuningOptions\": {\"maxConnections\": "+NegMaxConnections+"}}}", "--type=merge", "-n", ingctrl.namespace).Output() + o.Expect(err2).To(o.HaveOccurred()) + o.Expect(output).To(o.ContainSubstring("Unsupported value: " + NegMaxConnections + ": supported values: \"-1\", \"0\"")) + + compat_otp.By("14: Try to patch the ingress-controller with tuningOptions/maxConnections 2000001, which is a larger than the max 2000000") + NegMaxConnections = "2000001" + output, err2 = oc.AsAdmin().WithoutNamespace().Run("patch").Args(ingctrlResource, "-p", "{\"spec\": {\"tuningOptions\": {\"maxConnections\": "+NegMaxConnections+"}}}", "--type=merge", "-n", ingctrl.namespace).Output() + o.Expect(err2).To(o.HaveOccurred()) + o.Expect(output).To(o.ContainSubstring("Unsupported value: " + NegMaxConnections + ": supported values: \"-1\", \"0\"")) + + compat_otp.By("15: Try to patch the ingress-controller with tuningOptions/maxConnections abc, which is a string") + NegMaxConnections = "abc" + output, err2 = oc.AsAdmin().WithoutNamespace().Run("patch").Args(ingctrlResource, "-p", "{\"spec\": {\"tuningOptions\": {\"maxConnections\": \""+NegMaxConnections+"\"}}}", "--type=merge", "-n", ingctrl.namespace).Output() + o.Expect(err2).To(o.HaveOccurred()) + o.Expect(output).To(o.ContainSubstring("Invalid value: \"string\": spec.tuningOptions.maxConnections in body must be of type integer")) + }) + + // incorporate OCP-53605 and OCP-53608 + // Test case creater: shudili@redhat.com - OCP-53605 Expose a Configurable Reload Interval in HAproxy + // Test case creater: shudili@redhat.com - OCP-53608 Negative Test of Expose a Configurable Reload Interval in HAproxy + g.It("Author:shudili-High-53605-Expose a Configurable Reload Interval in HAproxy", func() { + buildPruningBaseDir := testdata.FixturePath("router") + baseTemp := filepath.Join(buildPruningBaseDir, "ingresscontroller-np.yaml") + extraParas := " tuningOptions:\n reloadInterval: 15s\n" + customTemp1 := addExtraParametersToYamlFile(baseTemp, "spec:", extraParas) + defer os.Remove(customTemp1) + extraParas = " tuningOptions:\n reloadInterval: 120s\n" + customTemp2 := addExtraParametersToYamlFile(baseTemp, "spec:", extraParas) + defer os.Remove(customTemp2) + ingctrl1 := ingressControllerDescription{ + name: "ocp53605one", + namespace: "openshift-ingress-operator", + domain: "", + template: customTemp1, + } + ingctrl2 := ingressControllerDescription{ + name: "ocp53605two", + namespace: "openshift-ingress-operator", + domain: "", + template: customTemp2, + } + ingctrlResource1 := "ingresscontrollers/" + ingctrl1.name + ingctrlResource2 := "ingresscontrollers/" + ingctrl2.name + + compat_otp.By("1: Create two custom ICs for testing router reload interval") + baseDomain := getBaseDomain(oc) + ingctrl1.domain = ingctrl1.name + "." + baseDomain + ingctrl2.domain = ingctrl2.name + "." + baseDomain + defer ingctrl1.delete(oc) + ingctrl1.create(oc) + defer ingctrl2.delete(oc) + ingctrl2.create(oc) + ensureRouterDeployGenerationIs(oc, ingctrl1.name, "1") + ensureRouterDeployGenerationIs(oc, ingctrl2.name, "1") + + compat_otp.By("2: Check RELOAD_INTERVAL env in a route pod of IC ocp53605one, which should be 15s") + podname1 := getOneNewRouterPodFromRollingUpdate(oc, ingctrl1.name) + riSearch := readRouterPodEnv(oc, podname1, "RELOAD_INTERVAL") + o.Expect(riSearch).To(o.ContainSubstring("RELOAD_INTERVAL=15s")) + + compat_otp.By("3: Check RELOAD_INTERVAL env in a route pod of IC ocp53605two, which should be 2m") + podname2 := getOneNewRouterPodFromRollingUpdate(oc, ingctrl2.name) + riSearch = readRouterPodEnv(oc, podname2, "RELOAD_INTERVAL") + o.Expect(riSearch).To(o.ContainSubstring("RELOAD_INTERVAL=2m")) + + compat_otp.By("4: Patch tuningOptions/reloadInterval with other valid unit m, for exmpale 1m to IC ocp53605one, while patch it with 0s to IC ocp53605two") + patchResourceAsAdmin(oc, ingctrl1.namespace, ingctrlResource1, "{\"spec\": {\"tuningOptions\": {\"reloadInterval\": \"1m\"}}}") + patchResourceAsAdmin(oc, ingctrl2.namespace, ingctrlResource2, "{\"spec\": {\"tuningOptions\": {\"reloadInterval\": \"0s\"}}}") + ensureRouterDeployGenerationIs(oc, ingctrl1.name, "2") + ensureRouterDeployGenerationIs(oc, ingctrl2.name, "2") + + compat_otp.By("5: Check RELOAD_INTERVAL env in a route pod of IC ocp53605one which should be 1m") + podname1 = getOneNewRouterPodFromRollingUpdate(oc, ingctrl1.name) + riSearch = readRouterPodEnv(oc, podname1, "RELOAD_INTERVAL") + o.Expect(riSearch).To(o.ContainSubstring("RELOAD_INTERVAL=1m")) + + compat_otp.By("6: Check RELOAD_INTERVAL env in a route pod of IC ocp53605two, which is the default 5s") + podname2 = getOneNewRouterPodFromRollingUpdate(oc, ingctrl2.name) + riSearch = readRouterPodEnv(oc, podname2, "RELOAD_INTERVAL") + o.Expect(riSearch).To(o.ContainSubstring("RELOAD_INTERVAL=5s")) + + // OCP-53608 Negative Test of Expose a Configurable Reload Interval in HAproxy + compat_otp.By("7: Try to patch tuningOptions/reloadInterval 121s which is larger than the max 120s, to the ingress-controller") + NegReloadInterval := "121s" + patchResourceAsAdmin(oc, ingctrl1.namespace, ingctrlResource1, "{\"spec\": {\"tuningOptions\": {\"reloadInterval\": \""+NegReloadInterval+"\"}}}") + ensureRouterDeployGenerationIs(oc, ingctrl1.name, "3") + + compat_otp.By("8: Check RELOAD_INTERVAL env in a route pod which should be the max 2m") + podname := getOneNewRouterPodFromRollingUpdate(oc, ingctrl1.name) + riSearch = readRouterPodEnv(oc, podname, "RELOAD_INTERVAL") + o.Expect(riSearch).To(o.ContainSubstring("RELOAD_INTERVAL=2m")) + + compat_otp.By("9: Try to patch tuningOptions/reloadInterval 0.5s which is less than the min 1s, to the ingress-controller") + NegReloadInterval = "0.5s" + patchResourceAsAdmin(oc, ingctrl1.namespace, ingctrlResource1, "{\"spec\": {\"tuningOptions\": {\"reloadInterval\": \""+NegReloadInterval+"\"}}}") + ensureRouterDeployGenerationIs(oc, ingctrl1.name, "4") + + compat_otp.By("10: Check RELOAD_INTERVAL env in a route pod which should be the min 1s") + podname = getOneNewRouterPodFromRollingUpdate(oc, ingctrl1.name) + riSearch = readRouterPodEnv(oc, podname, "RELOAD_INTERVAL") + o.Expect(riSearch).To(o.ContainSubstring("RELOAD_INTERVAL=1s")) + + compat_otp.By("11: Try to patch tuningOptions/reloadInterval -1s which is a minus value, to the ingress-controller") + NegReloadInterval = "-1s" + output, errCfg := patchResourceAsAdminAndGetLog(oc, ingctrl1.namespace, ingctrlResource1, "{\"spec\": {\"tuningOptions\": {\"reloadInterval\": \""+NegReloadInterval+"\"}}}") + o.Expect(errCfg).To(o.HaveOccurred()) + o.Expect(output).To(o.ContainSubstring("Invalid value: \"" + NegReloadInterval + "\"")) + + compat_otp.By("12: Try to patch tuningOptions/reloadInterval 1abc which is a string, to the ingress-controller") + NegReloadInterval = "1abc" + output, errCfg = patchResourceAsAdminAndGetLog(oc, ingctrl1.namespace, ingctrlResource1, "{\"spec\": {\"tuningOptions\": {\"reloadInterval\": \""+NegReloadInterval+"\"}}}") + o.Expect(errCfg).To(o.HaveOccurred()) + o.Expect(output).To(o.ContainSubstring("Invalid value: \"" + NegReloadInterval + "\"")) + + compat_otp.By("13: Try to patch tuningOptions/reloadInterval 012 s which contains a space character, to the ingress-controller") + NegReloadInterval = "012 s" + output, errCfg = patchResourceAsAdminAndGetLog(oc, ingctrl1.namespace, ingctrlResource1, "{\"spec\": {\"tuningOptions\": {\"reloadInterval\": \""+NegReloadInterval+"\"}}}") + o.Expect(errCfg).To(o.HaveOccurred()) + o.Expect(output).To(o.ContainSubstring("Invalid value: \"" + NegReloadInterval + "\"")) + }) + + // author: shudili@redhat.com + g.It("Author:shudili-LEVEL0-High-55367-Default HAProxy maxconn value to 50000 for OCP 4.12 and later", func() { + var ( + buildPruningBaseDir = testdata.FixturePath("router") + customTemp = filepath.Join(buildPruningBaseDir, "ingresscontroller-np.yaml") + ingctrl = ingressControllerDescription{ + name: "ocp55367", + namespace: "openshift-ingress-operator", + domain: "", + template: customTemp, + } + ) + + compat_otp.By("Create an custom ingresscontroller for testing ROUTER_MAX_CONNECTIONS") + baseDomain := getBaseDomain(oc) + ingctrl.domain = ingctrl.name + "." + baseDomain + ingctrlResource := "ingresscontrollers/" + ingctrl.name + defer ingctrl.delete(oc) + ingctrl.create(oc) + ensureRouterDeployGenerationIs(oc, ingctrl.name, "1") + + compat_otp.By("Check default value of ROUTER_MAX_CONNECTIONS env in a route pod, which shouldn't appear in it") + podname := getOneNewRouterPodFromRollingUpdate(oc, ingctrl.name) + cmd := fmt.Sprintf("/usr/bin/env | grep %s", "ROUTER_MAX_CONNECTIONS") + _, err := oc.AsAdmin().WithoutNamespace().Run("exec").Args("-n", "openshift-ingress", podname, "--", "bash", "-c", cmd).Output() + o.Expect(err).To(o.HaveOccurred()) + + compat_otp.By("Check maxconn in haproxy.config which should be 50000") + ensureHaproxyBlockConfigContains(oc, podname, "maxconn", []string{"maxconn 50000"}) + + compat_otp.By("Patch tuningOptions/maxConnections with null to the ingress-controller") + maxConnections := "null" + jpath := "{.status.observedGeneration}" + observedGen1 := getByJsonPath(oc, "openshift-ingress", "deployment.apps/router-default", jpath) + patchResourceAsAdmin(oc, ingctrl.namespace, ingctrlResource, "{\"spec\": {\"tuningOptions\": {\"maxConnections\": "+maxConnections+"}}}") + observedGen2 := getByJsonPath(oc, "openshift-ingress", "deployment.apps/router-default", jpath) + o.Expect(observedGen1).To(o.ContainSubstring(observedGen2)) + + compat_otp.By("Check ROUTER_MAX_CONNECTIONS env in a route pod which shouldn't appear in it by default") + podname = getOneRouterPodNameByIC(oc, ingctrl.name) + _, err = oc.AsAdmin().WithoutNamespace().Run("exec").Args("-n", "openshift-ingress", podname, "--", "bash", "-c", cmd).Output() + o.Expect(err).To(o.HaveOccurred()) + + compat_otp.By("Check maxconn in haproxy.config which should be 50000") + ensureHaproxyBlockConfigContains(oc, podname, "maxconn", []string{"maxconn 50000"}) + + compat_otp.By("Patch tuningOptions/maxConnections 50000 to the ingress-controller") + maxConnections = "500000" + patchResourceAsAdmin(oc, ingctrl.namespace, ingctrlResource, "{\"spec\": {\"tuningOptions\": {\"maxConnections\": "+maxConnections+"}}}") + ensureRouterDeployGenerationIs(oc, ingctrl.name, "2") + + compat_otp.By("Check ROUTER_MAX_CONNECTIONS env in a route pod which should be " + maxConnections) + podname = getOneNewRouterPodFromRollingUpdate(oc, ingctrl.name) + maxConnSearch := readRouterPodEnv(oc, podname, "ROUTER_MAX_CONNECTIONS") + o.Expect(maxConnSearch).To(o.ContainSubstring("ROUTER_MAX_CONNECTIONS=" + maxConnections)) + + compat_otp.By("Check maxconn in haproxy.config which should be 50000") + ensureHaproxyBlockConfigContains(oc, podname, "maxconn", []string{"maxconn 50000"}) + }) + + // OCPBUGS-61858 + // author: shudili@redhat.com + g.It("Author:shudili-ROSA-OSD_CCS-ARO-High-86153-Supporting HTTPKeepAliveTimeout tuning option", func() { + var ( + buildPruningBaseDir = testdata.FixturePath("router") + customTemp = filepath.Join(buildPruningBaseDir, "ingresscontroller-np.yaml") + ingctrl = ingressControllerDescription{ + name: "ocp86153", + namespace: "openshift-ingress-operator", + domain: "", + template: customTemp, + } + ingctrlResource = "ingresscontrollers/" + ingctrl.name + ) + + compat_otp.By("1.0: Create an custom ingresscontroller for the testing") + baseDomain := getBaseDomain(oc) + ingctrl.domain = ingctrl.name + "." + baseDomain + defer ingctrl.delete(oc) + ingctrl.create(oc) + ensureRouterDeployGenerationIs(oc, ingctrl.name, "1") + + compat_otp.By("2.0: Check the default http-keep-alive in haproxy.config which should be 300s") + podname := getOneNewRouterPodFromRollingUpdate(oc, ingctrl.name) + ensureHaproxyBlockConfigContains(oc, podname, "defaults", []string{"http-keep-alive 300s"}) + + compat_otp.By("2.1: Check default value of ROUTER_SLOWLORIS_HTTP_KEEPALIVE env in a route pod, which shouldn't appear") + cmd := fmt.Sprintf("/usr/bin/env | grep %s", "ROUTER_SLOWLORIS_HTTP_KEEPALIVE") + _, err := oc.AsAdmin().WithoutNamespace().Run("exec").Args("-n", "openshift-ingress", podname, "--", "bash", "-c", cmd).Output() + o.Expect(err).To(o.HaveOccurred()) + + compat_otp.By("3.0: Patch tuningOptions/httpKeepAliveTimeout with 50s to the ingress-controller") + patchResourceAsAdmin(oc, ingctrl.namespace, ingctrlResource, `{"spec": {"tuningOptions": {"httpKeepAliveTimeout": "50s"}}}`) + ensureRouterDeployGenerationIs(oc, ingctrl.name, "2") + + compat_otp.By("3.1: Check the tuningOptions.httpKeepAliveTimeout in the ingress-controller which should be 50s") + tuningOptionsKeepAlive := getByJsonPath(oc, ingctrl.namespace, ingctrlResource, "{.spec.tuningOptions.httpKeepAliveTimeout}") + o.Expect(tuningOptionsKeepAlive).To(o.ContainSubstring("50s")) + + compat_otp.By("3.2: Check http-keep-alive in haproxy.config which should be 50s") + podname = getOneNewRouterPodFromRollingUpdate(oc, ingctrl.name) + ensureHaproxyBlockConfigContains(oc, podname, "defaults", []string{"http-keep-alive 50s"}) + + compat_otp.By("3.3: Check ROUTER_SLOWLORIS_HTTP_KEEPALIVE env in a route pod which should be 50s") + podname = getOneNewRouterPodFromRollingUpdate(oc, ingctrl.name) + keepAlive := readRouterPodEnv(oc, podname, "ROUTER_SLOWLORIS_HTTP_KEEPALIVE") + o.Expect(keepAlive).To(o.ContainSubstring("ROUTER_SLOWLORIS_HTTP_KEEPALIVE=50s")) + + compat_otp.By("4.0: Try to patch tuningOptions/httpKeepAliveTimeout with 50m, which is larger than the max 15m to the ingress-controller") + output := patchResourceAsAdminWithErrorOutput(oc, ingctrl.namespace, ingctrlResource, `{"spec": {"tuningOptions": {"httpKeepAliveTimeout": "50m"}}}`) + o.Expect(output).To(o.ContainSubstring("httpKeepAliveTimeout must be less than or equal to 15 minutes")) + + compat_otp.By("5.0: Try to patch tuningOptions/httpKeepAliveTimeout with 1h, the unit of which is not supported") + output = patchResourceAsAdminWithErrorOutput(oc, ingctrl.namespace, ingctrlResource, `{"spec": {"tuningOptions": {"httpKeepAliveTimeout": "1h"}}}`) + o.Expect(output).To(o.ContainSubstring("httpKeepAliveTimeout must be a valid duration string composed of an unsigned integer value, optionally followed by a decimal fraction and a unit suffix (ms, s, m)")) + }) +}) diff --git a/tests-extension/test/e2e/util.go b/tests-extension/test/e2e/util.go new file mode 100644 index 000000000..b35d4edd3 --- /dev/null +++ b/tests-extension/test/e2e/util.go @@ -0,0 +1,2700 @@ +package router + +import ( + "context" + "fmt" + "io/ioutil" + "math/rand" + "net/http" + "net/netip" + "os" + "os/exec" + "path/filepath" + "reflect" + "regexp" + "slices" + "sort" + "strconv" + "strings" + "time" + + g "github.com/onsi/ginkgo/v2" + o "github.com/onsi/gomega" + exutil "github.com/openshift/origin/test/extended/util" + compat_otp "github.com/openshift/origin/test/extended/util/compat_otp" + "github.com/openshift/router-tests-extension/test/testdata" + clusterinfra "github.com/openshift/origin/test/extended/util/compat_otp/clusterinfra" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/wait" + e2e "k8s.io/kubernetes/test/e2e/framework" + e2eoutput "k8s.io/kubernetes/test/e2e/framework/pod/output" +) + +type ingressControllerDescription struct { + name string + namespace string + defaultCert string + domain string + shard string + replicas int + template string +} + +type ingctrlHostPortDescription struct { + name string + namespace string + defaultCert string + domain string + httpport int + httpsport int + statsport int + replicas int + template string +} + +type ipfailoverDescription struct { + name string + namespace string + image string + vip string + HAInterface string + template string +} + +type routeDescription struct { + name string + namespace string + domain string + subDomain string + template string +} + +type ingressDescription struct { + name string + namespace string + domain string + serviceName string + template string +} + +type webServerDeployDescription struct { + deployName string + svcSecureName string + svcUnsecureName string + template string + namespace string +} + +type gatewayDescription struct { + name string + namespace string + hostname string + template string +} + +type httpRouteDescription struct { + name string + namespace string + gwName string + hostname string + template string +} + +func (ingctrl *ingressControllerDescription) create(oc *exutil.CLI) { + availableWorkerNode, _ := exactNodeDetails(oc) + if availableWorkerNode < 1 { + g.Skip("Skipping as there is no enough worker nodes") + } + err := createResourceFromTemplate(oc, "--ignore-unknown-parameters=true", "-f", ingctrl.template, "-p", "NAME="+ingctrl.name, "NAMESPACE="+ingctrl.namespace, "DOMAIN="+ingctrl.domain, "SHARD="+ingctrl.shard) + o.Expect(err).NotTo(o.HaveOccurred()) +} + +func (ingctrl *ingressControllerDescription) delete(oc *exutil.CLI) error { + return oc.AsAdmin().WithoutNamespace().Run("delete").Args("--ignore-not-found", "-n", ingctrl.namespace, "ingresscontroller", ingctrl.name).Execute() +} + +// Function to create hostnetwork type ingresscontroller with custom http/https/stat ports +func (ingctrl *ingctrlHostPortDescription) create(oc *exutil.CLI) { + err := createResourceFromTemplate(oc, "--ignore-unknown-parameters=true", "-f", ingctrl.template, "-p", "NAME="+ingctrl.name, "NAMESPACE="+ingctrl.namespace, "DOMAIN="+ingctrl.domain, "HTTPPORT="+strconv.Itoa(ingctrl.httpport), "HTTPSPORT="+strconv.Itoa(ingctrl.httpsport), "STATSPORT="+strconv.Itoa(ingctrl.statsport)) + o.Expect(err).NotTo(o.HaveOccurred()) +} + +// Function to delete hostnetwork type ingresscontroller +func (ingctrl *ingctrlHostPortDescription) delete(oc *exutil.CLI) error { + return oc.AsAdmin().WithoutNamespace().Run("delete").Args("--ignore-not-found", "-n", ingctrl.namespace, "ingresscontroller", ingctrl.name).Execute() +} + +// create route object from template. +func (route *routeDescription) create(oc *exutil.CLI) { + err := createResourceToNsFromTemplate(oc, route.namespace, "--ignore-unknown-parameters=true", "-f", route.template, "-p", "SUBDOMAIN_NAME="+route.subDomain, "NAMESPACE="+route.namespace, "DOMAIN="+route.domain) + o.Expect(err).NotTo(o.HaveOccurred()) +} + +// create ingress object from template. +func (ing *ingressDescription) create(oc *exutil.CLI) { + err := createResourceToNsFromTemplate(oc, ing.namespace, "--ignore-unknown-parameters=true", "-f", ing.template, "-p", "NAME="+ing.name, "NAMESPACE="+ing.namespace, "DOMAIN="+ing.domain, "SERVICE_NAME="+ing.serviceName) + o.Expect(err).NotTo(o.HaveOccurred()) +} + +// create web server deployment from template +func (websrvdeploy *webServerDeployDescription) create(oc *exutil.CLI) { + err := createResourceToNsFromTemplate(oc, websrvdeploy.namespace, "--ignore-unknown-parameters=true", "-f", websrvdeploy.template, "-p", "DEPLOY_NAME="+websrvdeploy.deployName, "SVC_SECURE_NAME="+websrvdeploy.svcSecureName, "SVC_UNSECURE_NAME="+websrvdeploy.svcUnsecureName) + o.Expect(err).NotTo(o.HaveOccurred()) +} + +func (websrvdeploy *webServerDeployDescription) delete(oc *exutil.CLI) error { + return oc.AsAdmin().WithoutNamespace().Run("delete").Args("-n", websrvdeploy.namespace, "deployment", websrvdeploy.deployName).Execute() +} + +func (gw *gatewayDescription) create(oc *exutil.CLI) { + err := createResourceToNsFromTemplate(oc, gw.namespace, "--ignore-unknown-parameters=true", "-f", gw.template, "-p", "NAME="+gw.name, "NAMESPACE="+gw.namespace, "HOSTNAME="+gw.hostname) + o.Expect(err).NotTo(o.HaveOccurred()) +} + +func (gw *gatewayDescription) delete(oc *exutil.CLI) error { + return oc.AsAdmin().WithoutNamespace().Run("delete").Args("-n", gw.namespace, "gateway", gw.name).Execute() +} + +func (httpRoute *httpRouteDescription) userCreate(oc *exutil.CLI) { + output, err := createUserResourceToNsFromTemplate(oc, httpRoute.namespace, "--ignore-unknown-parameters=true", "-f", httpRoute.template, "-p", "NAME="+httpRoute.name, "NAMESPACE="+httpRoute.namespace, "GWNAME="+httpRoute.gwName, "HOSTNAME="+httpRoute.hostname) + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(output).To(o.ContainSubstring(`httproute.gateway.networking.k8s.io/` + httpRoute.name + ` created`)) +} + +func createResourceFromTemplate(oc *exutil.CLI, parameters ...string) error { + jsonCfg := parseToJSON(oc, parameters) + return oc.AsAdmin().WithoutNamespace().Run("create").Args("-f", jsonCfg).Execute() +} + +func createResourceToNsFromTemplate(oc *exutil.CLI, ns string, parameters ...string) error { + jsonCfg := parseToJSON(oc, parameters) + return oc.AsAdmin().WithoutNamespace().Run("create").Args("-n", ns, "-f", jsonCfg).Execute() +} + +func createUserResourceToNsFromTemplate(oc *exutil.CLI, ns string, parameters ...string) (string, error) { + jsonCfg := parseToJSON(oc, parameters) + return oc.WithoutNamespace().Run("create").Args("-n", ns, "-f", jsonCfg).Output() +} + +func getRandomString() string { + chars := "abcdefghijklmnopqrstuvwxyz0123456789" + seed := rand.New(rand.NewSource(time.Now().UnixNano())) + buffer := make([]byte, 8) + for index := range buffer { + buffer[index] = chars[seed.Intn(len(chars))] + } + return string(buffer) +} + +func getFixedLengthRandomString(length int) string { + const chars = "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ" + letterBytes := []byte(chars) + result := make([]byte, length) + for i := range result { + result[i] = letterBytes[rand.Intn(len(letterBytes))] + } + return string(result) +} + +func getBaseDomain(oc *exutil.CLI) string { + var basedomain string + + basedomain, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("dns.config/cluster", "-o=jsonpath={.spec.baseDomain}").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + e2e.Logf("the base domain of the cluster: %v", basedomain) + return basedomain +} + +func getSAToken(oc *exutil.CLI, sa, ns string) (string, error) { + e2e.Logf("Getting a token assgined to specific serviceaccount from %s namespace...", ns) + token, err := oc.AsAdmin().WithoutNamespace().Run("create").Args("token", sa, "-n", ns).Output() + if err != nil { + if strings.Contains(token, "unknown command") { + e2e.Logf("oc create token is not supported by current client, use oc sa get-token instead") + token, err = oc.AsAdmin().WithoutNamespace().Run("sa").Args("get-token", sa, "-n", ns).Output() + } else { + return "", err + } + } + + return token, err +} + +// extractGCPZoneName extracts the zone name from GCP zone ID +// In OpenShift 4.20+, GCP zone IDs are in the format: projects/{projectID}/managedZones/{zoneName} +// In OpenShift 4.19 and earlier, zone IDs are just the zone name +// This function handles both formats and returns just the zone name +func extractGCPZoneName(zoneID string) string { + // Check if zoneID is in the new full path format + if strings.Contains(zoneID, "managedZones/") { + // Extract zone name from the full path format + parts := strings.Split(zoneID, "managedZones/") + if len(parts) == 2 { + zoneName := parts[1] + e2e.Logf("Extracted GCP zone name '%s' from full path '%s'", zoneName, zoneID) + return zoneName + } else { + e2e.Failf("Cannot find GCP zone name from full path '%s'", zoneID) + } + } + // Return as-is for the old format (just the zone name) + e2e.Logf("Using GCP zone ID as-is: %s", zoneID) + return zoneID +} + +// to exact available linux worker node count and details +// for debugging: also list non linux or workers with special labels +func exactNodeDetails(oc *exutil.CLI) (int, string) { + linuxWorkerDetails, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("nodes", "--selector=node-role.kubernetes.io/worker=,kubernetes.io/os=linux").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + nodeCount := int(strings.Count(linuxWorkerDetails, "Ready")) - (int(strings.Count(linuxWorkerDetails, "SchedulingDisabled")) + int(strings.Count(linuxWorkerDetails, "NotReady"))) + e2e.Logf("Linux worker node details are:\n%v", linuxWorkerDetails) + e2e.Logf("Available linux worker node count is: %v", nodeCount) + // checking other type workers for debugging + nonLinuxWorker, _ := oc.AsAdmin().WithoutNamespace().Run("get").Args("nodes", "--selector=node-role.kubernetes.io/worker=,kubernetes.io/os!=linux").Output() + if !strings.Contains(nonLinuxWorker, "No resources found") { + e2e.Logf("Found non linux worker nodes and details are:\n%v", nonLinuxWorker) + } + remoteWorker, _ := oc.AsAdmin().WithoutNamespace().Run("get").Args("nodes", "--selector=node.openshift.io/remote-worker").Output() + if !strings.Contains(remoteWorker, "No resources found") { + e2e.Logf("Found remote worker nodes and details are:\n%v", remoteWorker) + } + outpostWorker, _ := oc.AsAdmin().WithoutNamespace().Run("get").Args("nodes", "--selector=topology.ebs.csi.aws.com/outpost-id").Output() + if !strings.Contains(outpostWorker, "No resources found") { + e2e.Logf("Found outpost worker nodes and details are:\n%v", outpostWorker) + } + localZoneWorker, _ := oc.AsAdmin().WithoutNamespace().Run("get").Args("nodes", "--selector=node-role.kubernetes.io/edge").Output() + if !strings.Contains(localZoneWorker, "No resources found") { + e2e.Logf("Found local zone worker nodes and details are:\n%v", localZoneWorker) + } + return nodeCount, linuxWorkerDetails +} + +// parse the yaml file to json. +func parseToJSON(oc *exutil.CLI, parameters []string) string { + var jsonCfg string + err := wait.Poll(3*time.Second, 15*time.Second, func() (bool, error) { + output, err := oc.AsAdmin().Run("process").Args(parameters...).OutputToFile(getRandomString() + "-temp-resource.json") + if err != nil { + e2e.Logf("the err:%v, and try next round", err) + return false, nil + } + jsonCfg = output + return true, nil + }) + compat_otp.AssertWaitPollNoErr(err, fmt.Sprintf("fail to process %v", parameters)) + e2e.Logf("the file of resource is %s", jsonCfg) + return jsonCfg +} + +func waitForCustomIngressControllerAvailable(oc *exutil.CLI, icname string) error { + e2e.Logf("check ingresscontroller if available") + return wait.Poll(5*time.Second, 3*time.Minute, func() (bool, error) { + status, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("ingresscontroller", icname, "--namespace=openshift-ingress-operator", `-ojsonpath={.status.conditions[?(@.type=="Available")].status}`).Output() + e2e.Logf("the status of ingresscontroller is %v", status) + if err != nil || status == "" { + e2e.Logf("failed to get ingresscontroller %s: %v, retrying...", icname, err) + return false, nil + } + if strings.Contains(status, "False") { + e2e.Logf("ingresscontroller %s conditions not available, retrying...", icname) + return false, nil + } + return true, nil + }) +} + +func ensureCustomIngressControllerAvailable(oc *exutil.CLI, icName string) { + ns := "openshift-ingress-operator" + err := waitForCustomIngressControllerAvailable(oc, icName) + // print custom ingresscontroller description for debugging purpose if err + if err != nil { + output, _ := oc.AsAdmin().WithoutNamespace().Run("describe").Args("-n", ns, "ingresscontroller", icName).Output() + e2e.Logf("The description of ingresscontroller %v is:\n%v", icName, output) + } + compat_otp.AssertWaitPollNoErr(err, fmt.Sprintf("max time reached but ingresscontroller %v is not available", icName)) +} + +func ensureRouteIsAdmittedByIngressController(oc *exutil.CLI, ns, routeName, icName string) { + jsonPath := fmt.Sprintf(`{.status.ingress[?(@.routerName=="%s")].conditions[?(@.type=="Admitted")].status}`, icName) + waitForOutputEquals(oc, ns, "route/"+routeName, jsonPath, "True") +} + +func ensureRouteIsNotAdmittedByIngressController(oc *exutil.CLI, ns, routeName, icName string) { + jsonPath := fmt.Sprintf(`{.status.ingress[?(@.routerName=="%s")].conditions[?(@.type=="Admitted")].status}`, icName) + waitForOutputEquals(oc, ns, "route/"+routeName, jsonPath, "False") +} + +func getOnePodNameByLabel(oc *exutil.CLI, ns, label string) string { + podName, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("pods", "-l", label, "-o=jsonpath={.items[0].metadata.name}", "-n", ns).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + e2e.Logf("the one pod with label %v is %v", label, podName) + return podName +} + +// getOneNewRouterPodFromRollingUpdate immediatly after/during deployment rolling update, don't care the previous pod status +func getOneNewRouterPodFromRollingUpdate(oc *exutil.CLI, icName string) string { + ns := "openshift-ingress" + deployName := "deployment/router-" + icName + rsLabel := "" + re := regexp.MustCompile(`NewReplicaSet:\s+router-.+-([a-z0-9]+)\s+`) + waitErr := wait.PollImmediate(3*time.Second, 15*time.Second, func() (bool, error) { + output, _ := oc.AsAdmin().WithoutNamespace().Run("describe").Args(deployName, "-n", ns).Output() + hash := re.FindStringSubmatch(output) + if len(hash) > 1 { + rsLabel = "pod-template-hash=" + hash[1] + return true, nil + } + return false, nil + }) + compat_otp.AssertWaitPollNoErr(waitErr, fmt.Sprintf("reached max time allowed but NewReplicaSet not found")) + e2e.Logf("the new ReplicaSet labels is %s", rsLabel) + err := waitForPodWithLabelReady(oc, ns, rsLabel) + if err != nil { + output, _ := oc.AsAdmin().WithoutNamespace().Run("get").Args("pods", "-n", ns).Output() + e2e.Logf("All current router pods are:\n%v", output) + } + compat_otp.AssertWaitPollNoErr(err, "the new router pod failed to be ready within allowed time!") + return getOnePodNameByLabel(oc, ns, rsLabel) +} + +func ensureRouterDeployGenerationIs(oc *exutil.CLI, icName, expectGeneration string) { + ns := "openshift-ingress" + deployName := "deployment/router-" + icName + actualGeneration := "0" + + waitErr := wait.PollImmediate(3*time.Second, 30*time.Second, func() (bool, error) { + actualGeneration, _ = oc.AsAdmin().WithoutNamespace().Run("get").Args(deployName, "-n", ns, "-o=jsonpath={.metadata.generation}").Output() + e2e.Logf("Get the deployment generation is: %v", actualGeneration) + if actualGeneration == expectGeneration { + e2e.Logf("The router deployment generation is updated to %v", actualGeneration) + return true, nil + } + return false, nil + }) + compat_otp.AssertWaitPollNoErr(waitErr, fmt.Sprintf("max time reached and the expected deployment generation is %v but got %v", expectGeneration, actualGeneration)) +} + +func waitForPodWithLabelReady(oc *exutil.CLI, ns, label string) error { + return wait.Poll(5*time.Second, 3*time.Minute, func() (bool, error) { + status, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("pod", "-n", ns, "-l", label, `-ojsonpath={.items[*].status.conditions[?(@.type=="Ready")].status}`).Output() + e2e.Logf("the Ready status of pod is %v", status) + if err != nil || status == "" { + e2e.Logf("failed to get pod status: %v, retrying...", err) + return false, nil + } + if strings.Contains(status, "False") { + e2e.Logf("the pod Ready status not met; wanted True but got %v, retrying...", status) + return false, nil + } + return true, nil + }) +} + +func ensurePodWithLabelReady(oc *exutil.CLI, ns, label string) { + err := waitForPodWithLabelReady(oc, ns, label) + // print pod status and logs for debugging purpose if err + if err != nil { + output, _ := oc.AsAdmin().WithoutNamespace().Run("get").Args("pod", "-n", ns, "-l", label).Output() + e2e.Logf("All pods with label %v are:\n%v", label, output) + logs, _ := oc.AsAdmin().WithoutNamespace().Run("logs").Args("-n", ns, "-l", label, "--tail=10").Output() + e2e.Logf("The logs of all labeled pods are:\n%v", logs) + } + compat_otp.AssertWaitPollNoErr(err, fmt.Sprintf("max time reached but the pods with label %v are not ready", label)) +} + +func waitForPodWithLabelAppear(oc *exutil.CLI, ns, label string) error { + return wait.Poll(5*time.Second, 3*time.Minute, func() (bool, error) { + podList, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("pod", "-n", ns, "-l", label).Output() + e2e.Logf("the pod list is %v", podList) + // add check for OCPQE-17360: pod list is "No resources found in xxx namespace" + podFlag := 1 + if strings.Contains(podList, "No resources found") { + podFlag = 0 + } + if err != nil || len(podList) < 1 || podFlag == 0 { + e2e.Logf("failed to get pod: %v, retrying...", err) + return false, nil + } + return true, nil + }) +} + +// wait for the named resource is disappeared, e.g. used while router deployment rolled out +func waitForResourceToDisappear(oc *exutil.CLI, ns, rsname string) error { + return wait.Poll(20*time.Second, 5*time.Minute, func() (bool, error) { + status, err := oc.AsAdmin().WithoutNamespace().Run("get").Args(rsname, "-n", ns).Output() + e2e.Logf("check resource %v and got: %v", rsname, status) + primary := false + if err != nil { + if strings.Contains(status, "NotFound") { + e2e.Logf("the resource is disappeared!") + primary = true + } else { + e2e.Logf("failed to get the resource: %v, retrying...", err) + } + } else { + e2e.Logf("the resource is still there, retrying...") + } + return primary, nil + }) +} + +// For normal user to create resources in the specified namespace from the file (not template) +func createResourceFromFile(oc *exutil.CLI, ns, file string) { + err := oc.WithoutNamespace().Run("create").Args("-f", file, "-n", ns).Execute() + o.Expect(err).NotTo(o.HaveOccurred()) +} + +// to use createResourceFromFile function to create resources from files like web-server-rc.yaml and web-server-signed-deploy.yaml +func createResourceFromWebServer(oc *exutil.CLI, ns, file, srvrcInfo string) []string { + createResourceFromFile(oc, ns, file) + err := waitForPodWithLabelReady(oc, ns, "name="+srvrcInfo) + compat_otp.AssertWaitPollNoErr(err, "backend server pod failed to be ready state within allowed time!") + srvPodList := getPodListByLabel(oc, ns, "name="+srvrcInfo) + return srvPodList +} + +// For admin user to create/delete resources in the specified namespace from the file (not template) +// oper, should be create or delete +func operateResourceFromFile(oc *exutil.CLI, oper, ns, file string) { + err := oc.AsAdmin().WithoutNamespace().Run(oper).Args("-f", file, "-n", ns).Execute() + o.Expect(err).NotTo(o.HaveOccurred()) +} + +// For normal user to patch a resource in the specified namespace +func patchResourceAsUser(oc *exutil.CLI, ns, resource, patch string) { + err := oc.WithoutNamespace().Run("patch").Args(resource, "-p", patch, "--type=merge", "-n", ns).Execute() + o.Expect(err).NotTo(o.HaveOccurred()) +} + +// For Admin to patch a resource in the specified namespace with 'merge' type +func patchResourceAsAdmin(oc *exutil.CLI, ns, resource, patch string) { + patchResourceAsAdminAnyType(oc, ns, resource, patch, "merge") +} + +// For Admin to patch a resource in the specified namespace with any TYPE +// Type can be any like 'merge', 'json' etc +func patchResourceAsAdminAnyType(oc *exutil.CLI, ns, resource, patch, typ string) { + err := oc.AsAdmin().WithoutNamespace().Run("patch").Args(resource, "-p", patch, "--type="+typ, "-n", ns).Execute() + o.Expect(err).NotTo(o.HaveOccurred()) +} + +func patchResourceAsAdminWithErrorOutput(oc *exutil.CLI, ns, resource, patch string) string { + output, err := oc.AsAdmin().WithoutNamespace().Run("patch").Args(resource, "-p", patch, "--type=merge", "-n", ns).Output() + o.Expect(err).To(o.HaveOccurred()) + return output +} + +// To patch global resources as Admin. Can used for patching resources such as ingresses or CVO +func patchGlobalResourceAsAdmin(oc *exutil.CLI, resource, patch string) { + patchOut, err := oc.AsAdmin().WithoutNamespace().Run("patch").Args(resource, "--patch="+patch, "--type=json").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + e2e.Logf("The output from the patch is:- %q ", patchOut) +} + +// For Admin to patch a resource in the specified namespace, and then return the output after the patching operation +func patchResourceAsAdminAndGetLog(oc *exutil.CLI, ns, resource, patch string) (string, error) { + outPut, err := oc.AsAdmin().WithoutNamespace().Run("patch").Args(resource, "-p", patch, "--type=merge", "-n", ns).Output() + return outPut, err +} + +func createRoute(oc *exutil.CLI, ns, routeType, routeName, serviceName string, extraParas []string) { + if routeType == "http" { + cmd := []string{"-n", ns, "service", serviceName, "--name=" + routeName} + cmd = append(cmd, extraParas...) + _, err := oc.Run("expose").Args(cmd...).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + } else { + cmd := []string{"-n", ns, "route", routeType, routeName, "--service=" + serviceName} + cmd = append(cmd, extraParas...) + _, err := oc.WithoutNamespace().Run("create").Args(cmd...).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + } +} + +func setAnnotation(oc *exutil.CLI, ns, resource, annotation string) { + err := oc.Run("annotate").Args("-n", ns, resource, annotation, "--overwrite").Execute() + o.Expect(err).NotTo(o.HaveOccurred()) +} + +// this function will set the annotation for the given resource +func setAnnotationAsAdmin(oc *exutil.CLI, ns, resource, annotation string) { + err := oc.AsAdmin().WithoutNamespace().Run("annotate").Args("-n", ns, resource, annotation, "--overwrite").Execute() + o.Expect(err).NotTo(o.HaveOccurred()) +} + +// this function will read the annotation from the given resource +func getAnnotation(oc *exutil.CLI, ns, resource, resourceName string) string { + findAnnotation, err := oc.AsAdmin().WithoutNamespace().Run("get").Args( + resource, resourceName, "-n", ns, "-o=jsonpath={.metadata.annotations}").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + return findAnnotation +} + +// this function will add the label(key and the value) for the given resource +func addLabelAsAdmin(oc *exutil.CLI, ns, resource, label string) { + err := oc.AsAdmin().WithoutNamespace().Run("label").Args("-n", ns, resource, label, "--overwrite").Execute() + o.Expect(err).NotTo(o.HaveOccurred()) +} + +func setEnvVariable(oc *exutil.CLI, ns, resource, envstring string) { + err := oc.AsAdmin().WithoutNamespace().Run("set").Args("env", "-n", ns, resource, envstring).Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + time.Sleep(10 * time.Second) +} + +// Generic function to collect resource values with jsonpath option +func getByJsonPath(oc *exutil.CLI, ns, resource, jsonPath string) string { + output, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("-n", ns, resource, "-o=jsonpath="+jsonPath).Output() + if err != nil { + e2e.Logf("the error is: %v", err.Error()) + } + e2e.Logf("the output filtered by jsonpath is: %v", output) + return output +} + +// this function get resource using label and filtered by jsonpath +func getByLabelAndJsonPath(oc *exutil.CLI, ns, resource, label, jsonPath string) string { + output, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("-n", ns, resource, "-l", label, "-ojsonpath="+jsonPath).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + e2e.Logf("the output filtered by label and jsonpath is: %v", output) + return output +} + +// getNodeNameByPod gets the pod located node's name +func getNodeNameByPod(oc *exutil.CLI, namespace string, podName string) string { + nodeName, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("pod", podName, "-n", namespace, "-o=jsonpath={.spec.nodeName}").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + e2e.Logf("The nodename for pod %s in namespace %s is %s", podName, namespace, nodeName) + return nodeName +} + +// Collect pod describe command details: +func describePodResource(oc *exutil.CLI, podName, namespace string) string { + podDescribe, err := oc.AsAdmin().WithoutNamespace().Run("describe").Args("pod", podName, "-n", namespace).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + return podDescribe +} + +// for collecting a single pod name for general use. +// usage example: podname := getOneRouterPodNameByIC(oc, "default/labelname") +// note: it might get wrong pod which will be terminated during deployment rolling update +func getOneRouterPodNameByIC(oc *exutil.CLI, icname string) string { + podName, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("pods", "-l", "ingresscontroller.operator.openshift.io/deployment-ingresscontroller="+icname, "-o=jsonpath={.items[0].metadata.name}", "-n", "openshift-ingress").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + e2e.Logf("the result of router pod name: %v", podName) + return podName +} + +// For collecting env details with grep from router pod [usage example: readRouterPodEnv(oc, podname, "search string")] . +// NOTE: This requires getOneRouterPodNameByIC function to collect the podname variable first! +func readRouterPodEnv(oc *exutil.CLI, routername, envname string) string { + ns := "openshift-ingress" + output := readPodEnv(oc, routername, ns, envname) + return output +} + +// For collecting env details with grep [usage example: readPodEnv(oc, namespace, podname, "search string")] +func readPodEnv(oc *exutil.CLI, routername, ns string, envname string) string { + cmd := fmt.Sprintf("/usr/bin/env | grep %s", envname) + output, err := oc.AsAdmin().WithoutNamespace().Run("exec").Args("-n", ns, routername, "--", "bash", "-c", cmd).Output() + if err != nil { + output = "NotFound" + } + e2e.Logf("the matched Env are:\n%v", output) + return output +} + +// to check the route data is present in the haproxy.config +// blockCfgStart is the used to get the bulk config from the getBlockConfig function +// searchList is used to locate the specified route config +func ensureHaproxyBlockConfigContains(oc *exutil.CLI, routerPodName string, blockCfgStart string, searchList []string) string { + var ( + haproxyCfg string + j = 0 + ) + + e2e.Logf("Polling and search haproxy config file") + waitErr := wait.Poll(5*time.Second, 60*time.Second, func() (bool, error) { + haproxyCfg = getBlockConfig(oc, routerPodName, blockCfgStart) + for i := j; i < len(searchList); i++ { + if strings.Contains(haproxyCfg, searchList[i]) { + e2e.Logf("Found the given string %v in haproxy.config", searchList[i]) + j++ + if j == len(searchList) { + e2e.Logf("All the given strings are found in haproxy.config") + return true, nil + } + } else { + e2e.Logf("The given string %v is still not found in haproxy.config, retrying...", searchList[i]) + return false, nil + } + } + return false, nil + }) + + compat_otp.AssertWaitPollNoErr(waitErr, fmt.Sprintf("Reached max time allowed but the given string was not found in haproxy.config")) + e2e.Logf("The part of haproxy.config that matching \"%s\" is:\n%v", blockCfgStart, haproxyCfg) + return haproxyCfg +} + +// similar to ensureHaproxyBlockConfigContains function, this function uses regexp searching not string searching +func ensureHaproxyBlockConfigMatchRegexp(oc *exutil.CLI, routerPodName string, blockCfgStart string, searchList []string) string { + var ( + haproxyCfg string + j = 0 + ) + + e2e.Logf("Polling and search haproxy config file") + waitErr := wait.Poll(5*time.Second, 60*time.Second, func() (bool, error) { + haproxyCfg = getBlockConfig(oc, routerPodName, blockCfgStart) + for i := j; i < len(searchList); i++ { + searchInfo := regexp.MustCompile(searchList[i]).FindStringSubmatch(haproxyCfg) + if len(searchInfo) > 0 { + e2e.Logf("Found the given string %v in haproxy.config", searchList[i]) + j++ + if j == len(searchList) { + e2e.Logf("All the given strings are found in haproxy.config") + return true, nil + } + } else { + e2e.Logf("The given string %v is still not found in haproxy.config, retrying...", searchList[i]) + return false, nil + } + } + return false, nil + }) + + compat_otp.AssertWaitPollNoErr(waitErr, fmt.Sprintf("Reached max time allowed but expected string was not found in haproxy.config")) + e2e.Logf("The part of haproxy.config that matching \"%s\" is:\n%v", blockCfgStart, haproxyCfg) + return haproxyCfg +} + +// to check the route data is not present in the haproxy.config +// blockCfgStart is the used to get the bulk config from the getBlockConfig function +// searchList is used to locate the specified route config +func ensureHaproxyBlockConfigNotContains(oc *exutil.CLI, routerPodName string, blockCfgStart string, searchList []string) string { + var ( + haproxyCfg string + j = 0 + ) + + e2e.Logf("Polling and search haproxy config file") + waitErr := wait.Poll(5*time.Second, 30*time.Second, func() (bool, error) { + haproxyCfg = getBlockConfig(oc, routerPodName, blockCfgStart) + for i := j; i < len(searchList); i++ { + if !strings.Contains(haproxyCfg, searchList[i]) { + e2e.Logf("Could not found the given string %v in haproxy.config as expected", searchList[i]) + j++ + if j == len(searchList) { + e2e.Logf("Could not found all given strings in haproxy.config as expected") + return true, nil + } + } else { + e2e.Logf("The given string %v is still present in haproxy.config, retrying...", searchList[i]) + return false, nil + } + } + return false, nil + }) + + compat_otp.AssertWaitPollNoErr(waitErr, fmt.Sprintf("Reached max time allowed but given string is still present in haproxy.config")) + e2e.Logf("The part of haproxy.config that matching \"%s\" is:\n%v", blockCfgStart, haproxyCfg) + return haproxyCfg +} + +// similar to ensureHaproxyBlockConfigNotContains function, this function uses regexp searching not string searching +func ensureHaproxyBlockConfigNotMatchRegexp(oc *exutil.CLI, routerPodName string, blockCfgStart string, searchList []string) string { + var ( + haproxyCfg string + j = 0 + ) + + e2e.Logf("Polling and search haproxy config file") + waitErr := wait.Poll(5*time.Second, 30*time.Second, func() (bool, error) { + haproxyCfg = getBlockConfig(oc, routerPodName, blockCfgStart) + for i := j; i < len(searchList); i++ { + searchInfo := regexp.MustCompile(searchList[i]).FindStringSubmatch(haproxyCfg) + if len(searchInfo) == 0 { + e2e.Logf("Could not found the given string %v in haproxy.config as expected", searchList[i]) + j++ + if j == len(searchList) { + e2e.Logf("Could not found all given strings in haproxy.config as expected") + return true, nil + } + } else { + e2e.Logf("The given string %v is still present in haproxy.config, retrying...", searchList[i]) + return false, nil + } + } + return false, nil + }) + + compat_otp.AssertWaitPollNoErr(waitErr, fmt.Sprintf("Reached max time allowed but given string is still present in haproxy.config")) + e2e.Logf("The part of haproxy.config that matching \"%s\" is:\n%v", blockCfgStart, haproxyCfg) + return haproxyCfg +} + +// used to get block content of haproxy.conf, for example, get one route's whole backend's configuration specified by searchString(for exmpale: "be_edge_http:" + ns + ":r1-edg") +func getBlockConfig(oc *exutil.CLI, routerPodName, searchString string) string { + output, err := oc.AsAdmin().WithoutNamespace().Run("exec").Args("-n", "openshift-ingress", routerPodName, "--", "bash", "-c", "cat haproxy.config").Output() + o.Expect(err).NotTo(o.HaveOccurred(), "get the content of haproxy.config failed") + result := "" + flag := 0 + startIndex := 0 + for _, line := range strings.Split(output, "\n") { + if strings.Contains(line, searchString) { + result = result + line + "\n" + flag = 1 + startIndex = len(line) - len(strings.TrimLeft(line, " ")) + } else if flag == 1 { + lineLen := len(line) + if lineLen == 0 { + result = result + "\n" + } else { + currentIndex := len(line) - len(strings.TrimLeft(line, " ")) + if currentIndex > startIndex { + result = result + line + "\n" + } else { + flag = 2 + } + } + + } else if flag == 2 { + break + } + } + e2e.Logf("The block configuration in haproxy that matching \"%s\" is:\n%v", searchString, result) + return result +} + +// this function is used to get haproxy's version +func getHAProxyVersion(oc *exutil.CLI) string { + var proxyVersion = "notFound" + routerpod := getOneRouterPodNameByIC(oc, "default") + haproxyOutput, err := oc.AsAdmin().WithoutNamespace().Run("exec").Args("-n", "openshift-ingress", routerpod, "--", "bash", "-c", "haproxy -v | grep version").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + haproxyRe := regexp.MustCompile("([0-9\\.]+)-([0-9a-z]+)") + haproxyInfo := haproxyRe.FindStringSubmatch(haproxyOutput) + if len(haproxyInfo) > 0 { + proxyVersion = haproxyInfo[0] + } + return proxyVersion +} + +func getHAProxyRPMVersion(oc *exutil.CLI) string { + routerpod := getOneRouterPodNameByIC(oc, "default") + haproxyOutput, err := oc.AsAdmin().WithoutNamespace().Run("exec").Args("-n", "openshift-ingress", routerpod, "--", "bash", "-c", "rpm -qa | grep haproxy").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + return haproxyOutput +} + +// this function is used to check whether a route pod has the specified certification of a route and returns its details +// ns: the route's namespace; routeName: the route's name +// certType: certs or cacerts +// option: --hasCert(means should have the certification) or --noCert(means shouldn't has the certification) +func checkRouteCertificationInRouterPod(oc *exutil.CLI, ns, routeName, routerpod, certType, option string) string { + var certCfg, cmd string + var err error + certName := fmt.Sprintf("%s:%s.pem", ns, routeName) + if option == "--noCert" && certType == "cacerts" { + cmd = "ls /var/lib/haproxy/router/cacerts/" + } else if option != "--noCert" && certType != "cacerts" { + cmd = "ls --full-time /var/lib/haproxy/router/certs/" + certName + } else if option != "--noCert" && certType == "cacerts" { + cmd = "ls --full-time /var/lib/haproxy/router/cacerts/" + certName + } else { + cmd = "ls /var/lib/haproxy/router/certs/" + } + + waitErr := wait.Poll(3*time.Second, 120*time.Second, func() (bool, error) { + flag := false + certCfg, err = oc.AsAdmin().WithoutNamespace().Run("exec").Args("-n", "openshift-ingress", routerpod, "--", "bash", "-c", cmd).Output() + if err != nil { + // make sure the certification is present + e2e.Logf("Tried to get the certification configuration, but got error(trying...):\n%v", err) + return false, nil + } + if option == "--hasCert" && strings.Contains(certCfg, certName) { + flag = true + } + if option == "--noCert" && !strings.Contains(certCfg, certName) { + flag = true + } + return flag, nil + }) + compat_otp.AssertWaitPollNoErr(waitErr, fmt.Sprintf("reached max time allowed but the certification with seaching option %s not matched", option)) + return certCfg +} + +func getImagePullSpecFromPayload(oc *exutil.CLI, image string) string { + var pullspec string + baseDir := testdata.FixturePath("router") + indexTmpPath := filepath.Join(baseDir, getRandomString()) + dockerconfigjsonpath := filepath.Join(indexTmpPath, ".dockerconfigjson") + defer exec.Command("rm", "-rf", indexTmpPath).Output() + err := os.MkdirAll(indexTmpPath, 0755) + o.Expect(err).NotTo(o.HaveOccurred()) + _, err = oc.AsAdmin().WithoutNamespace().Run("extract").Args("secret/pull-secret", "-n", "openshift-config", "--confirm", "--to="+indexTmpPath).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + pullspec, err = oc.AsAdmin().WithoutNamespace().Run("adm").Args("release", "info", "--image-for="+image, "-a", dockerconfigjsonpath).Output() + if err != nil { + g.Skip("Skipping as failed to get image pull spec from the payload") + } + e2e.Logf("the pull spec of image %v is: %v", image, pullspec) + return pullspec +} + +func (ipf *ipfailoverDescription) create(oc *exutil.CLI, ns string) { + // create ServiceAccount and add it to related SCC + _, err := oc.AsAdmin().WithoutNamespace().Run("create").Args("sa", "ipfailover", "-n", ns).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + _, err = oc.AsAdmin().WithoutNamespace().Run("adm").Args("policy", "add-scc-to-user", "privileged", "-z", "ipfailover", "-n", ns).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + // create the ipfailover deployment + err = createResourceFromTemplate(oc, "--ignore-unknown-parameters=true", "-f", ipf.template, "-p", "NAME="+ipf.name, "NAMESPACE="+ipf.namespace, "IMAGE="+ipf.image, "HAINTERFACE="+ipf.HAInterface) + o.Expect(err).NotTo(o.HaveOccurred()) +} + +func ensureLogsContainString(oc *exutil.CLI, ns, label, match string) { + waitErr := wait.Poll(3*time.Second, 90*time.Second, func() (bool, error) { + log, err := oc.AsAdmin().WithoutNamespace().Run("logs").Args("-n", ns, "-l", label, "--tail=20").Output() + // for debugging only + // e2e.Logf("the logs of labeled pods are: %v", log) + if err != nil || log == "" { + e2e.Logf("Failed to get logs: %v, retrying...", err) + return false, nil + } + if !strings.Contains(log, match) { + e2e.Logf("Cannot find the expected string in the logs, retrying...") + return false, nil + } + e2e.Logf("Find the expected string '%v' in the logs.", match) + return true, nil + }) + compat_otp.AssertWaitPollNoErr(waitErr, fmt.Sprintf("Reached max time allowed but cannot find the expected string '%v' in the logs.", match)) +} + +// This function will identify the master and backup pod of the ipfailover pods +func ensureIpfailoverMasterBackup(oc *exutil.CLI, ns string, podList []string) (string, string) { + var masterPod, backupPod string + // The sleep is given for the election process to finish + time.Sleep(10 * time.Second) + waitErr := wait.Poll(3*time.Second, 90*time.Second, func() (bool, error) { + podLogs1, err1 := compat_otp.GetSpecificPodLogs(oc, ns, "", podList[0], "Entering") + o.Expect(err1).NotTo(o.HaveOccurred()) + logList1 := strings.Split((strings.TrimSpace(podLogs1)), "\n") + e2e.Logf("The first pod log's failover status:- %v", podLogs1) + podLogs2, err2 := compat_otp.GetSpecificPodLogs(oc, ns, "", podList[1], "Entering") + o.Expect(err2).NotTo(o.HaveOccurred()) + logList2 := strings.Split((strings.TrimSpace(podLogs2)), "\n") + e2e.Logf("The second pod log's failover status:- %v", podLogs2) + + // Checking whether the first pod is failover state master and second pod backup + if strings.Contains(logList1[len(logList1)-1], "(ipfailover_VIP_1) Entering MASTER STATE") { + if strings.Contains(logList2[len(logList2)-1], "(ipfailover_VIP_1) Entering BACKUP STATE") { + masterPod = podList[0] + backupPod = podList[1] + return true, nil + } + // Checking whether the second pod is failover state master and first pod backup + } else if strings.Contains(logList1[len(logList1)-1], "(ipfailover_VIP_1) Entering BACKUP STATE") { + if strings.Contains(logList2[len(logList2)-1], "(ipfailover_VIP_1) Entering MASTER STATE") { + masterPod = podList[1] + backupPod = podList[0] + return true, nil + } + } + e2e.Logf("The ipfailover seems not yet converged, retrying again...") + return false, nil + }) + compat_otp.AssertWaitPollNoErr(waitErr, fmt.Sprintf("Reached max time allowed but IPfailover seems not working as expected.")) + e2e.Logf("The Master pod is %v and Backup pod is %v", masterPod, backupPod) + return masterPod, backupPod +} + +// For collecting information from router pod [usage example: readRouterPodData(oc, podname, executeCmd, "search string")] . +// NOTE: This requires getOneRouterPodNameByIC function to collect the podname variable first! +func readRouterPodData(oc *exutil.CLI, routername, executeCmd string, searchString string) string { + output := readPodData(oc, routername, "openshift-ingress", executeCmd, searchString) + return output +} + +func createConfigMapFromFile(oc *exutil.CLI, ns, name, cmFile string) { + _, err := oc.AsAdmin().WithoutNamespace().Run("create").Args("configmap", name, "--from-file="+cmFile, "-n", ns).Output() + o.Expect(err).NotTo(o.HaveOccurred()) +} + +func deleteConfigMap(oc *exutil.CLI, ns, name string) { + _, err := oc.AsAdmin().WithoutNamespace().Run("delete").Args("configmap", name, "-n", ns).Output() + o.Expect(err).NotTo(o.HaveOccurred()) +} + +// check if a configmap is created in specific namespace [usage: checkConfigMap(oc, namesapce, configmapName)] +func checkConfigMap(oc *exutil.CLI, ns, configmapName string) error { + return wait.Poll(5*time.Second, 3*time.Minute, func() (bool, error) { + searchOutput, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("cm", "-n", ns).Output() + if err != nil { + e2e.Logf("failed to get configmap: %v", err) + return false, nil + } + if o.Expect(searchOutput).To(o.ContainSubstring(configmapName)) { + e2e.Logf("configmap %v found", configmapName) + return true, nil + } + return false, nil + }) +} + +// To Collect ingresscontroller domain name +func getIngressctlDomain(oc *exutil.CLI, icname string) string { + var ingressctldomain string + ingressctldomain, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("ingresscontroller", icname, "--namespace=openshift-ingress-operator", "-o=jsonpath={.status.domain}").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + e2e.Logf("the domain for the ingresscontroller is : %v", ingressctldomain) + return ingressctldomain +} + +// this function helps to get the ipv4 address of the given pod +func getPodv4Address(oc *exutil.CLI, podName, namespace string) string { + podIPv4, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("pod", podName, "-n", namespace, "-o=jsonpath={.status.podIP}").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + e2e.Logf("IP of the %s pod in namespace %s is %q ", podName, namespace, podIPv4) + return podIPv4 +} + +// this function will replace the octate of the ipaddress with the given value +func replaceIPOctet(ipaddress []string, octet int, octetValue string) string { + ipv4address := ipaddress[0] + if strings.Count(ipaddress[0], ":") >= 2 { + ipv4address = ipaddress[1] + } + ipList := strings.Split(ipv4address, ".") + ipList[octet] = octetValue + vip := strings.Join(ipList, ".") + e2e.Logf("The modified ipaddress is %s ", vip) + return vip +} + +// this function is to obtain the pod name based on the particular label +func getPodListByLabel(oc *exutil.CLI, namespace string, label string) []string { + var podList []string + podNameAll, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("-n", namespace, "pod", "-l", label, "-ojsonpath={.items..metadata.name}").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + podList = strings.Split(podNameAll, " ") + e2e.Logf("The pod list is %v", podList) + return podList +} + +func getDNSPodName(oc *exutil.CLI) string { + ns := "openshift-dns" + podName, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("-n", ns, "pods", "-l", "dns.operator.openshift.io/daemonset-dns=default", "-o=jsonpath={.items[0].metadata.name}").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + e2e.Logf("The DNS pod name is: %v", podName) + return podName +} + +// to read the Corefile content in DNS pod +// searchString is to locate the specified section since Corefile might has multiple zones +// that containing same config strings +// grepOptions can specify the lines of the context, e.g. "-A20" or "-C10" +func readDNSCorefile(oc *exutil.CLI, dnsPodName, searchString, grepOption string) string { + ns := "openshift-dns" + cmd := fmt.Sprintf("grep \"%s\" /etc/coredns/Corefile %s", searchString, grepOption) + output, err := oc.AsAdmin().WithoutNamespace().Run("exec").Args("-n", ns, dnsPodName, "--", "bash", "-c", cmd).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + e2e.Logf("the part of Corefile that matching \"%s\" is: %v", searchString, output) + return output +} + +// coredns introduced reload plugin to update the Corefile without receating dns-default pod +// similar to readHaproxyConfig(), use wait.Poll to wait the searchString2 to be updated. +// searchString1 can locate the specified zone section since Corefile might has multiple zones +// grepOptions can specify the lines of the context, e.g. "-A20" or "-C10" +// searchString2 is the config to be checked, it might exist in multiple zones so searchString1 is required +func pollReadDnsCorefile(oc *exutil.CLI, dnsPodName, searchString1, grepOption, searchString2 string) string { + e2e.Logf("Polling and search dns Corefile") + ns := "openshift-dns" + cmd1 := fmt.Sprintf("grep \"%s\" /etc/coredns/Corefile %s | grep \"%s\"", searchString1, grepOption, searchString2) + cmd2 := fmt.Sprintf("grep \"%s\" /etc/coredns/Corefile %s", searchString1, grepOption) + + waitErr := wait.PollImmediate(5*time.Second, 120*time.Second, func() (bool, error) { + // trigger an immediately refresh configmap by updating pod's annotations + hackAnnotatePod(oc, ns, dnsPodName) + _, err := oc.AsAdmin().WithoutNamespace().Run("exec").Args("-n", ns, dnsPodName, "--", "bash", "-c", cmd1).Output() + if err != nil { + e2e.Logf("string not found, wait and try again...") + return false, nil + } + return true, nil + }) + // print all dns pods and one Corefile for debugging (normally the content is less than 20 lines) + if waitErr != nil { + output, _ := oc.AsAdmin().WithoutNamespace().Run("get").Args("pods", "-n", ns, "-l", "dns.operator.openshift.io/daemonset-dns=default").Output() + e2e.Logf("All current dns pods are:\n%v", output) + output, _ = oc.AsAdmin().WithoutNamespace().Run("exec").Args("-n", ns, dnsPodName, "--", "bash", "-c", "cat /etc/coredns/Corefile").Output() + e2e.Logf("The existing Corefile is: %v", output) + } + compat_otp.AssertWaitPollNoErr(waitErr, fmt.Sprintf("reached max time allowed but Corefile is not updated")) + output, err := oc.AsAdmin().WithoutNamespace().Run("exec").Args("-n", ns, dnsPodName, "--", "bash", "-c", cmd2).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + e2e.Logf("the part of Corefile that matching \"%s\" is: %v", searchString1, output) + return output +} + +// to trigger the configmap refresh immediately +// see https://kubernetes.io/docs/tasks/configure-pod-container/configure-pod-configmap/#mounted-configmaps-are-updated-automatically +func hackAnnotatePod(oc *exutil.CLI, ns, podName string) { + hackAnnotation := "ne-testing-hack=" + getRandomString() + oc.AsAdmin().WithoutNamespace().Run("annotate").Args("pod", podName, "-n", ns, hackAnnotation, "--overwrite").Execute() +} + +// this function get all cluster's operators +func getClusterOperators(oc *exutil.CLI) []string { + outputOps, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("clusteroperators", "-o=jsonpath={.items[*].metadata.name}").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + opList := strings.Split(outputOps, " ") + return opList +} + +// wait for "Progressing" is True +func ensureClusterOperatorProgress(oc *exutil.CLI, coName string) { + e2e.Logf("waiting for CO %v to start rolling update......", coName) + jsonPath := "-o=jsonpath={.status.conditions[?(@.type==\"Progressing\")].status}" + waitErr := wait.PollImmediate(6*time.Second, 180*time.Second, func() (bool, error) { + status, _ := oc.AsAdmin().WithoutNamespace().Run("get").Args("co/"+coName, jsonPath).Output() + primary := false + if strings.Compare(status, "True") == 0 { + e2e.Logf("Progressing status is True.") + primary = true + } else { + e2e.Logf("Progressing status is not True, wait and try again...") + } + return primary, nil + }) + compat_otp.AssertWaitPollNoErr(waitErr, fmt.Sprintf( + "reached max time allowed but CO %v didn't goto Progressing status.", coName)) +} + +// wait for the cluster operator back to normal status ("True False False") +// wait until get the specified number of successive normal status, which is defined by healthyThreshold and totalWaitTime +// healthyThreshold: max rounds for checking an CO, int type,and no less than 1 +// totalWaitTime: total checking time, time.Durationshould type, and no less than 1 +func ensureClusterOperatorNormal(oc *exutil.CLI, coName string, healthyThreshold int, totalWaitTime time.Duration) { + count := 0 + printCount := 0 + jsonPath := `{.status.conditions[?(@.type=="Available")].status}{.status.conditions[?(@.type=="Progressing")].status}{.status.conditions[?(@.type=="Degraded")].status}` + + e2e.Logf("waiting for CO %v back to normal status......", coName) + waitErr := wait.PollImmediate(5*time.Second, totalWaitTime*time.Second, func() (bool, error) { + status := getByJsonPath(oc, "default", "co/"+coName, jsonPath) + primary := false + printCount++ + if strings.Compare(status, "TrueFalseFalse") == 0 { + count++ + if count == healthyThreshold { + e2e.Logf("got %v successive good status (%v), the CO is stable!", count, status) + primary = true + } else { + e2e.Logf("got %v successive good status (%v), try again...", count, status) + } + } else { + count = 0 + if printCount%10 == 1 { + e2e.Logf("CO status is still abnormal (%v), wait and try again...", status) + } + } + return primary, nil + }) + // for debugging: print all messages in co status.conditions + if waitErr != nil { + output := getByJsonPath(oc, "default", "co/"+coName, "{.status.conditions}") + e2e.Logf("The co %v is abnormal and here is status: %v", coName, output) + if coName == "ingress" { + output, _ = oc.AsAdmin().WithoutNamespace().Run("describe").Args("-n", "openshift-ingress", "service", "router-default").Output() + e2e.Logf("The output of describe router-default service: %v", output) + } + } + compat_otp.AssertWaitPollNoErr(waitErr, fmt.Sprintf("reached max time allowed but CO %v is still abnoraml.", coName)) +} + +// this function ensure all cluster's operators become normal +func ensureAllClusterOperatorsNormal(oc *exutil.CLI, waitTime time.Duration) { + opList := getClusterOperators(oc) + for _, operator := range opList { + ensureClusterOperatorNormal(oc, operator, 1, waitTime) + } +} + +// this function pick up those cluster operators in bad status +func checkAllClusterOperatorsStatus(oc *exutil.CLI) []string { + badOpList := []string{} + opList := getClusterOperators(oc) + jsonPath := `-o=jsonpath={.status.conditions[?(@.type=="Available")].status}{.status.conditions[?(@.type=="Progressing")].status}{.status.conditions[?(@.type=="Degraded")].status}` + for _, operator := range opList { + searchLine, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("clusteroperator", operator, jsonPath).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + if !strings.Contains(searchLine, "TrueFalseFalse") { + badOpList = append(badOpList, operator) + } + } + return badOpList +} + +// to speed up the dns/coredns testing, just force only one dns-default pod in the cluster during the test +// find random linux node and add label "ne-dns-testing=true" to it, then patch spec.nodePlacement.nodeSelector +// please use func deleteDnsOperatorToRestore() for clear up. +func forceOnlyOneDnsPodExist(oc *exutil.CLI) string { + ns := "openshift-dns" + dnsPodLabel := "dns.operator.openshift.io/daemonset-dns=default" + dnsNodeSelector := `[{"op":"replace", "path":"/spec/nodePlacement/nodeSelector", "value":{"ne-dns-testing":"true"}}]` + // ensure no node with the label "ne-dns-testing=true" + oc.AsAdmin().WithoutNamespace().Run("label").Args("node", "-l", "ne-dns-testing=true", "ne-dns-testing-").Execute() + podList := getAllDNSPodsNames(oc) + if len(podList) == 1 { + e2e.Logf("Found only one dns-default pod and it looks like SNO cluster. Continue the test...") + } else { + dnsPodName := getRandomElementFromList(podList) + nodeName, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("pods", dnsPodName, "-o=jsonpath={.spec.nodeName}", "-n", ns).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + e2e.Logf("Find random dns pod '%s' and its node '%s' which will be used for the following testing", dnsPodName, nodeName) + // add special label "ne-dns-testing=true" to the node and force only one dns pod running on it + _, err = oc.AsAdmin().WithoutNamespace().Run("label").Args("node", nodeName, "ne-dns-testing=true").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + patchGlobalResourceAsAdmin(oc, "dnses.operator.openshift.io/default", dnsNodeSelector) + err1 := waitForResourceToDisappear(oc, ns, "pod/"+dnsPodName) + if err1 != nil { + output, _ := oc.AsAdmin().WithoutNamespace().Run("get").Args("pods", "-n", ns, "-l", dnsPodLabel).Output() + e2e.Logf("All current dns pods are:\n%v", output) + } + compat_otp.AssertWaitPollNoErr(err1, fmt.Sprintf("max time reached but pod %s is not terminated", dnsPodName)) + err2 := waitForPodWithLabelReady(oc, ns, dnsPodLabel) + if err2 != nil { + output, _ := oc.AsAdmin().WithoutNamespace().Run("get").Args("pods", "-n", ns, "-l", dnsPodLabel).Output() + e2e.Logf("All current dns pods are:\n%v", output) + } + compat_otp.AssertWaitPollNoErr(err2, fmt.Sprintf("max time reached but no dns pod ready")) + } + return getDNSPodName(oc) +} + +func deleteDnsOperatorToRestore(oc *exutil.CLI) { + _, err := oc.AsAdmin().WithoutNamespace().Run("delete").Args("dnses.operator.openshift.io/default").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + ensureClusterOperatorNormal(oc, "dns", 2, 120) + // remove special label "ne-dns-testing=true" from the node + oc.AsAdmin().WithoutNamespace().Run("label").Args("node", "-l", "ne-dns-testing=true", "ne-dns-testing-").Execute() +} + +// this function is to get all dns pods' names +func getAllDNSPodsNames(oc *exutil.CLI) []string { + ns := "openshift-dns" + label := "dns.operator.openshift.io/daemonset-dns=default" + dnsPods := getByLabelAndJsonPath(oc, ns, "pod", label, "{.items[*].metadata.name}") + return strings.Split(dnsPods, " ") +} + +// this function returns an element randomly from the given list +func getRandomElementFromList(list []string) string { + seed := rand.New(rand.NewSource(time.Now().UnixNano())) + index := seed.Intn(len(list)) + return list[index] +} + +// this function is to check whether the given resource pod's are deleted or not +func waitForRangeOfPodsToDisappear(oc *exutil.CLI, resource string, podList []string) { + for _, podName := range podList { + err := waitForResourceToDisappear(oc, resource, "pod/"+podName) + compat_otp.AssertWaitPollNoErr(err, fmt.Sprintf("%s pod %s is NOT deleted", resource, podName)) + } +} + +// this function is to wait for the expStr appearing in the corefile of the coredns under all dns pods +func keepSearchInAllDNSPods(oc *exutil.CLI, podList []string, expStr string) { + cmd := fmt.Sprintf("grep \"%s\" /etc/coredns/Corefile", expStr) + o.Expect(podList).NotTo(o.BeEmpty()) + for _, podName := range podList { + count := 0 + waitErr := wait.Poll(15*time.Second, 360*time.Second, func() (bool, error) { + output, _ := oc.AsAdmin().WithoutNamespace().Run("exec").Args("-n", "openshift-dns", podName, "-c", "dns", "--", "bash", "-c", cmd).Output() + count++ + primary := false + if strings.Contains(output, expStr) { + e2e.Logf("find " + expStr + " in the Corefile of pod " + podName) + primary = true + } else { + // reduce the logs + if count%2 == 1 { + e2e.Logf("can't find " + expStr + " in the Corefile of pod " + podName + ", wait and try again...") + } + } + return primary, nil + }) + compat_otp.AssertWaitPollNoErr(waitErr, "can't find "+expStr+" in the Corefile of pod "+podName) + } +} + +// this function is to get desired logs from all dns pods +func searchLogFromDNSPods(oc *exutil.CLI, podList []string, searchStr string) string { + o.Expect(podList).NotTo(o.BeEmpty()) + for _, podName := range podList { + output, _ := oc.AsAdmin().WithoutNamespace().Run("logs").Args(podName, "-c", "dns", "-n", "openshift-dns").Output() + outputList := strings.Split(output, "\n") + for _, line := range outputList { + if strings.Contains(line, searchStr) { + return line + } + } + } + return "none" +} + +func waitRouterLogsAppear(oc *exutil.CLI, routerpod, searchStr string) string { + result := "" + containerName := getByJsonPath(oc, "openshift-ingress", "pod/"+routerpod, "{.spec.containers[*].name}") + logCmd := []string{routerpod, "-n", "openshift-ingress"} + if strings.Contains(containerName, "logs") { + logCmd = []string{routerpod, "-c", strings.Split(containerName, " ")[1], "-n", "openshift-ingress"} + } + err := wait.Poll(10*time.Second, 300*time.Second, func() (bool, error) { + output, err := oc.AsAdmin().WithoutNamespace().Run("logs").Args(logCmd...).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + primary := false + outputList := strings.Split(output, "\n") + for _, line := range outputList { + if strings.Contains(line, searchStr) { + primary = true + result = line + e2e.Logf("the searchline has result:%v", line) + break + } + } + return primary, nil + }) + compat_otp.AssertWaitPollNoErr(err, fmt.Sprintf("expected string \"%s\" is not found in the router pod's logs", searchStr)) + return result +} + +// this function to get one dns pod's Corefile info related to the modified time, it looks like {{"dns-default-0001", "2021-12-30 18.011111 Modified"}} +func getOneCorefileStat(oc *exutil.CLI, dnspodname string) [][]string { + attrList := [][]string{} + cmd := "stat /etc/coredns/..data/Corefile | grep Modify" + output, err := oc.AsAdmin().WithoutNamespace().Run("exec").Args("-n", "openshift-dns", dnspodname, "-c", "dns", "--", "bash", "-c", cmd).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + return append(attrList, []string{dnspodname, output}) +} + +// replace the coredns image that specified by co/dns, currently only for replacement of coreDNS-pod.yaml +func replaceCoreDnsImage(oc *exutil.CLI, file string) { + coreDnsImage, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("co/dns", "-o=jsonpath={.status.versions[?(.name == \"coredns\")].version}").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + result, err := exec.Command("bash", "-c", fmt.Sprintf(`grep "image: " %s`, file)).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + e2e.Logf("the result of grep command is: %s", result) + if strings.Contains(string(result), coreDnsImage) { + e2e.Logf("the image has been updated, no action and continue") + } else { + // use "|" as delimiter here since the image looks like + // "quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:xxxxx" + sedCmd := fmt.Sprintf(`sed -i'' -e 's|replaced-at-runtime|%s|g' %s`, coreDnsImage, file) + _, err := exec.Command("bash", "-c", sedCmd).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + } +} + +// this fucntion will return the master pod who has the virtual ip +func getVipOwnerPod(oc *exutil.CLI, ns string, podname []string, vip string) string { + cmd := fmt.Sprintf("ip address |grep %s", vip) + var primaryNode string + for i := 0; i < len(podname); i++ { + output, err := oc.AsAdmin().WithoutNamespace().Run("exec").Args("-n", ns, podname[i], "--", "bash", "-c", cmd).Output() + if len(podname) == 1 && output == "command terminated with exit code 1" { + e2e.Failf("The given pod is not master") + } + if output == "command terminated with exit code 1" { + e2e.Logf("This Pod %v does not have the VIP", podname[i]) + } else if strings.Contains(output, vip) { + e2e.Logf("The pod owning the VIP is %v", podname[i]) + primaryNode = podname[i] + break + } else { + o.Expect(err).NotTo(o.HaveOccurred()) + } + } + return primaryNode +} + +// this function will remove the given element from the slice +func slicingElement(element string, podList []string) []string { + var newPodList []string + for index, pod := range podList { + if pod == element { + newPodList = append(podList[:index], podList[index+1:]...) + break + } + } + e2e.Logf("The remaining pod/s in the list is %v", newPodList) + return newPodList +} + +// this function checks whether given pod becomes primary or not +func waitForPrimaryPod(oc *exutil.CLI, ns string, pod string, vip string) { + cmd := fmt.Sprintf("ip address |grep %s", vip) + waitErr := wait.Poll(5*time.Second, 50*time.Second, func() (bool, error) { + output, _ := oc.AsAdmin().WithoutNamespace().Run("exec").Args("-n", ns, pod, "--", "bash", "-c", cmd).Output() + primary := false + if strings.Contains(output, vip) { + e2e.Logf("The new pod %v is the master", pod) + primary = true + } else { + e2e.Logf("pod failed to become master yet, retrying...the error is %v", output) + } + return primary, nil + }) + // for debugging + output1, _ := oc.AsAdmin().WithoutNamespace().Run("exec").Args("-n", ns, pod, "--", "bash", "-c", "ip address").Output() + compat_otp.AssertWaitPollNoErr(waitErr, fmt.Sprintf("max time reached, pod failed to become master and the entire ip details of the pod is %v", output1)) +} + +// this function will search the specific data from the given pod +func readPodData(oc *exutil.CLI, podname string, ns string, executeCmd string, searchString string) string { + cmd := fmt.Sprintf("%s | grep \"%s\"", executeCmd, searchString) + output, err := oc.AsAdmin().WithoutNamespace().Run("exec").Args("-n", ns, podname, "--", "bash", "-c", cmd).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + e2e.Logf("the matching part is: %s", output) + return output +} + +// this function is a wrapper for polling `readPodData` function +func pollReadPodData(oc *exutil.CLI, ns, routername, executeCmd, searchString string) string { + cmd := fmt.Sprintf("%s | grep \"%s\"", executeCmd, searchString) + var output string + var err error + waitErr := wait.Poll(5*time.Second, 60*time.Second, func() (bool, error) { + output, err = oc.AsAdmin().WithoutNamespace().Run("exec").Args("-n", ns, routername, "--", "bash", "-c", cmd).Output() + if err != nil { + e2e.Logf("failed to get search string: %v, retrying...", err) + return false, nil + } + return true, nil + }) + e2e.Logf("the matching part is: %s", output) + compat_otp.AssertWaitPollNoErr(waitErr, fmt.Sprintf("reached max time allowed but cannot find the search string.")) + return output +} + +// addPrivilegedLabelToNamespace adds all three privileged security labels to a specified namespace +func addPrivilegedLabelToNamespace(oc *exutil.CLI, ns string) { + enforceLabel := "pod-security.kubernetes.io/enforce=privileged" + auditLabel := "pod-security.kubernetes.io/audit=privileged" + warnLabel := "pod-security.kubernetes.io/warn=privileged" + err := oc.AsAdmin().WithoutNamespace().Run("label").Args("namespace", ns, enforceLabel, auditLabel, warnLabel, "--overwrite").Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + e2e.Logf("Successfully added privileged labels (enforce, audit, warn) to namespace %s", ns) +} + +// removePrivilegedLabelFromNamespace removes all three privileged security labels from a specified namespace +func removePrivilegedLabelFromNamespace(oc *exutil.CLI, ns string) { + err := oc.AsAdmin().WithoutNamespace().Run("label").Args("namespace", ns, "pod-security.kubernetes.io/enforce-", "pod-security.kubernetes.io/audit-", "pod-security.kubernetes.io/warn-").Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + e2e.Logf("Successfully removed privileged labels (enforce, audit, warn) from namespace %s", ns) +} + +// createExternalDNSCatalogSource creates the catalogsource with Konflux FBC build +func createExternalDNSCatalogSource(oc *exutil.CLI) { + var ( + buildPruningBaseDir = testdata.FixturePath("router/extdns") + nsOperator = filepath.Join(buildPruningBaseDir, "ns-external-dns-operator.yaml") + catSrcTemplate = filepath.Join(buildPruningBaseDir, "catalog-source.yaml") + operatorNamespace = "external-dns-operator" + ) + + msg, err := oc.AsAdmin().WithoutNamespace().Run("apply").Args("-f", nsOperator).Output() + e2e.Logf("err %v, msg %v", err, msg) + defer removePrivilegedLabelFromNamespace(oc, operatorNamespace) + addPrivilegedLabelToNamespace(oc, operatorNamespace) + err = createResourceToNsFromTemplate(oc, operatorNamespace, "--ignore-unknown-parameters=true", "-f", catSrcTemplate) + e2e.Logf("The error message is %v", err) + ensurePodWithLabelReady(oc, operatorNamespace, "olm.catalogSource=extdns-konflux-fbc") +} + +// createAWSLoadBalancerCatalogSource creates the catalogsource with Konflux FBC build +func createAWSLoadBalancerCatalogSource(oc *exutil.CLI) { + var ( + buildPruningBaseDir = testdata.FixturePath("router/awslb") + namespaceFile = filepath.Join(buildPruningBaseDir, "namespace.yaml") + catSrcTemplate = filepath.Join(buildPruningBaseDir, "catalog-source.yaml") + operatorNamespace = "aws-load-balancer-operator" + ) + + msg, err := oc.AsAdmin().WithoutNamespace().Run("apply").Args("-f", namespaceFile).Output() + e2e.Logf("err %v, msg %v", err, msg) + defer removePrivilegedLabelFromNamespace(oc, operatorNamespace) + addPrivilegedLabelToNamespace(oc, operatorNamespace) + err = createResourceToNsFromTemplate(oc, operatorNamespace, "--ignore-unknown-parameters=true", "-f", catSrcTemplate) + e2e.Logf("The error message is %v", err) + ensurePodWithLabelReady(oc, operatorNamespace, "olm.catalogSource=albo-konflux-fbc") +} + +// this function create external dns operator +func createExternalDNSOperator(oc *exutil.CLI) { + buildPruningBaseDir := testdata.FixturePath("router/extdns") + operatorGroup := filepath.Join(buildPruningBaseDir, "operatorgroup.yaml") + subscription := filepath.Join(buildPruningBaseDir, "subscription.yaml") + operatorNamespace := "external-dns-operator" + + msg, err := oc.AsAdmin().WithoutNamespace().Run("apply").Args("-f", operatorGroup).Output() + e2e.Logf("err %v, msg %v", err, msg) + msg, err = oc.AsAdmin().WithoutNamespace().Run("apply").Args("-f", subscription).Output() + e2e.Logf("err %v, msg %v", err, msg) + + // checking subscription status + errCheck := wait.Poll(10*time.Second, 180*time.Second, func() (bool, error) { + subState, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("sub", "external-dns-operator", "-n", operatorNamespace, "-o=jsonpath={.status.state}").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + if strings.Compare(subState, "AtLatestKnown") == 0 { + return true, nil + } + return false, nil + }) + compat_otp.AssertWaitPollNoErr(errCheck, fmt.Sprintf("subscription external-dns-operator is not correct status")) + + // checking csv status + csvName, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("sub", "external-dns-operator", "-n", operatorNamespace, "-o=jsonpath={.status.installedCSV}").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(csvName).NotTo(o.BeEmpty()) + errCheck = wait.Poll(10*time.Second, 180*time.Second, func() (bool, error) { + csvState, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("csv", csvName, "-n", operatorNamespace, "-o=jsonpath={.status.phase}").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + if strings.Compare(csvState, "Succeeded") == 0 { + e2e.Logf("CSV check complete!!!") + return true, nil + } + return false, nil + + }) + compat_otp.AssertWaitPollNoErr(errCheck, fmt.Sprintf("csv %v is not correct status", csvName)) +} + +func deleteNamespace(oc *exutil.CLI, ns string) { + err := oc.AdminKubeClient().CoreV1().Namespaces().Delete(context.Background(), ns, metav1.DeleteOptions{}) + if err != nil { + if apierrors.IsNotFound(err) { + err = nil + } + } + o.Expect(err).NotTo(o.HaveOccurred()) + err = wait.PollUntilContextTimeout(context.Background(), 5*time.Second, 180*time.Second, true, func(context.Context) (done bool, err error) { + _, err = oc.AdminKubeClient().CoreV1().Namespaces().Get(context.Background(), ns, metav1.GetOptions{}) + if err != nil { + if apierrors.IsNotFound(err) { + return true, nil + } + return false, err + } + return false, nil + }) + compat_otp.AssertWaitPollNoErr(err, fmt.Sprintf("Namespace %s is not deleted in 3 minutes", ns)) +} + +// Get OIDC from STS cluster +func getOidc(oc *exutil.CLI) string { + oidc, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("authentication.config", "cluster", "-o=jsonpath={.spec.serviceAccountIssuer}").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + oidc = strings.TrimPrefix(oidc, "https://") + e2e.Logf("The OIDC of STS cluster is: %v", oidc) + return oidc +} + +// this function create aws-load-balancer-operator +func createAWSLoadBalancerOperator(oc *exutil.CLI) { + buildPruningBaseDir := testdata.FixturePath("router/awslb") + operatorGroup := filepath.Join(buildPruningBaseDir, "operatorgroup.yaml") + subscription := filepath.Join(buildPruningBaseDir, "subscription-src-qe.yaml") + subSTS := filepath.Join(buildPruningBaseDir, "subscription-src-qe-sts.yaml") + ns := "aws-load-balancer-operator" + deployName := "deployment/aws-load-balancer-operator-controller-manager" + + if compat_otp.IsSTSCluster(oc) { + e2e.Logf("This is STS cluster, create ALB operator and controller secrets via AWS SDK") + prepareAllForStsCluster(oc) + } + + msg, err := oc.AsAdmin().WithoutNamespace().Run("apply").Args("-f", operatorGroup).Output() + e2e.Logf("err %v, msg %v", err, msg) + + if compat_otp.IsSTSCluster(oc) { + e2e.Logf("Updating and applying subcripton with Role ARN on STS cluster") + sedCmd := fmt.Sprintf(`sed -i'' -e 's|fakeARN-for-albo|%s|g' %s`, os.Getenv("ALBO_ROLE_ARN"), subSTS) + _, err := exec.Command("bash", "-c", sedCmd).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + msg, err = oc.AsAdmin().WithoutNamespace().Run("apply").Args("-f", subSTS).Output() + e2e.Logf("err %v, msg %v", err, msg) + } else { + msg, err = oc.AsAdmin().WithoutNamespace().Run("apply").Args("-f", subscription).Output() + e2e.Logf("err %v, msg %v", err, msg) + } + + // checking subscription status + errCheck := wait.Poll(10*time.Second, 180*time.Second, func() (bool, error) { + subState, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("sub", "aws-load-balancer-operator", "-n", ns, "-o=jsonpath={.status.state}").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + if strings.Compare(subState, "AtLatestKnown") == 0 { + return true, nil + } + return false, nil + }) + compat_otp.AssertWaitPollNoErr(errCheck, fmt.Sprintf("subscription aws-load-balancer-operator is not correct status")) + + // checking csv status + csvName, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("sub", "aws-load-balancer-operator", "-n", ns, "-o=jsonpath={.status.installedCSV}").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(csvName).NotTo(o.BeEmpty()) + errCheck = wait.Poll(10*time.Second, 180*time.Second, func() (bool, error) { + csvState, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("csv", csvName, "-n", ns, "-o=jsonpath={.status.phase}").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + if strings.Compare(csvState, "Succeeded") == 0 { + e2e.Logf("CSV check complete!!!") + return true, nil + } + return false, nil + }) + // output log of deployment/aws-load-balancer-operator-controller-manager for debugging + if errCheck != nil { + output, _ := oc.AsAdmin().WithoutNamespace().Run("logs").Args(deployName, "-n", ns, "--tail=10").Output() + e2e.Logf("The logs of albo deployment: %v", output) + } + compat_otp.AssertWaitPollNoErr(errCheck, fmt.Sprintf("csv %v is not correct status", csvName)) +} + +func patchAlbControllerWithRoleArn(oc *exutil.CLI, ns string) { + e2e.Logf("patching the ALB Controller with Role ARN on STS cluster") + jsonPatch := fmt.Sprintf(`[{"op":"add","path":"/spec/credentialsRequestConfig","value":{"stsIAMRoleARN":%s}}]`, os.Getenv("ALBC_ROLE_ARN")) + _, err := oc.AsAdmin().WithoutNamespace().Run("patch").Args("-n", ns, "awsloadbalancercontroller/cluster", "-p", jsonPatch, "--type=json").Output() + o.Expect(err).NotTo(o.HaveOccurred()) +} + +// get AWS outposts subnet so we can add annonation to ingress +func getOutpostSubnetId(oc *exutil.CLI) string { + machineSet := clusterinfra.GetOneOutpostMachineSet(oc) + subnetId, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("machineset", machineSet, "-n", "openshift-machine-api", "-o=jsonpath={.spec.template.spec.providerSpec.value.subnet.id}").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + e2e.Logf("the outpost subnet is %v", subnetId) + return subnetId +} + +// this function check if the load balancer provisioned +func waitForLoadBalancerProvision(oc *exutil.CLI, ns string, ingressName string) { + waitErr := wait.Poll(5*time.Second, 180*time.Second, func() (bool, error) { + output, _ := oc.AsAdmin().WithoutNamespace().Run("get").Args("-n", ns, "ingress", ingressName, "-o=jsonpath={.status.loadBalancer.ingress}").Output() + if output != "" && strings.Contains(output, "k8s-") { + e2e.Logf("The load balancer is provisoned: %v", output) + return true, nil + } + return false, nil + }) + compat_otp.AssertWaitPollNoErr(waitErr, fmt.Sprintf("max time reached but the Load Balancer is not provisioned")) +} + +// openssl generate the ca.key and ca.crt +func opensslNewCa(caKey, caCrt, caSubj string) { + opensslCmd := fmt.Sprintf(`openssl req -x509 -newkey rsa:4096 -sha256 -days 365 -keyout %v -out %v -nodes -subj "%v"`, caKey, caCrt, caSubj) + e2e.Logf("the openssl command is: %v", opensslCmd) + _, err := exec.Command("bash", "-c", opensslCmd).Output() + o.Expect(err).NotTo(o.HaveOccurred()) +} + +// openssl generate the server CSR +func opensslNewCsr(serverKey, serverCsr, serverSubj string) { + opensslCmd := fmt.Sprintf(`openssl req -newkey rsa:4096 -nodes -sha256 -keyout %v -out %v -subj "%v"`, serverKey, serverCsr, serverSubj) + e2e.Logf("the openssl command is: %v", opensslCmd) + _, err := exec.Command("bash", "-c", opensslCmd).Output() + o.Expect(err).NotTo(o.HaveOccurred()) +} + +// openssl sign the server CSR and generate server.crt +func opensslSignCsr(extfile, serverCsr, caCrt, caKey, serverCrt string) { + opensslCmd := fmt.Sprintf(`openssl x509 -req -extfile <(printf "%v") -days 30 -in %v -CA %v -CAcreateserial -CAkey %v -out %v`, extfile, serverCsr, caCrt, caKey, serverCrt) + e2e.Logf("the openssl command is: %v", opensslCmd) + _, err := exec.Command("bash", "-c", opensslCmd).Output() + o.Expect(err).NotTo(o.HaveOccurred()) +} + +// wait until curling route returns expected output (check error as well) +// curl is executed on client outside the cluster +func waitForOutsideCurlContains(url string, curlOptions string, expected string) string { + var output []byte + cmd := fmt.Sprintf(`curl --connect-timeout 10 -s %s %s 2>&1`, curlOptions, url) + e2e.Logf("the command is: %s", cmd) + waitErr := wait.Poll(5*time.Second, 30*time.Second, func() (bool, error) { + result, err := exec.Command("bash", "-c", cmd).Output() + e2e.Logf("the result is: %s", result) + output = result + if err != nil { + e2e.Logf("the error is: %v", err.Error()) + if strings.Contains(err.Error(), expected) { + e2e.Logf("the expected string is included in err: %v", err) + return true, nil + } else { + // route timeout case, curl returns an execution error which is expected + if strings.Contains(err.Error(), expected) { + e2e.Logf("Execution Error expected: %v", err) + return true, nil + } + e2e.Logf("hit execution error: %v, retrying...", err) + return false, nil + } + } + if !strings.Contains(string(result), expected) { + e2e.Logf("no expected string in the curl response: %s, retrying...", result) + return false, nil + } + return true, nil + }) + // for debugging: print verbose result of curl if timeout + if waitErr != nil { + debug_cmd := fmt.Sprintf(`curl --connect-time 10 -s -v %s %s 2>&1`, curlOptions, url) + e2e.Logf("the debug command is: %s", debug_cmd) + result, err := exec.Command("bash", "-c", debug_cmd).Output() + e2e.Logf("debug: the result of curl is %s and err is %v", result, err) + } + compat_otp.AssertWaitPollNoErr(waitErr, fmt.Sprintf("max time reached but not get expected string")) + return string(output) +} + +// curl command with poll +func waitForCurl(oc *exutil.CLI, podName, baseDomain string, routestring string, searchWord string, controllerIP string) { + e2e.Logf("Polling for curl command") + var output string + var err error + waitErr := wait.Poll(5*time.Second, 30*time.Second, func() (bool, error) { + if controllerIP != "" { + route := routestring + baseDomain + ":80" + toDst := routestring + baseDomain + ":80:" + controllerIP + output, err = oc.Run("exec").Args(podName, "--", "curl", "-v", "http://"+route, "--resolve", toDst, "--connect-timeout", "10").Output() + } else { + curlCmd2 := routestring + baseDomain + output, err = oc.Run("exec").Args(podName, "--", "curl", "-v", "http://"+curlCmd2, "--connect-timeout", "10").Output() + } + if err != nil { + e2e.Logf("curl is not yet resolving, retrying...") + return false, nil + } + if !strings.Contains(output, searchWord) { + e2e.Logf("retrying...cannot find the searchWord '%s' in the output:- %v ", searchWord, output) + return false, nil + } + return true, nil + }) + compat_otp.AssertWaitPollNoErr(waitErr, fmt.Sprintf("max time reached but the route is not reachable")) +} + +// used to send the nslookup command until the desired dns logs appear +func nslookupsAndWaitForDNSlog(oc *exutil.CLI, podName, searchLog string, dnsPodList []string, nslookupCmdPara ...string) string { + e2e.Logf("Polling for executing nslookupCmd and waiting the dns logs appear") + output := "" + cmd := append([]string{podName, "--", "nslookup"}, nslookupCmdPara...) + waitErr := wait.Poll(5*time.Second, 300*time.Second, func() (bool, error) { + oc.Run("exec").Args(cmd...).Execute() + output = searchLogFromDNSPods(oc, dnsPodList, searchLog) + primary := false + if len(output) > 1 && output != "none" { + primary = true + } + return primary, nil + }) + compat_otp.AssertWaitPollNoErr(waitErr, fmt.Sprintf("max time reached,but expected string \"%s\" is not found in the dns logs", searchLog)) + return output +} + +// this function will get the route hostname +func getRouteHost(oc *exutil.CLI, ns, routeName string) string { + host, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("route", routeName, "-n", ns, `-ojsonpath={.spec.host}`).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + e2e.Logf("the host of the route %v is %v.", routeName, host) + return host +} + +// this function will get the route detail +func getRoutes(oc *exutil.CLI, ns string) string { + output, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("route", "-n", ns).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + e2e.Logf("oc get route: %v", output) + return output +} + +// this function will get the ingress detail +func getIngress(oc *exutil.CLI, ns string) string { + output, err := oc.Run("get").Args("ingress", "-n", ns).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + e2e.Logf("oc get ingress: %v", output) + return output +} + +// this function will help to create Opaque secret using cert file and its key_name +func createGenericSecret(oc *exutil.CLI, ns, name, keyName, certFile string) { + cmd := fmt.Sprintf(`--from-file=%s=%v`, keyName, certFile) + _, err := oc.AsAdmin().WithoutNamespace().Run("create").Args( + "secret", "generic", name, cmd, "-n", ns).Output() + o.Expect(err).NotTo(o.HaveOccurred()) +} + +// this function is to obtain the resource name like ingress's,route's name +func getResourceName(oc *exutil.CLI, namespace, resourceName string) []string { + var resourceList []string + resourceNames, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("-n", namespace, resourceName, + "-ojsonpath={.items..metadata.name}").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + resourceList = strings.Split(resourceNames, " ") + e2e.Logf("The resource '%s' names are %v ", resourceName, resourceList) + return resourceList +} + +// this function is used to check whether proxy is configured or not +func checkProxy(oc *exutil.CLI) bool { + httpProxy, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("proxy", "cluster", "-o=jsonpath={.status.httpProxy}").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + httpsProxy, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("proxy", "cluster", "-o=jsonpath={.status.httpsProxy}").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + if httpProxy != "" || httpsProxy != "" { + return true + } + return false +} + +// this function will advertise unicast peers for Nutanix +func unicastIPFailover(oc *exutil.CLI, ns, failoverName string) { + platformtype := compat_otp.CheckPlatform(oc) + + if platformtype == "nutanix" || platformtype == "none" { + getPodListByLabel(oc, oc.Namespace(), "ipfailover=hello-openshift") + workerIPAddress, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("nodes", "--selector=node-role.kubernetes.io/worker=", "-ojsonpath={.items[*].status.addresses[0].address}").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + modifiedIPList := strings.Split(workerIPAddress, " ") + if len(modifiedIPList) < 2 { + e2e.Failf("There is not enough IP addresses to add as unicast peer") + } + ipList := strings.Join(modifiedIPList, ",") + cmd := fmt.Sprintf("OPENSHIFT_HA_UNICAST_PEERS=%v", ipList) + setEnvVariable(oc, ns, "deploy/"+failoverName, "OPENSHIFT_HA_USE_UNICAST=true") + setEnvVariable(oc, ns, "deploy/"+failoverName, cmd) + } +} + +// this function is to retrieve the status of the route after using RouteSelectors +func checkRouteDetailsRemoved(oc *exutil.CLI, namespace, routeName, ingresscontrollerName string) { + e2e.Logf("polling for route details") + jsonPath := fmt.Sprintf(`{.status.ingress[?(@.routerName=="%s")]}`, ingresscontrollerName) + waitErr := wait.Poll(5*time.Second, 150*time.Second, func() (bool, error) { + status, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("route", "-n", namespace, routeName, + "-ojsonpath="+jsonPath).Output() + if err != nil { + e2e.Logf("there is some execution error and it is %v, retrying...", err) + return false, nil + } + if strings.Contains(status, "Admitted") { + e2e.Logf("the matched string is still in the logs, retrying...") + return false, nil + } + e2e.Logf("The route status is cleared!") + return true, nil + }) + o.Expect(waitErr).NotTo(o.HaveOccurred(), "The route %s yielded unexpected results", routeName) +} + +// used to execute a command on the internal or external client for the desired times +// the return was the output of the last successfully executed command, and a list of counters for the expected output: +// for example, if one expected item is matched for one time, the mathcing counter will be increased by 1, which is useful to test http cookie cases +// support checking an expected error when executed a command and the error occur +func repeatCmdOnClient(oc *exutil.CLI, cmd, expectOutput interface{}, duration time.Duration, repeatTimes int) (string, []int) { + var ( + clientType = "Internal" + matchedTimesList = []int{} + successCurlCount = 0 + matchedCount = 0 + expectOutputList = []string{} + output = "" + ) + + cmdStr, ok := cmd.(string) + if ok { + clientType = "External" + } + cmdList, _ := cmd.([]string) + + expStr, ok := expectOutput.(string) + if ok { + expectOutputList = append(expectOutputList, expStr) + } + expList, ok := expectOutput.([]string) + if ok { + expectOutputList = expList + } + + for i := 0; i < len(expectOutputList); i++ { + matchedTimesList = append(matchedTimesList, 0) + } + + e2e.Logf("Using client type: %v", clientType) + e2e.Logf("The cmdStr (used by External client) is '%v' and cmdList (used by Internal client) is %v", cmdStr, cmdList) + e2e.Logf("The expectOutputList is %v and initial matchedTimesList is %v", expectOutputList, matchedTimesList) + + waitErr := wait.Poll(1*time.Second, duration*time.Second, func() (bool, error) { + isMatch := false + if clientType == "Internal" { + info, err := oc.AsAdmin().WithoutNamespace().Run("exec").Args(cmdList...).Output() + if err != nil { + e2e.Logf("The error is: %v", err.Error()) + searchInfo := regexp.MustCompile(expectOutputList[0]).FindStringSubmatch(err.Error()) + if len(searchInfo) > 0 { + e2e.Logf("The expected string is included in err: %v", err) + return true, nil + } else { + e2e.Logf("Failed to execute cmd and got err %v, retrying...", err.Error()) + return false, nil + } + } + output = info + } else { + info, err := exec.Command("bash", "-c", cmdStr).CombinedOutput() + if err != nil { + e2e.Logf("The error is: %v", err.Error()) + searchInfo := regexp.MustCompile(expectOutputList[0]).FindStringSubmatch(err.Error()) + if len(searchInfo) > 0 { + e2e.Logf("The expected string is included in err: %v", err) + return true, nil + } else { + e2e.Logf("Failed to execute cmd and got err %v, retrying...", err.Error()) + return false, nil + } + } + output = string(info) + } + + successCurlCount++ + e2e.Logf("Executed cmd for %v times on the client and got output: %s", successCurlCount, output) + + for i := 0; i < len(expectOutputList); i++ { + searchInfo := regexp.MustCompile(expectOutputList[i]).FindStringSubmatch(output) + if len(searchInfo) > 0 { + isMatch = true + matchedCount++ + matchedTimesList[i] = matchedTimesList[i] + 1 + break + } + } + + if isMatch { + e2e.Logf("Successfully executed cmd for %v times on the client, expecting %v times", matchedCount, repeatTimes) + if matchedCount == repeatTimes { + return true, nil + } else { + return false, nil + } + } else { + // if after executed the cmd, but could not get a output in the expectOutput list, decrease the successfully executed times + successCurlCount-- + e2e.Logf("Failed to find a match in the output, retrying...") + return false, nil + } + }) + + e2e.Logf("The matchedTimesList is: %v", matchedTimesList) + compat_otp.AssertWaitPollNoErr(waitErr, fmt.Sprintf("max time reached but can't execute the cmd successfully for the desired times")) + + // return the last succecessful curl output and the succecessful curl times list for the expected list + return output, matchedTimesList +} + +// used to execute a command on the internal or external client repeatly until the expected error occurs +// the return was the whole output of executing the command with the error occuring +func waitForErrorOccur(oc *exutil.CLI, cmd interface{}, expectedErrorInfo string, duration time.Duration) string { + var ( + clientType = "Internal" + output = "" + ) + + cmdStr, ok := cmd.(string) + if ok { + clientType = "External" + } + cmdList, _ := cmd.([]string) + + e2e.Logf("the command is: %v", cmd) + waitErr := wait.Poll(3*time.Second, duration*time.Second, func() (bool, error) { + if clientType == "Internal" { + info, err := oc.AsAdmin().WithoutNamespace().Run("exec").Args(cmdList...).Output() + output = info + if err == nil { + e2e.Logf("expected error %v not happened, retrying...", expectedErrorInfo) + return false, nil + } + } else { + info, err := exec.Command("bash", "-c", cmdStr).Output() + output = string(info) + if err == nil { + e2e.Logf("expected error %v not happened, retrying...", expectedErrorInfo) + return false, nil + } + } + + searchInfo := regexp.MustCompile(expectedErrorInfo).FindStringSubmatch(output) + if len(searchInfo) > 0 { + return true, nil + } else { + e2e.Logf("expected error %v not happened, retrying...", expectedErrorInfo) + return false, nil + } + }) + + compat_otp.AssertWaitPollNoErr(waitErr, fmt.Sprintf("max time reached but can't execute the cmd successfully in the desired time duration")) + return output +} + +// this function is to check whether given string is present or not in a list +func checkGivenStringPresentOrNot(shouldContain bool, iterateObject []string, searchString string) { + if shouldContain { + o.Expect(iterateObject).To(o.ContainElement(o.ContainSubstring(searchString))) + } else { + o.Expect(iterateObject).NotTo(o.ContainElement(o.ContainSubstring(searchString))) + } +} + +// this function is pollinng to check output which should contain the expected string +func waitForOutputContains(oc *exutil.CLI, ns, resourceName, jsonPath, expected string, args ...interface{}) { + waitDuration := 180 * time.Second + for _, arg := range args { + duration, ok := arg.(time.Duration) + if ok { + waitDuration = duration + } + } + + waitErr := wait.PollImmediate(5*time.Second, waitDuration, func() (bool, error) { + output := getByJsonPath(oc, ns, resourceName, jsonPath) + if strings.Contains(output, expected) { + return true, nil + } + e2e.Logf("The output of jsonpath does NOT contain the expected string: %v, retrying...", expected) + return false, nil + }) + compat_otp.AssertWaitPollNoErr(waitErr, fmt.Sprintf("max time reached but cannot find the expected string")) +} + +// this function is pollinng to check output which should equal the expected string +func waitForOutputEquals(oc *exutil.CLI, ns, resourceName, jsonPath, expected string, args ...interface{}) { + waitDuration := 180 * time.Second + for _, arg := range args { + duration, ok := arg.(time.Duration) + if ok { + waitDuration = duration + } + } + + waitErr := wait.PollImmediate(5*time.Second, waitDuration, func() (bool, error) { + output := getByJsonPath(oc, ns, resourceName, jsonPath) + if output == expected { + return true, nil + } + e2e.Logf("The output of jsonpath does NOT equal the expected string: %v, retrying...", expected) + return false, nil + }) + compat_otp.AssertWaitPollNoErr(waitErr, fmt.Sprintf("max time reached but cannot find the expected string")) +} + +// this function keep checking util the searching for the regular expression matches +func waitForOutputMatchRegexp(oc *exutil.CLI, ns, resourceName, jsonPath, regExpress string, args ...interface{}) string { + result := "NotMatch" + waitDuration := 180 * time.Second + for _, arg := range args { + duration, ok := arg.(time.Duration) + if ok { + waitDuration = duration + } + } + + wait.Poll(5*time.Second, waitDuration, func() (bool, error) { + sourceRange := getByJsonPath(oc, ns, resourceName, jsonPath) + searchRe := regexp.MustCompile(regExpress) + searchInfo := searchRe.FindStringSubmatch(sourceRange) + if len(searchInfo) > 0 { + result = searchInfo[0] + return true, nil + } + return false, nil + }) + return result +} + +// this function is pollinng to check output which should NOT contain the expected string +func waitForOutputNotContains(oc *exutil.CLI, ns, resourceName, jsonPath, expected string, args ...interface{}) { + waitDuration := 180 * time.Second + for _, arg := range args { + duration, ok := arg.(time.Duration) + if ok { + waitDuration = duration + } + } + + waitErr := wait.PollImmediate(5*time.Second, waitDuration, func() (bool, error) { + output := getByJsonPath(oc, ns, resourceName, jsonPath) + if !strings.Contains(output, expected) { + return true, nil + } + e2e.Logf("The output of jsonpath contained the expected string: %v, retrying...", expected) + return false, nil + }) + compat_otp.AssertWaitPollNoErr(waitErr, fmt.Sprintf("max time reached but still contained the expected string")) +} + +// this function check output of oc describe command is polled +func waitForDescriptionContains(oc *exutil.CLI, ns, resourceName, value string) { + n := 0 + waitErr := wait.PollImmediate(5*time.Second, 180*time.Second, func() (bool, error) { + output, err := oc.AsAdmin().WithoutNamespace().Run("describe").Args("-n", ns, resourceName).Output() + if err != nil { + return false, err + } + n++ + if n%10 == 1 { + e2e.Logf("the description is: %v", output) + } + if strings.Contains(output, value) { + return true, nil + } + return false, nil + }) + compat_otp.AssertWaitPollNoErr(waitErr, fmt.Sprintf("max time reached but the desired searchString does not appear")) +} + +// this function will search in the polled and described resource details +func searchInDescribeResource(oc *exutil.CLI, resource, resourceName, match string) string { + var output string + var err error + waitErr := wait.Poll(10*time.Second, 180*time.Second, func() (bool, error) { + output, err = oc.AsAdmin().WithoutNamespace().Run("describe").Args(resource, resourceName).Output() + if err != nil || output == "" { + e2e.Logf("failed to get describe output: %v, retrying...", err) + return false, nil + } + if !strings.Contains(output, match) { + e2e.Logf("cannot find the matched string in the output, retrying...") + return false, nil + } + return true, nil + }) + compat_otp.AssertWaitPollNoErr(waitErr, fmt.Sprintf("reached max time allowed but cannot find the search string.")) + return output +} + +// this function is to add taint to resource +func addTaint(oc *exutil.CLI, resource, resourceName, taint string) { + output, err := oc.AsAdmin().WithoutNamespace().Run("adm").Args("taint", resource, resourceName, taint).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(output).To(o.ContainSubstring(resource + "/" + resourceName + " tainted")) +} + +// this function is to remove the configured taint +func deleteTaint(oc *exutil.CLI, resource, resourceName, taint string) { + output, err := oc.AsAdmin().WithoutNamespace().Run("adm").Args("taint", resource, resourceName, taint).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(output).To(o.ContainSubstring(resource + "/" + resourceName + " untainted")) +} + +func waitCoBecomes(oc *exutil.CLI, coName string, waitTime int, expectedStatus map[string]string) error { + return wait.Poll(10*time.Second, time.Duration(waitTime)*time.Second, func() (bool, error) { + gottenStatus := getCoStatus(oc, coName, expectedStatus) + eq := reflect.DeepEqual(expectedStatus, gottenStatus) + if eq { + e2e.Logf("Given operator %s becomes %s", coName, gottenStatus) + return true, nil + } + return false, nil + }) +} + +func getCoStatus(oc *exutil.CLI, coName string, statusToCompare map[string]string) map[string]string { + newStatusToCompare := make(map[string]string) + for key := range statusToCompare { + args := fmt.Sprintf(`-o=jsonpath={.status.conditions[?(.type == '%s')].status}`, key) + status, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("co", args, coName).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + newStatusToCompare[key] = status + } + return newStatusToCompare +} + +// this function will check the status of dns record in ingress operator +func checkDnsRecordStatusOfIngressOperator(oc *exutil.CLI, dnsRecordsName, statusToSearch, stringToCheck string) []string { + jsonPath := fmt.Sprintf(`{.status.zones[*].conditions[*].%s}`, statusToSearch) + status := getByJsonPath(oc, "openshift-ingress-operator", "dnsrecords/"+dnsRecordsName, jsonPath) + statusList := strings.Split(status, " ") + for _, line := range statusList { + o.Expect(stringToCheck).To(o.ContainSubstring(line)) + } + return statusList +} + +// this function is to check whether the DNS Zone details are present in ingresss operator records +func checkDnsRecordsInIngressOperator(oc *exutil.CLI, recordName, privateZoneId, publicZoneId string) { + // Collecting zone details from ingress operator + Zones := getByJsonPath(oc, "openshift-ingress-operator", "dnsrecords/"+recordName, "{.status.zones[*].dnsZone}") + // check the private and public zone detail are matching + o.Expect(Zones).To(o.ContainSubstring(privateZoneId)) + if publicZoneId != "" { + o.Expect(Zones).To(o.ContainSubstring(publicZoneId)) + } +} + +// retrieve the IPV6 or IPV4 public client address of a cluster +func getClientIP(oc *exutil.CLI, clusterType string) string { + if strings.Contains(clusterType, "ipv6single") || strings.Contains(clusterType, "dualstack") { + res, _ := http.Get("https://api64.ipify.org") + result, _ := ioutil.ReadAll(res.Body) + return string(result) + } else { + res, _ := http.Get("https://api.ipify.org") + result, _ := ioutil.ReadAll(res.Body) + return string(result) + } +} + +// This function checks the cookie file generated through curl command and confirms that the file contains what is expected +func checkCookieFile(fileDir string, expectedString string) { + output, err := ioutil.ReadFile(fileDir) + o.Expect(err).NotTo(o.HaveOccurred()) + e2e.Logf("the cookie file content is: %s", output) + o.Expect(strings.Contains(string(output), expectedString)).To(o.BeTrue()) +} + +func checkIPStackType(oc *exutil.CLI) string { + svcNetwork, err := oc.WithoutNamespace().AsAdmin().Run("get").Args("network.operator", "cluster", "-o=jsonpath={.spec.serviceNetwork}").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + if strings.Count(svcNetwork, ":") >= 2 && strings.Count(svcNetwork, ".") >= 2 { + return "dualstack" + } else if strings.Count(svcNetwork, ":") >= 2 { + return "ipv6single" + } else if strings.Count(svcNetwork, ".") >= 2 { + return "ipv4single" + } + return "" +} + +// based on the orignal yaml file, this function is used to add some extra parameters behind the specified parameter, the return is the new file name +func addExtraParametersToYamlFile(originalFile, flagPara, AddedContent string) string { + filePath, _ := filepath.Split(originalFile) + newFile := filePath + getRandomString() + originalFileContent, err := os.ReadFile(originalFile) + o.Expect(err).NotTo(o.HaveOccurred()) + newFileContent := "" + for _, line := range strings.Split(string(originalFileContent), "\n") { + newFileContent = newFileContent + line + "\n" + if strings.Contains(line, flagPara) { + newFileContent = newFileContent + AddedContent + } + } + os.WriteFile(newFile, []byte(newFileContent), 0644) + return newFile +} + +// based on the orignal file, this function is used to add some content into the file behind the desired specified parameter, the return is the new file name +// if there are quite few file lines include the specified parameter, use the seq parameter to choose the desired one +func addContenToFileWithMatchedOrder(originalFile, flagPara, AddedContent string, seq int) string { + filePath, _ := filepath.Split(originalFile) + newFile := filePath + getRandomString() + originalFileContent, err := os.ReadFile(originalFile) + o.Expect(err).NotTo(o.HaveOccurred()) + newFileContent := "" + matchedTime := 0 + for _, line := range strings.Split(string(originalFileContent), "\n") { + newFileContent = newFileContent + line + "\n" + if strings.Contains(line, flagPara) { + matchedTime++ + if matchedTime == seq { + newFileContent = newFileContent + AddedContent + } + } + } + os.WriteFile(newFile, []byte(newFileContent), 0644) + return newFile +} + +// this function is to check whether the ingress canary route could be accessible from outside +func isCanaryRouteAvailable(oc *exutil.CLI) bool { + routehost := getByJsonPath(oc, "openshift-ingress-canary", "route/canary", "{.status.ingress[0].host}") + curlCmd := fmt.Sprintf(`curl https://%s -skI --connect-timeout 10`, routehost) + _, matchedTimes := repeatCmdOnClient(oc, curlCmd, "200", 60, 1) + if matchedTimes[0] == 1 { + return true + } else { + return false + } +} + +func updateFilebySedCmd(file, toBeReplaced, newContent string) { + sedCmd := fmt.Sprintf(`sed -i'' -e 's|%s|%s|g' %s`, toBeReplaced, newContent, file) + _, err := exec.Command("bash", "-c", sedCmd).Output() + o.Expect(err).NotTo(o.HaveOccurred()) +} + +// this function returns IPv6 and IPv4 on dual stack and main IP in case of single stack (v4 or v6) +func getPodIP(oc *exutil.CLI, namespace string, podName string) []string { + ipStack := checkIPStackType(oc) + var podIp []string + if (ipStack == "ipv6single") || (ipStack == "ipv4single") { + podIp1, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("pod", "-n", namespace, podName, "-o=jsonpath={.status.podIPs[0].ip}").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + e2e.Logf("The pod %s IP in namespace %s is %q", podName, namespace, podIp1) + podIp = append(podIp, podIp1) + } else if ipStack == "dualstack" { + podIp1, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("pod", "-n", namespace, podName, "-o=jsonpath={.status.podIPs[0].ip}").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + e2e.Logf("The pod's %s 1st IP in namespace %s is %q", podName, namespace, podIp1) + podIp2, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("pod", "-n", namespace, podName, "-o=jsonpath={.status.podIPs[1].ip}").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + e2e.Logf("The pod's %s 2nd IP in namespace %s is %q", podName, namespace, podIp2) + podIp = append(podIp, podIp1, podIp2) + } + return podIp +} + +// used to get a microshift node's valid host interfaces and host IPs for router-default load balancer service +func getValidInterfacesAndIPs(addressInfo string) (intList, IPList []string) { + intList = []string{} + IPList = []string{} + for _, line := range strings.Split(addressInfo, "\n") { + // loopback address and link-local address will not used for the load balancer's IPs(updated for OCPBUGS-32946) + if !strings.Contains(line, "inet 127") && !strings.Contains(line, "inet 169") { + intIPList := strings.Split(strings.TrimRight(line, " "), " ") + intName := intIPList[len(intIPList)-1] + hostIP := regexp.MustCompile("([0-9\\.]+)/[0-9]+").FindStringSubmatch(line)[1] + intList = append(intList, intName) + IPList = append(IPList, hostIP) + } + } + e2e.Logf("The valid host interfaces are %v", intList) + e2e.Logf("The valid host IPs are %v", IPList) + return intList, IPList +} + +// used to get a microshift node's valid host IPv6 addresses +func getValidIPv6Addresses(addressInfo string) (IPList []string) { + IPList = []string{} + ipv6Re := regexp.MustCompile("([0-9a-zA-Z]+:[0-9a-zA-Z:]+)") + for _, line := range strings.Split(addressInfo, "\n") { + if !strings.Contains(line, "deprecated") { + ipv6Info := ipv6Re.FindStringSubmatch(line) + if len(ipv6Info) > 0 { + IPList = append(IPList, ipv6Info[1]) + } + } + } + return IPList +} + +// used to backup the config.yaml file under a microshift node before the testing +func backupConfigYaml(oc *exutil.CLI, ns, caseID, nodeName string) { + backupConfig := fmt.Sprintf(` +if test -f /etc/microshift/config.yaml ; then + cp /etc/microshift/config.yaml /etc/microshift/config.yaml.backup%s +else + touch /etc/microshift/config.yaml.no%s +fi +`, caseID, caseID) + _, err := oc.AsAdmin().WithoutNamespace().Run("debug").Args("-n", ns, "--quiet=true", "node/"+nodeName, "--", "chroot", "/host", "bash", "-c", backupConfig).Output() + o.Expect(err).NotTo(o.HaveOccurred()) +} + +// used to restore the config.yaml file under a microshift node after the testing +func restoreConfigYaml(oc *exutil.CLI, ns, caseID, nodeName string) { + recoverCmd := fmt.Sprintf(` +if test -f /etc/microshift/config.yaml.no%s; then + rm -f /etc/microshift/config.yaml + rm -f /etc/microshift/config.yaml.no%s +elif test -f /etc/microshift/config.yaml.backup%s ; then + rm -f /etc/microshift/config.yaml + cp /etc/microshift/config.yaml.backup%s /etc/microshift/config.yaml + rm -f /etc/microshift/config.yaml.backup%s +fi +`, caseID, caseID, caseID, caseID, caseID) + defer func() { + _, err := oc.AsAdmin().WithoutNamespace().Run("debug").Args("-n", ns, "--quiet=true", "node/"+nodeName, "--", "chroot", "/host", "bash", "-c", recoverCmd).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + restartMicroshiftService(oc, ns, nodeName) + }() +} + +// used to append ingress configurtion to the original config.yaml file under a microshift node during the testing +func appendIngressToConfigYaml(oc *exutil.CLI, ns, caseID, nodeName, ingressConfig string) { + customConfig := fmt.Sprintf(` +rm /etc/microshift/config.yaml -f +if test -f /etc/microshift/config.yaml.backup%s ; then + cp /etc/microshift/config.yaml.backup%s /etc/microshift/config.yaml +fi +cat >> /etc/microshift/config.yaml << EOF +%s +EOF`, caseID, caseID, ingressConfig) + _, err := oc.AsAdmin().WithoutNamespace().Run("debug").Args("-n", ns, "--quiet=true", "node/"+nodeName, "--", "chroot", "/host", "bash", "-c", customConfig).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + output, err := oc.AsAdmin().WithoutNamespace().Run("debug").Args("-n", ns, "--quiet=true", "node/"+nodeName, "--", "chroot", "/host", "bash", "-c", "microshift show-config").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(output).NotTo(o.ContainSubstring("error: invalid configuration")) + e2e.Logf("microshift show-config is: \n%v", output) + restartMicroshiftService(oc, ns, nodeName) +} + +func appendInvalidIngressConfigToConfigYaml(oc *exutil.CLI, ns, caseID, nodeName, ingressConfig string) string { + customConfig := fmt.Sprintf(` +rm /etc/microshift/config.yaml -f +if test -f /etc/microshift/config.yaml.backup%s ; then + cp /etc/microshift/config.yaml.backup%s /etc/microshift/config.yaml +fi +cat >> /etc/microshift/config.yaml << EOF +%s +EOF`, caseID, caseID, ingressConfig) + _, err := oc.AsAdmin().WithoutNamespace().Run("debug").Args("-n", ns, "--quiet=true", "node/"+nodeName, "--", "chroot", "/host", "bash", "-c", customConfig).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + output, err := oc.AsAdmin().WithoutNamespace().Run("debug").Args("-n", ns, "--quiet=true", "node/"+nodeName, "--", "chroot", "/host", "bash", "-c", "microshift show-config").Output() + o.Expect(err).To(o.HaveOccurred()) + e2e.Logf("microshift show-config is: \n%v", output) + return output +} + +func getRouterDeploymentGeneration(oc *exutil.CLI, deploymentName string) int { + actualGen, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("deployment/"+deploymentName, "-n", "openshift-ingress", "-o=jsonpath={.metadata.generation}").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + actualGenerationInt, _ := strconv.Atoi(actualGen) + return actualGenerationInt +} + +// Convert the given IPv6 string to IPv6 PTR record +// ie, from "fd03::a" to "a.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.3.0.d.f.ip6.arpa" +func convertV6AddressToPTR(ipv6Address string) string { + addr, _ := netip.ParseAddr(ipv6Address) + // expand the ipv6 '::' string with zeros and remove the semicolon + v6AddWithoutSemicolon := strings.Join(strings.Split(addr.StringExpanded(), ":"), "") + reversedString := reverseString(v6AddWithoutSemicolon) + // split the string with dots and add another string (.ip6.arpa) + PtrString := strings.Join(strings.SplitAfter(reversedString, ""), ".") + ".ip6.arpa" + e2e.Logf("The PTR record is %s", PtrString) + return PtrString +} + +// this function is polling to check the output of the cmd executed on a debug node, which should contain the expected string +func waitForDebugNodeOutputContains(oc *exutil.CLI, ns, node string, cmdList []string, expectedString string, args ...interface{}) string { + var output string + count := 0 + waitDuration := 180 * time.Second + for _, arg := range args { + duration, ok := arg.(time.Duration) + if ok { + waitDuration = duration + } + } + + e2e.Logf("The expected string is: \n%s", expectedString) + waitErr := wait.Poll(10*time.Second, waitDuration*time.Second, func() (bool, error) { + // hostOutput, err := compat_otp.DebugNodeRetryWithOptionsAndChroot(oc, node, []string{}, "cat", "/etc/hosts") + output, err := compat_otp.DebugNodeRetryWithOptionsAndChroot(oc, node, []string{"--quiet=true", "--to-namespace=" + ns}, cmdList...) + o.Expect(err).NotTo(o.HaveOccurred()) + if count%5 == 0 { + e2e.Logf("The output of the cmd executed on the debug node is:\n%s", output) + } + count++ + + // Comparing the output + if !strings.Contains(output, expectedString) { + e2e.Logf("Failed to find the expected string, retring...") + return false, nil + } + + e2e.Logf(`Find the expected string in the debug node's output: %s`, output) + return true, nil + }) + + compat_otp.AssertWaitPollNoErr(waitErr, fmt.Sprintf("Max time reached, but the expected string was not found by checking the output of cmd executed on the debug node")) + return output +} + +// Get clusterIP of a service +func getSvcClusterIPByName(oc *exutil.CLI, ns, serviceName string) string { + clusterIP, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("-n", ns, "svc", serviceName, "-o=jsonpath={.spec.clusterIP}").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + e2e.Logf("The '%s' service's clusterIP of '%s' namespace is: %v", serviceName, ns, clusterIP) + return clusterIP +} + +// Function to reverse a string +func reverseString(str string) (result string) { + // iterate over str and prepend to result + for _, i := range str { + result = string(i) + result + } + return +} + +// used to sort string type of slice or string which can be transformed to the slice +func getSortedString(obj interface{}) string { + objList := []string{} + str, ok := obj.(string) + if ok { + objList = strings.Split(str, " ") + } + strList, ok := obj.([]string) + if ok { + objList = strList + } + sort.Strings(objList) + return strings.Join(objList, " ") +} + +// due to OCPBUGS-45192, used to configure the firewall on the microshfit node to permit fd01::/48 for the load balancer +// bug fixed PR: https://github.com/openshift/microshift/pull/4268(permit fd01::/48) +// the fixed not working for all deployment for some reasons, so just added the rules explicitly here +// since the fw rules won't take effect immediately after configured(and this function is for the microshift traffic testing only), just extend the duration of the curl request(this is also a methond to check whether the fw rules take effect) +func configFwForLB(oc *exutil.CLI, ns, nodeName, ip string) { + if strings.Contains(strings.ToLower(ip), "fd01:") { + ip6tablesCmd := fmt.Sprintf(` +ip6tables -A FORWARD -s fd01::/48 -j ACCEPT +ip6tables -A FORWARD -d fd01::/48 -j ACCEPT +ip6tables -A OUTPUT -s fd01::/48 -j ACCEPT +ip6tables -A OUTPUT -d fd01::/48 -j ACCEPT +`) + + output, err := oc.AsAdmin().WithoutNamespace().Run("debug").Args("-n", ns, "--quiet=true", "node/"+nodeName, "--", "chroot", "/host", "bash", "-c", "ip6tables -L").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + if !strings.Contains(strings.ToLower(output), "fd01:") { + _, err := oc.AsAdmin().WithoutNamespace().Run("debug").Args("-n", ns, "--quiet=true", "node/"+nodeName, "--", "chroot", "/host", "bash", "-c", ip6tablesCmd).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + } + } +} + +func checkNodeStatus(oc *exutil.CLI, nodeName string, expectedStatus string) { + var expectedStatus1 string + var statusOutput string + var err error + if expectedStatus == "Ready" { + expectedStatus1 = "True" + } else if expectedStatus == "NotReady" { + expectedStatus1 = "Unknown" + } else { + err1 := fmt.Errorf("TBD supported node status") + o.Expect(err1).NotTo(o.HaveOccurred()) + } + errWait := wait.Poll(15*time.Second, 15*time.Minute, func() (bool, error) { + statusOutput, err = oc.AsAdmin().WithoutNamespace().Run("get").Args("nodes", nodeName, "-ojsonpath={.status.conditions[-1].status}").Output() + if err != nil { + return false, nil + } + if statusOutput != expectedStatus1 { + return false, nil + } + return true, nil + }) + if errWait != nil { + e2e.Logf("Expect Node %s in state %v, kubelet status is %s with error", nodeName, expectedStatus, statusOutput, err.Error()) + } + compat_otp.AssertWaitPollNoErr(errWait, fmt.Sprintf("Node %s is not in expected status %s", nodeName, expectedStatus)) +} + +func restartMicroshiftService(oc *exutil.CLI, ns, nodeName string) { + // As restart the microshift service, the debug node pod will quit with error + // debug pod in the default namespace won't be deleted automatically, so debug the node in another namepace + oc.AsAdmin().WithoutNamespace().Run("debug").Args("-n", ns, "--quiet=true", "node/"+nodeName, "--", "chroot", "/host", "bash", "-c", "sudo systemctl restart microshift").Output() + exec.Command("bash", "-c", "sleep 60").Output() + checkNodeStatus(oc, nodeName, "Ready") +} + +// the function will provide enough time for the egressfirewall to get applied +func waitEgressFirewallApplied(oc *exutil.CLI, efName, ns string) string { + var output string + checkErr := wait.Poll(10*time.Second, 60*time.Second, func() (bool, error) { + output, efErr := oc.AsAdmin().WithoutNamespace().Run("get").Args("egressfirewall", "-n", ns, efName).Output() + if efErr != nil { + e2e.Logf("Failed to get egressfirewall %v, error: %s. Trying again", efName, efErr) + return false, nil + } + if !strings.Contains(output, "EgressFirewall Rules applied") { + e2e.Logf("The egressfirewall was not applied, trying again. \n %s", output) + return false, nil + } + return true, nil + }) + compat_otp.AssertWaitPollNoErr(checkErr, fmt.Sprintf("reached max time allowed but cannot find the egressfirewall details.")) + return output +} + +func checkDomainReachability(oc *exutil.CLI, podName, ns, domainName string, passOrFail bool) { + curlCmd := fmt.Sprintf("curl -s -I %s --connect-timeout 5 ", domainName) + if passOrFail { + _, err := e2eoutput.RunHostCmdWithRetries(ns, podName, curlCmd, 10*time.Second, 20*time.Second) + o.Expect(err).NotTo(o.HaveOccurred()) + ipStackType := checkIPStackType(oc) + if ipStackType == "dualstack" { + curlCmd = fmt.Sprintf("curl -s -6 -I %s --connect-timeout 5", domainName) + _, err := e2eoutput.RunHostCmdWithRetries(ns, podName, curlCmd, 10*time.Second, 20*time.Second) + o.Expect(err).NotTo(o.HaveOccurred()) + } + + } else { + o.Eventually(func() error { + _, err := e2eoutput.RunHostCmd(ns, podName, curlCmd) + return err + }, "20s", "10s").Should(o.HaveOccurred()) + } +} + +// Check whether the given cluster is a byo vpc cluster +func isByoVpcCluster(oc *exutil.CLI) bool { + jsonPath := `{.items[*].spec.template.spec.providerSpec.value.subnet.id}` + subnet, _ := oc.AsAdmin().WithoutNamespace().Run("get").Args( + "-n", "openshift-machine-api", "machinesets.machine.openshift.io", "-o=jsonpath="+jsonPath).Output() + if len(subnet) > 0 { + e2e.Logf("This is a byo vpc cluster") + return true + } + return false +} + +// get list of all private subnets from machinesets, it might contains dup subnets or name without "private" +// this function will not work on Hypershift cluster +func getRawPrivateSubnetList(oc *exutil.CLI) []string { + var jsonPath string + if isByoVpcCluster(oc) { + jsonPath = `{.items[*].spec.template.spec.providerSpec.value.subnet.id}` + } else { + jsonPath = `{.items[*].spec.template.spec.providerSpec.value.subnet.filters[].values[]}` + } + privateSubnets := getByJsonPath(oc, "openshift-machine-api", "machinesets.machine.openshift.io", jsonPath) + privateSubnetList := strings.Split(privateSubnets, " ") + e2e.Logf("The raw private subnet list from machinesets is: %v", privateSubnetList) + return privateSubnetList +} + +// convert private subnet list to public subnet list +// this func is used by AWS subnets/EIPs feature, ignore uncommon and duplicated subnets +func getPublicSubnetList(oc *exutil.CLI) []string { + var publicSubnetList []string + for _, subnet := range getRawPrivateSubnetList(oc) { + if !strings.Contains(subnet, "subnet-private") { + e2e.Logf("Warning: found subnet without private keyword: %v, ignore it", subnet) + continue + } + // example subnet: ci-op-iip84q8t-3ca97-8fqzp-subnet-private-us-east-1a + if matched, _ := regexp.MatchString(".*-private-([a-z]+)-([a-z]+)-([0-9a-z]+)$", subnet); !matched { + e2e.Logf("Warning: found uncommon subnet: %v, ignore it", subnet) + continue + } + publicSubnet := fmt.Sprintf(`"%s"`, strings.Replace(subnet, "subnet-private", "subnet-public", -1)) + if !slices.Contains(publicSubnetList, publicSubnet) { + e2e.Logf("Got new valid public subnet: %v, append it", publicSubnet) + publicSubnetList = append(publicSubnetList, publicSubnet) + } else { + e2e.Logf("Warning: found duplicated public subnet: %v, ignore it", publicSubnet) + } + } + e2e.Logf("The public subnet list generated from private is: %v", publicSubnetList) + return publicSubnetList +} + +// for DCM testing, Check whether a new static endpoint is added to the backend +func isNewStaticEPAdded(initSrvStates, currentSrvStates string) bool { + upEpReg := regexp.MustCompile("([0-9\\.a-zA-Z:]+ UP)") + initUpEps := "" + for _, entry := range strings.Split(initSrvStates, "\n") { + if len(upEpReg.FindStringSubmatch(entry)) > 1 { + initUpEps = initUpEps + upEpReg.FindStringSubmatch(entry)[1] + " " + } + } + + for _, entry := range strings.Split(currentSrvStates, "\n") { + if len(upEpReg.FindStringSubmatch(entry)) > 1 && !strings.Contains(entry, "dynamic-pod") { + if !strings.Contains(initUpEps, upEpReg.FindStringSubmatch(entry)[1]) { + e2e.Logf("new static endpoint %s is added to the backend", upEpReg.FindStringSubmatch(entry)[1]) + return true + } + } + } + e2e.Logf("no new static endpoint is added to the backend") + return false +} + +// for DCM testing, scale Deployment +func scaleDeploy(oc *exutil.CLI, ns, deployName string, num int) []string { + expReplicas := strconv.Itoa(num) + if num == 0 { + expReplicas = "" + } + _, err := oc.AsAdmin().WithoutNamespace().Run("scale").Args("-n", ns, "deployment/"+deployName, "--replicas="+strconv.Itoa(num)).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + waitForOutputEquals(oc, ns, "deployment/"+deployName, "{.status.availableReplicas}", expReplicas) + podList, err := compat_otp.GetAllPodsWithLabel(oc, ns, "name="+deployName) + o.Expect(err).NotTo(o.HaveOccurred()) + return podList +} + +// for DCM testing, check route's backend configuration +func checkDcmBackendCfg(oc *exutil.CLI, routerpod, backend string) { + dynamicPod := `server-template _dynamic-pod- 1-1.+check disabled` + if strings.Contains(backend, "be_secure") { + dynamicPod = `server _dynamic-pod-1.+disabled check.+verifyhost service.+` + } + + backendCfg := getBlockConfig(oc, routerpod, backend) + o.Expect(backendCfg).Should(o.And( + o.MatchRegexp(`server pod:.+`), + o.MatchRegexp(dynamicPod), + o.MatchRegexp(`dynamic-cookie-key [0-9a-zA-A]+`))) + + // passthrough route hasn't the dynamic cookie + if !strings.Contains(backend, "be_tcp") { + o.Expect(backendCfg).To(o.MatchRegexp(`cookie.+dynamic`)) + } +} + +// for DCM testing, check UP endpoint of the deployment +func checkDcmUpEndpoints(oc *exutil.CLI, routerpod, socatCmd string, replicasNum int) string { + currentSrvStates, err := oc.AsAdmin().WithoutNamespace().Run("exec").Args("-n", "openshift-ingress", routerpod, "--", "bash", "-c", socatCmd).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + upEpNum := strings.Count(currentSrvStates, "UP") + o.Expect(upEpNum).To(o.Equal(replicasNum)) + return currentSrvStates +} + +// for DCM testing, check whether there are router reloaded logs as expected +func checkRouterReloadedLogs(oc *exutil.CLI, routerpod string, initReloadedNum int, initSrvStates, currentSrvStates string) int { + isNewEPAdded := isNewStaticEPAdded(initSrvStates, currentSrvStates) + log, err := oc.AsAdmin().WithoutNamespace().Run("logs").Args("-n", "openshift-ingress", routerpod).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + currentReloadedNum := strings.Count(log, `"msg"="router reloaded" "logger"="template" "output"=`) + if isNewEPAdded { + o.Expect(currentReloadedNum > initReloadedNum).To(o.BeTrue()) + } else { + e2e.Logf("initReloadedNum is: %v", initReloadedNum) + e2e.Logf("currentReloadedNum is: %v", currentReloadedNum) + o.Expect(currentReloadedNum-initReloadedNum <= 1).To(o.BeTrue()) + } + return currentReloadedNum +} + +// for DCM testing, check whether all the deployment pods are accessible or not +func checkDcmServersAccessible(oc *exutil.CLI, curlCmd, podList []string, duration time.Duration, repeatTimes int) { + _, result := repeatCmdOnClient(oc, curlCmd, podList, duration, repeatTimes) + for i := 0; i < len(podList); i++ { + o.Expect(result[i] > 0).To(o.BeTrue()) + } +} + +// check if default ingresscontroller using internal LB, before calling this need to check if Cloud platforms +func isInternalLBScopeInDefaultIngresscontroller(oc *exutil.CLI) bool { + lbScope := getByJsonPath(oc, "openshift-ingress-operator", "ingresscontroller/default", "{.status.endpointPublishingStrategy.loadBalancer.scope}") + if strings.Compare(lbScope, "Internal") == 0 { + e2e.Logf("The default ingresscontroller LB scope is Internal") + return true + } + e2e.Logf("The default ingresscontroller LB scope is %v", lbScope) + return false +} diff --git a/tests-extension/test/testdata/fixtures.go b/tests-extension/test/testdata/fixtures.go new file mode 100644 index 000000000..3b3b4c76f --- /dev/null +++ b/tests-extension/test/testdata/fixtures.go @@ -0,0 +1,228 @@ +package testdata + +import ( + "fmt" + "io/ioutil" + "os" + "path/filepath" + "sort" + "strings" +) + +var ( + // fixtureDir is where extracted fixtures are stored + fixtureDir string +) + +// init sets up the temporary directory for fixtures +func init() { + var err error + fixtureDir, err = ioutil.TempDir("", "testdata-fixtures-") + if err != nil { + panic(fmt.Sprintf("failed to create fixture directory: %v", err)) + } +} + +// FixturePath returns the filesystem path to a test fixture file. +// This replaces functions like compat_otp.FixturePath(). +// +// The file is extracted from embedded bindata to the filesystem on first access. +// Files are extracted to a temporary directory that persists for the test run. +// +// Example: +// configPath := testdata.FixturePath("router/config.yaml") +// data, err := os.ReadFile(configPath) +func FixturePath(relativePath string) string { + targetPath := filepath.Join(fixtureDir, relativePath) + + // Check if already extracted + if _, err := os.Stat(targetPath); err == nil { + return targetPath + } + + // Create parent directory + if err := os.MkdirAll(filepath.Dir(targetPath), 0755); err != nil { + panic(fmt.Sprintf("failed to create directory for %s: %v", relativePath, err)) + } + + // Bindata stores assets with "testdata/" prefix + // e.g., bindata has "testdata/router/file.yaml" but tests call FixturePath("router/file.yaml") + bindataPath := filepath.Join("testdata", relativePath) + + // Extract to temp directory first to handle path mismatch + tempDir, err := os.MkdirTemp("", "bindata-extract-") + if err != nil { + panic(fmt.Sprintf("failed to create temp directory: %v", err)) + } + defer os.RemoveAll(tempDir) + + // Try to restore single asset or directory to temp location + if err := RestoreAsset(tempDir, bindataPath); err != nil { + // If single file fails, try restoring as directory + if err := RestoreAssets(tempDir, bindataPath); err != nil { + panic(fmt.Sprintf("failed to restore fixture %s: %v", relativePath, err)) + } + } + + // Move extracted files from temp location to target location + extractedPath := filepath.Join(tempDir, bindataPath) + if err := os.Rename(extractedPath, targetPath); err != nil { + panic(fmt.Sprintf("failed to move extracted files from %s to %s: %v", extractedPath, targetPath, err)) + } + + // Set appropriate permissions for directories + if info, err := os.Stat(targetPath); err == nil && info.IsDir() { + filepath.Walk(targetPath, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + if info.IsDir() { + os.Chmod(path, 0755) + } else { + os.Chmod(path, 0644) + } + return nil + }) + } + + return targetPath +} + +// CleanupFixtures removes all extracted fixture files. +// Call this in test cleanup (e.g., AfterAll hook). +func CleanupFixtures() error { + if fixtureDir != "" { + return os.RemoveAll(fixtureDir) + } + return nil +} + +// GetFixtureData reads and returns the contents of a fixture file directly from bindata. +// Use this for small files that don't need to be written to disk. +// +// Example: +// data, err := testdata.GetFixtureData("config.yaml") +func GetFixtureData(relativePath string) ([]byte, error) { + // Normalize path - bindata uses "testdata/" prefix + cleanPath := relativePath + if len(cleanPath) > 0 && cleanPath[0] == '/' { + cleanPath = cleanPath[1:] + } + + return Asset(filepath.Join("testdata", cleanPath)) +} + +// MustGetFixtureData is like GetFixtureData but panics on error. +// Useful in test initialization code. +func MustGetFixtureData(relativePath string) []byte { + data, err := GetFixtureData(relativePath) + if err != nil { + panic(fmt.Sprintf("failed to get fixture data for %s: %v", relativePath, err)) + } + return data +} + +// Component-specific helper functions + +// FixtureExists checks if a fixture exists in the embedded bindata. +// Use this to validate fixtures before accessing them. +// +// Example: +// if testdata.FixtureExists("router/deployment.yaml") { +// path := testdata.FixturePath("router/deployment.yaml") +// } +func FixtureExists(relativePath string) bool { + cleanPath := relativePath + if len(cleanPath) > 0 && cleanPath[0] == '/' { + cleanPath = cleanPath[1:] + } + _, err := Asset(filepath.Join("testdata", cleanPath)) + return err == nil +} + +// ListFixtures returns all available fixture paths in the embedded bindata. +// Useful for debugging and test discovery. +// +// Example: +// fixtures := testdata.ListFixtures() +// fmt.Printf("Available fixtures: %v\n", fixtures) +func ListFixtures() []string { + names := AssetNames() + fixtures := make([]string, 0, len(names)) + for _, name := range names { + // Remove "testdata/" prefix for cleaner paths + if strings.HasPrefix(name, "testdata/") { + fixtures = append(fixtures, strings.TrimPrefix(name, "testdata/")) + } + } + sort.Strings(fixtures) + return fixtures +} + +// ListFixturesInDir returns all fixtures within a specific directory. +// +// Example: +// manifests := testdata.ListFixturesInDir("router") +// // Returns: ["router/deployment.yaml", "router/service.yaml", ...] +func ListFixturesInDir(dir string) []string { + allFixtures := ListFixtures() + var matching []string + prefix := dir + if !strings.HasSuffix(prefix, "/") { + prefix = prefix + "/" + } + for _, fixture := range allFixtures { + if strings.HasPrefix(fixture, prefix) { + matching = append(matching, fixture) + } + } + return matching +} + +// GetManifest is a convenience function for accessing manifest files. +// Equivalent to FixturePath("router/" + name). +// +// Example: +// deploymentPath := testdata.GetManifest("deployment.yaml") +func GetManifest(name string) string { + return FixturePath(filepath.Join("router", name)) +} + +// GetConfig is a convenience function for accessing config files. +// Equivalent to FixturePath("router/" + name). +// +// Example: +// configPath := testdata.GetConfig("settings.yaml") +func GetConfig(name string) string { + return FixturePath(filepath.Join("router", name)) +} + +// ValidateFixtures checks that all expected fixtures are present in bindata. +// Call this in BeforeAll to catch missing testdata early. +// +// Example: +// required := []string{"router/deployment.yaml", "router/config.yaml"} +// if err := testdata.ValidateFixtures(required); err != nil { +// panic(err) +// } +func ValidateFixtures(required []string) error { + var missing []string + for _, fixture := range required { + if !FixtureExists(fixture) { + missing = append(missing, fixture) + } + } + if len(missing) > 0 { + return fmt.Errorf("missing required fixtures: %v", missing) + } + return nil +} + +// GetFixtureDir returns the temporary directory where fixtures are extracted. +// Use this if you need to pass a directory path to external tools. +// +// Example: +// fixtureRoot := testdata.GetFixtureDir() +func GetFixtureDir() string { + return fixtureDir +} diff --git a/tests-extension/test/testdata/router/49802-route.yaml b/tests-extension/test/testdata/router/49802-route.yaml new file mode 100644 index 000000000..c81702075 --- /dev/null +++ b/tests-extension/test/testdata/router/49802-route.yaml @@ -0,0 +1,31 @@ +apiVersion: template.openshift.io/v1 +kind: Template +objects: +- apiVersion: route.openshift.io/v1 + kind: Route + metadata: + labels: + name: service-unsecure + name: hello-pod + namespace: ${NAMESPACE} + spec: + tls: + termination: edge + insecureEdgeTerminationPolicy: Redirect + to: + kind: Service + name: service-unsecure +- apiVersion: route.openshift.io/v1 + kind: Route + metadata: + labels: + name: service-unsecure + name: hello-pod-http + namespace: ${NAMESPACE} + spec: + path: "/test" + to: + kind: Service + name: service-unsecure +parameters: +- name: NAMESPACE diff --git a/tests-extension/test/testdata/router/OWNERS b/tests-extension/test/testdata/router/OWNERS new file mode 100644 index 000000000..0df8b4e14 --- /dev/null +++ b/tests-extension/test/testdata/router/OWNERS @@ -0,0 +1,5 @@ +approvers: + - lihongan + - melvinjoseph86 + - ShudiLi + - rhamini3 diff --git a/tests-extension/test/testdata/router/awslb/awslbcontroller.yaml b/tests-extension/test/testdata/router/awslb/awslbcontroller.yaml new file mode 100644 index 000000000..69e31d64d --- /dev/null +++ b/tests-extension/test/testdata/router/awslb/awslbcontroller.yaml @@ -0,0 +1,9 @@ +kind: AWSLoadBalancerController +apiVersion: networking.olm.openshift.io/v1 +metadata: + name: cluster +spec: + config: + replicas: 1 + ingressClass: alb + subnetTagging: Auto diff --git a/tests-extension/test/testdata/router/awslb/catalog-source.yaml b/tests-extension/test/testdata/router/awslb/catalog-source.yaml new file mode 100644 index 000000000..abfb45042 --- /dev/null +++ b/tests-extension/test/testdata/router/awslb/catalog-source.yaml @@ -0,0 +1,21 @@ +apiVersion: template.openshift.io/v1 +kind: Template +objects: +- apiVersion: operators.coreos.com/v1alpha1 + kind: CatalogSource + metadata: + name: "${CATALOG_NAME}" + namespace: "${NAMESPACE}" + spec: + displayName: Konflux ALBO + image: "${IMAGE}" + sourceType: grpc + grpcPodConfig: + securityContextConfig: legacy +parameters: +- name: CATALOG_NAME + value: "albo-konflux-fbc" +- name: IMAGE + value: "quay.io/redhat-user-workloads/aws-load-balancer-operator-tenant/aws-lb-optr-fbc-v4-21/aws-lb-fbc-container-aws-lb-optr-fbc-v4-21:candidate" +- name: NAMESPACE + value: aws-load-balancer-operator diff --git a/tests-extension/test/testdata/router/awslb/ingress-test.yaml b/tests-extension/test/testdata/router/awslb/ingress-test.yaml new file mode 100644 index 000000000..c3b0f9029 --- /dev/null +++ b/tests-extension/test/testdata/router/awslb/ingress-test.yaml @@ -0,0 +1,20 @@ +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: ingress-test + annotations: + alb.ingress.kubernetes.io/scheme: internal + alb.ingress.kubernetes.io/target-type: instance +spec: + ingressClassName: alb + rules: + - host: foo.bar.com + http: + paths: + - backend: + service: + name: service-unsecure + port: + number: 27017 + path: / + pathType: Exact diff --git a/tests-extension/test/testdata/router/awslb/namespace.yaml b/tests-extension/test/testdata/router/awslb/namespace.yaml new file mode 100644 index 000000000..37800efc0 --- /dev/null +++ b/tests-extension/test/testdata/router/awslb/namespace.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: aws-load-balancer-operator diff --git a/tests-extension/test/testdata/router/awslb/operatorgroup.yaml b/tests-extension/test/testdata/router/awslb/operatorgroup.yaml new file mode 100644 index 000000000..713665395 --- /dev/null +++ b/tests-extension/test/testdata/router/awslb/operatorgroup.yaml @@ -0,0 +1,7 @@ +apiVersion: operators.coreos.com/v1 +kind: OperatorGroup +metadata: + name: aws-lb-operatorgroup + namespace: aws-load-balancer-operator +spec: + upgradeStrategy: Default diff --git a/tests-extension/test/testdata/router/awslb/podsvc.yaml b/tests-extension/test/testdata/router/awslb/podsvc.yaml new file mode 100644 index 000000000..1225f8e7d --- /dev/null +++ b/tests-extension/test/testdata/router/awslb/podsvc.yaml @@ -0,0 +1,29 @@ +apiVersion: v1 +kind: Pod +metadata: + labels: + name: web-server + name: web-server-1 +spec: + containers: + - image: quay.io/openshifttest/nginx-alpine@sha256:cee6930776b92dc1e93b73f9e5965925d49cff3d2e91e1d071c2f0ff72cbca29 + name: nginx + ports: + - containerPort: 8080 + - containerPort: 8443 +--- +apiVersion: v1 +kind: Service +metadata: + labels: + name: service-unsecure + name: service-unsecure +spec: + ports: + - name: http + port: 27017 + protocol: TCP + targetPort: 8080 + selector: + name: web-server + type: NodePort diff --git a/tests-extension/test/testdata/router/awslb/sts-albc-perms-policy.json b/tests-extension/test/testdata/router/awslb/sts-albc-perms-policy.json new file mode 100644 index 000000000..25293bfb8 --- /dev/null +++ b/tests-extension/test/testdata/router/awslb/sts-albc-perms-policy.json @@ -0,0 +1,241 @@ +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "iam:CreateServiceLinkedRole" + ], + "Resource": "*", + "Condition": { + "StringEquals": { + "iam:AWSServiceName": "elasticloadbalancing.amazonaws.com" + } + } + }, + { + "Effect": "Allow", + "Action": [ + "ec2:DescribeAccountAttributes", + "ec2:DescribeAddresses", + "ec2:DescribeAvailabilityZones", + "ec2:DescribeInternetGateways", + "ec2:DescribeVpcs", + "ec2:DescribeVpcPeeringConnections", + "ec2:DescribeSubnets", + "ec2:DescribeSecurityGroups", + "ec2:DescribeInstances", + "ec2:DescribeNetworkInterfaces", + "ec2:DescribeTags", + "ec2:GetCoipPoolUsage", + "ec2:DescribeCoipPools", + "elasticloadbalancing:DescribeLoadBalancers", + "elasticloadbalancing:DescribeLoadBalancerAttributes", + "elasticloadbalancing:DescribeListeners", + "elasticloadbalancing:DescribeListenerCertificates", + "elasticloadbalancing:DescribeSSLPolicies", + "elasticloadbalancing:DescribeRules", + "elasticloadbalancing:DescribeTargetGroups", + "elasticloadbalancing:DescribeTargetGroupAttributes", + "elasticloadbalancing:DescribeTargetHealth", + "elasticloadbalancing:DescribeTags" + ], + "Resource": "*" + }, + { + "Effect": "Allow", + "Action": [ + "cognito-idp:DescribeUserPoolClient", + "acm:ListCertificates", + "acm:DescribeCertificate", + "iam:ListServerCertificates", + "iam:GetServerCertificate", + "waf-regional:GetWebACL", + "waf-regional:GetWebACLForResource", + "waf-regional:AssociateWebACL", + "waf-regional:DisassociateWebACL", + "wafv2:GetWebACL", + "wafv2:GetWebACLForResource", + "wafv2:AssociateWebACL", + "wafv2:DisassociateWebACL", + "shield:GetSubscriptionState", + "shield:DescribeProtection", + "shield:CreateProtection", + "shield:DeleteProtection" + ], + "Resource": "*" + }, + { + "Effect": "Allow", + "Action": [ + "ec2:AuthorizeSecurityGroupIngress", + "ec2:RevokeSecurityGroupIngress" + ], + "Resource": "*" + }, + { + "Effect": "Allow", + "Action": [ + "ec2:CreateSecurityGroup" + ], + "Resource": "*" + }, + { + "Effect": "Allow", + "Action": [ + "ec2:CreateTags" + ], + "Resource": "arn:aws:ec2:*:*:security-group/*", + "Condition": { + "StringEquals": { + "ec2:CreateAction": "CreateSecurityGroup" + }, + "Null": { + "aws:RequestTag/elbv2.k8s.aws/cluster": "false" + } + } + }, + { + "Effect": "Allow", + "Action": [ + "ec2:CreateTags", + "ec2:DeleteTags" + ], + "Resource": "arn:aws:ec2:*:*:security-group/*", + "Condition": { + "Null": { + "aws:RequestTag/elbv2.k8s.aws/cluster": "true", + "aws:ResourceTag/elbv2.k8s.aws/cluster": "false" + } + } + }, + { + "Effect": "Allow", + "Action": [ + "ec2:AuthorizeSecurityGroupIngress", + "ec2:RevokeSecurityGroupIngress", + "ec2:DeleteSecurityGroup" + ], + "Resource": "*", + "Condition": { + "Null": { + "aws:ResourceTag/elbv2.k8s.aws/cluster": "false" + } + } + }, + { + "Effect": "Allow", + "Action": [ + "elasticloadbalancing:CreateLoadBalancer", + "elasticloadbalancing:CreateTargetGroup" + ], + "Resource": "*", + "Condition": { + "Null": { + "aws:RequestTag/elbv2.k8s.aws/cluster": "false" + } + } + }, + { + "Effect": "Allow", + "Action": [ + "elasticloadbalancing:CreateListener", + "elasticloadbalancing:DeleteListener", + "elasticloadbalancing:CreateRule", + "elasticloadbalancing:DeleteRule" + ], + "Resource": "*" + }, + { + "Effect": "Allow", + "Action": [ + "elasticloadbalancing:AddTags", + "elasticloadbalancing:RemoveTags" + ], + "Resource": [ + "arn:aws:elasticloadbalancing:*:*:targetgroup/*/*", + "arn:aws:elasticloadbalancing:*:*:loadbalancer/net/*/*", + "arn:aws:elasticloadbalancing:*:*:loadbalancer/app/*/*" + ], + "Condition": { + "Null": { + "aws:RequestTag/elbv2.k8s.aws/cluster": "true", + "aws:ResourceTag/elbv2.k8s.aws/cluster": "false" + } + } + }, + { + "Effect": "Allow", + "Action": [ + "elasticloadbalancing:AddTags", + "elasticloadbalancing:RemoveTags" + ], + "Resource": [ + "arn:aws:elasticloadbalancing:*:*:listener/net/*/*/*", + "arn:aws:elasticloadbalancing:*:*:listener/app/*/*/*", + "arn:aws:elasticloadbalancing:*:*:listener-rule/net/*/*/*", + "arn:aws:elasticloadbalancing:*:*:listener-rule/app/*/*/*" + ] + }, + { + "Effect": "Allow", + "Action": [ + "elasticloadbalancing:AddTags" + ], + "Resource": [ + "arn:aws:elasticloadbalancing:*:*:targetgroup/*/*", + "arn:aws:elasticloadbalancing:*:*:loadbalancer/net/*/*", + "arn:aws:elasticloadbalancing:*:*:loadbalancer/app/*/*" + ], + "Condition": { + "StringEquals": { + "elasticloadbalancing:CreateAction": [ + "CreateTargetGroup", + "CreateLoadBalancer" + ] + }, + "Null": { + "aws:RequestTag/elbv2.k8s.aws/cluster": "false" + } + } + }, + { + "Effect": "Allow", + "Action": [ + "elasticloadbalancing:ModifyLoadBalancerAttributes", + "elasticloadbalancing:SetIpAddressType", + "elasticloadbalancing:SetSecurityGroups", + "elasticloadbalancing:SetSubnets", + "elasticloadbalancing:DeleteLoadBalancer", + "elasticloadbalancing:ModifyTargetGroup", + "elasticloadbalancing:ModifyTargetGroupAttributes", + "elasticloadbalancing:DeleteTargetGroup" + ], + "Resource": "*", + "Condition": { + "Null": { + "aws:ResourceTag/elbv2.k8s.aws/cluster": "false" + } + } + }, + { + "Effect": "Allow", + "Action": [ + "elasticloadbalancing:RegisterTargets", + "elasticloadbalancing:DeregisterTargets" + ], + "Resource": "arn:aws:elasticloadbalancing:*:*:targetgroup/*/*" + }, + { + "Effect": "Allow", + "Action": [ + "elasticloadbalancing:SetWebAcl", + "elasticloadbalancing:ModifyListener", + "elasticloadbalancing:AddListenerCertificates", + "elasticloadbalancing:RemoveListenerCertificates", + "elasticloadbalancing:ModifyRule" + ], + "Resource": "*" + } + ] +} diff --git a/tests-extension/test/testdata/router/awslb/sts-albo-perms-policy.json b/tests-extension/test/testdata/router/awslb/sts-albo-perms-policy.json new file mode 100644 index 000000000..f8ff9dd09 --- /dev/null +++ b/tests-extension/test/testdata/router/awslb/sts-albo-perms-policy.json @@ -0,0 +1,27 @@ +{ + "Version": "2012-10-17", + "Statement": [ + { + "Action": [ + "ec2:DescribeSubnets" + ], + "Effect": "Allow", + "Resource": "*" + }, + { + "Action": [ + "ec2:CreateTags", + "ec2:DeleteTags" + ], + "Effect": "Allow", + "Resource": "arn:aws:ec2:*:*:subnet/*" + }, + { + "Action": [ + "ec2:DescribeVpcs" + ], + "Effect": "Allow", + "Resource": "*" + } + ] +} diff --git a/tests-extension/test/testdata/router/awslb/subscription-src-qe-sts.yaml b/tests-extension/test/testdata/router/awslb/subscription-src-qe-sts.yaml new file mode 100644 index 000000000..3268227dc --- /dev/null +++ b/tests-extension/test/testdata/router/awslb/subscription-src-qe-sts.yaml @@ -0,0 +1,15 @@ +apiVersion: operators.coreos.com/v1alpha1 +kind: Subscription +metadata: + name: aws-load-balancer-operator + namespace: aws-load-balancer-operator +spec: + channel: stable-v1 + config: + env: + - name: ROLEARN + value: fakeARN-for-albo + installPlanApproval: Automatic + name: aws-load-balancer-operator + source: albo-konflux-fbc + sourceNamespace: aws-load-balancer-operator diff --git a/tests-extension/test/testdata/router/awslb/subscription-src-qe.yaml b/tests-extension/test/testdata/router/awslb/subscription-src-qe.yaml new file mode 100644 index 000000000..0d21ab010 --- /dev/null +++ b/tests-extension/test/testdata/router/awslb/subscription-src-qe.yaml @@ -0,0 +1,11 @@ +apiVersion: operators.coreos.com/v1alpha1 +kind: Subscription +metadata: + name: aws-load-balancer-operator + namespace: aws-load-balancer-operator +spec: + channel: stable-v1 + installPlanApproval: Automatic + name: aws-load-balancer-operator + source: albo-konflux-fbc + sourceNamespace: aws-load-balancer-operator diff --git a/tests-extension/test/testdata/router/bug1826225-proh2-deploy.yaml b/tests-extension/test/testdata/router/bug1826225-proh2-deploy.yaml new file mode 100644 index 000000000..490476d84 --- /dev/null +++ b/tests-extension/test/testdata/router/bug1826225-proh2-deploy.yaml @@ -0,0 +1,37 @@ +apiVersion: v1 +kind: List +items: +- apiVersion: apps/v1 + kind: Deployment + metadata: + name: web-server-deploy + labels: + name: web-server-deploy + spec: + replicas: 1 + selector: + matchLabels: + name: web-server-deploy + template: + metadata: + labels: + name: web-server-deploy + spec: + containers: + - name: nginx + image: quay.io/openshifttest/nginx-alpine@sha256:cee6930776b92dc1e93b73f9e5965925d49cff3d2e91e1d071c2f0ff72cbca29 +- apiVersion: v1 + kind: Service + metadata: + labels: + name: service-h2c-57001 + name: service-h2c-57001 + spec: + ports: + - appProtocol: h2c + name: h2c + port: 1110 + protocol: TCP + targetPort: 8080 + selector: + name: web-server-deploy diff --git a/tests-extension/test/testdata/router/bug2013004-lb-services.yaml b/tests-extension/test/testdata/router/bug2013004-lb-services.yaml new file mode 100644 index 000000000..6b176f09d --- /dev/null +++ b/tests-extension/test/testdata/router/bug2013004-lb-services.yaml @@ -0,0 +1,31 @@ +apiVersion: v1 +kind: List +items: +- apiVersion: v1 + kind: Service + metadata: + name: external-lb-57089 + spec: + ports: + - name: https + port: 28443 + protocol: TCP + targetPort: 8443 + selector: + name: web-server-rc + type: LoadBalancer +- apiVersion: v1 + kind: Service + metadata: + name: internal-lb-57089 + annotations: + service.beta.kubernetes.io/azure-load-balancer-internal: "true" + spec: + ports: + - name: https + port: 29443 + protocol: TCP + targetPort: 8443 + selector: + name: web-server-rc + type: LoadBalancer diff --git a/tests-extension/test/testdata/router/ca-bundle.pem b/tests-extension/test/testdata/router/ca-bundle.pem new file mode 100644 index 000000000..fcfcf2a8c --- /dev/null +++ b/tests-extension/test/testdata/router/ca-bundle.pem @@ -0,0 +1,30 @@ +-----BEGIN CERTIFICATE----- +MIIFDTCCAvWgAwIBAgIUDtkk2PPE+b/uQ8STNJNyuacqLq8wDQYJKoZIhvcNAQEL +BQAwFTETMBEGA1UEAwwKTkUtVGVzdC1DQTAgFw0yNTAxMTcwMzA0NDJaGA8yMDc1 +MDExNzAzMDQ0MlowFTETMBEGA1UEAwwKTkUtVGVzdC1DQTCCAiIwDQYJKoZIhvcN +AQEBBQADggIPADCCAgoCggIBAJMUI8/l6x4IDw0uQtJo0f9kV3ogWkgi4yU6UD4c +Wln8TMyg4kwLDg/sADd9o7dGnReLpc0ojnZpsJwBe7AyqmYTv7Q7IfWBmo1Cic5G +FdUbFS77YcIdyU77e7S1iWIt642wJShodU6i6suqvbraR9rNv7YuDs37GRsECHlj +IDRP2GStQqVtyPWLZ8+yG+3COIq+eUD9atsqmGrAYCe4IVy1lDi37g7nupveIFTM +FqQP4NFhTKCEW0ZhnaTHukOOOKRsnec5KV1rbdtkFBH035ECrkIbUemPwuZv+hMH +WNq4cfE8osT6RsY0rxKdGrteUAb7LZJZ3EiK+f7agekO9rvtekq63tMPsbJz4JOj +vNeF0oifvrQYeL4Lv4OGxjI37pwCmbc/f95Dg9P6I9on9/h/Yyyejhlkd7nFNjDL +RhtikKR/at20QF2QKvAb+1CTexa9xIhzMY9ZxtCvQ/PJM5PRHxYv+kSDhfJqFoHc +be6SP2oXZEiXhVlWKCTnuUHjSa6sBgJlZDoT0k5mORWn64WQqBG+L+ywzvbqZX0J +08+mHvpDRQwzOOJg+3IQhsMQDaQ6w7nE9CEHv9Oh1mgoGDSPDr1NIYPRdZ5os1h5 +w4sGjnxAzCTX36xU3XqkGLuDsOiBxkf+skIqZ88fr331KvQSbBw2Ia662q2SddDy +ArO3AgMBAAGjUzBRMB0GA1UdDgQWBBTopSzVEFFIz8Y6w270BvrEDDBPUDAfBgNV +HSMEGDAWgBTopSzVEFFIz8Y6w270BvrEDDBPUDAPBgNVHRMBAf8EBTADAQH/MA0G +CSqGSIb3DQEBCwUAA4ICAQCAD17WmP/yOv5I0QUXYgEYiJDVZvmDvOewWhKiTZDI +7hC6edDd4lx/hLmxzSOPHSj4K0wEA+0cVVArYCDp/uiWcV96c1531DdaYu/r2snd +FtUa8YhLB1jvBfxuTqJ1Q9+235PXDrOMprf4g7UMUk00M07D10ujIUorv0dccs3i +hmA+N83nHt9pGE2A2Cnwr++vmsy4jo9kqx5oXjAUovEP1xlOzP+tV6XjLy2tS4l1 +rqfF48nnPzhK1nPUR/S7voHcvCfEV+LOCUp/PAZg0SbQDPTUBo0vajOtU4ChB84n +L5c/T2/NOHMR8kV+2d+b4rvoYaHaF9nFd0pC5d+vjAgn1BJRXXGP2tCSjxsMkwVO +C7KF8S1gXN+nXYCQbcrcRsAdCscSI+MJNv9IyMfxeZ/CtjBOQz4AYibrykp+EYL7 +o0T9IW/1fjgBhC3qP/YXHLzkzUxdkktOsNdx7RzZNc0XACPcusnXx1Bw+YhQQFJT +sak7Fc8f0+4lFrQ6I2oVJ9CHWNDhSzxpXfY+JExOk8fytpvoFi4PyHyhWZOZOBSr +sshzq+cLKvLqR0rmUdrcC6PtXhM9got8xa/7cW5sbo393oZ720YEYhB4InpbmtDz +IawnZiugprouq6eaQ40KRv4xAJJXyX/yOHA0iGpjvqo8DLg5jzC8kIjulSUEIrgw +4w== +-----END CERTIFICATE----- diff --git a/tests-extension/test/testdata/router/coreDNS-pod.yaml b/tests-extension/test/testdata/router/coreDNS-pod.yaml new file mode 100644 index 000000000..4827231f8 --- /dev/null +++ b/tests-extension/test/testdata/router/coreDNS-pod.yaml @@ -0,0 +1,74 @@ +apiVersion: v1 +kind: Pod +metadata: + labels: + name: test-coredns + name: test-coredns +spec: + securityContext: + runAsNonRoot: false + seccompProfile: + type: RuntimeDefault + containers: + - args: + - -conf + - /etc/coredns/Corefile + command: + - coredns + image: replaced-at-runtime + imagePullPolicy: IfNotPresent + name: coredns + securityContext: + allowPrivilegeEscalation: true + capabilities: + add: + - NET_BIND_SERVICE + drop: + - ALL + volumeMounts: + - mountPath: /etc/coredns + name: config-volume + readOnly: true + nodeSelector: + kubernetes.io/os: linux + volumes: + - configMap: + defaultMode: 420 + items: + - key: Corefile + path: Corefile + name: test-coredns-cm + name: config-volume +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: test-coredns-cm +data: + Corefile: | + .:53 { + bufsize 512 + errors + log . { + class error + } + health { + lameduck 20s + } + ready + template ANY MX { + rcode NXDOMAIN + } + hosts { + 192.168.11.241 www.myocp-test.com + 2000::abcd myocp-testv6.com + } + prometheus 127.0.0.1:9153 + forward . 8.8.8.8 { + policy random + } + cache 900 { + denial 9984 30 + } + reload + } diff --git a/tests-extension/test/testdata/router/egressfirewall-multiDomain.yaml b/tests-extension/test/testdata/router/egressfirewall-multiDomain.yaml new file mode 100644 index 000000000..4c6b95bd4 --- /dev/null +++ b/tests-extension/test/testdata/router/egressfirewall-multiDomain.yaml @@ -0,0 +1,21 @@ +kind: EgressFirewall +apiVersion: k8s.ovn.org/v1 +metadata: + name: default +spec: + egress: + - type: Allow + to: + dnsName: registry-1.docker.io + - type: Allow + to: + dnsName: www.facebook.com + ports: + - protocol: TCP + port: 80 + - type: Deny + to: + cidrSelector: 0.0.0.0/0 + - type: Deny + to: + cidrSelector: ::/0 diff --git a/tests-extension/test/testdata/router/egressfirewall-wildcard.yaml b/tests-extension/test/testdata/router/egressfirewall-wildcard.yaml new file mode 100644 index 000000000..f2c9f3374 --- /dev/null +++ b/tests-extension/test/testdata/router/egressfirewall-wildcard.yaml @@ -0,0 +1,15 @@ +kind: EgressFirewall +apiVersion: k8s.ovn.org/v1 +metadata: + name: default +spec: + egress: + - type: Allow + to: + dnsName: "*.google.com" + - type: Deny + to: + cidrSelector: 0.0.0.0/0 + - type: Deny + to: + cidrSelector: ::/0 diff --git a/tests-extension/test/testdata/router/error-page-404.http b/tests-extension/test/testdata/router/error-page-404.http new file mode 100644 index 000000000..9cce4eddd --- /dev/null +++ b/tests-extension/test/testdata/router/error-page-404.http @@ -0,0 +1,10 @@ +HTTP/1.0 404 Not Found +Connection: close +Content-Type: text/html + + +Custom:Not Found + +

Custom error page:The requested document was not found.

+ + diff --git a/tests-extension/test/testdata/router/error-page-503.http b/tests-extension/test/testdata/router/error-page-503.http new file mode 100644 index 000000000..90c349904 --- /dev/null +++ b/tests-extension/test/testdata/router/error-page-503.http @@ -0,0 +1,10 @@ +HTTP/1.0 503 Service Unavailable +Connection: close +Content-Type: text/html + + +Custom:Application Unavailable + +

Custom error page:The requested application is not available.

+ + diff --git a/tests-extension/test/testdata/router/error-page2-404.http b/tests-extension/test/testdata/router/error-page2-404.http new file mode 100644 index 000000000..d82e7065f --- /dev/null +++ b/tests-extension/test/testdata/router/error-page2-404.http @@ -0,0 +1,10 @@ +HTTP/1.0 404 Not Found +Connection: close +Content-Type: text/html + + +Custom:Not Found + +

Custom error page:THE REQUESTED DOCUMENT WAS NOT FOUND YET!

+ + diff --git a/tests-extension/test/testdata/router/error-page2-503.http b/tests-extension/test/testdata/router/error-page2-503.http new file mode 100644 index 000000000..b89829a5f --- /dev/null +++ b/tests-extension/test/testdata/router/error-page2-503.http @@ -0,0 +1,10 @@ +HTTP/1.0 503 Service Unavailable +Connection: close +Content-Type: text/html + + +Custom:Application Unavailable + +

Custom error page:THE REQUESTED APPLICATION IS NOT AVAILABLE YET!

+ + diff --git a/tests-extension/test/testdata/router/extdns/aws-sts-creds-secret.yaml b/tests-extension/test/testdata/router/extdns/aws-sts-creds-secret.yaml new file mode 100644 index 000000000..9bd252c36 --- /dev/null +++ b/tests-extension/test/testdata/router/extdns/aws-sts-creds-secret.yaml @@ -0,0 +1,11 @@ +apiVersion: v1 +kind: Secret +metadata: + name: aws-sts-creds + namespace: external-dns-operator +stringData: + credentials: |- + [default] + sts_regional_endpoints = regional + role_arn = external-dns-role-arn + web_identity_token_file = /var/run/secrets/openshift/serviceaccount/token diff --git a/tests-extension/test/testdata/router/extdns/catalog-source.yaml b/tests-extension/test/testdata/router/extdns/catalog-source.yaml new file mode 100644 index 000000000..54d3dc764 --- /dev/null +++ b/tests-extension/test/testdata/router/extdns/catalog-source.yaml @@ -0,0 +1,21 @@ +apiVersion: template.openshift.io/v1 +kind: Template +objects: +- apiVersion: operators.coreos.com/v1alpha1 + kind: CatalogSource + metadata: + name: "${CATALOG_NAME}" + namespace: "${NAMESPACE}" + spec: + displayName: Konflux ExtDNS + image: "${IMAGE}" + sourceType: grpc + grpcPodConfig: + securityContextConfig: legacy +parameters: +- name: CATALOG_NAME + value: "extdns-konflux-fbc" +- name: IMAGE + value: "quay.io/redhat-user-workloads/external-dns-operator-tenant/ext-dns-optr-fbc-v4-21/external-dns-fbc-container-ext-dns-optr-fbc-v4-21:candidate" +- name: NAMESPACE + value: external-dns-operator diff --git a/tests-extension/test/testdata/router/extdns/ns-external-dns-operator.yaml b/tests-extension/test/testdata/router/extdns/ns-external-dns-operator.yaml new file mode 100644 index 000000000..b8730d508 --- /dev/null +++ b/tests-extension/test/testdata/router/extdns/ns-external-dns-operator.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: external-dns-operator diff --git a/tests-extension/test/testdata/router/extdns/operatorgroup.yaml b/tests-extension/test/testdata/router/extdns/operatorgroup.yaml new file mode 100644 index 000000000..5e39de1f5 --- /dev/null +++ b/tests-extension/test/testdata/router/extdns/operatorgroup.yaml @@ -0,0 +1,8 @@ +apiVersion: operators.coreos.com/v1 +kind: OperatorGroup +metadata: + name: external-dns-group + namespace: external-dns-operator +spec: + targetNamespaces: + - external-dns-operator diff --git a/tests-extension/test/testdata/router/extdns/sample-aws-rt.yaml b/tests-extension/test/testdata/router/extdns/sample-aws-rt.yaml new file mode 100644 index 000000000..4304e39fb --- /dev/null +++ b/tests-extension/test/testdata/router/extdns/sample-aws-rt.yaml @@ -0,0 +1,18 @@ +apiVersion: externaldns.olm.openshift.io/v1beta1 +kind: ExternalDNS +metadata: + name: sample-aws-rt +spec: + domains: + - filterType: Include + matchType: Exact + name: basedomain + provider: + type: AWS + source: + openshiftRouteOptions: + routerName: default + type: OpenShiftRoute + labelFilter: + matchLabels: + external-dns.mydomain.org/publish: "yes" diff --git a/tests-extension/test/testdata/router/extdns/sample-aws-sharedvpc-rt.yaml b/tests-extension/test/testdata/router/extdns/sample-aws-sharedvpc-rt.yaml new file mode 100644 index 000000000..edf3f1ba2 --- /dev/null +++ b/tests-extension/test/testdata/router/extdns/sample-aws-sharedvpc-rt.yaml @@ -0,0 +1,21 @@ +apiVersion: externaldns.olm.openshift.io/v1beta1 +kind: ExternalDNS +metadata: + name: sample-aws-sharedvpc-rt +spec: + domains: + - filterType: Include + matchType: Exact + name: basedomain + provider: + type: AWS + aws: + assumeRole: + arn: privatezoneiamrole + source: + openshiftRouteOptions: + routerName: default + type: OpenShiftRoute + labelFilter: + matchLabels: + external-dns.mydomain.org/publish: "yes" diff --git a/tests-extension/test/testdata/router/extdns/sample-aws-sts-rt.yaml b/tests-extension/test/testdata/router/extdns/sample-aws-sts-rt.yaml new file mode 100644 index 000000000..9d2c7987d --- /dev/null +++ b/tests-extension/test/testdata/router/extdns/sample-aws-sts-rt.yaml @@ -0,0 +1,23 @@ +apiVersion: externaldns.olm.openshift.io/v1beta1 +kind: ExternalDNS +metadata: + name: sample-aws-sts-rt +spec: + domains: + - filterType: Include + matchType: Exact + name: basedomain + provider: + type: AWS + aws: + credentials: + name: aws-sts-creds + zones: + - privatezone + source: + labelFilter: + matchLabels: + external-dns.mydomain.org/publish: "yes" + openshiftRouteOptions: + routerName: default + type: OpenShiftRoute diff --git a/tests-extension/test/testdata/router/extdns/sample-azure-rt.yaml b/tests-extension/test/testdata/router/extdns/sample-azure-rt.yaml new file mode 100644 index 000000000..ac278eba2 --- /dev/null +++ b/tests-extension/test/testdata/router/extdns/sample-azure-rt.yaml @@ -0,0 +1,16 @@ +apiVersion: externaldns.olm.openshift.io/v1beta1 +kind: ExternalDNS +metadata: + name: sample-azure-rt +spec: + provider: + type: Azure + source: + labelFilter: + matchLabels: + external-dns.mydomain.org/publish: "yes" + openshiftRouteOptions: + routerName: default + type: OpenShiftRoute + zones: + - "mydomain.org" diff --git a/tests-extension/test/testdata/router/extdns/sample-gcp-svc.yaml b/tests-extension/test/testdata/router/extdns/sample-gcp-svc.yaml new file mode 100644 index 000000000..f1e666eca --- /dev/null +++ b/tests-extension/test/testdata/router/extdns/sample-gcp-svc.yaml @@ -0,0 +1,20 @@ +apiVersion: externaldns.olm.openshift.io/v1beta1 +kind: ExternalDNS +metadata: + name: sample-gcp-svc +spec: + provider: + type: GCP + source: + labelFilter: + matchLabels: + external-dns.mydomain.org/publish: "yes" + service: + serviceType: + - LoadBalancer + - ClusterIP + type: Service + fqdnTemplate: + - "{{.Name}}.mydomain.org" + zones: + - "mydomain.org" diff --git a/tests-extension/test/testdata/router/extdns/sts-exdns-perms-policy.json b/tests-extension/test/testdata/router/extdns/sts-exdns-perms-policy.json new file mode 100644 index 000000000..312e29430 --- /dev/null +++ b/tests-extension/test/testdata/router/extdns/sts-exdns-perms-policy.json @@ -0,0 +1,25 @@ +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "route53:ChangeResourceRecordSets" + ], + "Resource": [ + "arn:aws:route53:::hostedzone/*" + ] + }, + { + "Effect": "Allow", + "Action": [ + "route53:ListHostedZones", + "route53:ListResourceRecordSets", + "route53:ListTagsForResource" + ], + "Resource": [ + "*" + ] + } + ] +} diff --git a/tests-extension/test/testdata/router/extdns/subscription.yaml b/tests-extension/test/testdata/router/extdns/subscription.yaml new file mode 100644 index 000000000..439738210 --- /dev/null +++ b/tests-extension/test/testdata/router/extdns/subscription.yaml @@ -0,0 +1,11 @@ +apiVersion: operators.coreos.com/v1alpha1 +kind: Subscription +metadata: + name: external-dns-operator + namespace: external-dns-operator +spec: + channel: stable-v1 + installPlanApproval: Automatic + name: external-dns-operator + source: extdns-konflux-fbc + sourceNamespace: external-dns-operator diff --git a/tests-extension/test/testdata/router/gateway.yaml b/tests-extension/test/testdata/router/gateway.yaml new file mode 100644 index 000000000..fa516366d --- /dev/null +++ b/tests-extension/test/testdata/router/gateway.yaml @@ -0,0 +1,28 @@ +apiVersion: template.openshift.io/v1 +kind: Template +objects: +- apiVersion: gateway.networking.k8s.io/v1 + kind: Gateway + metadata: + name: ${NAME} + namespace: ${NAMESPACE} + spec: + gatewayClassName: openshift-default + listeners: + - name: demo + hostname: ${HOSTNAME} + port: 80 + protocol: HTTP + allowedRoutes: + namespaces: + from: Selector + selector: + matchLabels: + app: gwapi +parameters: +- name: NAME + value: gateway-default +- name: NAMESPACE + value: openshift-ingress +- name: HOSTNAME + value: gwapi.example.com \ No newline at end of file diff --git a/tests-extension/test/testdata/router/gatewayclass.yaml b/tests-extension/test/testdata/router/gatewayclass.yaml new file mode 100644 index 000000000..c18472477 --- /dev/null +++ b/tests-extension/test/testdata/router/gatewayclass.yaml @@ -0,0 +1,6 @@ +apiVersion: gateway.networking.k8s.io/v1 +kind: GatewayClass +metadata: + name: openshift-default +spec: + controllerName: openshift.io/gateway-controller/v1 diff --git a/tests-extension/test/testdata/router/httpbin-deploy.yaml b/tests-extension/test/testdata/router/httpbin-deploy.yaml new file mode 100644 index 000000000..086f44308 --- /dev/null +++ b/tests-extension/test/testdata/router/httpbin-deploy.yaml @@ -0,0 +1,56 @@ +apiVersion: v1 +kind: List +items: +- apiVersion: apps/v1 + kind: Deployment + metadata: + name: httpbin-pod + labels: + name: httpbin-pod + spec: + replicas: 1 + selector: + matchLabels: + name: httpbin-pod + template: + metadata: + labels: + name: httpbin-pod + spec: + containers: + - name: httpbin-http + image: quay.io/openshifttest/httpbin@sha256:cc44fbd857f4148d8aad8359acc03efa719517e01d390b152e4f3830ad871c9f + ports: + - containerPort: 8080 + - name: httpbin-https + image: quay.io/openshifttest/httpbin@sha256:f57f4e682e05bcdadb103c93ae5ab9be166f79bcbbccaf45d92a2cad18da8d64 + ports: + - containerPort: 8443 +- apiVersion: v1 + kind: Service + metadata: + labels: + name: httpbin-svc-insecure + name: httpbin-svc-insecure + spec: + ports: + - name: http + port: 27017 + protocol: TCP + targetPort: 8080 + selector: + name: httpbin-pod +- apiVersion: v1 + kind: Service + metadata: + labels: + name: httpbin-svc-secure + name: httpbin-svc-secure + spec: + ports: + - name: https + port: 27443 + protocol: TCP + targetPort: 8443 + selector: + name: httpbin-pod diff --git a/tests-extension/test/testdata/router/httpbin-pod-withprivilege.json b/tests-extension/test/testdata/router/httpbin-pod-withprivilege.json new file mode 100644 index 000000000..ca44a951a --- /dev/null +++ b/tests-extension/test/testdata/router/httpbin-pod-withprivilege.json @@ -0,0 +1,36 @@ +{ + "kind": "Pod", + "apiVersion":"v1", + "metadata": { + "name": "httpbin-pod", + "labels": { + "name": "httpbin-pod" + } + }, + "spec": { + "containers": [{ + "name": "httpbin-http", + "image": "quay.io/openshifttest/httpbin@sha256:cc44fbd857f4148d8aad8359acc03efa719517e01d390b152e4f3830ad871c9f", + "ports": [ + { + "containerPort": 8080, + "securityContext": { + "privileged": "true" + } + } + ] + }, + { + "name": "httpbin-https", + "image": "quay.io/openshifttest/httpbin@sha256:f57f4e682e05bcdadb103c93ae5ab9be166f79bcbbccaf45d92a2cad18da8d64", + "ports": [ + { + "containerPort": 8443, + "securityContext": { + "privileged": "true" + } + } + ] + }] + } +} diff --git a/tests-extension/test/testdata/router/httpbin-service_secure.json b/tests-extension/test/testdata/router/httpbin-service_secure.json new file mode 100644 index 000000000..3e0847a60 --- /dev/null +++ b/tests-extension/test/testdata/router/httpbin-service_secure.json @@ -0,0 +1,20 @@ +{ + "kind": "Service", + "apiVersion": "v1", + "metadata": { + "name": "httpbin-svc-secure" + }, + "spec": { + "ports": [ + { + "name": "https", + "protocol": "TCP", + "port": 27443, + "targetPort": 8443 + } + ], + "selector": { + "name": "httpbin-pod" + } + } +} diff --git a/tests-extension/test/testdata/router/httproute.yaml b/tests-extension/test/testdata/router/httproute.yaml new file mode 100644 index 000000000..9fc68047a --- /dev/null +++ b/tests-extension/test/testdata/router/httproute.yaml @@ -0,0 +1,22 @@ +apiVersion: template.openshift.io/v1 +kind: Template +objects: +- apiVersion: gateway.networking.k8s.io/v1 + kind: HTTPRoute + metadata: + name: ${NAME} + namespace: ${NAMESPACE} + spec: + parentRefs: + - name: ${GWNAME} + namespace: openshift-ingress + hostnames: ["${HOSTNAME}"] + rules: + - backendRefs: + - name: service-unsecure + port: 27017 +parameters: +- name: NAME +- name: NAMESPACE +- name: GWNAME +- name: HOSTNAME diff --git a/tests-extension/test/testdata/router/ingress-destCA.yaml b/tests-extension/test/testdata/router/ingress-destCA.yaml new file mode 100644 index 000000000..a1c9f9886 --- /dev/null +++ b/tests-extension/test/testdata/router/ingress-destCA.yaml @@ -0,0 +1,28 @@ +apiVersion: template.openshift.io/v1 +kind: Template +objects: +- apiVersion: networking.k8s.io/v1 + kind: Ingress + metadata: + name: ${NAME} + annotations: + route.openshift.io/destination-ca-certificate-secret: service-secret + route.openshift.io/termination: reencrypt + namespace: ${NAMESPACE} + spec: + rules: + - host: ${SERVICE_NAME}-${NAMESPACE}.${DOMAIN} + http: + paths: + - backend: + service: + name: ${SERVICE_NAME} + port: + number: 27443 + path: "/" + pathType: Prefix +parameters: +- name: NAME +- name: NAMESPACE +- name: DOMAIN +- name: SERVICE_NAME diff --git a/tests-extension/test/testdata/router/ingress-resource.yaml b/tests-extension/test/testdata/router/ingress-resource.yaml new file mode 100644 index 000000000..df835e881 --- /dev/null +++ b/tests-extension/test/testdata/router/ingress-resource.yaml @@ -0,0 +1,86 @@ +kind: List +items: +- apiVersion: networking.k8s.io/v1 + kind: Ingress + metadata: + name: ingress-edge + annotations: + route.openshift.io/termination: "edge" + spec: + rules: + - host: edgehostname + http: + paths: + - path: "/" + backend: + service: + name: service-unsecure + port: + number: 27017 + pathType: "Prefix" + tls: + - hosts: + - hostname1 + secretName: ingress-secret +- apiVersion: networking.k8s.io/v1 + kind: Ingress + metadata: + name: ingress-passth + annotations: + route.openshift.io/termination: passthrough + spec: + rules: + - host: passhostname + http: + paths: + - path: '' + pathType: ImplementationSpecific + backend: + service: + name: service-secure + port: + number: 27443 +- apiVersion: networking.k8s.io/v1 + kind: Ingress + metadata: + name: ingress-reencrypt + annotations: + route.openshift.io/termination: "reencrypt" + spec: + rules: + - host: reenhostname + http: + paths: + - path: "/" + backend: + service: + name: service-secure + port: + number: 27443 + pathType: "Prefix" + tls: + - hosts: + - hostname3 + secretName: ingress-secret +- apiVersion: networking.k8s.io/v1 + kind: Ingress + metadata: + name: ingress-random + annotations: + route.openshift.io/termination: "abcd" + spec: + rules: + - host: randomhostname + http: + paths: + - path: "/" + backend: + service: + name: service-unsecure + port: + number: 27017 + pathType: "Prefix" + tls: + - hosts: + - randomhostname + secretName: ingress-secret \ No newline at end of file diff --git a/tests-extension/test/testdata/router/ingress-with-class.yaml b/tests-extension/test/testdata/router/ingress-with-class.yaml new file mode 100644 index 000000000..096858bae --- /dev/null +++ b/tests-extension/test/testdata/router/ingress-with-class.yaml @@ -0,0 +1,17 @@ +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: ingress-with-class +spec: + ingressClassName: mytest + rules: + - host: foo.bar.com + http: + paths: + - backend: + service: + name: service-unsecure + port: + number: 27017 + path: /test + pathType: ImplementationSpecific diff --git a/tests-extension/test/testdata/router/ingresscontroller-IBMproxy.yaml b/tests-extension/test/testdata/router/ingresscontroller-IBMproxy.yaml new file mode 100644 index 000000000..8862b0a70 --- /dev/null +++ b/tests-extension/test/testdata/router/ingresscontroller-IBMproxy.yaml @@ -0,0 +1,25 @@ +apiVersion: template.openshift.io/v1 +kind: Template +objects: +- apiVersion: operator.openshift.io/v1 + kind: IngressController + metadata: + name: ${NAME} + namespace: ${NAMESPACE} + spec: + domain: ${DOMAIN} + replicas: 1 + endpointPublishingStrategy: + loadBalancer: + dnsManagementPolicy: Managed + providerParameters: + ibm: + protocol: PROXY + type: IBM + scope: External + type: LoadBalancerService +parameters: +- name: NAME +- name: NAMESPACE + value: openshift-ingress-operator +- name: DOMAIN diff --git a/tests-extension/test/testdata/router/ingresscontroller-azure-cidr.yaml b/tests-extension/test/testdata/router/ingresscontroller-azure-cidr.yaml new file mode 100644 index 000000000..5bb94a5d1 --- /dev/null +++ b/tests-extension/test/testdata/router/ingresscontroller-azure-cidr.yaml @@ -0,0 +1,4017 @@ +apiVersion: template.openshift.io/v1 +kind: Template +objects: +- apiVersion: operator.openshift.io/v1 + kind: IngressController + metadata: + name: ${NAME} + namespace: ${NAMESPACE} + spec: + domain: ${DOMAIN} + replicas: 1 + endpointPublishingStrategy: + type: LoadBalancerService + loadBalancer: + DNSManagementPolicy: Managed + scope: External + allowedSourceRanges: + - 1.1.1.2/32 + - 1.1.1.4/32 + - 1.1.1.6/32 + - 1.1.1.8/32 + - 1.1.1.10/32 + - 1.1.1.12/32 + - 1.1.1.14/32 + - 1.1.1.16/32 + - 1.1.1.18/32 + - 1.1.1.20/32 + - 1.1.1.22/32 + - 1.1.1.24/32 + - 1.1.1.26/32 + - 1.1.1.28/32 + - 1.1.1.30/32 + - 1.1.1.32/32 + - 1.1.1.34/32 + - 1.1.1.36/32 + - 1.1.1.38/32 + - 1.1.1.40/32 + - 1.1.1.42/32 + - 1.1.1.44/32 + - 1.1.1.46/32 + - 1.1.1.48/32 + - 1.1.1.50/32 + - 1.1.1.52/32 + - 1.1.1.54/32 + - 1.1.1.56/32 + - 1.1.1.58/32 + - 1.1.1.60/32 + - 1.1.1.62/32 + - 1.1.1.64/32 + - 1.1.1.66/32 + - 1.1.1.68/32 + - 1.1.1.70/32 + - 1.1.1.72/32 + - 1.1.1.74/32 + - 1.1.1.76/32 + - 1.1.1.78/32 + - 1.1.1.80/32 + - 1.1.1.82/32 + - 1.1.1.84/32 + - 1.1.1.86/32 + - 1.1.1.88/32 + - 1.1.1.90/32 + - 1.1.1.92/32 + - 1.1.1.94/32 + - 1.1.1.96/32 + - 1.1.1.98/32 + - 1.1.1.100/32 + - 1.1.1.102/32 + - 1.1.1.104/32 + - 1.1.1.106/32 + - 1.1.1.108/32 + - 1.1.1.110/32 + - 1.1.1.112/32 + - 1.1.1.114/32 + - 1.1.1.116/32 + - 1.1.1.118/32 + - 1.1.1.120/32 + - 1.1.1.122/32 + - 1.1.1.124/32 + - 1.1.1.126/32 + - 1.1.1.128/32 + - 1.1.1.130/32 + - 1.1.1.132/32 + - 1.1.1.134/32 + - 1.1.1.136/32 + - 1.1.1.138/32 + - 1.1.1.140/32 + - 1.1.1.142/32 + - 1.1.1.144/32 + - 1.1.1.146/32 + - 1.1.1.148/32 + - 1.1.1.150/32 + - 1.1.1.152/32 + - 1.1.1.154/32 + - 1.1.1.156/32 + - 1.1.1.158/32 + - 1.1.1.160/32 + - 1.1.1.162/32 + - 1.1.1.164/32 + - 1.1.1.166/32 + - 1.1.1.168/32 + - 1.1.1.170/32 + - 1.1.1.172/32 + - 1.1.1.174/32 + - 1.1.1.176/32 + - 1.1.1.178/32 + - 1.1.1.180/32 + - 1.1.1.182/32 + - 1.1.1.184/32 + - 1.1.1.186/32 + - 1.1.1.188/32 + - 1.1.1.190/32 + - 1.1.1.192/32 + - 1.1.1.194/32 + - 1.1.1.196/32 + - 1.1.1.198/32 + - 1.1.1.200/32 + - 1.1.1.202/32 + - 1.1.1.204/32 + - 1.1.1.206/32 + - 1.1.1.208/32 + - 1.1.1.210/32 + - 1.1.1.212/32 + - 1.1.1.214/32 + - 1.1.1.216/32 + - 1.1.1.218/32 + - 1.1.1.220/32 + - 1.1.1.222/32 + - 1.1.1.224/32 + - 1.1.1.226/32 + - 1.1.1.228/32 + - 1.1.1.230/32 + - 1.1.1.232/32 + - 1.1.1.234/32 + - 1.1.1.236/32 + - 1.1.1.238/32 + - 1.1.1.240/32 + - 1.1.1.242/32 + - 1.1.1.244/32 + - 1.1.1.246/32 + - 1.1.1.248/32 + - 1.1.1.250/32 + - 1.1.1.252/32 + - 1.1.1.254/32 + - 1.1.2.2/32 + - 1.1.2.4/32 + - 1.1.2.6/32 + - 1.1.2.8/32 + - 1.1.2.10/32 + - 1.1.2.12/32 + - 1.1.2.14/32 + - 1.1.2.16/32 + - 1.1.2.18/32 + - 1.1.2.20/32 + - 1.1.2.22/32 + - 1.1.2.24/32 + - 1.1.2.26/32 + - 1.1.2.28/32 + - 1.1.2.30/32 + - 1.1.2.32/32 + - 1.1.2.34/32 + - 1.1.2.36/32 + - 1.1.2.38/32 + - 1.1.2.40/32 + - 1.1.2.42/32 + - 1.1.2.44/32 + - 1.1.2.46/32 + - 1.1.2.48/32 + - 1.1.2.50/32 + - 1.1.2.52/32 + - 1.1.2.54/32 + - 1.1.2.56/32 + - 1.1.2.58/32 + - 1.1.2.60/32 + - 1.1.2.62/32 + - 1.1.2.64/32 + - 1.1.2.66/32 + - 1.1.2.68/32 + - 1.1.2.70/32 + - 1.1.2.72/32 + - 1.1.2.74/32 + - 1.1.2.76/32 + - 1.1.2.78/32 + - 1.1.2.80/32 + - 1.1.2.82/32 + - 1.1.2.84/32 + - 1.1.2.86/32 + - 1.1.2.88/32 + - 1.1.2.90/32 + - 1.1.2.92/32 + - 1.1.2.94/32 + - 1.1.2.96/32 + - 1.1.2.98/32 + - 1.1.2.100/32 + - 1.1.2.102/32 + - 1.1.2.104/32 + - 1.1.2.106/32 + - 1.1.2.108/32 + - 1.1.2.110/32 + - 1.1.2.112/32 + - 1.1.2.114/32 + - 1.1.2.116/32 + - 1.1.2.118/32 + - 1.1.2.120/32 + - 1.1.2.122/32 + - 1.1.2.124/32 + - 1.1.2.126/32 + - 1.1.2.128/32 + - 1.1.2.130/32 + - 1.1.2.132/32 + - 1.1.2.134/32 + - 1.1.2.136/32 + - 1.1.2.138/32 + - 1.1.2.140/32 + - 1.1.2.142/32 + - 1.1.2.144/32 + - 1.1.2.146/32 + - 1.1.2.148/32 + - 1.1.2.150/32 + - 1.1.2.152/32 + - 1.1.2.154/32 + - 1.1.2.156/32 + - 1.1.2.158/32 + - 1.1.2.160/32 + - 1.1.2.162/32 + - 1.1.2.164/32 + - 1.1.2.166/32 + - 1.1.2.168/32 + - 1.1.2.170/32 + - 1.1.2.172/32 + - 1.1.2.174/32 + - 1.1.2.176/32 + - 1.1.2.178/32 + - 1.1.2.180/32 + - 1.1.2.182/32 + - 1.1.2.184/32 + - 1.1.2.186/32 + - 1.1.2.188/32 + - 1.1.2.190/32 + - 1.1.2.192/32 + - 1.1.2.194/32 + - 1.1.2.196/32 + - 1.1.2.198/32 + - 1.1.2.200/32 + - 1.1.2.202/32 + - 1.1.2.204/32 + - 1.1.2.206/32 + - 1.1.2.208/32 + - 1.1.2.210/32 + - 1.1.2.212/32 + - 1.1.2.214/32 + - 1.1.2.216/32 + - 1.1.2.218/32 + - 1.1.2.220/32 + - 1.1.2.222/32 + - 1.1.2.224/32 + - 1.1.2.226/32 + - 1.1.2.228/32 + - 1.1.2.230/32 + - 1.1.2.232/32 + - 1.1.2.234/32 + - 1.1.2.236/32 + - 1.1.2.238/32 + - 1.1.2.240/32 + - 1.1.2.242/32 + - 1.1.2.244/32 + - 1.1.2.246/32 + - 1.1.2.248/32 + - 1.1.2.250/32 + - 1.1.2.252/32 + - 1.1.2.254/32 + - 1.1.3.2/32 + - 1.1.3.4/32 + - 1.1.3.6/32 + - 1.1.3.8/32 + - 1.1.3.10/32 + - 1.1.3.12/32 + - 1.1.3.14/32 + - 1.1.3.16/32 + - 1.1.3.18/32 + - 1.1.3.20/32 + - 1.1.3.22/32 + - 1.1.3.24/32 + - 1.1.3.26/32 + - 1.1.3.28/32 + - 1.1.3.30/32 + - 1.1.3.32/32 + - 1.1.3.34/32 + - 1.1.3.36/32 + - 1.1.3.38/32 + - 1.1.3.40/32 + - 1.1.3.42/32 + - 1.1.3.44/32 + - 1.1.3.46/32 + - 1.1.3.48/32 + - 1.1.3.50/32 + - 1.1.3.52/32 + - 1.1.3.54/32 + - 1.1.3.56/32 + - 1.1.3.58/32 + - 1.1.3.60/32 + - 1.1.3.62/32 + - 1.1.3.64/32 + - 1.1.3.66/32 + - 1.1.3.68/32 + - 1.1.3.70/32 + - 1.1.3.72/32 + - 1.1.3.74/32 + - 1.1.3.76/32 + - 1.1.3.78/32 + - 1.1.3.80/32 + - 1.1.3.82/32 + - 1.1.3.84/32 + - 1.1.3.86/32 + - 1.1.3.88/32 + - 1.1.3.90/32 + - 1.1.3.92/32 + - 1.1.3.94/32 + - 1.1.3.96/32 + - 1.1.3.98/32 + - 1.1.3.100/32 + - 1.1.3.102/32 + - 1.1.3.104/32 + - 1.1.3.106/32 + - 1.1.3.108/32 + - 1.1.3.110/32 + - 1.1.3.112/32 + - 1.1.3.114/32 + - 1.1.3.116/32 + - 1.1.3.118/32 + - 1.1.3.120/32 + - 1.1.3.122/32 + - 1.1.3.124/32 + - 1.1.3.126/32 + - 1.1.3.128/32 + - 1.1.3.130/32 + - 1.1.3.132/32 + - 1.1.3.134/32 + - 1.1.3.136/32 + - 1.1.3.138/32 + - 1.1.3.140/32 + - 1.1.3.142/32 + - 1.1.3.144/32 + - 1.1.3.146/32 + - 1.1.3.148/32 + - 1.1.3.150/32 + - 1.1.3.152/32 + - 1.1.3.154/32 + - 1.1.3.156/32 + - 1.1.3.158/32 + - 1.1.3.160/32 + - 1.1.3.162/32 + - 1.1.3.164/32 + - 1.1.3.166/32 + - 1.1.3.168/32 + - 1.1.3.170/32 + - 1.1.3.172/32 + - 1.1.3.174/32 + - 1.1.3.176/32 + - 1.1.3.178/32 + - 1.1.3.180/32 + - 1.1.3.182/32 + - 1.1.3.184/32 + - 1.1.3.186/32 + - 1.1.3.188/32 + - 1.1.3.190/32 + - 1.1.3.192/32 + - 1.1.3.194/32 + - 1.1.3.196/32 + - 1.1.3.198/32 + - 1.1.3.200/32 + - 1.1.3.202/32 + - 1.1.3.204/32 + - 1.1.3.206/32 + - 1.1.3.208/32 + - 1.1.3.210/32 + - 1.1.3.212/32 + - 1.1.3.214/32 + - 1.1.3.216/32 + - 1.1.3.218/32 + - 1.1.3.220/32 + - 1.1.3.222/32 + - 1.1.3.224/32 + - 1.1.3.226/32 + - 1.1.3.228/32 + - 1.1.3.230/32 + - 1.1.3.232/32 + - 1.1.3.234/32 + - 1.1.3.236/32 + - 1.1.3.238/32 + - 1.1.3.240/32 + - 1.1.3.242/32 + - 1.1.3.244/32 + - 1.1.3.246/32 + - 1.1.3.248/32 + - 1.1.3.250/32 + - 1.1.3.252/32 + - 1.1.3.254/32 + - 1.1.4.2/32 + - 1.1.4.4/32 + - 1.1.4.6/32 + - 1.1.4.8/32 + - 1.1.4.10/32 + - 1.1.4.12/32 + - 1.1.4.14/32 + - 1.1.4.16/32 + - 1.1.4.18/32 + - 1.1.4.20/32 + - 1.1.4.22/32 + - 1.1.4.24/32 + - 1.1.4.26/32 + - 1.1.4.28/32 + - 1.1.4.30/32 + - 1.1.4.32/32 + - 1.1.4.34/32 + - 1.1.4.36/32 + - 1.1.4.38/32 + - 1.1.4.40/32 + - 1.1.4.42/32 + - 1.1.4.44/32 + - 1.1.4.46/32 + - 1.1.4.48/32 + - 1.1.4.50/32 + - 1.1.4.52/32 + - 1.1.4.54/32 + - 1.1.4.56/32 + - 1.1.4.58/32 + - 1.1.4.60/32 + - 1.1.4.62/32 + - 1.1.4.64/32 + - 1.1.4.66/32 + - 1.1.4.68/32 + - 1.1.4.70/32 + - 1.1.4.72/32 + - 1.1.4.74/32 + - 1.1.4.76/32 + - 1.1.4.78/32 + - 1.1.4.80/32 + - 1.1.4.82/32 + - 1.1.4.84/32 + - 1.1.4.86/32 + - 1.1.4.88/32 + - 1.1.4.90/32 + - 1.1.4.92/32 + - 1.1.4.94/32 + - 1.1.4.96/32 + - 1.1.4.98/32 + - 1.1.4.100/32 + - 1.1.4.102/32 + - 1.1.4.104/32 + - 1.1.4.106/32 + - 1.1.4.108/32 + - 1.1.4.110/32 + - 1.1.4.112/32 + - 1.1.4.114/32 + - 1.1.4.116/32 + - 1.1.4.118/32 + - 1.1.4.120/32 + - 1.1.4.122/32 + - 1.1.4.124/32 + - 1.1.4.126/32 + - 1.1.4.128/32 + - 1.1.4.130/32 + - 1.1.4.132/32 + - 1.1.4.134/32 + - 1.1.4.136/32 + - 1.1.4.138/32 + - 1.1.4.140/32 + - 1.1.4.142/32 + - 1.1.4.144/32 + - 1.1.4.146/32 + - 1.1.4.148/32 + - 1.1.4.150/32 + - 1.1.4.152/32 + - 1.1.4.154/32 + - 1.1.4.156/32 + - 1.1.4.158/32 + - 1.1.4.160/32 + - 1.1.4.162/32 + - 1.1.4.164/32 + - 1.1.4.166/32 + - 1.1.4.168/32 + - 1.1.4.170/32 + - 1.1.4.172/32 + - 1.1.4.174/32 + - 1.1.4.176/32 + - 1.1.4.178/32 + - 1.1.4.180/32 + - 1.1.4.182/32 + - 1.1.4.184/32 + - 1.1.4.186/32 + - 1.1.4.188/32 + - 1.1.4.190/32 + - 1.1.4.192/32 + - 1.1.4.194/32 + - 1.1.4.196/32 + - 1.1.4.198/32 + - 1.1.4.200/32 + - 1.1.4.202/32 + - 1.1.4.204/32 + - 1.1.4.206/32 + - 1.1.4.208/32 + - 1.1.4.210/32 + - 1.1.4.212/32 + - 1.1.4.214/32 + - 1.1.4.216/32 + - 1.1.4.218/32 + - 1.1.4.220/32 + - 1.1.4.222/32 + - 1.1.4.224/32 + - 1.1.4.226/32 + - 1.1.4.228/32 + - 1.1.4.230/32 + - 1.1.4.232/32 + - 1.1.4.234/32 + - 1.1.4.236/32 + - 1.1.4.238/32 + - 1.1.4.240/32 + - 1.1.4.242/32 + - 1.1.4.244/32 + - 1.1.4.246/32 + - 1.1.4.248/32 + - 1.1.4.250/32 + - 1.1.4.252/32 + - 1.1.4.254/32 + - 1.1.5.2/32 + - 1.1.5.4/32 + - 1.1.5.6/32 + - 1.1.5.8/32 + - 1.1.5.10/32 + - 1.1.5.12/32 + - 1.1.5.14/32 + - 1.1.5.16/32 + - 1.1.5.18/32 + - 1.1.5.20/32 + - 1.1.5.22/32 + - 1.1.5.24/32 + - 1.1.5.26/32 + - 1.1.5.28/32 + - 1.1.5.30/32 + - 1.1.5.32/32 + - 1.1.5.34/32 + - 1.1.5.36/32 + - 1.1.5.38/32 + - 1.1.5.40/32 + - 1.1.5.42/32 + - 1.1.5.44/32 + - 1.1.5.46/32 + - 1.1.5.48/32 + - 1.1.5.50/32 + - 1.1.5.52/32 + - 1.1.5.54/32 + - 1.1.5.56/32 + - 1.1.5.58/32 + - 1.1.5.60/32 + - 1.1.5.62/32 + - 1.1.5.64/32 + - 1.1.5.66/32 + - 1.1.5.68/32 + - 1.1.5.70/32 + - 1.1.5.72/32 + - 1.1.5.74/32 + - 1.1.5.76/32 + - 1.1.5.78/32 + - 1.1.5.80/32 + - 1.1.5.82/32 + - 1.1.5.84/32 + - 1.1.5.86/32 + - 1.1.5.88/32 + - 1.1.5.90/32 + - 1.1.5.92/32 + - 1.1.5.94/32 + - 1.1.5.96/32 + - 1.1.5.98/32 + - 1.1.5.100/32 + - 1.1.5.102/32 + - 1.1.5.104/32 + - 1.1.5.106/32 + - 1.1.5.108/32 + - 1.1.5.110/32 + - 1.1.5.112/32 + - 1.1.5.114/32 + - 1.1.5.116/32 + - 1.1.5.118/32 + - 1.1.5.120/32 + - 1.1.5.122/32 + - 1.1.5.124/32 + - 1.1.5.126/32 + - 1.1.5.128/32 + - 1.1.5.130/32 + - 1.1.5.132/32 + - 1.1.5.134/32 + - 1.1.5.136/32 + - 1.1.5.138/32 + - 1.1.5.140/32 + - 1.1.5.142/32 + - 1.1.5.144/32 + - 1.1.5.146/32 + - 1.1.5.148/32 + - 1.1.5.150/32 + - 1.1.5.152/32 + - 1.1.5.154/32 + - 1.1.5.156/32 + - 1.1.5.158/32 + - 1.1.5.160/32 + - 1.1.5.162/32 + - 1.1.5.164/32 + - 1.1.5.166/32 + - 1.1.5.168/32 + - 1.1.5.170/32 + - 1.1.5.172/32 + - 1.1.5.174/32 + - 1.1.5.176/32 + - 1.1.5.178/32 + - 1.1.5.180/32 + - 1.1.5.182/32 + - 1.1.5.184/32 + - 1.1.5.186/32 + - 1.1.5.188/32 + - 1.1.5.190/32 + - 1.1.5.192/32 + - 1.1.5.194/32 + - 1.1.5.196/32 + - 1.1.5.198/32 + - 1.1.5.200/32 + - 1.1.5.202/32 + - 1.1.5.204/32 + - 1.1.5.206/32 + - 1.1.5.208/32 + - 1.1.5.210/32 + - 1.1.5.212/32 + - 1.1.5.214/32 + - 1.1.5.216/32 + - 1.1.5.218/32 + - 1.1.5.220/32 + - 1.1.5.222/32 + - 1.1.5.224/32 + - 1.1.5.226/32 + - 1.1.5.228/32 + - 1.1.5.230/32 + - 1.1.5.232/32 + - 1.1.5.234/32 + - 1.1.5.236/32 + - 1.1.5.238/32 + - 1.1.5.240/32 + - 1.1.5.242/32 + - 1.1.5.244/32 + - 1.1.5.246/32 + - 1.1.5.248/32 + - 1.1.5.250/32 + - 1.1.5.252/32 + - 1.1.5.254/32 + - 1.1.6.2/32 + - 1.1.6.4/32 + - 1.1.6.6/32 + - 1.1.6.8/32 + - 1.1.6.10/32 + - 1.1.6.12/32 + - 1.1.6.14/32 + - 1.1.6.16/32 + - 1.1.6.18/32 + - 1.1.6.20/32 + - 1.1.6.22/32 + - 1.1.6.24/32 + - 1.1.6.26/32 + - 1.1.6.28/32 + - 1.1.6.30/32 + - 1.1.6.32/32 + - 1.1.6.34/32 + - 1.1.6.36/32 + - 1.1.6.38/32 + - 1.1.6.40/32 + - 1.1.6.42/32 + - 1.1.6.44/32 + - 1.1.6.46/32 + - 1.1.6.48/32 + - 1.1.6.50/32 + - 1.1.6.52/32 + - 1.1.6.54/32 + - 1.1.6.56/32 + - 1.1.6.58/32 + - 1.1.6.60/32 + - 1.1.6.62/32 + - 1.1.6.64/32 + - 1.1.6.66/32 + - 1.1.6.68/32 + - 1.1.6.70/32 + - 1.1.6.72/32 + - 1.1.6.74/32 + - 1.1.6.76/32 + - 1.1.6.78/32 + - 1.1.6.80/32 + - 1.1.6.82/32 + - 1.1.6.84/32 + - 1.1.6.86/32 + - 1.1.6.88/32 + - 1.1.6.90/32 + - 1.1.6.92/32 + - 1.1.6.94/32 + - 1.1.6.96/32 + - 1.1.6.98/32 + - 1.1.6.100/32 + - 1.1.6.102/32 + - 1.1.6.104/32 + - 1.1.6.106/32 + - 1.1.6.108/32 + - 1.1.6.110/32 + - 1.1.6.112/32 + - 1.1.6.114/32 + - 1.1.6.116/32 + - 1.1.6.118/32 + - 1.1.6.120/32 + - 1.1.6.122/32 + - 1.1.6.124/32 + - 1.1.6.126/32 + - 1.1.6.128/32 + - 1.1.6.130/32 + - 1.1.6.132/32 + - 1.1.6.134/32 + - 1.1.6.136/32 + - 1.1.6.138/32 + - 1.1.6.140/32 + - 1.1.6.142/32 + - 1.1.6.144/32 + - 1.1.6.146/32 + - 1.1.6.148/32 + - 1.1.6.150/32 + - 1.1.6.152/32 + - 1.1.6.154/32 + - 1.1.6.156/32 + - 1.1.6.158/32 + - 1.1.6.160/32 + - 1.1.6.162/32 + - 1.1.6.164/32 + - 1.1.6.166/32 + - 1.1.6.168/32 + - 1.1.6.170/32 + - 1.1.6.172/32 + - 1.1.6.174/32 + - 1.1.6.176/32 + - 1.1.6.178/32 + - 1.1.6.180/32 + - 1.1.6.182/32 + - 1.1.6.184/32 + - 1.1.6.186/32 + - 1.1.6.188/32 + - 1.1.6.190/32 + - 1.1.6.192/32 + - 1.1.6.194/32 + - 1.1.6.196/32 + - 1.1.6.198/32 + - 1.1.6.200/32 + - 1.1.6.202/32 + - 1.1.6.204/32 + - 1.1.6.206/32 + - 1.1.6.208/32 + - 1.1.6.210/32 + - 1.1.6.212/32 + - 1.1.6.214/32 + - 1.1.6.216/32 + - 1.1.6.218/32 + - 1.1.6.220/32 + - 1.1.6.222/32 + - 1.1.6.224/32 + - 1.1.6.226/32 + - 1.1.6.228/32 + - 1.1.6.230/32 + - 1.1.6.232/32 + - 1.1.6.234/32 + - 1.1.6.236/32 + - 1.1.6.238/32 + - 1.1.6.240/32 + - 1.1.6.242/32 + - 1.1.6.244/32 + - 1.1.6.246/32 + - 1.1.6.248/32 + - 1.1.6.250/32 + - 1.1.6.252/32 + - 1.1.6.254/32 + - 1.1.7.2/32 + - 1.1.7.4/32 + - 1.1.7.6/32 + - 1.1.7.8/32 + - 1.1.7.10/32 + - 1.1.7.12/32 + - 1.1.7.14/32 + - 1.1.7.16/32 + - 1.1.7.18/32 + - 1.1.7.20/32 + - 1.1.7.22/32 + - 1.1.7.24/32 + - 1.1.7.26/32 + - 1.1.7.28/32 + - 1.1.7.30/32 + - 1.1.7.32/32 + - 1.1.7.34/32 + - 1.1.7.36/32 + - 1.1.7.38/32 + - 1.1.7.40/32 + - 1.1.7.42/32 + - 1.1.7.44/32 + - 1.1.7.46/32 + - 1.1.7.48/32 + - 1.1.7.50/32 + - 1.1.7.52/32 + - 1.1.7.54/32 + - 1.1.7.56/32 + - 1.1.7.58/32 + - 1.1.7.60/32 + - 1.1.7.62/32 + - 1.1.7.64/32 + - 1.1.7.66/32 + - 1.1.7.68/32 + - 1.1.7.70/32 + - 1.1.7.72/32 + - 1.1.7.74/32 + - 1.1.7.76/32 + - 1.1.7.78/32 + - 1.1.7.80/32 + - 1.1.7.82/32 + - 1.1.7.84/32 + - 1.1.7.86/32 + - 1.1.7.88/32 + - 1.1.7.90/32 + - 1.1.7.92/32 + - 1.1.7.94/32 + - 1.1.7.96/32 + - 1.1.7.98/32 + - 1.1.7.100/32 + - 1.1.7.102/32 + - 1.1.7.104/32 + - 1.1.7.106/32 + - 1.1.7.108/32 + - 1.1.7.110/32 + - 1.1.7.112/32 + - 1.1.7.114/32 + - 1.1.7.116/32 + - 1.1.7.118/32 + - 1.1.7.120/32 + - 1.1.7.122/32 + - 1.1.7.124/32 + - 1.1.7.126/32 + - 1.1.7.128/32 + - 1.1.7.130/32 + - 1.1.7.132/32 + - 1.1.7.134/32 + - 1.1.7.136/32 + - 1.1.7.138/32 + - 1.1.7.140/32 + - 1.1.7.142/32 + - 1.1.7.144/32 + - 1.1.7.146/32 + - 1.1.7.148/32 + - 1.1.7.150/32 + - 1.1.7.152/32 + - 1.1.7.154/32 + - 1.1.7.156/32 + - 1.1.7.158/32 + - 1.1.7.160/32 + - 1.1.7.162/32 + - 1.1.7.164/32 + - 1.1.7.166/32 + - 1.1.7.168/32 + - 1.1.7.170/32 + - 1.1.7.172/32 + - 1.1.7.174/32 + - 1.1.7.176/32 + - 1.1.7.178/32 + - 1.1.7.180/32 + - 1.1.7.182/32 + - 1.1.7.184/32 + - 1.1.7.186/32 + - 1.1.7.188/32 + - 1.1.7.190/32 + - 1.1.7.192/32 + - 1.1.7.194/32 + - 1.1.7.196/32 + - 1.1.7.198/32 + - 1.1.7.200/32 + - 1.1.7.202/32 + - 1.1.7.204/32 + - 1.1.7.206/32 + - 1.1.7.208/32 + - 1.1.7.210/32 + - 1.1.7.212/32 + - 1.1.7.214/32 + - 1.1.7.216/32 + - 1.1.7.218/32 + - 1.1.7.220/32 + - 1.1.7.222/32 + - 1.1.7.224/32 + - 1.1.7.226/32 + - 1.1.7.228/32 + - 1.1.7.230/32 + - 1.1.7.232/32 + - 1.1.7.234/32 + - 1.1.7.236/32 + - 1.1.7.238/32 + - 1.1.7.240/32 + - 1.1.7.242/32 + - 1.1.7.244/32 + - 1.1.7.246/32 + - 1.1.7.248/32 + - 1.1.7.250/32 + - 1.1.7.252/32 + - 1.1.7.254/32 + - 1.1.8.2/32 + - 1.1.8.4/32 + - 1.1.8.6/32 + - 1.1.8.8/32 + - 1.1.8.10/32 + - 1.1.8.12/32 + - 1.1.8.14/32 + - 1.1.8.16/32 + - 1.1.8.18/32 + - 1.1.8.20/32 + - 1.1.8.22/32 + - 1.1.8.24/32 + - 1.1.8.26/32 + - 1.1.8.28/32 + - 1.1.8.30/32 + - 1.1.8.32/32 + - 1.1.8.34/32 + - 1.1.8.36/32 + - 1.1.8.38/32 + - 1.1.8.40/32 + - 1.1.8.42/32 + - 1.1.8.44/32 + - 1.1.8.46/32 + - 1.1.8.48/32 + - 1.1.8.50/32 + - 1.1.8.52/32 + - 1.1.8.54/32 + - 1.1.8.56/32 + - 1.1.8.58/32 + - 1.1.8.60/32 + - 1.1.8.62/32 + - 1.1.8.64/32 + - 1.1.8.66/32 + - 1.1.8.68/32 + - 1.1.8.70/32 + - 1.1.8.72/32 + - 1.1.8.74/32 + - 1.1.8.76/32 + - 1.1.8.78/32 + - 1.1.8.80/32 + - 1.1.8.82/32 + - 1.1.8.84/32 + - 1.1.8.86/32 + - 1.1.8.88/32 + - 1.1.8.90/32 + - 1.1.8.92/32 + - 1.1.8.94/32 + - 1.1.8.96/32 + - 1.1.8.98/32 + - 1.1.8.100/32 + - 1.1.8.102/32 + - 1.1.8.104/32 + - 1.1.8.106/32 + - 1.1.8.108/32 + - 1.1.8.110/32 + - 1.1.8.112/32 + - 1.1.8.114/32 + - 1.1.8.116/32 + - 1.1.8.118/32 + - 1.1.8.120/32 + - 1.1.8.122/32 + - 1.1.8.124/32 + - 1.1.8.126/32 + - 1.1.8.128/32 + - 1.1.8.130/32 + - 1.1.8.132/32 + - 1.1.8.134/32 + - 1.1.8.136/32 + - 1.1.8.138/32 + - 1.1.8.140/32 + - 1.1.8.142/32 + - 1.1.8.144/32 + - 1.1.8.146/32 + - 1.1.8.148/32 + - 1.1.8.150/32 + - 1.1.8.152/32 + - 1.1.8.154/32 + - 1.1.8.156/32 + - 1.1.8.158/32 + - 1.1.8.160/32 + - 1.1.8.162/32 + - 1.1.8.164/32 + - 1.1.8.166/32 + - 1.1.8.168/32 + - 1.1.8.170/32 + - 1.1.8.172/32 + - 1.1.8.174/32 + - 1.1.8.176/32 + - 1.1.8.178/32 + - 1.1.8.180/32 + - 1.1.8.182/32 + - 1.1.8.184/32 + - 1.1.8.186/32 + - 1.1.8.188/32 + - 1.1.8.190/32 + - 1.1.8.192/32 + - 1.1.8.194/32 + - 1.1.8.196/32 + - 1.1.8.198/32 + - 1.1.8.200/32 + - 1.1.8.202/32 + - 1.1.8.204/32 + - 1.1.8.206/32 + - 1.1.8.208/32 + - 1.1.8.210/32 + - 1.1.8.212/32 + - 1.1.8.214/32 + - 1.1.8.216/32 + - 1.1.8.218/32 + - 1.1.8.220/32 + - 1.1.8.222/32 + - 1.1.8.224/32 + - 1.1.8.226/32 + - 1.1.8.228/32 + - 1.1.8.230/32 + - 1.1.8.232/32 + - 1.1.8.234/32 + - 1.1.8.236/32 + - 1.1.8.238/32 + - 1.1.8.240/32 + - 1.1.8.242/32 + - 1.1.8.244/32 + - 1.1.8.246/32 + - 1.1.8.248/32 + - 1.1.8.250/32 + - 1.1.8.252/32 + - 1.1.8.254/32 + - 1.1.9.2/32 + - 1.1.9.4/32 + - 1.1.9.6/32 + - 1.1.9.8/32 + - 1.1.9.10/32 + - 1.1.9.12/32 + - 1.1.9.14/32 + - 1.1.9.16/32 + - 1.1.9.18/32 + - 1.1.9.20/32 + - 1.1.9.22/32 + - 1.1.9.24/32 + - 1.1.9.26/32 + - 1.1.9.28/32 + - 1.1.9.30/32 + - 1.1.9.32/32 + - 1.1.9.34/32 + - 1.1.9.36/32 + - 1.1.9.38/32 + - 1.1.9.40/32 + - 1.1.9.42/32 + - 1.1.9.44/32 + - 1.1.9.46/32 + - 1.1.9.48/32 + - 1.1.9.50/32 + - 1.1.9.52/32 + - 1.1.9.54/32 + - 1.1.9.56/32 + - 1.1.9.58/32 + - 1.1.9.60/32 + - 1.1.9.62/32 + - 1.1.9.64/32 + - 1.1.9.66/32 + - 1.1.9.68/32 + - 1.1.9.70/32 + - 1.1.9.72/32 + - 1.1.9.74/32 + - 1.1.9.76/32 + - 1.1.9.78/32 + - 1.1.9.80/32 + - 1.1.9.82/32 + - 1.1.9.84/32 + - 1.1.9.86/32 + - 1.1.9.88/32 + - 1.1.9.90/32 + - 1.1.9.92/32 + - 1.1.9.94/32 + - 1.1.9.96/32 + - 1.1.9.98/32 + - 1.1.9.100/32 + - 1.1.9.102/32 + - 1.1.9.104/32 + - 1.1.9.106/32 + - 1.1.9.108/32 + - 1.1.9.110/32 + - 1.1.9.112/32 + - 1.1.9.114/32 + - 1.1.9.116/32 + - 1.1.9.118/32 + - 1.1.9.120/32 + - 1.1.9.122/32 + - 1.1.9.124/32 + - 1.1.9.126/32 + - 1.1.9.128/32 + - 1.1.9.130/32 + - 1.1.9.132/32 + - 1.1.9.134/32 + - 1.1.9.136/32 + - 1.1.9.138/32 + - 1.1.9.140/32 + - 1.1.9.142/32 + - 1.1.9.144/32 + - 1.1.9.146/32 + - 1.1.9.148/32 + - 1.1.9.150/32 + - 1.1.9.152/32 + - 1.1.9.154/32 + - 1.1.9.156/32 + - 1.1.9.158/32 + - 1.1.9.160/32 + - 1.1.9.162/32 + - 1.1.9.164/32 + - 1.1.9.166/32 + - 1.1.9.168/32 + - 1.1.9.170/32 + - 1.1.9.172/32 + - 1.1.9.174/32 + - 1.1.9.176/32 + - 1.1.9.178/32 + - 1.1.9.180/32 + - 1.1.9.182/32 + - 1.1.9.184/32 + - 1.1.9.186/32 + - 1.1.9.188/32 + - 1.1.9.190/32 + - 1.1.9.192/32 + - 1.1.9.194/32 + - 1.1.9.196/32 + - 1.1.9.198/32 + - 1.1.9.200/32 + - 1.1.9.202/32 + - 1.1.9.204/32 + - 1.1.9.206/32 + - 1.1.9.208/32 + - 1.1.9.210/32 + - 1.1.9.212/32 + - 1.1.9.214/32 + - 1.1.9.216/32 + - 1.1.9.218/32 + - 1.1.9.220/32 + - 1.1.9.222/32 + - 1.1.9.224/32 + - 1.1.9.226/32 + - 1.1.9.228/32 + - 1.1.9.230/32 + - 1.1.9.232/32 + - 1.1.9.234/32 + - 1.1.9.236/32 + - 1.1.9.238/32 + - 1.1.9.240/32 + - 1.1.9.242/32 + - 1.1.9.244/32 + - 1.1.9.246/32 + - 1.1.9.248/32 + - 1.1.9.250/32 + - 1.1.9.252/32 + - 1.1.9.254/32 + - 1.1.10.2/32 + - 1.1.10.4/32 + - 1.1.10.6/32 + - 1.1.10.8/32 + - 1.1.10.10/32 + - 1.1.10.12/32 + - 1.1.10.14/32 + - 1.1.10.16/32 + - 1.1.10.18/32 + - 1.1.10.20/32 + - 1.1.10.22/32 + - 1.1.10.24/32 + - 1.1.10.26/32 + - 1.1.10.28/32 + - 1.1.10.30/32 + - 1.1.10.32/32 + - 1.1.10.34/32 + - 1.1.10.36/32 + - 1.1.10.38/32 + - 1.1.10.40/32 + - 1.1.10.42/32 + - 1.1.10.44/32 + - 1.1.10.46/32 + - 1.1.10.48/32 + - 1.1.10.50/32 + - 1.1.10.52/32 + - 1.1.10.54/32 + - 1.1.10.56/32 + - 1.1.10.58/32 + - 1.1.10.60/32 + - 1.1.10.62/32 + - 1.1.10.64/32 + - 1.1.10.66/32 + - 1.1.10.68/32 + - 1.1.10.70/32 + - 1.1.10.72/32 + - 1.1.10.74/32 + - 1.1.10.76/32 + - 1.1.10.78/32 + - 1.1.10.80/32 + - 1.1.10.82/32 + - 1.1.10.84/32 + - 1.1.10.86/32 + - 1.1.10.88/32 + - 1.1.10.90/32 + - 1.1.10.92/32 + - 1.1.10.94/32 + - 1.1.10.96/32 + - 1.1.10.98/32 + - 1.1.10.100/32 + - 1.1.10.102/32 + - 1.1.10.104/32 + - 1.1.10.106/32 + - 1.1.10.108/32 + - 1.1.10.110/32 + - 1.1.10.112/32 + - 1.1.10.114/32 + - 1.1.10.116/32 + - 1.1.10.118/32 + - 1.1.10.120/32 + - 1.1.10.122/32 + - 1.1.10.124/32 + - 1.1.10.126/32 + - 1.1.10.128/32 + - 1.1.10.130/32 + - 1.1.10.132/32 + - 1.1.10.134/32 + - 1.1.10.136/32 + - 1.1.10.138/32 + - 1.1.10.140/32 + - 1.1.10.142/32 + - 1.1.10.144/32 + - 1.1.10.146/32 + - 1.1.10.148/32 + - 1.1.10.150/32 + - 1.1.10.152/32 + - 1.1.10.154/32 + - 1.1.10.156/32 + - 1.1.10.158/32 + - 1.1.10.160/32 + - 1.1.10.162/32 + - 1.1.10.164/32 + - 1.1.10.166/32 + - 1.1.10.168/32 + - 1.1.10.170/32 + - 1.1.10.172/32 + - 1.1.10.174/32 + - 1.1.10.176/32 + - 1.1.10.178/32 + - 1.1.10.180/32 + - 1.1.10.182/32 + - 1.1.10.184/32 + - 1.1.10.186/32 + - 1.1.10.188/32 + - 1.1.10.190/32 + - 1.1.10.192/32 + - 1.1.10.194/32 + - 1.1.10.196/32 + - 1.1.10.198/32 + - 1.1.10.200/32 + - 1.1.10.202/32 + - 1.1.10.204/32 + - 1.1.10.206/32 + - 1.1.10.208/32 + - 1.1.10.210/32 + - 1.1.10.212/32 + - 1.1.10.214/32 + - 1.1.10.216/32 + - 1.1.10.218/32 + - 1.1.10.220/32 + - 1.1.10.222/32 + - 1.1.10.224/32 + - 1.1.10.226/32 + - 1.1.10.228/32 + - 1.1.10.230/32 + - 1.1.10.232/32 + - 1.1.10.234/32 + - 1.1.10.236/32 + - 1.1.10.238/32 + - 1.1.10.240/32 + - 1.1.10.242/32 + - 1.1.10.244/32 + - 1.1.10.246/32 + - 1.1.10.248/32 + - 1.1.10.250/32 + - 1.1.10.252/32 + - 1.1.10.254/32 + - 1.1.11.2/32 + - 1.1.11.4/32 + - 1.1.11.6/32 + - 1.1.11.8/32 + - 1.1.11.10/32 + - 1.1.11.12/32 + - 1.1.11.14/32 + - 1.1.11.16/32 + - 1.1.11.18/32 + - 1.1.11.20/32 + - 1.1.11.22/32 + - 1.1.11.24/32 + - 1.1.11.26/32 + - 1.1.11.28/32 + - 1.1.11.30/32 + - 1.1.11.32/32 + - 1.1.11.34/32 + - 1.1.11.36/32 + - 1.1.11.38/32 + - 1.1.11.40/32 + - 1.1.11.42/32 + - 1.1.11.44/32 + - 1.1.11.46/32 + - 1.1.11.48/32 + - 1.1.11.50/32 + - 1.1.11.52/32 + - 1.1.11.54/32 + - 1.1.11.56/32 + - 1.1.11.58/32 + - 1.1.11.60/32 + - 1.1.11.62/32 + - 1.1.11.64/32 + - 1.1.11.66/32 + - 1.1.11.68/32 + - 1.1.11.70/32 + - 1.1.11.72/32 + - 1.1.11.74/32 + - 1.1.11.76/32 + - 1.1.11.78/32 + - 1.1.11.80/32 + - 1.1.11.82/32 + - 1.1.11.84/32 + - 1.1.11.86/32 + - 1.1.11.88/32 + - 1.1.11.90/32 + - 1.1.11.92/32 + - 1.1.11.94/32 + - 1.1.11.96/32 + - 1.1.11.98/32 + - 1.1.11.100/32 + - 1.1.11.102/32 + - 1.1.11.104/32 + - 1.1.11.106/32 + - 1.1.11.108/32 + - 1.1.11.110/32 + - 1.1.11.112/32 + - 1.1.11.114/32 + - 1.1.11.116/32 + - 1.1.11.118/32 + - 1.1.11.120/32 + - 1.1.11.122/32 + - 1.1.11.124/32 + - 1.1.11.126/32 + - 1.1.11.128/32 + - 1.1.11.130/32 + - 1.1.11.132/32 + - 1.1.11.134/32 + - 1.1.11.136/32 + - 1.1.11.138/32 + - 1.1.11.140/32 + - 1.1.11.142/32 + - 1.1.11.144/32 + - 1.1.11.146/32 + - 1.1.11.148/32 + - 1.1.11.150/32 + - 1.1.11.152/32 + - 1.1.11.154/32 + - 1.1.11.156/32 + - 1.1.11.158/32 + - 1.1.11.160/32 + - 1.1.11.162/32 + - 1.1.11.164/32 + - 1.1.11.166/32 + - 1.1.11.168/32 + - 1.1.11.170/32 + - 1.1.11.172/32 + - 1.1.11.174/32 + - 1.1.11.176/32 + - 1.1.11.178/32 + - 1.1.11.180/32 + - 1.1.11.182/32 + - 1.1.11.184/32 + - 1.1.11.186/32 + - 1.1.11.188/32 + - 1.1.11.190/32 + - 1.1.11.192/32 + - 1.1.11.194/32 + - 1.1.11.196/32 + - 1.1.11.198/32 + - 1.1.11.200/32 + - 1.1.11.202/32 + - 1.1.11.204/32 + - 1.1.11.206/32 + - 1.1.11.208/32 + - 1.1.11.210/32 + - 1.1.11.212/32 + - 1.1.11.214/32 + - 1.1.11.216/32 + - 1.1.11.218/32 + - 1.1.11.220/32 + - 1.1.11.222/32 + - 1.1.11.224/32 + - 1.1.11.226/32 + - 1.1.11.228/32 + - 1.1.11.230/32 + - 1.1.11.232/32 + - 1.1.11.234/32 + - 1.1.11.236/32 + - 1.1.11.238/32 + - 1.1.11.240/32 + - 1.1.11.242/32 + - 1.1.11.244/32 + - 1.1.11.246/32 + - 1.1.11.248/32 + - 1.1.11.250/32 + - 1.1.11.252/32 + - 1.1.11.254/32 + - 1.1.12.2/32 + - 1.1.12.4/32 + - 1.1.12.6/32 + - 1.1.12.8/32 + - 1.1.12.10/32 + - 1.1.12.12/32 + - 1.1.12.14/32 + - 1.1.12.16/32 + - 1.1.12.18/32 + - 1.1.12.20/32 + - 1.1.12.22/32 + - 1.1.12.24/32 + - 1.1.12.26/32 + - 1.1.12.28/32 + - 1.1.12.30/32 + - 1.1.12.32/32 + - 1.1.12.34/32 + - 1.1.12.36/32 + - 1.1.12.38/32 + - 1.1.12.40/32 + - 1.1.12.42/32 + - 1.1.12.44/32 + - 1.1.12.46/32 + - 1.1.12.48/32 + - 1.1.12.50/32 + - 1.1.12.52/32 + - 1.1.12.54/32 + - 1.1.12.56/32 + - 1.1.12.58/32 + - 1.1.12.60/32 + - 1.1.12.62/32 + - 1.1.12.64/32 + - 1.1.12.66/32 + - 1.1.12.68/32 + - 1.1.12.70/32 + - 1.1.12.72/32 + - 1.1.12.74/32 + - 1.1.12.76/32 + - 1.1.12.78/32 + - 1.1.12.80/32 + - 1.1.12.82/32 + - 1.1.12.84/32 + - 1.1.12.86/32 + - 1.1.12.88/32 + - 1.1.12.90/32 + - 1.1.12.92/32 + - 1.1.12.94/32 + - 1.1.12.96/32 + - 1.1.12.98/32 + - 1.1.12.100/32 + - 1.1.12.102/32 + - 1.1.12.104/32 + - 1.1.12.106/32 + - 1.1.12.108/32 + - 1.1.12.110/32 + - 1.1.12.112/32 + - 1.1.12.114/32 + - 1.1.12.116/32 + - 1.1.12.118/32 + - 1.1.12.120/32 + - 1.1.12.122/32 + - 1.1.12.124/32 + - 1.1.12.126/32 + - 1.1.12.128/32 + - 1.1.12.130/32 + - 1.1.12.132/32 + - 1.1.12.134/32 + - 1.1.12.136/32 + - 1.1.12.138/32 + - 1.1.12.140/32 + - 1.1.12.142/32 + - 1.1.12.144/32 + - 1.1.12.146/32 + - 1.1.12.148/32 + - 1.1.12.150/32 + - 1.1.12.152/32 + - 1.1.12.154/32 + - 1.1.12.156/32 + - 1.1.12.158/32 + - 1.1.12.160/32 + - 1.1.12.162/32 + - 1.1.12.164/32 + - 1.1.12.166/32 + - 1.1.12.168/32 + - 1.1.12.170/32 + - 1.1.12.172/32 + - 1.1.12.174/32 + - 1.1.12.176/32 + - 1.1.12.178/32 + - 1.1.12.180/32 + - 1.1.12.182/32 + - 1.1.12.184/32 + - 1.1.12.186/32 + - 1.1.12.188/32 + - 1.1.12.190/32 + - 1.1.12.192/32 + - 1.1.12.194/32 + - 1.1.12.196/32 + - 1.1.12.198/32 + - 1.1.12.200/32 + - 1.1.12.202/32 + - 1.1.12.204/32 + - 1.1.12.206/32 + - 1.1.12.208/32 + - 1.1.12.210/32 + - 1.1.12.212/32 + - 1.1.12.214/32 + - 1.1.12.216/32 + - 1.1.12.218/32 + - 1.1.12.220/32 + - 1.1.12.222/32 + - 1.1.12.224/32 + - 1.1.12.226/32 + - 1.1.12.228/32 + - 1.1.12.230/32 + - 1.1.12.232/32 + - 1.1.12.234/32 + - 1.1.12.236/32 + - 1.1.12.238/32 + - 1.1.12.240/32 + - 1.1.12.242/32 + - 1.1.12.244/32 + - 1.1.12.246/32 + - 1.1.12.248/32 + - 1.1.12.250/32 + - 1.1.12.252/32 + - 1.1.12.254/32 + - 1.1.13.2/32 + - 1.1.13.4/32 + - 1.1.13.6/32 + - 1.1.13.8/32 + - 1.1.13.10/32 + - 1.1.13.12/32 + - 1.1.13.14/32 + - 1.1.13.16/32 + - 1.1.13.18/32 + - 1.1.13.20/32 + - 1.1.13.22/32 + - 1.1.13.24/32 + - 1.1.13.26/32 + - 1.1.13.28/32 + - 1.1.13.30/32 + - 1.1.13.32/32 + - 1.1.13.34/32 + - 1.1.13.36/32 + - 1.1.13.38/32 + - 1.1.13.40/32 + - 1.1.13.42/32 + - 1.1.13.44/32 + - 1.1.13.46/32 + - 1.1.13.48/32 + - 1.1.13.50/32 + - 1.1.13.52/32 + - 1.1.13.54/32 + - 1.1.13.56/32 + - 1.1.13.58/32 + - 1.1.13.60/32 + - 1.1.13.62/32 + - 1.1.13.64/32 + - 1.1.13.66/32 + - 1.1.13.68/32 + - 1.1.13.70/32 + - 1.1.13.72/32 + - 1.1.13.74/32 + - 1.1.13.76/32 + - 1.1.13.78/32 + - 1.1.13.80/32 + - 1.1.13.82/32 + - 1.1.13.84/32 + - 1.1.13.86/32 + - 1.1.13.88/32 + - 1.1.13.90/32 + - 1.1.13.92/32 + - 1.1.13.94/32 + - 1.1.13.96/32 + - 1.1.13.98/32 + - 1.1.13.100/32 + - 1.1.13.102/32 + - 1.1.13.104/32 + - 1.1.13.106/32 + - 1.1.13.108/32 + - 1.1.13.110/32 + - 1.1.13.112/32 + - 1.1.13.114/32 + - 1.1.13.116/32 + - 1.1.13.118/32 + - 1.1.13.120/32 + - 1.1.13.122/32 + - 1.1.13.124/32 + - 1.1.13.126/32 + - 1.1.13.128/32 + - 1.1.13.130/32 + - 1.1.13.132/32 + - 1.1.13.134/32 + - 1.1.13.136/32 + - 1.1.13.138/32 + - 1.1.13.140/32 + - 1.1.13.142/32 + - 1.1.13.144/32 + - 1.1.13.146/32 + - 1.1.13.148/32 + - 1.1.13.150/32 + - 1.1.13.152/32 + - 1.1.13.154/32 + - 1.1.13.156/32 + - 1.1.13.158/32 + - 1.1.13.160/32 + - 1.1.13.162/32 + - 1.1.13.164/32 + - 1.1.13.166/32 + - 1.1.13.168/32 + - 1.1.13.170/32 + - 1.1.13.172/32 + - 1.1.13.174/32 + - 1.1.13.176/32 + - 1.1.13.178/32 + - 1.1.13.180/32 + - 1.1.13.182/32 + - 1.1.13.184/32 + - 1.1.13.186/32 + - 1.1.13.188/32 + - 1.1.13.190/32 + - 1.1.13.192/32 + - 1.1.13.194/32 + - 1.1.13.196/32 + - 1.1.13.198/32 + - 1.1.13.200/32 + - 1.1.13.202/32 + - 1.1.13.204/32 + - 1.1.13.206/32 + - 1.1.13.208/32 + - 1.1.13.210/32 + - 1.1.13.212/32 + - 1.1.13.214/32 + - 1.1.13.216/32 + - 1.1.13.218/32 + - 1.1.13.220/32 + - 1.1.13.222/32 + - 1.1.13.224/32 + - 1.1.13.226/32 + - 1.1.13.228/32 + - 1.1.13.230/32 + - 1.1.13.232/32 + - 1.1.13.234/32 + - 1.1.13.236/32 + - 1.1.13.238/32 + - 1.1.13.240/32 + - 1.1.13.242/32 + - 1.1.13.244/32 + - 1.1.13.246/32 + - 1.1.13.248/32 + - 1.1.13.250/32 + - 1.1.13.252/32 + - 1.1.13.254/32 + - 1.1.14.2/32 + - 1.1.14.4/32 + - 1.1.14.6/32 + - 1.1.14.8/32 + - 1.1.14.10/32 + - 1.1.14.12/32 + - 1.1.14.14/32 + - 1.1.14.16/32 + - 1.1.14.18/32 + - 1.1.14.20/32 + - 1.1.14.22/32 + - 1.1.14.24/32 + - 1.1.14.26/32 + - 1.1.14.28/32 + - 1.1.14.30/32 + - 1.1.14.32/32 + - 1.1.14.34/32 + - 1.1.14.36/32 + - 1.1.14.38/32 + - 1.1.14.40/32 + - 1.1.14.42/32 + - 1.1.14.44/32 + - 1.1.14.46/32 + - 1.1.14.48/32 + - 1.1.14.50/32 + - 1.1.14.52/32 + - 1.1.14.54/32 + - 1.1.14.56/32 + - 1.1.14.58/32 + - 1.1.14.60/32 + - 1.1.14.62/32 + - 1.1.14.64/32 + - 1.1.14.66/32 + - 1.1.14.68/32 + - 1.1.14.70/32 + - 1.1.14.72/32 + - 1.1.14.74/32 + - 1.1.14.76/32 + - 1.1.14.78/32 + - 1.1.14.80/32 + - 1.1.14.82/32 + - 1.1.14.84/32 + - 1.1.14.86/32 + - 1.1.14.88/32 + - 1.1.14.90/32 + - 1.1.14.92/32 + - 1.1.14.94/32 + - 1.1.14.96/32 + - 1.1.14.98/32 + - 1.1.14.100/32 + - 1.1.14.102/32 + - 1.1.14.104/32 + - 1.1.14.106/32 + - 1.1.14.108/32 + - 1.1.14.110/32 + - 1.1.14.112/32 + - 1.1.14.114/32 + - 1.1.14.116/32 + - 1.1.14.118/32 + - 1.1.14.120/32 + - 1.1.14.122/32 + - 1.1.14.124/32 + - 1.1.14.126/32 + - 1.1.14.128/32 + - 1.1.14.130/32 + - 1.1.14.132/32 + - 1.1.14.134/32 + - 1.1.14.136/32 + - 1.1.14.138/32 + - 1.1.14.140/32 + - 1.1.14.142/32 + - 1.1.14.144/32 + - 1.1.14.146/32 + - 1.1.14.148/32 + - 1.1.14.150/32 + - 1.1.14.152/32 + - 1.1.14.154/32 + - 1.1.14.156/32 + - 1.1.14.158/32 + - 1.1.14.160/32 + - 1.1.14.162/32 + - 1.1.14.164/32 + - 1.1.14.166/32 + - 1.1.14.168/32 + - 1.1.14.170/32 + - 1.1.14.172/32 + - 1.1.14.174/32 + - 1.1.14.176/32 + - 1.1.14.178/32 + - 1.1.14.180/32 + - 1.1.14.182/32 + - 1.1.14.184/32 + - 1.1.14.186/32 + - 1.1.14.188/32 + - 1.1.14.190/32 + - 1.1.14.192/32 + - 1.1.14.194/32 + - 1.1.14.196/32 + - 1.1.14.198/32 + - 1.1.14.200/32 + - 1.1.14.202/32 + - 1.1.14.204/32 + - 1.1.14.206/32 + - 1.1.14.208/32 + - 1.1.14.210/32 + - 1.1.14.212/32 + - 1.1.14.214/32 + - 1.1.14.216/32 + - 1.1.14.218/32 + - 1.1.14.220/32 + - 1.1.14.222/32 + - 1.1.14.224/32 + - 1.1.14.226/32 + - 1.1.14.228/32 + - 1.1.14.230/32 + - 1.1.14.232/32 + - 1.1.14.234/32 + - 1.1.14.236/32 + - 1.1.14.238/32 + - 1.1.14.240/32 + - 1.1.14.242/32 + - 1.1.14.244/32 + - 1.1.14.246/32 + - 1.1.14.248/32 + - 1.1.14.250/32 + - 1.1.14.252/32 + - 1.1.14.254/32 + - 1.1.15.2/32 + - 1.1.15.4/32 + - 1.1.15.6/32 + - 1.1.15.8/32 + - 1.1.15.10/32 + - 1.1.15.12/32 + - 1.1.15.14/32 + - 1.1.15.16/32 + - 1.1.15.18/32 + - 1.1.15.20/32 + - 1.1.15.22/32 + - 1.1.15.24/32 + - 1.1.15.26/32 + - 1.1.15.28/32 + - 1.1.15.30/32 + - 1.1.15.32/32 + - 1.1.15.34/32 + - 1.1.15.36/32 + - 1.1.15.38/32 + - 1.1.15.40/32 + - 1.1.15.42/32 + - 1.1.15.44/32 + - 1.1.15.46/32 + - 1.1.15.48/32 + - 1.1.15.50/32 + - 1.1.15.52/32 + - 1.1.15.54/32 + - 1.1.15.56/32 + - 1.1.15.58/32 + - 1.1.15.60/32 + - 1.1.15.62/32 + - 1.1.15.64/32 + - 1.1.15.66/32 + - 1.1.15.68/32 + - 1.1.15.70/32 + - 1.1.15.72/32 + - 1.1.15.74/32 + - 1.1.15.76/32 + - 1.1.15.78/32 + - 1.1.15.80/32 + - 1.1.15.82/32 + - 1.1.15.84/32 + - 1.1.15.86/32 + - 1.1.15.88/32 + - 1.1.15.90/32 + - 1.1.15.92/32 + - 1.1.15.94/32 + - 1.1.15.96/32 + - 1.1.15.98/32 + - 1.1.15.100/32 + - 1.1.15.102/32 + - 1.1.15.104/32 + - 1.1.15.106/32 + - 1.1.15.108/32 + - 1.1.15.110/32 + - 1.1.15.112/32 + - 1.1.15.114/32 + - 1.1.15.116/32 + - 1.1.15.118/32 + - 1.1.15.120/32 + - 1.1.15.122/32 + - 1.1.15.124/32 + - 1.1.15.126/32 + - 1.1.15.128/32 + - 1.1.15.130/32 + - 1.1.15.132/32 + - 1.1.15.134/32 + - 1.1.15.136/32 + - 1.1.15.138/32 + - 1.1.15.140/32 + - 1.1.15.142/32 + - 1.1.15.144/32 + - 1.1.15.146/32 + - 1.1.15.148/32 + - 1.1.15.150/32 + - 1.1.15.152/32 + - 1.1.15.154/32 + - 1.1.15.156/32 + - 1.1.15.158/32 + - 1.1.15.160/32 + - 1.1.15.162/32 + - 1.1.15.164/32 + - 1.1.15.166/32 + - 1.1.15.168/32 + - 1.1.15.170/32 + - 1.1.15.172/32 + - 1.1.15.174/32 + - 1.1.15.176/32 + - 1.1.15.178/32 + - 1.1.15.180/32 + - 1.1.15.182/32 + - 1.1.15.184/32 + - 1.1.15.186/32 + - 1.1.15.188/32 + - 1.1.15.190/32 + - 1.1.15.192/32 + - 1.1.15.194/32 + - 1.1.15.196/32 + - 1.1.15.198/32 + - 1.1.15.200/32 + - 1.1.15.202/32 + - 1.1.15.204/32 + - 1.1.15.206/32 + - 1.1.15.208/32 + - 1.1.15.210/32 + - 1.1.15.212/32 + - 1.1.15.214/32 + - 1.1.15.216/32 + - 1.1.15.218/32 + - 1.1.15.220/32 + - 1.1.15.222/32 + - 1.1.15.224/32 + - 1.1.15.226/32 + - 1.1.15.228/32 + - 1.1.15.230/32 + - 1.1.15.232/32 + - 1.1.15.234/32 + - 1.1.15.236/32 + - 1.1.15.238/32 + - 1.1.15.240/32 + - 1.1.15.242/32 + - 1.1.15.244/32 + - 1.1.15.246/32 + - 1.1.15.248/32 + - 1.1.15.250/32 + - 1.1.15.252/32 + - 1.1.15.254/32 + - 1.1.16.2/32 + - 1.1.16.4/32 + - 1.1.16.6/32 + - 1.1.16.8/32 + - 1.1.16.10/32 + - 1.1.16.12/32 + - 1.1.16.14/32 + - 1.1.16.16/32 + - 1.1.16.18/32 + - 1.1.16.20/32 + - 1.1.16.22/32 + - 1.1.16.24/32 + - 1.1.16.26/32 + - 1.1.16.28/32 + - 1.1.16.30/32 + - 1.1.16.32/32 + - 1.1.16.34/32 + - 1.1.16.36/32 + - 1.1.16.38/32 + - 1.1.16.40/32 + - 1.1.16.42/32 + - 1.1.16.44/32 + - 1.1.16.46/32 + - 1.1.16.48/32 + - 1.1.16.50/32 + - 1.1.16.52/32 + - 1.1.16.54/32 + - 1.1.16.56/32 + - 1.1.16.58/32 + - 1.1.16.60/32 + - 1.1.16.62/32 + - 1.1.16.64/32 + - 1.1.16.66/32 + - 1.1.16.68/32 + - 1.1.16.70/32 + - 1.1.16.72/32 + - 1.1.16.74/32 + - 1.1.16.76/32 + - 1.1.16.78/32 + - 1.1.16.80/32 + - 1.1.16.82/32 + - 1.1.16.84/32 + - 1.1.16.86/32 + - 1.1.16.88/32 + - 1.1.16.90/32 + - 1.1.16.92/32 + - 1.1.16.94/32 + - 1.1.16.96/32 + - 1.1.16.98/32 + - 1.1.16.100/32 + - 1.1.16.102/32 + - 1.1.16.104/32 + - 1.1.16.106/32 + - 1.1.16.108/32 + - 1.1.16.110/32 + - 1.1.16.112/32 + - 1.1.16.114/32 + - 1.1.16.116/32 + - 1.1.16.118/32 + - 1.1.16.120/32 + - 1.1.16.122/32 + - 1.1.16.124/32 + - 1.1.16.126/32 + - 1.1.16.128/32 + - 1.1.16.130/32 + - 1.1.16.132/32 + - 1.1.16.134/32 + - 1.1.16.136/32 + - 1.1.16.138/32 + - 1.1.16.140/32 + - 1.1.16.142/32 + - 1.1.16.144/32 + - 1.1.16.146/32 + - 1.1.16.148/32 + - 1.1.16.150/32 + - 1.1.16.152/32 + - 1.1.16.154/32 + - 1.1.16.156/32 + - 1.1.16.158/32 + - 1.1.16.160/32 + - 1.1.16.162/32 + - 1.1.16.164/32 + - 1.1.16.166/32 + - 1.1.16.168/32 + - 1.1.16.170/32 + - 1.1.16.172/32 + - 1.1.16.174/32 + - 1.1.16.176/32 + - 1.1.16.178/32 + - 1.1.16.180/32 + - 1.1.16.182/32 + - 1.1.16.184/32 + - 1.1.16.186/32 + - 1.1.16.188/32 + - 1.1.16.190/32 + - 1.1.16.192/32 + - 1.1.16.194/32 + - 1.1.16.196/32 + - 1.1.16.198/32 + - 1.1.16.200/32 + - 1.1.16.202/32 + - 1.1.16.204/32 + - 1.1.16.206/32 + - 1.1.16.208/32 + - 1.1.16.210/32 + - 1.1.16.212/32 + - 1.1.16.214/32 + - 1.1.16.216/32 + - 1.1.16.218/32 + - 1.1.16.220/32 + - 1.1.16.222/32 + - 1.1.16.224/32 + - 1.1.16.226/32 + - 1.1.16.228/32 + - 1.1.16.230/32 + - 1.1.16.232/32 + - 1.1.16.234/32 + - 1.1.16.236/32 + - 1.1.16.238/32 + - 1.1.16.240/32 + - 1.1.16.242/32 + - 1.1.16.244/32 + - 1.1.16.246/32 + - 1.1.16.248/32 + - 1.1.16.250/32 + - 1.1.16.252/32 + - 1.1.16.254/32 + - 1.1.17.2/32 + - 1.1.17.4/32 + - 1.1.17.6/32 + - 1.1.17.8/32 + - 1.1.17.10/32 + - 1.1.17.12/32 + - 1.1.17.14/32 + - 1.1.17.16/32 + - 1.1.17.18/32 + - 1.1.17.20/32 + - 1.1.17.22/32 + - 1.1.17.24/32 + - 1.1.17.26/32 + - 1.1.17.28/32 + - 1.1.17.30/32 + - 1.1.17.32/32 + - 1.1.17.34/32 + - 1.1.17.36/32 + - 1.1.17.38/32 + - 1.1.17.40/32 + - 1.1.17.42/32 + - 1.1.17.44/32 + - 1.1.17.46/32 + - 1.1.17.48/32 + - 1.1.17.50/32 + - 1.1.17.52/32 + - 1.1.17.54/32 + - 1.1.17.56/32 + - 1.1.17.58/32 + - 1.1.17.60/32 + - 1.1.17.62/32 + - 1.1.17.64/32 + - 1.1.17.66/32 + - 1.1.17.68/32 + - 1.1.17.70/32 + - 1.1.17.72/32 + - 1.1.17.74/32 + - 1.1.17.76/32 + - 1.1.17.78/32 + - 1.1.17.80/32 + - 1.1.17.82/32 + - 1.1.17.84/32 + - 1.1.17.86/32 + - 1.1.17.88/32 + - 1.1.17.90/32 + - 1.1.17.92/32 + - 1.1.17.94/32 + - 1.1.17.96/32 + - 1.1.17.98/32 + - 1.1.17.100/32 + - 1.1.17.102/32 + - 1.1.17.104/32 + - 1.1.17.106/32 + - 1.1.17.108/32 + - 1.1.17.110/32 + - 1.1.17.112/32 + - 1.1.17.114/32 + - 1.1.17.116/32 + - 1.1.17.118/32 + - 1.1.17.120/32 + - 1.1.17.122/32 + - 1.1.17.124/32 + - 1.1.17.126/32 + - 1.1.17.128/32 + - 1.1.17.130/32 + - 1.1.17.132/32 + - 1.1.17.134/32 + - 1.1.17.136/32 + - 1.1.17.138/32 + - 1.1.17.140/32 + - 1.1.17.142/32 + - 1.1.17.144/32 + - 1.1.17.146/32 + - 1.1.17.148/32 + - 1.1.17.150/32 + - 1.1.17.152/32 + - 1.1.17.154/32 + - 1.1.17.156/32 + - 1.1.17.158/32 + - 1.1.17.160/32 + - 1.1.17.162/32 + - 1.1.17.164/32 + - 1.1.17.166/32 + - 1.1.17.168/32 + - 1.1.17.170/32 + - 1.1.17.172/32 + - 1.1.17.174/32 + - 1.1.17.176/32 + - 1.1.17.178/32 + - 1.1.17.180/32 + - 1.1.17.182/32 + - 1.1.17.184/32 + - 1.1.17.186/32 + - 1.1.17.188/32 + - 1.1.17.190/32 + - 1.1.17.192/32 + - 1.1.17.194/32 + - 1.1.17.196/32 + - 1.1.17.198/32 + - 1.1.17.200/32 + - 1.1.17.202/32 + - 1.1.17.204/32 + - 1.1.17.206/32 + - 1.1.17.208/32 + - 1.1.17.210/32 + - 1.1.17.212/32 + - 1.1.17.214/32 + - 1.1.17.216/32 + - 1.1.17.218/32 + - 1.1.17.220/32 + - 1.1.17.222/32 + - 1.1.17.224/32 + - 1.1.17.226/32 + - 1.1.17.228/32 + - 1.1.17.230/32 + - 1.1.17.232/32 + - 1.1.17.234/32 + - 1.1.17.236/32 + - 1.1.17.238/32 + - 1.1.17.240/32 + - 1.1.17.242/32 + - 1.1.17.244/32 + - 1.1.17.246/32 + - 1.1.17.248/32 + - 1.1.17.250/32 + - 1.1.17.252/32 + - 1.1.17.254/32 + - 1.1.18.2/32 + - 1.1.18.4/32 + - 1.1.18.6/32 + - 1.1.18.8/32 + - 1.1.18.10/32 + - 1.1.18.12/32 + - 1.1.18.14/32 + - 1.1.18.16/32 + - 1.1.18.18/32 + - 1.1.18.20/32 + - 1.1.18.22/32 + - 1.1.18.24/32 + - 1.1.18.26/32 + - 1.1.18.28/32 + - 1.1.18.30/32 + - 1.1.18.32/32 + - 1.1.18.34/32 + - 1.1.18.36/32 + - 1.1.18.38/32 + - 1.1.18.40/32 + - 1.1.18.42/32 + - 1.1.18.44/32 + - 1.1.18.46/32 + - 1.1.18.48/32 + - 1.1.18.50/32 + - 1.1.18.52/32 + - 1.1.18.54/32 + - 1.1.18.56/32 + - 1.1.18.58/32 + - 1.1.18.60/32 + - 1.1.18.62/32 + - 1.1.18.64/32 + - 1.1.18.66/32 + - 1.1.18.68/32 + - 1.1.18.70/32 + - 1.1.18.72/32 + - 1.1.18.74/32 + - 1.1.18.76/32 + - 1.1.18.78/32 + - 1.1.18.80/32 + - 1.1.18.82/32 + - 1.1.18.84/32 + - 1.1.18.86/32 + - 1.1.18.88/32 + - 1.1.18.90/32 + - 1.1.18.92/32 + - 1.1.18.94/32 + - 1.1.18.96/32 + - 1.1.18.98/32 + - 1.1.18.100/32 + - 1.1.18.102/32 + - 1.1.18.104/32 + - 1.1.18.106/32 + - 1.1.18.108/32 + - 1.1.18.110/32 + - 1.1.18.112/32 + - 1.1.18.114/32 + - 1.1.18.116/32 + - 1.1.18.118/32 + - 1.1.18.120/32 + - 1.1.18.122/32 + - 1.1.18.124/32 + - 1.1.18.126/32 + - 1.1.18.128/32 + - 1.1.18.130/32 + - 1.1.18.132/32 + - 1.1.18.134/32 + - 1.1.18.136/32 + - 1.1.18.138/32 + - 1.1.18.140/32 + - 1.1.18.142/32 + - 1.1.18.144/32 + - 1.1.18.146/32 + - 1.1.18.148/32 + - 1.1.18.150/32 + - 1.1.18.152/32 + - 1.1.18.154/32 + - 1.1.18.156/32 + - 1.1.18.158/32 + - 1.1.18.160/32 + - 1.1.18.162/32 + - 1.1.18.164/32 + - 1.1.18.166/32 + - 1.1.18.168/32 + - 1.1.18.170/32 + - 1.1.18.172/32 + - 1.1.18.174/32 + - 1.1.18.176/32 + - 1.1.18.178/32 + - 1.1.18.180/32 + - 1.1.18.182/32 + - 1.1.18.184/32 + - 1.1.18.186/32 + - 1.1.18.188/32 + - 1.1.18.190/32 + - 1.1.18.192/32 + - 1.1.18.194/32 + - 1.1.18.196/32 + - 1.1.18.198/32 + - 1.1.18.200/32 + - 1.1.18.202/32 + - 1.1.18.204/32 + - 1.1.18.206/32 + - 1.1.18.208/32 + - 1.1.18.210/32 + - 1.1.18.212/32 + - 1.1.18.214/32 + - 1.1.18.216/32 + - 1.1.18.218/32 + - 1.1.18.220/32 + - 1.1.18.222/32 + - 1.1.18.224/32 + - 1.1.18.226/32 + - 1.1.18.228/32 + - 1.1.18.230/32 + - 1.1.18.232/32 + - 1.1.18.234/32 + - 1.1.18.236/32 + - 1.1.18.238/32 + - 1.1.18.240/32 + - 1.1.18.242/32 + - 1.1.18.244/32 + - 1.1.18.246/32 + - 1.1.18.248/32 + - 1.1.18.250/32 + - 1.1.18.252/32 + - 1.1.18.254/32 + - 1.1.19.2/32 + - 1.1.19.4/32 + - 1.1.19.6/32 + - 1.1.19.8/32 + - 1.1.19.10/32 + - 1.1.19.12/32 + - 1.1.19.14/32 + - 1.1.19.16/32 + - 1.1.19.18/32 + - 1.1.19.20/32 + - 1.1.19.22/32 + - 1.1.19.24/32 + - 1.1.19.26/32 + - 1.1.19.28/32 + - 1.1.19.30/32 + - 1.1.19.32/32 + - 1.1.19.34/32 + - 1.1.19.36/32 + - 1.1.19.38/32 + - 1.1.19.40/32 + - 1.1.19.42/32 + - 1.1.19.44/32 + - 1.1.19.46/32 + - 1.1.19.48/32 + - 1.1.19.50/32 + - 1.1.19.52/32 + - 1.1.19.54/32 + - 1.1.19.56/32 + - 1.1.19.58/32 + - 1.1.19.60/32 + - 1.1.19.62/32 + - 1.1.19.64/32 + - 1.1.19.66/32 + - 1.1.19.68/32 + - 1.1.19.70/32 + - 1.1.19.72/32 + - 1.1.19.74/32 + - 1.1.19.76/32 + - 1.1.19.78/32 + - 1.1.19.80/32 + - 1.1.19.82/32 + - 1.1.19.84/32 + - 1.1.19.86/32 + - 1.1.19.88/32 + - 1.1.19.90/32 + - 1.1.19.92/32 + - 1.1.19.94/32 + - 1.1.19.96/32 + - 1.1.19.98/32 + - 1.1.19.100/32 + - 1.1.19.102/32 + - 1.1.19.104/32 + - 1.1.19.106/32 + - 1.1.19.108/32 + - 1.1.19.110/32 + - 1.1.19.112/32 + - 1.1.19.114/32 + - 1.1.19.116/32 + - 1.1.19.118/32 + - 1.1.19.120/32 + - 1.1.19.122/32 + - 1.1.19.124/32 + - 1.1.19.126/32 + - 1.1.19.128/32 + - 1.1.19.130/32 + - 1.1.19.132/32 + - 1.1.19.134/32 + - 1.1.19.136/32 + - 1.1.19.138/32 + - 1.1.19.140/32 + - 1.1.19.142/32 + - 1.1.19.144/32 + - 1.1.19.146/32 + - 1.1.19.148/32 + - 1.1.19.150/32 + - 1.1.19.152/32 + - 1.1.19.154/32 + - 1.1.19.156/32 + - 1.1.19.158/32 + - 1.1.19.160/32 + - 1.1.19.162/32 + - 1.1.19.164/32 + - 1.1.19.166/32 + - 1.1.19.168/32 + - 1.1.19.170/32 + - 1.1.19.172/32 + - 1.1.19.174/32 + - 1.1.19.176/32 + - 1.1.19.178/32 + - 1.1.19.180/32 + - 1.1.19.182/32 + - 1.1.19.184/32 + - 1.1.19.186/32 + - 1.1.19.188/32 + - 1.1.19.190/32 + - 1.1.19.192/32 + - 1.1.19.194/32 + - 1.1.19.196/32 + - 1.1.19.198/32 + - 1.1.19.200/32 + - 1.1.19.202/32 + - 1.1.19.204/32 + - 1.1.19.206/32 + - 1.1.19.208/32 + - 1.1.19.210/32 + - 1.1.19.212/32 + - 1.1.19.214/32 + - 1.1.19.216/32 + - 1.1.19.218/32 + - 1.1.19.220/32 + - 1.1.19.222/32 + - 1.1.19.224/32 + - 1.1.19.226/32 + - 1.1.19.228/32 + - 1.1.19.230/32 + - 1.1.19.232/32 + - 1.1.19.234/32 + - 1.1.19.236/32 + - 1.1.19.238/32 + - 1.1.19.240/32 + - 1.1.19.242/32 + - 1.1.19.244/32 + - 1.1.19.246/32 + - 1.1.19.248/32 + - 1.1.19.250/32 + - 1.1.19.252/32 + - 1.1.19.254/32 + - 1.1.20.2/32 + - 1.1.20.4/32 + - 1.1.20.6/32 + - 1.1.20.8/32 + - 1.1.20.10/32 + - 1.1.20.12/32 + - 1.1.20.14/32 + - 1.1.20.16/32 + - 1.1.20.18/32 + - 1.1.20.20/32 + - 1.1.20.22/32 + - 1.1.20.24/32 + - 1.1.20.26/32 + - 1.1.20.28/32 + - 1.1.20.30/32 + - 1.1.20.32/32 + - 1.1.20.34/32 + - 1.1.20.36/32 + - 1.1.20.38/32 + - 1.1.20.40/32 + - 1.1.20.42/32 + - 1.1.20.44/32 + - 1.1.20.46/32 + - 1.1.20.48/32 + - 1.1.20.50/32 + - 1.1.20.52/32 + - 1.1.20.54/32 + - 1.1.20.56/32 + - 1.1.20.58/32 + - 1.1.20.60/32 + - 1.1.20.62/32 + - 1.1.20.64/32 + - 1.1.20.66/32 + - 1.1.20.68/32 + - 1.1.20.70/32 + - 1.1.20.72/32 + - 1.1.20.74/32 + - 1.1.20.76/32 + - 1.1.20.78/32 + - 1.1.20.80/32 + - 1.1.20.82/32 + - 1.1.20.84/32 + - 1.1.20.86/32 + - 1.1.20.88/32 + - 1.1.20.90/32 + - 1.1.20.92/32 + - 1.1.20.94/32 + - 1.1.20.96/32 + - 1.1.20.98/32 + - 1.1.20.100/32 + - 1.1.20.102/32 + - 1.1.20.104/32 + - 1.1.20.106/32 + - 1.1.20.108/32 + - 1.1.20.110/32 + - 1.1.20.112/32 + - 1.1.20.114/32 + - 1.1.20.116/32 + - 1.1.20.118/32 + - 1.1.20.120/32 + - 1.1.20.122/32 + - 1.1.20.124/32 + - 1.1.20.126/32 + - 1.1.20.128/32 + - 1.1.20.130/32 + - 1.1.20.132/32 + - 1.1.20.134/32 + - 1.1.20.136/32 + - 1.1.20.138/32 + - 1.1.20.140/32 + - 1.1.20.142/32 + - 1.1.20.144/32 + - 1.1.20.146/32 + - 1.1.20.148/32 + - 1.1.20.150/32 + - 1.1.20.152/32 + - 1.1.20.154/32 + - 1.1.20.156/32 + - 1.1.20.158/32 + - 1.1.20.160/32 + - 1.1.20.162/32 + - 1.1.20.164/32 + - 1.1.20.166/32 + - 1.1.20.168/32 + - 1.1.20.170/32 + - 1.1.20.172/32 + - 1.1.20.174/32 + - 1.1.20.176/32 + - 1.1.20.178/32 + - 1.1.20.180/32 + - 1.1.20.182/32 + - 1.1.20.184/32 + - 1.1.20.186/32 + - 1.1.20.188/32 + - 1.1.20.190/32 + - 1.1.20.192/32 + - 1.1.20.194/32 + - 1.1.20.196/32 + - 1.1.20.198/32 + - 1.1.20.200/32 + - 1.1.20.202/32 + - 1.1.20.204/32 + - 1.1.20.206/32 + - 1.1.20.208/32 + - 1.1.20.210/32 + - 1.1.20.212/32 + - 1.1.20.214/32 + - 1.1.20.216/32 + - 1.1.20.218/32 + - 1.1.20.220/32 + - 1.1.20.222/32 + - 1.1.20.224/32 + - 1.1.20.226/32 + - 1.1.20.228/32 + - 1.1.20.230/32 + - 1.1.20.232/32 + - 1.1.20.234/32 + - 1.1.20.236/32 + - 1.1.20.238/32 + - 1.1.20.240/32 + - 1.1.20.242/32 + - 1.1.20.244/32 + - 1.1.20.246/32 + - 1.1.20.248/32 + - 1.1.20.250/32 + - 1.1.20.252/32 + - 1.1.20.254/32 + - 1.1.21.2/32 + - 1.1.21.4/32 + - 1.1.21.6/32 + - 1.1.21.8/32 + - 1.1.21.10/32 + - 1.1.21.12/32 + - 1.1.21.14/32 + - 1.1.21.16/32 + - 1.1.21.18/32 + - 1.1.21.20/32 + - 1.1.21.22/32 + - 1.1.21.24/32 + - 1.1.21.26/32 + - 1.1.21.28/32 + - 1.1.21.30/32 + - 1.1.21.32/32 + - 1.1.21.34/32 + - 1.1.21.36/32 + - 1.1.21.38/32 + - 1.1.21.40/32 + - 1.1.21.42/32 + - 1.1.21.44/32 + - 1.1.21.46/32 + - 1.1.21.48/32 + - 1.1.21.50/32 + - 1.1.21.52/32 + - 1.1.21.54/32 + - 1.1.21.56/32 + - 1.1.21.58/32 + - 1.1.21.60/32 + - 1.1.21.62/32 + - 1.1.21.64/32 + - 1.1.21.66/32 + - 1.1.21.68/32 + - 1.1.21.70/32 + - 1.1.21.72/32 + - 1.1.21.74/32 + - 1.1.21.76/32 + - 1.1.21.78/32 + - 1.1.21.80/32 + - 1.1.21.82/32 + - 1.1.21.84/32 + - 1.1.21.86/32 + - 1.1.21.88/32 + - 1.1.21.90/32 + - 1.1.21.92/32 + - 1.1.21.94/32 + - 1.1.21.96/32 + - 1.1.21.98/32 + - 1.1.21.100/32 + - 1.1.21.102/32 + - 1.1.21.104/32 + - 1.1.21.106/32 + - 1.1.21.108/32 + - 1.1.21.110/32 + - 1.1.21.112/32 + - 1.1.21.114/32 + - 1.1.21.116/32 + - 1.1.21.118/32 + - 1.1.21.120/32 + - 1.1.21.122/32 + - 1.1.21.124/32 + - 1.1.21.126/32 + - 1.1.21.128/32 + - 1.1.21.130/32 + - 1.1.21.132/32 + - 1.1.21.134/32 + - 1.1.21.136/32 + - 1.1.21.138/32 + - 1.1.21.140/32 + - 1.1.21.142/32 + - 1.1.21.144/32 + - 1.1.21.146/32 + - 1.1.21.148/32 + - 1.1.21.150/32 + - 1.1.21.152/32 + - 1.1.21.154/32 + - 1.1.21.156/32 + - 1.1.21.158/32 + - 1.1.21.160/32 + - 1.1.21.162/32 + - 1.1.21.164/32 + - 1.1.21.166/32 + - 1.1.21.168/32 + - 1.1.21.170/32 + - 1.1.21.172/32 + - 1.1.21.174/32 + - 1.1.21.176/32 + - 1.1.21.178/32 + - 1.1.21.180/32 + - 1.1.21.182/32 + - 1.1.21.184/32 + - 1.1.21.186/32 + - 1.1.21.188/32 + - 1.1.21.190/32 + - 1.1.21.192/32 + - 1.1.21.194/32 + - 1.1.21.196/32 + - 1.1.21.198/32 + - 1.1.21.200/32 + - 1.1.21.202/32 + - 1.1.21.204/32 + - 1.1.21.206/32 + - 1.1.21.208/32 + - 1.1.21.210/32 + - 1.1.21.212/32 + - 1.1.21.214/32 + - 1.1.21.216/32 + - 1.1.21.218/32 + - 1.1.21.220/32 + - 1.1.21.222/32 + - 1.1.21.224/32 + - 1.1.21.226/32 + - 1.1.21.228/32 + - 1.1.21.230/32 + - 1.1.21.232/32 + - 1.1.21.234/32 + - 1.1.21.236/32 + - 1.1.21.238/32 + - 1.1.21.240/32 + - 1.1.21.242/32 + - 1.1.21.244/32 + - 1.1.21.246/32 + - 1.1.21.248/32 + - 1.1.21.250/32 + - 1.1.21.252/32 + - 1.1.21.254/32 + - 1.1.22.2/32 + - 1.1.22.4/32 + - 1.1.22.6/32 + - 1.1.22.8/32 + - 1.1.22.10/32 + - 1.1.22.12/32 + - 1.1.22.14/32 + - 1.1.22.16/32 + - 1.1.22.18/32 + - 1.1.22.20/32 + - 1.1.22.22/32 + - 1.1.22.24/32 + - 1.1.22.26/32 + - 1.1.22.28/32 + - 1.1.22.30/32 + - 1.1.22.32/32 + - 1.1.22.34/32 + - 1.1.22.36/32 + - 1.1.22.38/32 + - 1.1.22.40/32 + - 1.1.22.42/32 + - 1.1.22.44/32 + - 1.1.22.46/32 + - 1.1.22.48/32 + - 1.1.22.50/32 + - 1.1.22.52/32 + - 1.1.22.54/32 + - 1.1.22.56/32 + - 1.1.22.58/32 + - 1.1.22.60/32 + - 1.1.22.62/32 + - 1.1.22.64/32 + - 1.1.22.66/32 + - 1.1.22.68/32 + - 1.1.22.70/32 + - 1.1.22.72/32 + - 1.1.22.74/32 + - 1.1.22.76/32 + - 1.1.22.78/32 + - 1.1.22.80/32 + - 1.1.22.82/32 + - 1.1.22.84/32 + - 1.1.22.86/32 + - 1.1.22.88/32 + - 1.1.22.90/32 + - 1.1.22.92/32 + - 1.1.22.94/32 + - 1.1.22.96/32 + - 1.1.22.98/32 + - 1.1.22.100/32 + - 1.1.22.102/32 + - 1.1.22.104/32 + - 1.1.22.106/32 + - 1.1.22.108/32 + - 1.1.22.110/32 + - 1.1.22.112/32 + - 1.1.22.114/32 + - 1.1.22.116/32 + - 1.1.22.118/32 + - 1.1.22.120/32 + - 1.1.22.122/32 + - 1.1.22.124/32 + - 1.1.22.126/32 + - 1.1.22.128/32 + - 1.1.22.130/32 + - 1.1.22.132/32 + - 1.1.22.134/32 + - 1.1.22.136/32 + - 1.1.22.138/32 + - 1.1.22.140/32 + - 1.1.22.142/32 + - 1.1.22.144/32 + - 1.1.22.146/32 + - 1.1.22.148/32 + - 1.1.22.150/32 + - 1.1.22.152/32 + - 1.1.22.154/32 + - 1.1.22.156/32 + - 1.1.22.158/32 + - 1.1.22.160/32 + - 1.1.22.162/32 + - 1.1.22.164/32 + - 1.1.22.166/32 + - 1.1.22.168/32 + - 1.1.22.170/32 + - 1.1.22.172/32 + - 1.1.22.174/32 + - 1.1.22.176/32 + - 1.1.22.178/32 + - 1.1.22.180/32 + - 1.1.22.182/32 + - 1.1.22.184/32 + - 1.1.22.186/32 + - 1.1.22.188/32 + - 1.1.22.190/32 + - 1.1.22.192/32 + - 1.1.22.194/32 + - 1.1.22.196/32 + - 1.1.22.198/32 + - 1.1.22.200/32 + - 1.1.22.202/32 + - 1.1.22.204/32 + - 1.1.22.206/32 + - 1.1.22.208/32 + - 1.1.22.210/32 + - 1.1.22.212/32 + - 1.1.22.214/32 + - 1.1.22.216/32 + - 1.1.22.218/32 + - 1.1.22.220/32 + - 1.1.22.222/32 + - 1.1.22.224/32 + - 1.1.22.226/32 + - 1.1.22.228/32 + - 1.1.22.230/32 + - 1.1.22.232/32 + - 1.1.22.234/32 + - 1.1.22.236/32 + - 1.1.22.238/32 + - 1.1.22.240/32 + - 1.1.22.242/32 + - 1.1.22.244/32 + - 1.1.22.246/32 + - 1.1.22.248/32 + - 1.1.22.250/32 + - 1.1.22.252/32 + - 1.1.22.254/32 + - 1.1.23.2/32 + - 1.1.23.4/32 + - 1.1.23.6/32 + - 1.1.23.8/32 + - 1.1.23.10/32 + - 1.1.23.12/32 + - 1.1.23.14/32 + - 1.1.23.16/32 + - 1.1.23.18/32 + - 1.1.23.20/32 + - 1.1.23.22/32 + - 1.1.23.24/32 + - 1.1.23.26/32 + - 1.1.23.28/32 + - 1.1.23.30/32 + - 1.1.23.32/32 + - 1.1.23.34/32 + - 1.1.23.36/32 + - 1.1.23.38/32 + - 1.1.23.40/32 + - 1.1.23.42/32 + - 1.1.23.44/32 + - 1.1.23.46/32 + - 1.1.23.48/32 + - 1.1.23.50/32 + - 1.1.23.52/32 + - 1.1.23.54/32 + - 1.1.23.56/32 + - 1.1.23.58/32 + - 1.1.23.60/32 + - 1.1.23.62/32 + - 1.1.23.64/32 + - 1.1.23.66/32 + - 1.1.23.68/32 + - 1.1.23.70/32 + - 1.1.23.72/32 + - 1.1.23.74/32 + - 1.1.23.76/32 + - 1.1.23.78/32 + - 1.1.23.80/32 + - 1.1.23.82/32 + - 1.1.23.84/32 + - 1.1.23.86/32 + - 1.1.23.88/32 + - 1.1.23.90/32 + - 1.1.23.92/32 + - 1.1.23.94/32 + - 1.1.23.96/32 + - 1.1.23.98/32 + - 1.1.23.100/32 + - 1.1.23.102/32 + - 1.1.23.104/32 + - 1.1.23.106/32 + - 1.1.23.108/32 + - 1.1.23.110/32 + - 1.1.23.112/32 + - 1.1.23.114/32 + - 1.1.23.116/32 + - 1.1.23.118/32 + - 1.1.23.120/32 + - 1.1.23.122/32 + - 1.1.23.124/32 + - 1.1.23.126/32 + - 1.1.23.128/32 + - 1.1.23.130/32 + - 1.1.23.132/32 + - 1.1.23.134/32 + - 1.1.23.136/32 + - 1.1.23.138/32 + - 1.1.23.140/32 + - 1.1.23.142/32 + - 1.1.23.144/32 + - 1.1.23.146/32 + - 1.1.23.148/32 + - 1.1.23.150/32 + - 1.1.23.152/32 + - 1.1.23.154/32 + - 1.1.23.156/32 + - 1.1.23.158/32 + - 1.1.23.160/32 + - 1.1.23.162/32 + - 1.1.23.164/32 + - 1.1.23.166/32 + - 1.1.23.168/32 + - 1.1.23.170/32 + - 1.1.23.172/32 + - 1.1.23.174/32 + - 1.1.23.176/32 + - 1.1.23.178/32 + - 1.1.23.180/32 + - 1.1.23.182/32 + - 1.1.23.184/32 + - 1.1.23.186/32 + - 1.1.23.188/32 + - 1.1.23.190/32 + - 1.1.23.192/32 + - 1.1.23.194/32 + - 1.1.23.196/32 + - 1.1.23.198/32 + - 1.1.23.200/32 + - 1.1.23.202/32 + - 1.1.23.204/32 + - 1.1.23.206/32 + - 1.1.23.208/32 + - 1.1.23.210/32 + - 1.1.23.212/32 + - 1.1.23.214/32 + - 1.1.23.216/32 + - 1.1.23.218/32 + - 1.1.23.220/32 + - 1.1.23.222/32 + - 1.1.23.224/32 + - 1.1.23.226/32 + - 1.1.23.228/32 + - 1.1.23.230/32 + - 1.1.23.232/32 + - 1.1.23.234/32 + - 1.1.23.236/32 + - 1.1.23.238/32 + - 1.1.23.240/32 + - 1.1.23.242/32 + - 1.1.23.244/32 + - 1.1.23.246/32 + - 1.1.23.248/32 + - 1.1.23.250/32 + - 1.1.23.252/32 + - 1.1.23.254/32 + - 1.1.24.2/32 + - 1.1.24.4/32 + - 1.1.24.6/32 + - 1.1.24.8/32 + - 1.1.24.10/32 + - 1.1.24.12/32 + - 1.1.24.14/32 + - 1.1.24.16/32 + - 1.1.24.18/32 + - 1.1.24.20/32 + - 1.1.24.22/32 + - 1.1.24.24/32 + - 1.1.24.26/32 + - 1.1.24.28/32 + - 1.1.24.30/32 + - 1.1.24.32/32 + - 1.1.24.34/32 + - 1.1.24.36/32 + - 1.1.24.38/32 + - 1.1.24.40/32 + - 1.1.24.42/32 + - 1.1.24.44/32 + - 1.1.24.46/32 + - 1.1.24.48/32 + - 1.1.24.50/32 + - 1.1.24.52/32 + - 1.1.24.54/32 + - 1.1.24.56/32 + - 1.1.24.58/32 + - 1.1.24.60/32 + - 1.1.24.62/32 + - 1.1.24.64/32 + - 1.1.24.66/32 + - 1.1.24.68/32 + - 1.1.24.70/32 + - 1.1.24.72/32 + - 1.1.24.74/32 + - 1.1.24.76/32 + - 1.1.24.78/32 + - 1.1.24.80/32 + - 1.1.24.82/32 + - 1.1.24.84/32 + - 1.1.24.86/32 + - 1.1.24.88/32 + - 1.1.24.90/32 + - 1.1.24.92/32 + - 1.1.24.94/32 + - 1.1.24.96/32 + - 1.1.24.98/32 + - 1.1.24.100/32 + - 1.1.24.102/32 + - 1.1.24.104/32 + - 1.1.24.106/32 + - 1.1.24.108/32 + - 1.1.24.110/32 + - 1.1.24.112/32 + - 1.1.24.114/32 + - 1.1.24.116/32 + - 1.1.24.118/32 + - 1.1.24.120/32 + - 1.1.24.122/32 + - 1.1.24.124/32 + - 1.1.24.126/32 + - 1.1.24.128/32 + - 1.1.24.130/32 + - 1.1.24.132/32 + - 1.1.24.134/32 + - 1.1.24.136/32 + - 1.1.24.138/32 + - 1.1.24.140/32 + - 1.1.24.142/32 + - 1.1.24.144/32 + - 1.1.24.146/32 + - 1.1.24.148/32 + - 1.1.24.150/32 + - 1.1.24.152/32 + - 1.1.24.154/32 + - 1.1.24.156/32 + - 1.1.24.158/32 + - 1.1.24.160/32 + - 1.1.24.162/32 + - 1.1.24.164/32 + - 1.1.24.166/32 + - 1.1.24.168/32 + - 1.1.24.170/32 + - 1.1.24.172/32 + - 1.1.24.174/32 + - 1.1.24.176/32 + - 1.1.24.178/32 + - 1.1.24.180/32 + - 1.1.24.182/32 + - 1.1.24.184/32 + - 1.1.24.186/32 + - 1.1.24.188/32 + - 1.1.24.190/32 + - 1.1.24.192/32 + - 1.1.24.194/32 + - 1.1.24.196/32 + - 1.1.24.198/32 + - 1.1.24.200/32 + - 1.1.24.202/32 + - 1.1.24.204/32 + - 1.1.24.206/32 + - 1.1.24.208/32 + - 1.1.24.210/32 + - 1.1.24.212/32 + - 1.1.24.214/32 + - 1.1.24.216/32 + - 1.1.24.218/32 + - 1.1.24.220/32 + - 1.1.24.222/32 + - 1.1.24.224/32 + - 1.1.24.226/32 + - 1.1.24.228/32 + - 1.1.24.230/32 + - 1.1.24.232/32 + - 1.1.24.234/32 + - 1.1.24.236/32 + - 1.1.24.238/32 + - 1.1.24.240/32 + - 1.1.24.242/32 + - 1.1.24.244/32 + - 1.1.24.246/32 + - 1.1.24.248/32 + - 1.1.24.250/32 + - 1.1.24.252/32 + - 1.1.24.254/32 + - 1.1.25.2/32 + - 1.1.25.4/32 + - 1.1.25.6/32 + - 1.1.25.8/32 + - 1.1.25.10/32 + - 1.1.25.12/32 + - 1.1.25.14/32 + - 1.1.25.16/32 + - 1.1.25.18/32 + - 1.1.25.20/32 + - 1.1.25.22/32 + - 1.1.25.24/32 + - 1.1.25.26/32 + - 1.1.25.28/32 + - 1.1.25.30/32 + - 1.1.25.32/32 + - 1.1.25.34/32 + - 1.1.25.36/32 + - 1.1.25.38/32 + - 1.1.25.40/32 + - 1.1.25.42/32 + - 1.1.25.44/32 + - 1.1.25.46/32 + - 1.1.25.48/32 + - 1.1.25.50/32 + - 1.1.25.52/32 + - 1.1.25.54/32 + - 1.1.25.56/32 + - 1.1.25.58/32 + - 1.1.25.60/32 + - 1.1.25.62/32 + - 1.1.25.64/32 + - 1.1.25.66/32 + - 1.1.25.68/32 + - 1.1.25.70/32 + - 1.1.25.72/32 + - 1.1.25.74/32 + - 1.1.25.76/32 + - 1.1.25.78/32 + - 1.1.25.80/32 + - 1.1.25.82/32 + - 1.1.25.84/32 + - 1.1.25.86/32 + - 1.1.25.88/32 + - 1.1.25.90/32 + - 1.1.25.92/32 + - 1.1.25.94/32 + - 1.1.25.96/32 + - 1.1.25.98/32 + - 1.1.25.100/32 + - 1.1.25.102/32 + - 1.1.25.104/32 + - 1.1.25.106/32 + - 1.1.25.108/32 + - 1.1.25.110/32 + - 1.1.25.112/32 + - 1.1.25.114/32 + - 1.1.25.116/32 + - 1.1.25.118/32 + - 1.1.25.120/32 + - 1.1.25.122/32 + - 1.1.25.124/32 + - 1.1.25.126/32 + - 1.1.25.128/32 + - 1.1.25.130/32 + - 1.1.25.132/32 + - 1.1.25.134/32 + - 1.1.25.136/32 + - 1.1.25.138/32 + - 1.1.25.140/32 + - 1.1.25.142/32 + - 1.1.25.144/32 + - 1.1.25.146/32 + - 1.1.25.148/32 + - 1.1.25.150/32 + - 1.1.25.152/32 + - 1.1.25.154/32 + - 1.1.25.156/32 + - 1.1.25.158/32 + - 1.1.25.160/32 + - 1.1.25.162/32 + - 1.1.25.164/32 + - 1.1.25.166/32 + - 1.1.25.168/32 + - 1.1.25.170/32 + - 1.1.25.172/32 + - 1.1.25.174/32 + - 1.1.25.176/32 + - 1.1.25.178/32 + - 1.1.25.180/32 + - 1.1.25.182/32 + - 1.1.25.184/32 + - 1.1.25.186/32 + - 1.1.25.188/32 + - 1.1.25.190/32 + - 1.1.25.192/32 + - 1.1.25.194/32 + - 1.1.25.196/32 + - 1.1.25.198/32 + - 1.1.25.200/32 + - 1.1.25.202/32 + - 1.1.25.204/32 + - 1.1.25.206/32 + - 1.1.25.208/32 + - 1.1.25.210/32 + - 1.1.25.212/32 + - 1.1.25.214/32 + - 1.1.25.216/32 + - 1.1.25.218/32 + - 1.1.25.220/32 + - 1.1.25.222/32 + - 1.1.25.224/32 + - 1.1.25.226/32 + - 1.1.25.228/32 + - 1.1.25.230/32 + - 1.1.25.232/32 + - 1.1.25.234/32 + - 1.1.25.236/32 + - 1.1.25.238/32 + - 1.1.25.240/32 + - 1.1.25.242/32 + - 1.1.25.244/32 + - 1.1.25.246/32 + - 1.1.25.248/32 + - 1.1.25.250/32 + - 1.1.25.252/32 + - 1.1.25.254/32 + - 1.1.26.2/32 + - 1.1.26.4/32 + - 1.1.26.6/32 + - 1.1.26.8/32 + - 1.1.26.10/32 + - 1.1.26.12/32 + - 1.1.26.14/32 + - 1.1.26.16/32 + - 1.1.26.18/32 + - 1.1.26.20/32 + - 1.1.26.22/32 + - 1.1.26.24/32 + - 1.1.26.26/32 + - 1.1.26.28/32 + - 1.1.26.30/32 + - 1.1.26.32/32 + - 1.1.26.34/32 + - 1.1.26.36/32 + - 1.1.26.38/32 + - 1.1.26.40/32 + - 1.1.26.42/32 + - 1.1.26.44/32 + - 1.1.26.46/32 + - 1.1.26.48/32 + - 1.1.26.50/32 + - 1.1.26.52/32 + - 1.1.26.54/32 + - 1.1.26.56/32 + - 1.1.26.58/32 + - 1.1.26.60/32 + - 1.1.26.62/32 + - 1.1.26.64/32 + - 1.1.26.66/32 + - 1.1.26.68/32 + - 1.1.26.70/32 + - 1.1.26.72/32 + - 1.1.26.74/32 + - 1.1.26.76/32 + - 1.1.26.78/32 + - 1.1.26.80/32 + - 1.1.26.82/32 + - 1.1.26.84/32 + - 1.1.26.86/32 + - 1.1.26.88/32 + - 1.1.26.90/32 + - 1.1.26.92/32 + - 1.1.26.94/32 + - 1.1.26.96/32 + - 1.1.26.98/32 + - 1.1.26.100/32 + - 1.1.26.102/32 + - 1.1.26.104/32 + - 1.1.26.106/32 + - 1.1.26.108/32 + - 1.1.26.110/32 + - 1.1.26.112/32 + - 1.1.26.114/32 + - 1.1.26.116/32 + - 1.1.26.118/32 + - 1.1.26.120/32 + - 1.1.26.122/32 + - 1.1.26.124/32 + - 1.1.26.126/32 + - 1.1.26.128/32 + - 1.1.26.130/32 + - 1.1.26.132/32 + - 1.1.26.134/32 + - 1.1.26.136/32 + - 1.1.26.138/32 + - 1.1.26.140/32 + - 1.1.26.142/32 + - 1.1.26.144/32 + - 1.1.26.146/32 + - 1.1.26.148/32 + - 1.1.26.150/32 + - 1.1.26.152/32 + - 1.1.26.154/32 + - 1.1.26.156/32 + - 1.1.26.158/32 + - 1.1.26.160/32 + - 1.1.26.162/32 + - 1.1.26.164/32 + - 1.1.26.166/32 + - 1.1.26.168/32 + - 1.1.26.170/32 + - 1.1.26.172/32 + - 1.1.26.174/32 + - 1.1.26.176/32 + - 1.1.26.178/32 + - 1.1.26.180/32 + - 1.1.26.182/32 + - 1.1.26.184/32 + - 1.1.26.186/32 + - 1.1.26.188/32 + - 1.1.26.190/32 + - 1.1.26.192/32 + - 1.1.26.194/32 + - 1.1.26.196/32 + - 1.1.26.198/32 + - 1.1.26.200/32 + - 1.1.26.202/32 + - 1.1.26.204/32 + - 1.1.26.206/32 + - 1.1.26.208/32 + - 1.1.26.210/32 + - 1.1.26.212/32 + - 1.1.26.214/32 + - 1.1.26.216/32 + - 1.1.26.218/32 + - 1.1.26.220/32 + - 1.1.26.222/32 + - 1.1.26.224/32 + - 1.1.26.226/32 + - 1.1.26.228/32 + - 1.1.26.230/32 + - 1.1.26.232/32 + - 1.1.26.234/32 + - 1.1.26.236/32 + - 1.1.26.238/32 + - 1.1.26.240/32 + - 1.1.26.242/32 + - 1.1.26.244/32 + - 1.1.26.246/32 + - 1.1.26.248/32 + - 1.1.26.250/32 + - 1.1.26.252/32 + - 1.1.26.254/32 + - 1.1.27.2/32 + - 1.1.27.4/32 + - 1.1.27.6/32 + - 1.1.27.8/32 + - 1.1.27.10/32 + - 1.1.27.12/32 + - 1.1.27.14/32 + - 1.1.27.16/32 + - 1.1.27.18/32 + - 1.1.27.20/32 + - 1.1.27.22/32 + - 1.1.27.24/32 + - 1.1.27.26/32 + - 1.1.27.28/32 + - 1.1.27.30/32 + - 1.1.27.32/32 + - 1.1.27.34/32 + - 1.1.27.36/32 + - 1.1.27.38/32 + - 1.1.27.40/32 + - 1.1.27.42/32 + - 1.1.27.44/32 + - 1.1.27.46/32 + - 1.1.27.48/32 + - 1.1.27.50/32 + - 1.1.27.52/32 + - 1.1.27.54/32 + - 1.1.27.56/32 + - 1.1.27.58/32 + - 1.1.27.60/32 + - 1.1.27.62/32 + - 1.1.27.64/32 + - 1.1.27.66/32 + - 1.1.27.68/32 + - 1.1.27.70/32 + - 1.1.27.72/32 + - 1.1.27.74/32 + - 1.1.27.76/32 + - 1.1.27.78/32 + - 1.1.27.80/32 + - 1.1.27.82/32 + - 1.1.27.84/32 + - 1.1.27.86/32 + - 1.1.27.88/32 + - 1.1.27.90/32 + - 1.1.27.92/32 + - 1.1.27.94/32 + - 1.1.27.96/32 + - 1.1.27.98/32 + - 1.1.27.100/32 + - 1.1.27.102/32 + - 1.1.27.104/32 + - 1.1.27.106/32 + - 1.1.27.108/32 + - 1.1.27.110/32 + - 1.1.27.112/32 + - 1.1.27.114/32 + - 1.1.27.116/32 + - 1.1.27.118/32 + - 1.1.27.120/32 + - 1.1.27.122/32 + - 1.1.27.124/32 + - 1.1.27.126/32 + - 1.1.27.128/32 + - 1.1.27.130/32 + - 1.1.27.132/32 + - 1.1.27.134/32 + - 1.1.27.136/32 + - 1.1.27.138/32 + - 1.1.27.140/32 + - 1.1.27.142/32 + - 1.1.27.144/32 + - 1.1.27.146/32 + - 1.1.27.148/32 + - 1.1.27.150/32 + - 1.1.27.152/32 + - 1.1.27.154/32 + - 1.1.27.156/32 + - 1.1.27.158/32 + - 1.1.27.160/32 + - 1.1.27.162/32 + - 1.1.27.164/32 + - 1.1.27.166/32 + - 1.1.27.168/32 + - 1.1.27.170/32 + - 1.1.27.172/32 + - 1.1.27.174/32 + - 1.1.27.176/32 + - 1.1.27.178/32 + - 1.1.27.180/32 + - 1.1.27.182/32 + - 1.1.27.184/32 + - 1.1.27.186/32 + - 1.1.27.188/32 + - 1.1.27.190/32 + - 1.1.27.192/32 + - 1.1.27.194/32 + - 1.1.27.196/32 + - 1.1.27.198/32 + - 1.1.27.200/32 + - 1.1.27.202/32 + - 1.1.27.204/32 + - 1.1.27.206/32 + - 1.1.27.208/32 + - 1.1.27.210/32 + - 1.1.27.212/32 + - 1.1.27.214/32 + - 1.1.27.216/32 + - 1.1.27.218/32 + - 1.1.27.220/32 + - 1.1.27.222/32 + - 1.1.27.224/32 + - 1.1.27.226/32 + - 1.1.27.228/32 + - 1.1.27.230/32 + - 1.1.27.232/32 + - 1.1.27.234/32 + - 1.1.27.236/32 + - 1.1.27.238/32 + - 1.1.27.240/32 + - 1.1.27.242/32 + - 1.1.27.244/32 + - 1.1.27.246/32 + - 1.1.27.248/32 + - 1.1.27.250/32 + - 1.1.27.252/32 + - 1.1.27.254/32 + - 1.1.28.2/32 + - 1.1.28.4/32 + - 1.1.28.6/32 + - 1.1.28.8/32 + - 1.1.28.10/32 + - 1.1.28.12/32 + - 1.1.28.14/32 + - 1.1.28.16/32 + - 1.1.28.18/32 + - 1.1.28.20/32 + - 1.1.28.22/32 + - 1.1.28.24/32 + - 1.1.28.26/32 + - 1.1.28.28/32 + - 1.1.28.30/32 + - 1.1.28.32/32 + - 1.1.28.34/32 + - 1.1.28.36/32 + - 1.1.28.38/32 + - 1.1.28.40/32 + - 1.1.28.42/32 + - 1.1.28.44/32 + - 1.1.28.46/32 + - 1.1.28.48/32 + - 1.1.28.50/32 + - 1.1.28.52/32 + - 1.1.28.54/32 + - 1.1.28.56/32 + - 1.1.28.58/32 + - 1.1.28.60/32 + - 1.1.28.62/32 + - 1.1.28.64/32 + - 1.1.28.66/32 + - 1.1.28.68/32 + - 1.1.28.70/32 + - 1.1.28.72/32 + - 1.1.28.74/32 + - 1.1.28.76/32 + - 1.1.28.78/32 + - 1.1.28.80/32 + - 1.1.28.82/32 + - 1.1.28.84/32 + - 1.1.28.86/32 + - 1.1.28.88/32 + - 1.1.28.90/32 + - 1.1.28.92/32 + - 1.1.28.94/32 + - 1.1.28.96/32 + - 1.1.28.98/32 + - 1.1.28.100/32 + - 1.1.28.102/32 + - 1.1.28.104/32 + - 1.1.28.106/32 + - 1.1.28.108/32 + - 1.1.28.110/32 + - 1.1.28.112/32 + - 1.1.28.114/32 + - 1.1.28.116/32 + - 1.1.28.118/32 + - 1.1.28.120/32 + - 1.1.28.122/32 + - 1.1.28.124/32 + - 1.1.28.126/32 + - 1.1.28.128/32 + - 1.1.28.130/32 + - 1.1.28.132/32 + - 1.1.28.134/32 + - 1.1.28.136/32 + - 1.1.28.138/32 + - 1.1.28.140/32 + - 1.1.28.142/32 + - 1.1.28.144/32 + - 1.1.28.146/32 + - 1.1.28.148/32 + - 1.1.28.150/32 + - 1.1.28.152/32 + - 1.1.28.154/32 + - 1.1.28.156/32 + - 1.1.28.158/32 + - 1.1.28.160/32 + - 1.1.28.162/32 + - 1.1.28.164/32 + - 1.1.28.166/32 + - 1.1.28.168/32 + - 1.1.28.170/32 + - 1.1.28.172/32 + - 1.1.28.174/32 + - 1.1.28.176/32 + - 1.1.28.178/32 + - 1.1.28.180/32 + - 1.1.28.182/32 + - 1.1.28.184/32 + - 1.1.28.186/32 + - 1.1.28.188/32 + - 1.1.28.190/32 + - 1.1.28.192/32 + - 1.1.28.194/32 + - 1.1.28.196/32 + - 1.1.28.198/32 + - 1.1.28.200/32 + - 1.1.28.202/32 + - 1.1.28.204/32 + - 1.1.28.206/32 + - 1.1.28.208/32 + - 1.1.28.210/32 + - 1.1.28.212/32 + - 1.1.28.214/32 + - 1.1.28.216/32 + - 1.1.28.218/32 + - 1.1.28.220/32 + - 1.1.28.222/32 + - 1.1.28.224/32 + - 1.1.28.226/32 + - 1.1.28.228/32 + - 1.1.28.230/32 + - 1.1.28.232/32 + - 1.1.28.234/32 + - 1.1.28.236/32 + - 1.1.28.238/32 + - 1.1.28.240/32 + - 1.1.28.242/32 + - 1.1.28.244/32 + - 1.1.28.246/32 + - 1.1.28.248/32 + - 1.1.28.250/32 + - 1.1.28.252/32 + - 1.1.28.254/32 + - 1.1.29.2/32 + - 1.1.29.4/32 + - 1.1.29.6/32 + - 1.1.29.8/32 + - 1.1.29.10/32 + - 1.1.29.12/32 + - 1.1.29.14/32 + - 1.1.29.16/32 + - 1.1.29.18/32 + - 1.1.29.20/32 + - 1.1.29.22/32 + - 1.1.29.24/32 + - 1.1.29.26/32 + - 1.1.29.28/32 + - 1.1.29.30/32 + - 1.1.29.32/32 + - 1.1.29.34/32 + - 1.1.29.36/32 + - 1.1.29.38/32 + - 1.1.29.40/32 + - 1.1.29.42/32 + - 1.1.29.44/32 + - 1.1.29.46/32 + - 1.1.29.48/32 + - 1.1.29.50/32 + - 1.1.29.52/32 + - 1.1.29.54/32 + - 1.1.29.56/32 + - 1.1.29.58/32 + - 1.1.29.60/32 + - 1.1.29.62/32 + - 1.1.29.64/32 + - 1.1.29.66/32 + - 1.1.29.68/32 + - 1.1.29.70/32 + - 1.1.29.72/32 + - 1.1.29.74/32 + - 1.1.29.76/32 + - 1.1.29.78/32 + - 1.1.29.80/32 + - 1.1.29.82/32 + - 1.1.29.84/32 + - 1.1.29.86/32 + - 1.1.29.88/32 + - 1.1.29.90/32 + - 1.1.29.92/32 + - 1.1.29.94/32 + - 1.1.29.96/32 + - 1.1.29.98/32 + - 1.1.29.100/32 + - 1.1.29.102/32 + - 1.1.29.104/32 + - 1.1.29.106/32 + - 1.1.29.108/32 + - 1.1.29.110/32 + - 1.1.29.112/32 + - 1.1.29.114/32 + - 1.1.29.116/32 + - 1.1.29.118/32 + - 1.1.29.120/32 + - 1.1.29.122/32 + - 1.1.29.124/32 + - 1.1.29.126/32 + - 1.1.29.128/32 + - 1.1.29.130/32 + - 1.1.29.132/32 + - 1.1.29.134/32 + - 1.1.29.136/32 + - 1.1.29.138/32 + - 1.1.29.140/32 + - 1.1.29.142/32 + - 1.1.29.144/32 + - 1.1.29.146/32 + - 1.1.29.148/32 + - 1.1.29.150/32 + - 1.1.29.152/32 + - 1.1.29.154/32 + - 1.1.29.156/32 + - 1.1.29.158/32 + - 1.1.29.160/32 + - 1.1.29.162/32 + - 1.1.29.164/32 + - 1.1.29.166/32 + - 1.1.29.168/32 + - 1.1.29.170/32 + - 1.1.29.172/32 + - 1.1.29.174/32 + - 1.1.29.176/32 + - 1.1.29.178/32 + - 1.1.29.180/32 + - 1.1.29.182/32 + - 1.1.29.184/32 + - 1.1.29.186/32 + - 1.1.29.188/32 + - 1.1.29.190/32 + - 1.1.29.192/32 + - 1.1.29.194/32 + - 1.1.29.196/32 + - 1.1.29.198/32 + - 1.1.29.200/32 + - 1.1.29.202/32 + - 1.1.29.204/32 + - 1.1.29.206/32 + - 1.1.29.208/32 + - 1.1.29.210/32 + - 1.1.29.212/32 + - 1.1.29.214/32 + - 1.1.29.216/32 + - 1.1.29.218/32 + - 1.1.29.220/32 + - 1.1.29.222/32 + - 1.1.29.224/32 + - 1.1.29.226/32 + - 1.1.29.228/32 + - 1.1.29.230/32 + - 1.1.29.232/32 + - 1.1.29.234/32 + - 1.1.29.236/32 + - 1.1.29.238/32 + - 1.1.29.240/32 + - 1.1.29.242/32 + - 1.1.29.244/32 + - 1.1.29.246/32 + - 1.1.29.248/32 + - 1.1.29.250/32 + - 1.1.29.252/32 + - 1.1.29.254/32 + - 1.1.30.2/32 + - 1.1.30.4/32 + - 1.1.30.6/32 + - 1.1.30.8/32 + - 1.1.30.10/32 + - 1.1.30.12/32 + - 1.1.30.14/32 + - 1.1.30.16/32 + - 1.1.30.18/32 + - 1.1.30.20/32 + - 1.1.30.22/32 + - 1.1.30.24/32 + - 1.1.30.26/32 + - 1.1.30.28/32 + - 1.1.30.30/32 + - 1.1.30.32/32 + - 1.1.30.34/32 + - 1.1.30.36/32 + - 1.1.30.38/32 + - 1.1.30.40/32 + - 1.1.30.42/32 + - 1.1.30.44/32 + - 1.1.30.46/32 + - 1.1.30.48/32 + - 1.1.30.50/32 + - 1.1.30.52/32 + - 1.1.30.54/32 + - 1.1.30.56/32 + - 1.1.30.58/32 + - 1.1.30.60/32 + - 1.1.30.62/32 + - 1.1.30.64/32 + - 1.1.30.66/32 + - 1.1.30.68/32 + - 1.1.30.70/32 + - 1.1.30.72/32 + - 1.1.30.74/32 + - 1.1.30.76/32 + - 1.1.30.78/32 + - 1.1.30.80/32 + - 1.1.30.82/32 + - 1.1.30.84/32 + - 1.1.30.86/32 + - 1.1.30.88/32 + - 1.1.30.90/32 + - 1.1.30.92/32 + - 1.1.30.94/32 + - 1.1.30.96/32 + - 1.1.30.98/32 + - 1.1.30.100/32 + - 1.1.30.102/32 + - 1.1.30.104/32 + - 1.1.30.106/32 + - 1.1.30.108/32 + - 1.1.30.110/32 + - 1.1.30.112/32 + - 1.1.30.114/32 + - 1.1.30.116/32 + - 1.1.30.118/32 + - 1.1.30.120/32 + - 1.1.30.122/32 + - 1.1.30.124/32 + - 1.1.30.126/32 + - 1.1.30.128/32 + - 1.1.30.130/32 + - 1.1.30.132/32 + - 1.1.30.134/32 + - 1.1.30.136/32 + - 1.1.30.138/32 + - 1.1.30.140/32 + - 1.1.30.142/32 + - 1.1.30.144/32 + - 1.1.30.146/32 + - 1.1.30.148/32 + - 1.1.30.150/32 + - 1.1.30.152/32 + - 1.1.30.154/32 + - 1.1.30.156/32 + - 1.1.30.158/32 + - 1.1.30.160/32 + - 1.1.30.162/32 + - 1.1.30.164/32 + - 1.1.30.166/32 + - 1.1.30.168/32 + - 1.1.30.170/32 + - 1.1.30.172/32 + - 1.1.30.174/32 + - 1.1.30.176/32 + - 1.1.30.178/32 + - 1.1.30.180/32 + - 1.1.30.182/32 + - 1.1.30.184/32 + - 1.1.30.186/32 + - 1.1.30.188/32 + - 1.1.30.190/32 + - 1.1.30.192/32 + - 1.1.30.194/32 + - 1.1.30.196/32 + - 1.1.30.198/32 + - 1.1.30.200/32 + - 1.1.30.202/32 + - 1.1.30.204/32 + - 1.1.30.206/32 + - 1.1.30.208/32 + - 1.1.30.210/32 + - 1.1.30.212/32 + - 1.1.30.214/32 + - 1.1.30.216/32 + - 1.1.30.218/32 + - 1.1.30.220/32 + - 1.1.30.222/32 + - 1.1.30.224/32 + - 1.1.30.226/32 + - 1.1.30.228/32 + - 1.1.30.230/32 + - 1.1.30.232/32 + - 1.1.30.234/32 + - 1.1.30.236/32 + - 1.1.30.238/32 + - 1.1.30.240/32 + - 1.1.30.242/32 + - 1.1.30.244/32 + - 1.1.30.246/32 + - 1.1.30.248/32 + - 1.1.30.250/32 + - 1.1.30.252/32 + - 1.1.30.254/32 + - 1.1.31.2/32 + - 1.1.31.4/32 + - 1.1.31.6/32 + - 1.1.31.8/32 + - 1.1.31.10/32 + - 1.1.31.12/32 + - 1.1.31.14/32 + - 1.1.31.16/32 + - 1.1.31.18/32 + - 1.1.31.20/32 + - 1.1.31.22/32 + - 1.1.31.24/32 + - 1.1.31.26/32 + - 1.1.31.28/32 + - 1.1.31.30/32 + - 1.1.31.32/32 + - 1.1.31.34/32 + - 1.1.31.36/32 + - 1.1.31.38/32 + - 1.1.31.40/32 + - 1.1.31.42/32 + - 1.1.31.44/32 + - 1.1.31.46/32 + - 1.1.31.48/32 + - 1.1.31.50/32 + - 1.1.31.52/32 + - 1.1.31.54/32 + - 1.1.31.56/32 + - 1.1.31.58/32 + - 1.1.31.60/32 + - 1.1.31.62/32 + - 1.1.31.64/32 + - 1.1.31.66/32 + - 1.1.31.68/32 + - 1.1.31.70/32 + - 1.1.31.72/32 + - 1.1.31.74/32 + - 1.1.31.76/32 + - 1.1.31.78/32 + - 1.1.31.80/32 + - 1.1.31.82/32 + - 1.1.31.84/32 + - 1.1.31.86/32 + - 1.1.31.88/32 + - 1.1.31.90/32 + - 1.1.31.92/32 + - 1.1.31.94/32 + - 1.1.31.96/32 + - 1.1.31.98/32 + - 1.1.31.100/32 + - 1.1.31.102/32 + - 1.1.31.104/32 + - 1.1.31.106/32 + - 1.1.31.108/32 + - 1.1.31.110/32 + - 1.1.31.112/32 + - 1.1.31.114/32 + - 1.1.31.116/32 + - 1.1.31.118/32 + - 1.1.31.120/32 + - 1.1.31.122/32 + - 1.1.31.124/32 + - 1.1.31.126/32 + - 1.1.31.128/32 + - 1.1.31.130/32 + - 1.1.31.132/32 + - 1.1.31.134/32 + - 1.1.31.136/32 + - 1.1.31.138/32 + - 1.1.31.140/32 + - 1.1.31.142/32 + - 1.1.31.144/32 + - 1.1.31.146/32 + - 1.1.31.148/32 + - 1.1.31.150/32 + - 1.1.31.152/32 + - 1.1.31.154/32 + - 1.1.31.156/32 + - 1.1.31.158/32 + - 1.1.31.160/32 + - 1.1.31.162/32 + - 1.1.31.164/32 + - 1.1.31.166/32 + - 1.1.31.168/32 + - 1.1.31.170/32 + - 1.1.31.172/32 + - 1.1.31.174/32 + - 1.1.31.176/32 + - 1.1.31.178/32 + - 1.1.31.180/32 + - 1.1.31.182/32 + - 1.1.31.184/32 + - 1.1.31.186/32 + - 1.1.31.188/32 + - 1.1.31.190/32 + - 1.1.31.192/32 + - 1.1.31.194/32 + - 1.1.31.196/32 + - 1.1.31.198/32 + - 1.1.31.200/32 + - 1.1.31.202/32 + - 1.1.31.204/32 + - 1.1.31.206/32 + - 1.1.31.208/32 + - 1.1.31.210/32 + - 1.1.31.212/32 + - 1.1.31.214/32 + - 1.1.31.216/32 + - 1.1.31.218/32 + - 1.1.31.220/32 + - 1.1.31.222/32 + - 1.1.31.224/32 + - 1.1.31.226/32 + - 1.1.31.228/32 + - 1.1.31.230/32 + - 1.1.31.232/32 + - 1.1.31.234/32 + - 1.1.31.236/32 + - 1.1.31.238/32 + - 1.1.31.240/32 + - 1.1.31.242/32 + - 1.1.31.244/32 + - 1.1.31.246/32 + - 1.1.31.248/32 + - 1.1.31.250/32 + - 1.1.31.252/32 + - 1.1.31.254/32 + - 1.1.32.2/32 + - 1.1.32.4/32 + - 1.1.32.6/32 + - 1.1.32.8/32 + - 1.1.32.10/32 + - 1.1.32.12/32 + - 1.1.32.14/32 + - 1.1.32.16/32 + - 1.1.32.18/32 + - 1.1.32.20/32 + - 1.1.32.22/32 + - 1.1.32.24/32 + - 1.1.32.26/32 + - 1.1.32.28/32 + - 1.1.32.30/32 + - 1.1.32.32/32 + - 1.1.32.34/32 + - 1.1.32.36/32 + - 1.1.32.38/32 + - 1.1.32.40/32 + - 1.1.32.42/32 + - 1.1.32.44/32 + - 1.1.32.46/32 + - 1.1.32.48/32 + - 1.1.32.50/32 + - 1.1.32.52/32 + - 1.1.32.54/32 + - 1.1.32.56/32 + - 1.1.32.58/32 + - 1.1.32.60/32 + - 1.1.32.62/32 + - 1.1.32.64/32 + - 1.1.32.66/32 + - 1.1.32.68/32 + - 1.1.32.70/32 + - 1.1.32.72/32 + - 1.1.32.74/32 + - 1.1.32.76/32 + - 1.1.32.78/32 + - 1.1.32.80/32 + - 1.1.32.82/32 + - 1.1.32.84/32 + - 1.1.32.86/32 + - 1.1.32.88/32 + - 1.1.32.90/32 + - 1.1.32.92/32 + - 1.1.32.94/32 + - 1.1.32.96/32 + - 1.1.32.98/32 + - 1.1.32.100/32 + - 1.1.32.102/32 + - 1.1.32.104/32 + - 1.1.32.106/32 + - 1.1.32.108/32 + - 1.1.32.110/32 + - 1.1.32.112/32 + - 1.1.32.114/32 + - 1.1.32.116/32 +parameters: +- name: NAME +- name: NAMESPACE + value: openshift-ingress-operator +- name: DOMAIN diff --git a/tests-extension/test/testdata/router/ingresscontroller-clb.yaml b/tests-extension/test/testdata/router/ingresscontroller-clb.yaml new file mode 100644 index 000000000..036f66524 --- /dev/null +++ b/tests-extension/test/testdata/router/ingresscontroller-clb.yaml @@ -0,0 +1,24 @@ +apiVersion: template.openshift.io/v1 +kind: Template +objects: +- apiVersion: operator.openshift.io/v1 + kind: IngressController + metadata: + name: ${NAME} + namespace: ${NAMESPACE} + spec: + domain: ${DOMAIN} + replicas: 1 + endpointPublishingStrategy: + loadBalancer: + providerParameters: + aws: + type: Classic + type: AWS + scope: External + type: LoadBalancerService +parameters: +- name: NAME +- name: NAMESPACE + value: openshift-ingress-operator +- name: DOMAIN diff --git a/tests-extension/test/testdata/router/ingresscontroller-external.yaml b/tests-extension/test/testdata/router/ingresscontroller-external.yaml new file mode 100644 index 000000000..e2827d8ef --- /dev/null +++ b/tests-extension/test/testdata/router/ingresscontroller-external.yaml @@ -0,0 +1,20 @@ +apiVersion: template.openshift.io/v1 +kind: Template +objects: +- apiVersion: operator.openshift.io/v1 + kind: IngressController + metadata: + name: ${NAME} + namespace: ${NAMESPACE} + spec: + domain: ${DOMAIN} + replicas: 1 + endpointPublishingStrategy: + loadBalancer: + scope: External + type: LoadBalancerService +parameters: +- name: NAME +- name: NAMESPACE + value: openshift-ingress-operator +- name: DOMAIN diff --git a/tests-extension/test/testdata/router/ingresscontroller-hn-PROXY.yaml b/tests-extension/test/testdata/router/ingresscontroller-hn-PROXY.yaml new file mode 100644 index 000000000..2812fa436 --- /dev/null +++ b/tests-extension/test/testdata/router/ingresscontroller-hn-PROXY.yaml @@ -0,0 +1,25 @@ +apiVersion: template.openshift.io/v1 +kind: Template +objects: +- apiVersion: operator.openshift.io/v1 + kind: IngressController + metadata: + name: ${NAME} + namespace: ${NAMESPACE} + spec: + domain: ${DOMAIN} + replicas: 1 + endpointPublishingStrategy: + hostNetwork: + protocol: PROXY + type: HostNetwork + nodePlacement: + nodeSelector: + matchLabels: + kubernetes.io/os: linux + node-role.kubernetes.io/worker: "" +parameters: +- name: NAME +- name: NAMESPACE + value: openshift-ingress-operator +- name: DOMAIN diff --git a/tests-extension/test/testdata/router/ingresscontroller-hostnetwork-only.yaml b/tests-extension/test/testdata/router/ingresscontroller-hostnetwork-only.yaml new file mode 100644 index 000000000..217afdfbc --- /dev/null +++ b/tests-extension/test/testdata/router/ingresscontroller-hostnetwork-only.yaml @@ -0,0 +1,30 @@ +apiVersion: template.openshift.io/v1 +kind: Template +objects: +- apiVersion: operator.openshift.io/v1 + kind: IngressController + metadata: + name: ${NAME} + namespace: ${NAMESPACE} + spec: + domain: ${DOMAIN} + replicas: 1 + endpointPublishingStrategy: + type: HostNetwork + hostNetwork: + httpPort: ${{HTTPPORT}} + httpsPort: ${{HTTPSPORT}} + statsPort: ${{STATSPORT}} + nodePlacement: + nodeSelector: + matchLabels: + kubernetes.io/os: linux + node-role.kubernetes.io/worker: "" +parameters: +- name: NAME +- name: NAMESPACE + value: openshift-ingress-operator +- name: DOMAIN +- name: HTTPPORT +- name: HTTPSPORT +- name: STATSPORT diff --git a/tests-extension/test/testdata/router/ingresscontroller-np-PROXY.yaml b/tests-extension/test/testdata/router/ingresscontroller-np-PROXY.yaml new file mode 100644 index 000000000..a1ecfe958 --- /dev/null +++ b/tests-extension/test/testdata/router/ingresscontroller-np-PROXY.yaml @@ -0,0 +1,20 @@ +apiVersion: template.openshift.io/v1 +kind: Template +objects: +- apiVersion: operator.openshift.io/v1 + kind: IngressController + metadata: + name: ${NAME} + namespace: ${NAMESPACE} + spec: + domain: ${DOMAIN} + replicas: 1 + endpointPublishingStrategy: + nodePort: + protocol: PROXY + type: NodePortService +parameters: +- name: NAME +- name: NAMESPACE + value: openshift-ingress-operator +- name: DOMAIN diff --git a/tests-extension/test/testdata/router/ingresscontroller-np.yaml b/tests-extension/test/testdata/router/ingresscontroller-np.yaml new file mode 100644 index 000000000..0be7a7a04 --- /dev/null +++ b/tests-extension/test/testdata/router/ingresscontroller-np.yaml @@ -0,0 +1,21 @@ +apiVersion: template.openshift.io/v1 +kind: Template +objects: +- apiVersion: operator.openshift.io/v1 + kind: IngressController + metadata: + name: ${NAME} + namespace: ${NAMESPACE} + spec: + domain: ${DOMAIN} + replicas: 1 + endpointPublishingStrategy: + type: NodePortService + routeAdmission: + namespaceOwnership: Strict + wildcardPolicy: WildcardsDisallowed +parameters: +- name: NAME +- name: NAMESPACE + value: openshift-ingress-operator +- name: DOMAIN diff --git a/tests-extension/test/testdata/router/ingresscontroller-private.yaml b/tests-extension/test/testdata/router/ingresscontroller-private.yaml new file mode 100644 index 000000000..e9ff1e3bd --- /dev/null +++ b/tests-extension/test/testdata/router/ingresscontroller-private.yaml @@ -0,0 +1,18 @@ +apiVersion: template.openshift.io/v1 +kind: Template +objects: +- apiVersion: operator.openshift.io/v1 + kind: IngressController + metadata: + name: ${NAME} + namespace: ${NAMESPACE} + spec: + domain: ${DOMAIN} + replicas: 1 + endpointPublishingStrategy: + type: Private +parameters: +- name: NAME +- name: NAMESPACE + value: openshift-ingress-operator +- name: DOMAIN diff --git a/tests-extension/test/testdata/router/ingresscontroller-shard.yaml b/tests-extension/test/testdata/router/ingresscontroller-shard.yaml new file mode 100644 index 000000000..ea6d833d3 --- /dev/null +++ b/tests-extension/test/testdata/router/ingresscontroller-shard.yaml @@ -0,0 +1,22 @@ +apiVersion: template.openshift.io/v1 +kind: Template +objects: +- apiVersion: operator.openshift.io/v1 + kind: IngressController + metadata: + name: ${NAME} + namespace: ${NAMESPACE} + spec: + domain: ${DOMAIN} + replicas: 1 + routeSelector: + matchLabels: + shard: ${SHARD} + endpointPublishingStrategy: + type: NodePortService +parameters: +- name: NAME +- name: NAMESPACE + value: openshift-ingress-operator +- name: DOMAIN +- name: SHARD diff --git a/tests-extension/test/testdata/router/ingresscontroller-sidecar.yaml b/tests-extension/test/testdata/router/ingresscontroller-sidecar.yaml new file mode 100644 index 000000000..c1d17fccd --- /dev/null +++ b/tests-extension/test/testdata/router/ingresscontroller-sidecar.yaml @@ -0,0 +1,23 @@ +apiVersion: template.openshift.io/v1 +kind: Template +objects: +- apiVersion: operator.openshift.io/v1 + kind: IngressController + metadata: + name: ${NAME} + namespace: ${NAMESPACE} + spec: + domain: ${DOMAIN} + endpointPublishingStrategy: + type: NodePortService + replicas: 1 + logging: + access: + destination: + type: Container + container: {} +parameters: +- name: NAME +- name: NAMESPACE + value: openshift-ingress-operator +- name: DOMAIN diff --git a/tests-extension/test/testdata/router/ingresscontroller-syslog.yaml b/tests-extension/test/testdata/router/ingresscontroller-syslog.yaml new file mode 100755 index 000000000..618b61dec --- /dev/null +++ b/tests-extension/test/testdata/router/ingresscontroller-syslog.yaml @@ -0,0 +1,25 @@ +apiVersion: template.openshift.io/v1 +kind: Template +objects: +- apiVersion: operator.openshift.io/v1 + kind: IngressController + metadata: + name: ${NAME} + namespace: ${NAMESPACE} + spec: + domain: ${DOMAIN} + endpointPublishingStrategy: + type: NodePortService + replicas: 1 + logging: + access: + destination: + type: Syslog + syslog: + address: 1.2.3.4 + port: 514 +parameters: +- name: NAME +- name: NAMESPACE + value: openshift-ingress-operator +- name: DOMAIN diff --git a/tests-extension/test/testdata/router/ingresscontroller-tuning.yaml b/tests-extension/test/testdata/router/ingresscontroller-tuning.yaml new file mode 100644 index 000000000..8d8144f7f --- /dev/null +++ b/tests-extension/test/testdata/router/ingresscontroller-tuning.yaml @@ -0,0 +1,30 @@ +apiVersion: template.openshift.io/v1 +kind: Template +objects: +- apiVersion: operator.openshift.io/v1 + kind: IngressController + metadata: + name: ${NAME} + namespace: ${NAMESPACE} + spec: + tuningOptions: + headerBufferBytes: 16385 + headerBufferMaxRewriteBytes: 4097 + threadCount: 4 + maxConnections: -1 + domain: ${DOMAIN} + replicas: 1 + endpointPublishingStrategy: + type: NodePortService + nodePlacement: + nodeSelector: + matchLabels: + beta.kubernetes.io/os: linux + tolerations: + - effect: NoSchedule + operator: Exists +parameters: +- name: NAME +- name: NAMESPACE + value: openshift-ingress-operator +- name: DOMAIN diff --git a/tests-extension/test/testdata/router/ipfailover.yaml b/tests-extension/test/testdata/router/ipfailover.yaml new file mode 100644 index 000000000..0ef8bb6a5 --- /dev/null +++ b/tests-extension/test/testdata/router/ipfailover.yaml @@ -0,0 +1,95 @@ +apiVersion: template.openshift.io/v1 +kind: Template +objects: +- apiVersion: apps/v1 + kind: Deployment + metadata: + name: ${NAME} + namespace: ${NAMESPACE} + labels: + ipfailover: hello-openshift + spec: + strategy: + type: Recreate + replicas: 2 + selector: + matchLabels: + ipfailover: hello-openshift + template: + metadata: + labels: + ipfailover: hello-openshift + spec: + serviceAccountName: ipfailover + privileged: true + hostNetwork: true + nodeSelector: + node-role.kubernetes.io/worker: "" + containers: + - name: openshift-ipfailover + image: ${IMAGE} + ports: + - containerPort: 63000 + hostPort: 63000 + imagePullPolicy: IfNotPresent + securityContext: + privileged: true + volumeMounts: + - name: lib-modules + mountPath: /lib/modules + readOnly: true + - name: host-slash + mountPath: /host + readOnly: true + mountPropagation: HostToContainer + - name: etc-sysconfig + mountPath: /etc/sysconfig + readOnly: true + env: + - name: OPENSHIFT_HA_CONFIG_NAME + value: "ipfailover" + - name: OPENSHIFT_HA_VIRTUAL_IPS + value: ${VIRTUALIPS} + - name: OPENSHIFT_HA_VIP_GROUPS + value: "10" + - name: OPENSHIFT_HA_NETWORK_INTERFACE + value: ${HAINTERFACE} + - name: OPENSHIFT_HA_MONITOR_PORT + value: ${MONITORPORT} + - name: OPENSHIFT_HA_VRRP_ID_OFFSET + value: ${VRRP_ID_OFFSET} + - name: OPENSHIFT_HA_REPLICA_COUNT + value: "2" + - name: OPENSHIFT_HA_IPTABLES_CHAIN + value: "INPUT" + - name: OPENSHIFT_HA_PREEMPTION + value: "nopreempt" + - name: OPENSHIFT_HA_CHECK_INTERVAL + value: "5" + livenessProbe: + initialDelaySeconds: 10 + exec: + command: + - pgrep + - keepalived + volumes: + - name: lib-modules + hostPath: + path: /lib/modules + - name: host-slash + hostPath: + path: / + - name: etc-sysconfig + hostPath: + path: /etc/sysconfig +parameters: +- name: NAME +- name: NAMESPACE +- name: IMAGE +- name: VIRTUALIPS + value: "192.168.123.123" +- name: MONITORPORT + value: "22" +- name: HAINTERFACE +- name: VRRP_ID_OFFSET + value: "9" diff --git a/tests-extension/test/testdata/router/microshift-ingress-destca.yaml b/tests-extension/test/testdata/router/microshift-ingress-destca.yaml new file mode 100644 index 000000000..ac3452ff6 --- /dev/null +++ b/tests-extension/test/testdata/router/microshift-ingress-destca.yaml @@ -0,0 +1,19 @@ +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: ingress-ms-reen + annotations: + route.openshift.io/destination-ca-certificate-secret: service-secret + route.openshift.io/termination: reencrypt +spec: + rules: + - host: service-secure-test.example.com + http: + paths: + - backend: + service: + name: service-secure + port: + number: 27443 + path: "/" + pathType: Prefix diff --git a/tests-extension/test/testdata/router/microshift-ingress-http.yaml b/tests-extension/test/testdata/router/microshift-ingress-http.yaml new file mode 100644 index 000000000..16fdb304d --- /dev/null +++ b/tests-extension/test/testdata/router/microshift-ingress-http.yaml @@ -0,0 +1,16 @@ +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: ingress-on-microshift +spec: + rules: + - host: service-unsecure-test.example.com + http: + paths: + - backend: + service: + name: service-unsecure + port: + number: 27017 + path: "/" + pathType: Prefix diff --git a/tests-extension/test/testdata/router/ocp57404-stateful-set.yaml b/tests-extension/test/testdata/router/ocp57404-stateful-set.yaml new file mode 100644 index 000000000..9025caed9 --- /dev/null +++ b/tests-extension/test/testdata/router/ocp57404-stateful-set.yaml @@ -0,0 +1,38 @@ +apiVersion: v1 +kind: List +items: +- apiVersion: apps/v1 + kind: StatefulSet + metadata: + name: echoenv-sts + spec: + selector: + matchLabels: + app: echoenv-sts + replicas: 2 + template: + metadata: + labels: + app: echoenv-sts + spec: + terminationGracePeriodSeconds: 10 + containers: + - name: nginx + image: quay.io/openshifttest/nginx-alpine@sha256:cee6930776b92dc1e93b73f9e5965925d49cff3d2e91e1d071c2f0ff72cbca29 + ports: + - containerPort: 8080 + name: web +- apiVersion: v1 + kind: Service + metadata: + name: echoenv-statefulset-service + labels: + app: echoenv-sts + spec: + ports: + - port: 8080 + name: web + clusterIP: None + selector: + app: echoenv-sts + diff --git a/tests-extension/test/testdata/router/ocp73771-role.yaml b/tests-extension/test/testdata/router/ocp73771-role.yaml new file mode 100644 index 000000000..9d907035b --- /dev/null +++ b/tests-extension/test/testdata/router/ocp73771-role.yaml @@ -0,0 +1,23 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: secret-reader +rules: +- apiGroups: [""] + resources: ["secrets"] + verbs: ["get", "list", "watch"] + resourceNames: ["mytls"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: secret-reader-binding +subjects: +- kind: ServiceAccount + name: router + namespace: openshift-ingress +roleRef: + kind: Role + name: secret-reader + apiGroup: rbac.authorization.k8s.io +--- diff --git a/tests-extension/test/testdata/router/rsyslogd-pod.yaml b/tests-extension/test/testdata/router/rsyslogd-pod.yaml new file mode 100644 index 000000000..07ba1425b --- /dev/null +++ b/tests-extension/test/testdata/router/rsyslogd-pod.yaml @@ -0,0 +1,20 @@ +kind: Pod +apiVersion: v1 +metadata: + name: rsyslogd-pod + labels: + name: rsyslogd +spec: + containers: + - image: quay.io/openshifttest/rsyslogd-container@sha256:e806eb41f05d7cc6eec96bf09c7bcb692f97562d4a983cb019289bd048d9aee2 + name: rsyslogd-container + securityContext: + privileged: true + ports: + - containerPort: 514 + protocol: TCP + - containerPort: 514 + protocol: UDP + resources: + limits: + memory: 340Mi diff --git a/tests-extension/test/testdata/router/scc-bug2007246.json b/tests-extension/test/testdata/router/scc-bug2007246.json new file mode 100644 index 000000000..569db1fae --- /dev/null +++ b/tests-extension/test/testdata/router/scc-bug2007246.json @@ -0,0 +1,55 @@ +{ + "allowHostDirVolumePlugin": false, + "allowHostIPC": false, + "allowHostNetwork": false, + "allowHostPID": false, + "allowHostPorts": false, + "allowPrivilegeEscalation": false, + "allowPrivilegedContainer": false, + "allowedCapabilities": null, + "apiVersion": "security.openshift.io/v1", + "defaultAddCapabilities": null, + "fsGroup": { + "type": "MustRunAs" + }, + "groups": [ + "system:authenticated" + ], + "kind": "SecurityContextConstraints", + "metadata": { + "annotations": { + "include.release.openshift.io/ibm-cloud-managed": "true", + "include.release.openshift.io/self-managed-high-availability": "true", + "include.release.openshift.io/single-node-developer": "true", + "kubernetes.io/description": "restricted denies access to all host features and requires pods to be run with a UID, and SELinux context that are allocated to the namespace. This is the most restrictive SCC and it is used by default for authenticated users.", + "release.openshift.io/create-only": "true" + }, + "name": "custom-restricted" + }, + "priority": null, + "readOnlyRootFilesystem": false, + "requiredDropCapabilities": [ + "KILL", + "MKNOD", + "SETUID", + "SETGID" + ], + "runAsUser": { + "type": "MustRunAsRange" + }, + "seLinuxContext": { + "type": "MustRunAs" + }, + "supplementalGroups": { + "type": "RunAsAny" + }, + "users": [], + "volumes": [ + "configMap", + "downwardAPI", + "emptyDir", + "persistentVolumeClaim", + "projected", + "secret" + ] +} diff --git a/tests-extension/test/testdata/router/subdomain-routes/alpha-shard-route.yaml b/tests-extension/test/testdata/router/subdomain-routes/alpha-shard-route.yaml new file mode 100644 index 000000000..805bff00b --- /dev/null +++ b/tests-extension/test/testdata/router/subdomain-routes/alpha-shard-route.yaml @@ -0,0 +1,19 @@ +apiVersion: template.openshift.io/v1 +kind: Template +objects: +- apiVersion: route.openshift.io/v1 + kind: Route + metadata: + annotations: + labels: + shard: alpha + name: ${SUBDOMAIN_NAME}-unsecure + namespace: ${NAMESPACE} + spec: + subdomain: ${SUBDOMAIN_NAME} + to: + kind: Service + name: service-unsecure +parameters: +- name: NAMESPACE +- name: SUBDOMAIN_NAME diff --git a/tests-extension/test/testdata/router/subdomain-routes/ocp51148-route.yaml b/tests-extension/test/testdata/router/subdomain-routes/ocp51148-route.yaml new file mode 100644 index 000000000..8cf4095ed --- /dev/null +++ b/tests-extension/test/testdata/router/subdomain-routes/ocp51148-route.yaml @@ -0,0 +1,59 @@ +apiVersion: template.openshift.io/v1 +kind: Template +objects: +- apiVersion: route.openshift.io/v1 + kind: Route + metadata: + annotations: + labels: + name: service-unsecure + name: ${SUBDOMAIN_NAME}-unsecure1 + namespace: ${NAMESPACE} + spec: + subdomain: ${SUBDOMAIN_NAME} + to: + kind: Service + name: service-unsecure +- apiVersion: route.openshift.io/v1 + kind: Route + metadata: + annotations: + labels: + name: service-unsecure + name: ${SUBDOMAIN_NAME}-unsecure2 + namespace: ${NAMESPACE} + spec: + to: + kind: Service + name: service-unsecure +- apiVersion: route.openshift.io/v1 + kind: Route + metadata: + annotations: + labels: + name: service-unsecure + name: ${SUBDOMAIN_NAME}-unsecure3 + namespace: ${NAMESPACE} + spec: + host: man-${NAMESPACE}.${DOMAIN} + subdomain: ${SUBDOMAIN_NAME} + to: + kind: Service + name: service-unsecure +- apiVersion: route.openshift.io/v1 + kind: Route + metadata: + annotations: + labels: + name: service-unsecure + name: ${SUBDOMAIN_NAME}-unsecure4 + namespace: ${NAMESPACE} + spec: + host: bar-${NAMESPACE}.${DOMAIN} + to: + kind: Service + name: service-unsecure +parameters: +- name: NAMESPACE +- name: DOMAIN +- name: SUBDOMAIN_NAME diff --git a/tests-extension/test/testdata/router/subdomain-routes/route.yaml b/tests-extension/test/testdata/router/subdomain-routes/route.yaml new file mode 100644 index 000000000..d8f3763e2 --- /dev/null +++ b/tests-extension/test/testdata/router/subdomain-routes/route.yaml @@ -0,0 +1,19 @@ +apiVersion: template.openshift.io/v1 +kind: Template +objects: +- apiVersion: route.openshift.io/v1 + kind: Route + metadata: + annotations: + labels: + name: service-unsecure + name: ${SUBDOMAIN_NAME}-unsecure + namespace: ${NAMESPACE} + spec: + subdomain: ${SUBDOMAIN_NAME} + to: + kind: Service + name: service-unsecure +parameters: +- name: NAMESPACE +- name: SUBDOMAIN_NAME diff --git a/tests-extension/test/testdata/router/svc-additional-backend.yaml b/tests-extension/test/testdata/router/svc-additional-backend.yaml new file mode 100644 index 000000000..9e449c437 --- /dev/null +++ b/tests-extension/test/testdata/router/svc-additional-backend.yaml @@ -0,0 +1,33 @@ +apiVersion: v1 +kind: List +items: +- kind: Service + apiVersion: v1 + metadata: + labels: + name: service-secure2 + name: service-secure2 + spec: + ports: + - name: https + protocol: TCP + port: 27443 + targetPort: 8443 + selector: + name: web-server-rc +- kind: Service + apiVersion: v1 + metadata: + labels: + name: service-unsecure2 + name: service-unsecure2 + spec: + internalTrafficPolicy: Cluster + ports: + - name: http + port: 27017 + protocol: TCP + targetPort: 8080 + selector: + name: web-server-rc + type: ClusterIP diff --git a/tests-extension/test/testdata/router/template-web-server-deploy.yaml b/tests-extension/test/testdata/router/template-web-server-deploy.yaml new file mode 100644 index 000000000..85acf2cfc --- /dev/null +++ b/tests-extension/test/testdata/router/template-web-server-deploy.yaml @@ -0,0 +1,64 @@ +apiVersion: template.openshift.io/v1 +kind: Template +objects: + - apiVersion: apps/v1 + kind: Deployment + metadata: + name: ${DEPLOY_NAME} + labels: + name: ${DEPLOY_NAME} + spec: + replicas: 1 + selector: + matchLabels: + name: ${DEPLOY_NAME} + template: + metadata: + labels: + name: ${DEPLOY_NAME} + spec: + containers: + - image: quay.io/openshifttest/nginx-alpine@sha256:cee6930776b92dc1e93b73f9e5965925d49cff3d2e91e1d071c2f0ff72cbca29 + name: nginx + ports: + - containerPort: 8080 + name: http + protocol: TCP + - containerPort: 8443 + name: https + protocol: TCP + - apiVersion: v1 + kind: Service + metadata: + labels: + name: ${SVC_SECURE_NAME} + name: ${SVC_SECURE_NAME} + spec: + ports: + - name: https + port: 27443 + protocol: TCP + targetPort: 8443 + selector: + name: ${DEPLOY_NAME} + - apiVersion: v1 + kind: Service + metadata: + labels: + name: ${SVC_UNSECURE_NAME} + name: ${SVC_UNSECURE_NAME} + spec: + ports: + - name: http + port: 27017 + protocol: TCP + targetPort: 8080 + selector: + name: ${DEPLOY_NAME} +parameters: +- name: DEPLOY_NAME + value: web-server-deploy +- name: SVC_SECURE_NAME + value: service-secure +- name: SVC_UNSECURE_NAME + value: service-unsecure diff --git a/tests-extension/test/testdata/router/test-client-pod.yaml b/tests-extension/test/testdata/router/test-client-pod.yaml new file mode 100644 index 000000000..05c2e6638 --- /dev/null +++ b/tests-extension/test/testdata/router/test-client-pod.yaml @@ -0,0 +1,22 @@ +apiVersion: v1 +kind: Pod +metadata: + labels: + app: hello-pod + name: hello-pod +spec: + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault + containers: + - image: quay.io/openshifttest/nginx-alpine@sha256:cee6930776b92dc1e93b73f9e5965925d49cff3d2e91e1d071c2f0ff72cbca29 + name: hello-pod + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + ports: + - containerPort: 8080 + - containerPort: 8443 diff --git a/tests-extension/test/testdata/router/testpod-60350.yaml b/tests-extension/test/testdata/router/testpod-60350.yaml new file mode 100644 index 000000000..5924ac127 --- /dev/null +++ b/tests-extension/test/testdata/router/testpod-60350.yaml @@ -0,0 +1,24 @@ +apiVersion: v1 +kind: Pod +metadata: + name: testpod-60350 + labels: + app: testpod-60350 +spec: + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault + containers: + - name: myapp-container + image: quay.io/openshifttest/busybox@sha256:c5439d7db88ab5423999530349d327b04279ad3161d7596d2126dfb5b02bfd1f + name: testpod-60350 + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + command: ['sh', '-c', 'echo The app is running! && sleep 3600'] + dnsPolicy: ClusterFirst + dnsConfig: + searches: [8th.com, 9th.com, 10th.com, 11th.com, 12th.com, 13th.com, 14th.com, 15th.com, 16th.com, 17th.com, 18th.com, 19th.com, 20th.com, 21th.com, 22th.com, 23th.com, 24th.com, 25th.com, 26th.com, 27th.com, 28th.com, 29th.com, 30th.com, 31th.com, 32th.com] diff --git a/tests-extension/test/testdata/router/testpod-60492.yaml b/tests-extension/test/testdata/router/testpod-60492.yaml new file mode 100644 index 000000000..4363abb99 --- /dev/null +++ b/tests-extension/test/testdata/router/testpod-60492.yaml @@ -0,0 +1,25 @@ +apiVersion: v1 +kind: Pod +metadata: + name: testpod-60492 + labels: + app: testpod-60492 +spec: + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault + containers: + - name: myapp-container + image: quay.io/openshifttest/busybox@sha256:c5439d7db88ab5423999530349d327b04279ad3161d7596d2126dfb5b02bfd1f + name: testpod-60492 + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + command: ['sh', '-c', 'echo The app is running! && sleep 3600'] + dnsPolicy: ClusterFirst + dnsConfig: + searches: [t47x6d4lzz1zxm1bakrmiceb0tljzl9n8r19kqu9s3731ectkllp9mezn7cldozt25nlenyh5jus5b9rr687u2icimakjpyf4rsux3c66giulc0d2ipsa6bpa6dykgd0mc25r1m89hvzjcix73sdwfbu5q67t0c131i1fqne0o7we20ve2emh1046h9m854wfxo0spb2gv5d65v9x2ibuiti7rhr2y8u72hil5cutp63sbhi832kf3v4vuxa0] + diff --git a/tests-extension/test/testdata/router/web-server-deploy.yaml b/tests-extension/test/testdata/router/web-server-deploy.yaml new file mode 100644 index 000000000..9c31424f2 --- /dev/null +++ b/tests-extension/test/testdata/router/web-server-deploy.yaml @@ -0,0 +1,57 @@ +apiVersion: v1 +kind: List +items: +- apiVersion: apps/v1 + kind: Deployment + metadata: + name: web-server-deploy + labels: + app: web-server-deploy + spec: + replicas: 1 + selector: + matchLabels: + name: web-server-deploy + template: + metadata: + labels: + name: web-server-deploy + spec: + containers: + - name: nginx + image: quay.io/openshifttest/nginx-alpine@sha256:cee6930776b92dc1e93b73f9e5965925d49cff3d2e91e1d071c2f0ff72cbca29 + ports: + - containerPort: 8080 + name: http + protocol: TCP + - containerPort: 8443 + name: https + protocol: TCP +- kind: Service + apiVersion: v1 + metadata: + labels: + name: service-secure + name: service-secure + spec: + ports: + - name: https + protocol: TCP + port: 27443 + targetPort: 8443 + selector: + name: web-server-deploy +- apiVersion: v1 + kind: Service + metadata: + labels: + name: service-unsecure + name: service-unsecure + spec: + ports: + - name: http + port: 27017 + protocol: TCP + targetPort: 8080 + selector: + name: web-server-deploy \ No newline at end of file diff --git a/tests-extension/test/testdata/router/web-server-signed-deploy.yaml b/tests-extension/test/testdata/router/web-server-signed-deploy.yaml new file mode 100644 index 000000000..1a77bb7c0 --- /dev/null +++ b/tests-extension/test/testdata/router/web-server-signed-deploy.yaml @@ -0,0 +1,94 @@ +apiVersion: v1 +kind: List +items: +- apiVersion: v1 + kind: ConfigMap + metadata: + name: nginx-config + data: + nginx.conf: | + events { + worker_connections 1024; + } + + http { + server { + listen 8080; + listen [::]:8080; + location / { + root /data/http; + } + } + + server { + listen 8443 ssl http2 default; + listen [::]:8443 ssl http2 default; + server_name _; + ssl_certificate certs/tls.crt; + ssl_certificate_key certs/tls.key; + location / { + root /data/https-default; + } + } + } +- apiVersion: apps/v1 + kind: Deployment + metadata: + name: web-server-deploy + labels: + name: web-server-deploy + spec: + replicas: 1 + selector: + matchExpressions: + - {key: name, operator: In, values: [web-server-deploy]} + template: + metadata: + labels: + name: web-server-deploy + spec: + containers: + - name: nginx + image: quay.io/openshifttest/nginx-alpine@sha256:cee6930776b92dc1e93b73f9e5965925d49cff3d2e91e1d071c2f0ff72cbca29 + volumeMounts: + - name: service-secret + mountPath: /etc/nginx/certs/ + - name: nginx-config + mountPath: /etc/nginx/ + volumes: + - name: service-secret + secret: + secretName: service-secret + - name: nginx-config + configMap: + name: nginx-config +- kind: Service + apiVersion: v1 + metadata: + annotations: + service.beta.openshift.io/serving-cert-secret-name: service-secret + labels: + name: service-secure + name: service-secure + spec: + ports: + - name: https + protocol: TCP + port: 27443 + targetPort: 8443 + selector: + name: web-server-deploy +- apiVersion: v1 + kind: Service + metadata: + labels: + name: service-unsecure + name: service-unsecure + spec: + ports: + - name: http + port: 27017 + protocol: TCP + targetPort: 8080 + selector: + name: web-server-deploy diff --git a/tests-extension/test/testdata/router/web-server-v4v6rc.yaml b/tests-extension/test/testdata/router/web-server-v4v6rc.yaml new file mode 100644 index 000000000..1d6c3cff6 --- /dev/null +++ b/tests-extension/test/testdata/router/web-server-v4v6rc.yaml @@ -0,0 +1,49 @@ +apiVersion: v1 +kind: List +items: +- apiVersion: v1 + kind: ReplicationController + metadata: + labels: + name: web-server-v4v6rc + name: web-server-v4v6rc + spec: + replicas: 1 + template: + metadata: + labels: + name: web-server-v4v6rc + spec: + containers: + - image: quay.io/openshifttest/nginx-alpine@sha256:cee6930776b92dc1e93b73f9e5965925d49cff3d2e91e1d071c2f0ff72cbca29 + name: nginx +- apiVersion: v1 + kind: Service + metadata: + labels: + name: service-securev4v6 + name: service-securev4v6 + spec: + ipFamilyPolicy: PreferDualStack + ports: + - name: https + port: 27443 + protocol: TCP + targetPort: 8443 + selector: + name: web-server-v4v6rc +- apiVersion: v1 + kind: Service + metadata: + labels: + name: service-unsecurev4v6 + name: service-unsecurev4v6 + spec: + ipFamilyPolicy: PreferDualStack + ports: + - name: http + port: 27017 + protocol: TCP + targetPort: 8080 + selector: + name: web-server-v4v6rc diff --git a/tests-extension/test/testdata/router/websocket-deploy.yaml b/tests-extension/test/testdata/router/websocket-deploy.yaml new file mode 100644 index 000000000..b7ea6affd --- /dev/null +++ b/tests-extension/test/testdata/router/websocket-deploy.yaml @@ -0,0 +1,43 @@ +apiVersion: v1 +kind: List +items: +- apiVersion: apps/v1 + kind: Deployment + metadata: + name: websocket + labels: + app: hello-websocket + spec: + replicas: 1 + selector: + matchLabels: + name: hello-websocket + template: + metadata: + labels: + name: hello-websocket + spec: + containers: + - name: hello-websocket + image: quay.io/openshifttest/hello-websocket@sha256:d485f1fedd2bc929551e68288812c10957b99db3e833ec150071f00291ea77db + ports: + - containerPort: 9999 + name: http + protocol: TCP + - containerPort: 9443 + name: https + protocol: TCP +- apiVersion: v1 + kind: Service + metadata: + labels: + name: ws-unsecure + name: ws-unsecure + spec: + ports: + - name: http + port: 27017 + protocol: TCP + targetPort: 9999 + selector: + name: hello-websocket