From 205808148279d3f109f61a5329607bf13fa7f62d Mon Sep 17 00:00:00 2001 From: Sergey Shevchenko Date: Wed, 22 May 2024 11:08:11 +0300 Subject: [PATCH 1/5] feat: Add initial e2e tests --- Makefile | 12 +- go.mod | 18 +-- go.sum | 35 +---- test/e2e/e2e_test.go | 172 +++++++++++++++++++--- test/e2e/testdata/etcdcluster-3.3.yaml | 15 ++ test/e2e/testdata/etcdcluster-3.4.yaml | 15 ++ test/e2e/testdata/etcdcluster-3.5.yaml | 15 ++ test/e2e/testdata/etcdcluster-simple.yaml | 7 + 8 files changed, 220 insertions(+), 69 deletions(-) create mode 100644 test/e2e/testdata/etcdcluster-3.3.yaml create mode 100644 test/e2e/testdata/etcdcluster-3.4.yaml create mode 100644 test/e2e/testdata/etcdcluster-3.5.yaml create mode 100644 test/e2e/testdata/etcdcluster-simple.yaml diff --git a/Makefile b/Makefile index b324f38c..6aa29576 100644 --- a/Makefile +++ b/Makefile @@ -80,8 +80,8 @@ test: manifests generate fmt vet envtest ## Run tests. # Utilize Kind or modify the e2e tests to load the image locally, enabling compatibility with other vendors. .PHONY: test-e2e # Run the e2e tests against a Kind k8s instance that is spun up. -test-e2e: - go test ./test/e2e/ -v -ginkgo.v +test-e2e: ginkgo + $(GINKGO) -v ./test/e2e/ .PHONY: lint lint: golangci-lint ## Run golangci-lint linter & yamllint @@ -249,6 +249,7 @@ HELM ?= $(LOCALBIN)/helm HELM_DOCS ?= $(LOCALBIN)/helm-docs YQ = $(LOCALBIN)/yq CRD_REF_DOCS ?= $(LOCALBIN)/crd-ref-docs +GINKGO ?= $(LOCALBIN)/ginkgo ## Tool Versions # renovate: datasource=github-tags depName=kubernetes-sigs/kustomize @@ -268,6 +269,8 @@ HELM_SCHEMA_VERSION ?= v1.4.1 HELM_DOCS_VERSION ?= v1.13.1 # renovate: datasource=github-tags depName=mikefarah/yq YQ_VERSION ?= v4.44.1 +# renovate: datasource=github-tags depName=onsi/ginkgo +GINKGO_VERSION ?= v2.17.3 ## Tool install scripts KUSTOMIZE_INSTALL_SCRIPT ?= "https://raw.githubusercontent.com/kubernetes-sigs/kustomize/master/hack/install_kustomize.sh" @@ -298,6 +301,11 @@ golangci-lint: $(LOCALBIN) @test -x $(GOLANGCI_LINT) && $(GOLANGCI_LINT) version | grep -q $(GOLANGCI_LINT_VERSION) || \ GOBIN=$(LOCALBIN) go install github.com/golangci/golangci-lint/cmd/golangci-lint@$(GOLANGCI_LINT_VERSION) +.PHONY: ginkgo +ginkgo: $(LOCALBIN) + @test -x $(GINKGO) && $(GINKGO) version | grep -q $(GINKGO_VERSION) || \ + GOBIN=$(LOCALBIN) go install github.com/onsi/ginkgo/v2/ginkgo@$(GINKGO_VERSION) + .PHONY: nilaway nilaway: $(LOCALBIN) @test -x $(NILAWAY_LINT) || GOBIN=$(LOCALBIN) go install go.uber.org/nilaway/cmd/nilaway@latest diff --git a/go.mod b/go.mod index d74f4195..ac3f59d6 100644 --- a/go.mod +++ b/go.mod @@ -28,6 +28,7 @@ require ( github.com/coreos/go-systemd/v22 v22.5.0 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/emicklei/go-restful/v3 v3.11.0 // indirect + github.com/evanphx/json-patch v5.7.0+incompatible // indirect github.com/evanphx/json-patch/v5 v5.9.0 // indirect github.com/fsnotify/fsnotify v1.7.0 // indirect github.com/go-logr/zapr v1.3.0 // indirect @@ -42,10 +43,6 @@ require ( github.com/google/go-cmp v0.6.0 // indirect github.com/google/gofuzz v1.2.0 // indirect github.com/google/pprof v0.0.0-20240424215950-a892ee059fd6 // indirect - github.com/gorilla/websocket v1.5.0 // indirect - github.com/hashicorp/hcl v1.0.0 // indirect - github.com/imdario/mergo v0.3.6 // indirect - github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/magiconair/properties v1.8.7 // indirect @@ -72,19 +69,6 @@ require ( go.etcd.io/etcd/api/v3 v3.5.14 // indirect go.etcd.io/etcd/client/pkg/v3 v3.5.14 // indirect go.uber.org/multierr v1.11.0 // indirect - golang.org/x/exp v0.0.0-20230905200255-921286631fa9 // indirect - golang.org/x/net v0.25.0 // indirect - golang.org/x/oauth2 v0.18.0 // indirect - golang.org/x/sys v0.20.0 // indirect - golang.org/x/term v0.20.0 // indirect - golang.org/x/text v0.15.0 // indirect - golang.org/x/time v0.5.0 // indirect - golang.org/x/tools v0.21.0 // indirect - gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect - google.golang.org/appengine v1.6.8 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20240311132316-a219d84964c2 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20240314234333-6e1732d8331c // indirect - google.golang.org/grpc v1.62.1 // indirect google.golang.org/protobuf v1.33.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/ini.v1 v1.67.0 // indirect diff --git a/go.sum b/go.sum index a13159d3..5c989351 100644 --- a/go.sum +++ b/go.sum @@ -14,12 +14,10 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= -github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/emicklei/go-restful/v3 v3.11.0 h1:rAQeMHw1c7zTmncogyy8VvRZwtkmkZ4FxERmMY4rD+g= github.com/emicklei/go-restful/v3 v3.11.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= -github.com/evanphx/json-patch v4.12.0+incompatible h1:4onqiflcdA9EOZ4RxV643DvftH5pOlLGNtQ5lPWQu84= -github.com/evanphx/json-patch v4.12.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= +github.com/evanphx/json-patch v5.7.0+incompatible h1:vgGkfT/9f8zE6tvSCe74nfpAVDQ2tG6yudJd8LBksgI= +github.com/evanphx/json-patch v5.7.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/evanphx/json-patch/v5 v5.9.0 h1:kcBlZQbplgElYIlo/n1hJbls2z/1awpXxpRi0/FOJfg= github.com/evanphx/json-patch/v5 v5.9.0/go.mod h1:VNkHZ/282BpEyt/tObQO8s5CMPmYYq14uClGH4abBuQ= github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= @@ -60,15 +58,6 @@ github.com/google/pprof v0.0.0-20240424215950-a892ee059fd6 h1:k7nVchz72niMH6YLQN github.com/google/pprof v0.0.0-20240424215950-a892ee059fd6/go.mod h1:kf6iHlnVGwgKolg33glAes7Yg/8iWP8ukqeldJSO7jw= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= -github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc= -github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= -github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= -github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= -github.com/imdario/mergo v0.3.6 h1:xTNEAn+kxVO7dTZGu0CegyqKZmoWFI0rF8UxjlB2d28= -github.com/imdario/mergo v0.3.6/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= -github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= -github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= @@ -172,7 +161,6 @@ go.uber.org/zap/exp v0.2.0/go.mod h1:t0gqAIdh1MfKv9EwN/dLwfZnJxe9ITAZN78HEWPFWDQ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/exp v0.0.0-20230905200255-921286631fa9 h1:GoHiUyI/Tp2nVkLI2mCxVkOjsbSXD66ic0XW0js0R9g= golang.org/x/exp v0.0.0-20230905200255-921286631fa9/go.mod h1:S2oDrQGGwySpoQPVqRShND87VCbxmc6bL1Yd2oYrm6k= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= @@ -182,10 +170,6 @@ golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac= -golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= golang.org/x/oauth2 v0.18.0 h1:09qnuIAgzdx1XplqJvW6CQqMCtGZykZWcXzPMPUusvI= golang.org/x/oauth2 v0.18.0/go.mod h1:Wf7knwG0MPoWIMMBgFlEaSUDaKskp0dCfrlJRJXbBi8= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -207,10 +191,6 @@ golang.org/x/term v0.20.0 h1:VnkxpohqXaOBYJtBmEppKUG6mXpi+4O6purfc2+sMhw= golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= -golang.org/x/text v0.15.0 h1:h1V/4gjBv8v9cjcR6+AR5+/cIYK5N/WAgiv4xlsEtAk= -golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -226,16 +206,6 @@ golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gomodules.xyz/jsonpatch/v2 v2.4.0 h1:Ci3iUJyx9UeRx7CeFN8ARgGbkESwJK+KB9lLcWxY/Zw= gomodules.xyz/jsonpatch/v2 v2.4.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuBJT4moAY= -google.golang.org/appengine v1.6.8 h1:IhEN5q69dyKagZPYMSdIjS2HqprW324FRQZJcGqPAsM= -google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJffLiz/Ds= -google.golang.org/genproto/googleapis/api v0.0.0-20240311132316-a219d84964c2 h1:rIo7ocm2roD9DcFIX67Ym8icoGCKSARAiPljFhh5suQ= -google.golang.org/genproto/googleapis/api v0.0.0-20240311132316-a219d84964c2/go.mod h1:O1cOfN1Cy6QEYr7VxtjOyP5AdAuR0aJ/MYZaaof623Y= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240314234333-6e1732d8331c h1:lfpJ/2rWPa/kJgxyyXM8PrNnfCzcmxJ265mADgwmvLI= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240314234333-6e1732d8331c/go.mod h1:WtryC6hu0hhx87FDGxWCDptyssuo68sk10vYjF+T9fY= -google.golang.org/grpc v1.62.1 h1:B4n+nfKzOICUXMgyrNd19h/I9oH0L1pizfk1d4zSgTk= -google.golang.org/grpc v1.62.1/go.mod h1:IWTG0VlJLCh1SkC58F7np9ka9mx/WNkjl4PGJaiq+QE= -google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= -google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -249,6 +219,7 @@ gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= k8s.io/api v0.30.2 h1:+ZhRj+28QT4UOH+BKznu4CBgPWgkXO7XAvMcMl0qKvI= diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.go index c9967858..43de15fc 100644 --- a/test/e2e/e2e_test.go +++ b/test/e2e/e2e_test.go @@ -20,7 +20,9 @@ import ( "fmt" "os" "os/exec" + "strconv" "sync" + "time" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -31,6 +33,7 @@ import ( ) var _ = Describe("etcd-operator", Ordered, func() { + dir, _ := utils.GetProjectDir() BeforeAll(func() { var err error @@ -55,7 +58,7 @@ var _ = Describe("etcd-operator", Ordered, func() { By("wait while etcd-operator is ready", func() { cmd := exec.Command("kubectl", "wait", "--namespace", "etcd-operator-system", "deployment/etcd-operator-controller-manager", - "--for", "jsonpath={.status.availableReplicas}=1", "--timeout=5m") + "--for", "jsonpath={.status.readyReplicas}=1", "--timeout=5m") _, err = utils.Run(cmd) ExpectWithOffset(1, err).NotTo(HaveOccurred()) }) @@ -73,17 +76,12 @@ var _ = Describe("etcd-operator", Ordered, func() { Context("Simple", func() { It("should deploy etcd cluster", func() { - var err error const namespace = "test-simple-etcd-cluster" var wg sync.WaitGroup wg.Add(1) - By("create namespace", func() { - cmd := exec.Command("sh", "-c", - fmt.Sprintf("kubectl create namespace %s --dry-run=client -o yaml | kubectl apply -f -", namespace)) // nolint:lll - _, err = utils.Run(cmd) - ExpectWithOffset(1, err).NotTo(HaveOccurred()) - }) + CreateNamespace(namespace) + DeferCleanup(DeleteNamespaceCB(namespace)) By("apply simple etcd cluster manifest", func() { dir, _ := utils.GetProjectDir() @@ -95,16 +93,9 @@ var _ = Describe("etcd-operator", Ordered, func() { ExpectWithOffset(1, err).NotTo(HaveOccurred()) }) - By("wait for statefulset is ready", func() { - cmd := exec.Command("kubectl", "wait", - "statefulset/test", - "--for", "jsonpath={.status.readyReplicas}=3", - "--namespace", namespace, - "--timeout", "5m", - ) - _, err = utils.Run(cmd) - ExpectWithOffset(1, err).NotTo(HaveOccurred()) - }) + WaitSTSReady("statefulset/test", namespace) + + // CheckEtcdClusterHealthy("service/test", namespace) client, err := utils.GetEtcdClient(ctx, client.ObjectKey{Namespace: namespace, Name: "test"}) Expect(err).NotTo(HaveOccurred()) @@ -187,4 +178,149 @@ var _ = Describe("etcd-operator", Ordered, func() { }) }) + DescribeTable("Upgrade", + func(namespace string, fileName string, version string) { + CreateNamespace(namespace) + DeferCleanup(DeleteNamespaceCB(namespace)) + + By("apply upgrade etcd cluster manifest") + + cmd := exec.Command("kubectl", "apply", + "--filename", dir+fileName, + "--namespace", namespace, + ) + _, err := utils.Run(cmd) + + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + + WaitSTSReady("statefulset/test", namespace) + CheckEtcdClusterHealthy("service/test", namespace) + + By("upgrade etcd cluster to one patch version") + cmd = exec.Command("kubectl", "patch", + "etcdcluster/test", + "--type", "json", + // Strategic Merge Patch is not currently supported by the etcd-operator + // "--patch", + // fmt.Sprintf( + // "{\"spec\":{\"podTemplate\":{\"containers\":[{\"name\":\"etcd\",\"image\":\"quay.io/coreos/etcd:%s\"}]}}}", + // version, + // ), + "--patch", + fmt.Sprintf( + "[{\"op\": \"replace\", \"path\": \"/spec/podTemplate/spec/containers/0/image\", "+ + "\"value\":\"quay.io/coreos/etcd:%s\"}]", + version, + ), + "--namespace", namespace, + ) + _, err = utils.Run(cmd) + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + + // Give the operator some time to update the sts + time.Sleep(2 * time.Second) + + WaitSTSReady("statefulset/test", namespace) + CheckEtcdClusterHealthy("service/test", namespace) + }, + Entry( + "should upgrade etcd cluster patch version", + "test-upgrade-3-5-11--3-5-12", + "/test/e2e/testdata/etcdcluster-3.5.yaml", + "v3.5.12", + ), + Entry( + "should downgrade etcd cluster patch version", + "test-downgrade-3-5-11--3-5-10", + "/test/e2e/testdata/etcdcluster-3.5.yaml", + "v3.5.10", + ), + Entry( + "should upgrade etcd cluster to one minor version", + "test-upgrade-3-4-32--3-5-10", + "/test/e2e/testdata/etcdcluster-3.4.yaml", + "v3.5.10", + ), + Entry( + "should downgrade etcd cluster to one minor version", + "test-downgrade-3-5-11--3-4-32", + "/test/e2e/testdata/etcdcluster-3.5.yaml", + "v3.4.32", + ), + Entry( + "should upgrade etcd cluster to multiple minor version", + "test-upgrade-3-3-27--3-5-11", + "/test/e2e/testdata/etcdcluster-3.3.yaml", + "v3.5.11", + ), + Entry( + "should downgrade etcd cluster to multiple minor version", + "test-downgrade-3-5-11--3-3-27", + "/test/e2e/testdata/etcdcluster-3.5.yaml", + "v3.3.27", + ), + ) + }) + +func DeleteNamespaceCB(namespace string) func() error { + return func() error { + return DeleteNamespace(namespace) + } +} + +func DeleteNamespace(namespace string) error { + By("delete namespace") + + cmd := exec.Command("kubectl", "delete", "namespace", namespace) + _, err := utils.Run(cmd) + + return err +} + +func CreateNamespace(namespace string) { + By("create namespace") + + cmd := exec.Command("kubectl", "create", "namespace", namespace) + _, err := utils.Run(cmd) + ExpectWithOffset(1, err).NotTo(HaveOccurred()) +} + +func WaitSTSReady(stsName, namespace string) { + By("wait for statefulset is ready") + + cmd := exec.Command("kubectl", "wait", + stsName, + "--for", "jsonpath={.status.availableReplicas}=3", + "--namespace", namespace, + "--timeout", "5m", + ) + _, err := utils.Run(cmd) + + ExpectWithOffset(1, err).NotTo(HaveOccurred()) +} + +func CheckEtcdClusterHealthy(serviceName, namespace string) { + var wg sync.WaitGroup + wg.Add(1) + + By("port-forward service to localhost") + port, _ := utils.GetFreePort() + go func() { + defer GinkgoRecover() + defer wg.Done() + + cmd := exec.Command("kubectl", "port-forward", + serviceName, strconv.Itoa(port)+":2379", + "--namespace", namespace, + ) + _, err := utils.Run(cmd) + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + }() + + By("check etcd cluster is healthy") + endpoints := []string{"localhost:" + strconv.Itoa(port)} + for i := 0; i < 3; i++ { + Expect(utils.IsEtcdClusterHealthy(endpoints)).To(BeTrue()) + } +} diff --git a/test/e2e/testdata/etcdcluster-3.3.yaml b/test/e2e/testdata/etcdcluster-3.3.yaml new file mode 100644 index 00000000..d8bf5b9f --- /dev/null +++ b/test/e2e/testdata/etcdcluster-3.3.yaml @@ -0,0 +1,15 @@ +--- +apiVersion: etcd.aenix.io/v1alpha1 +kind: EtcdCluster +metadata: + name: test +spec: + replicas: 3 + podTemplate: + spec: + containers: + - name: etcd + image: "quay.io/coreos/etcd:v3.3.27" +# volumes: [] +# storage: +# emptyDir: {} diff --git a/test/e2e/testdata/etcdcluster-3.4.yaml b/test/e2e/testdata/etcdcluster-3.4.yaml new file mode 100644 index 00000000..9e80cce4 --- /dev/null +++ b/test/e2e/testdata/etcdcluster-3.4.yaml @@ -0,0 +1,15 @@ +--- +apiVersion: etcd.aenix.io/v1alpha1 +kind: EtcdCluster +metadata: + name: test +spec: + replicas: 3 + podTemplate: + spec: + containers: + - name: etcd + image: "quay.io/coreos/etcd:v3.4.32" +# volumes: [] +# storage: +# emptyDir: {} diff --git a/test/e2e/testdata/etcdcluster-3.5.yaml b/test/e2e/testdata/etcdcluster-3.5.yaml new file mode 100644 index 00000000..71bf3f38 --- /dev/null +++ b/test/e2e/testdata/etcdcluster-3.5.yaml @@ -0,0 +1,15 @@ +--- +apiVersion: etcd.aenix.io/v1alpha1 +kind: EtcdCluster +metadata: + name: test +spec: + replicas: 3 + podTemplate: + spec: + containers: + - name: etcd + image: "quay.io/coreos/etcd:v3.5.11" +# volumes: [] +# storage: +# emptyDir: {} diff --git a/test/e2e/testdata/etcdcluster-simple.yaml b/test/e2e/testdata/etcdcluster-simple.yaml new file mode 100644 index 00000000..f1bff9e4 --- /dev/null +++ b/test/e2e/testdata/etcdcluster-simple.yaml @@ -0,0 +1,7 @@ +--- +apiVersion: etcd.aenix.io/v1alpha1 +kind: EtcdCluster +metadata: + name: test +spec: + replicas: 3 From 94b1f601ca50ddf00d205aedcd99ad247f88154e Mon Sep 17 00:00:00 2001 From: Kirill Garbar Date: Tue, 18 Jun 2024 20:05:06 +0000 Subject: [PATCH 2/5] go mod tidy --- go.mod | 17 +++++++++++++++++ go.sum | 31 ++++++++++++++++++++++++++++++- 2 files changed, 47 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index ac3f59d6..95f6139b 100644 --- a/go.mod +++ b/go.mod @@ -43,6 +43,10 @@ require ( github.com/google/go-cmp v0.6.0 // indirect github.com/google/gofuzz v1.2.0 // indirect github.com/google/pprof v0.0.0-20240424215950-a892ee059fd6 // indirect + github.com/gorilla/websocket v1.5.0 // indirect + github.com/hashicorp/hcl v1.0.0 // indirect + github.com/imdario/mergo v0.3.6 // indirect + github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/magiconair/properties v1.8.7 // indirect @@ -69,6 +73,19 @@ require ( go.etcd.io/etcd/api/v3 v3.5.14 // indirect go.etcd.io/etcd/client/pkg/v3 v3.5.14 // indirect go.uber.org/multierr v1.11.0 // indirect + golang.org/x/exp v0.0.0-20230905200255-921286631fa9 // indirect + golang.org/x/net v0.25.0 // indirect + golang.org/x/oauth2 v0.18.0 // indirect + golang.org/x/sys v0.20.0 // indirect + golang.org/x/term v0.20.0 // indirect + golang.org/x/text v0.15.0 // indirect + golang.org/x/time v0.5.0 // indirect + golang.org/x/tools v0.21.0 // indirect + gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect + google.golang.org/appengine v1.6.8 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20240311132316-a219d84964c2 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240314234333-6e1732d8331c // indirect + google.golang.org/grpc v1.62.1 // indirect google.golang.org/protobuf v1.33.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/ini.v1 v1.67.0 // indirect diff --git a/go.sum b/go.sum index 5c989351..aaf91b21 100644 --- a/go.sum +++ b/go.sum @@ -14,6 +14,8 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/emicklei/go-restful/v3 v3.11.0 h1:rAQeMHw1c7zTmncogyy8VvRZwtkmkZ4FxERmMY4rD+g= github.com/emicklei/go-restful/v3 v3.11.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= github.com/evanphx/json-patch v5.7.0+incompatible h1:vgGkfT/9f8zE6tvSCe74nfpAVDQ2tG6yudJd8LBksgI= @@ -58,6 +60,15 @@ github.com/google/pprof v0.0.0-20240424215950-a892ee059fd6 h1:k7nVchz72niMH6YLQN github.com/google/pprof v0.0.0-20240424215950-a892ee059fd6/go.mod h1:kf6iHlnVGwgKolg33glAes7Yg/8iWP8ukqeldJSO7jw= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc= +github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= +github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= +github.com/imdario/mergo v0.3.6 h1:xTNEAn+kxVO7dTZGu0CegyqKZmoWFI0rF8UxjlB2d28= +github.com/imdario/mergo v0.3.6/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= +github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= +github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= @@ -161,6 +172,7 @@ go.uber.org/zap/exp v0.2.0/go.mod h1:t0gqAIdh1MfKv9EwN/dLwfZnJxe9ITAZN78HEWPFWDQ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/exp v0.0.0-20230905200255-921286631fa9 h1:GoHiUyI/Tp2nVkLI2mCxVkOjsbSXD66ic0XW0js0R9g= golang.org/x/exp v0.0.0-20230905200255-921286631fa9/go.mod h1:S2oDrQGGwySpoQPVqRShND87VCbxmc6bL1Yd2oYrm6k= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= @@ -170,6 +182,10 @@ golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac= +golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= golang.org/x/oauth2 v0.18.0 h1:09qnuIAgzdx1XplqJvW6CQqMCtGZykZWcXzPMPUusvI= golang.org/x/oauth2 v0.18.0/go.mod h1:Wf7knwG0MPoWIMMBgFlEaSUDaKskp0dCfrlJRJXbBi8= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -191,6 +207,10 @@ golang.org/x/term v0.20.0 h1:VnkxpohqXaOBYJtBmEppKUG6mXpi+4O6purfc2+sMhw= golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= +golang.org/x/text v0.15.0 h1:h1V/4gjBv8v9cjcR6+AR5+/cIYK5N/WAgiv4xlsEtAk= +golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -206,6 +226,16 @@ golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gomodules.xyz/jsonpatch/v2 v2.4.0 h1:Ci3iUJyx9UeRx7CeFN8ARgGbkESwJK+KB9lLcWxY/Zw= gomodules.xyz/jsonpatch/v2 v2.4.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuBJT4moAY= +google.golang.org/appengine v1.6.8 h1:IhEN5q69dyKagZPYMSdIjS2HqprW324FRQZJcGqPAsM= +google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJffLiz/Ds= +google.golang.org/genproto/googleapis/api v0.0.0-20240311132316-a219d84964c2 h1:rIo7ocm2roD9DcFIX67Ym8icoGCKSARAiPljFhh5suQ= +google.golang.org/genproto/googleapis/api v0.0.0-20240311132316-a219d84964c2/go.mod h1:O1cOfN1Cy6QEYr7VxtjOyP5AdAuR0aJ/MYZaaof623Y= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240314234333-6e1732d8331c h1:lfpJ/2rWPa/kJgxyyXM8PrNnfCzcmxJ265mADgwmvLI= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240314234333-6e1732d8331c/go.mod h1:WtryC6hu0hhx87FDGxWCDptyssuo68sk10vYjF+T9fY= +google.golang.org/grpc v1.62.1 h1:B4n+nfKzOICUXMgyrNd19h/I9oH0L1pizfk1d4zSgTk= +google.golang.org/grpc v1.62.1/go.mod h1:IWTG0VlJLCh1SkC58F7np9ka9mx/WNkjl4PGJaiq+QE= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -219,7 +249,6 @@ gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= k8s.io/api v0.30.2 h1:+ZhRj+28QT4UOH+BKznu4CBgPWgkXO7XAvMcMl0qKvI= From 96305219b56a32b214601910f0c7348d95b4c475 Mon Sep 17 00:00:00 2001 From: Kirill Garbar Date: Tue, 18 Jun 2024 20:36:51 +0000 Subject: [PATCH 3/5] adjust tests --- test/e2e/e2e_test.go | 160 +++++++++++++++++++++---------------------- 1 file changed, 79 insertions(+), 81 deletions(-) diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.go index 43de15fc..14654fed 100644 --- a/test/e2e/e2e_test.go +++ b/test/e2e/e2e_test.go @@ -20,7 +20,6 @@ import ( "fmt" "os" "os/exec" - "strconv" "sync" "time" @@ -89,23 +88,25 @@ var _ = Describe("etcd-operator", Ordered, func() { "--filename", dir+"/examples/manifests/etcdcluster-simple.yaml", "--namespace", namespace, ) - _, err = utils.Run(cmd) + _, err := utils.Run(cmd) ExpectWithOffset(1, err).NotTo(HaveOccurred()) }) WaitSTSReady("statefulset/test", namespace) - // CheckEtcdClusterHealthy("service/test", namespace) + var etcdClient *clientv3.Client - client, err := utils.GetEtcdClient(ctx, client.ObjectKey{Namespace: namespace, Name: "test"}) - Expect(err).NotTo(HaveOccurred()) - defer func() { - err := client.Close() + By("get etcd client", func() { + etcdClient, err := utils.GetEtcdClient(ctx, client.ObjectKey{Namespace: namespace, Name: "test"}) Expect(err).NotTo(HaveOccurred()) - }() + defer func() { + err := etcdClient.Close() + Expect(err).NotTo(HaveOccurred()) + }() + }) By("check etcd cluster is healthy", func() { - Expect(utils.IsEtcdClusterHealthy(ctx, client)).To(BeTrue()) + Expect(utils.IsEtcdClusterHealthy(ctx, etcdClient)).To(BeTrue()) }) }) @@ -145,18 +146,22 @@ var _ = Describe("etcd-operator", Ordered, func() { ExpectWithOffset(1, err).NotTo(HaveOccurred()) }) - client, err := utils.GetEtcdClient(ctx, client.ObjectKey{Namespace: namespace, Name: "test"}) - Expect(err).NotTo(HaveOccurred()) - defer func() { - err := client.Close() + var etcdClient *clientv3.Client + + By("get etcd client", func() { + etcdClient, err := utils.GetEtcdClient(ctx, client.ObjectKey{Namespace: namespace, Name: "test"}) Expect(err).NotTo(HaveOccurred()) - }() + defer func() { + err := etcdClient.Close() + Expect(err).NotTo(HaveOccurred()) + }() + }) By("check etcd cluster is healthy", func() { - Expect(utils.IsEtcdClusterHealthy(ctx, client)).To(BeTrue()) + Expect(utils.IsEtcdClusterHealthy(ctx, etcdClient)).To(BeTrue()) }) - auth := clientv3.NewAuth(client) + auth := clientv3.NewAuth(etcdClient) By("check root role is created", func() { _, err = auth.RoleGet(ctx, "root") @@ -183,45 +188,61 @@ var _ = Describe("etcd-operator", Ordered, func() { CreateNamespace(namespace) DeferCleanup(DeleteNamespaceCB(namespace)) - By("apply upgrade etcd cluster manifest") + By("apply upgrade etcd cluster manifest", func() { + cmd := exec.Command("kubectl", "apply", + "--filename", dir+fileName, + "--namespace", namespace, + ) + _, err := utils.Run(cmd) - cmd := exec.Command("kubectl", "apply", - "--filename", dir+fileName, - "--namespace", namespace, - ) - _, err := utils.Run(cmd) + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + }) - ExpectWithOffset(1, err).NotTo(HaveOccurred()) + var etcdClient *clientv3.Client + + By("get etcd client", func() { + etcdClient, err := utils.GetEtcdClient(ctx, client.ObjectKey{Namespace: namespace, Name: "test"}) + Expect(err).NotTo(HaveOccurred()) + defer func() { + err := etcdClient.Close() + Expect(err).NotTo(HaveOccurred()) + }() + }) WaitSTSReady("statefulset/test", namespace) - CheckEtcdClusterHealthy("service/test", namespace) - - By("upgrade etcd cluster to one patch version") - cmd = exec.Command("kubectl", "patch", - "etcdcluster/test", - "--type", "json", - // Strategic Merge Patch is not currently supported by the etcd-operator - // "--patch", - // fmt.Sprintf( - // "{\"spec\":{\"podTemplate\":{\"containers\":[{\"name\":\"etcd\",\"image\":\"quay.io/coreos/etcd:%s\"}]}}}", - // version, - // ), - "--patch", - fmt.Sprintf( - "[{\"op\": \"replace\", \"path\": \"/spec/podTemplate/spec/containers/0/image\", "+ - "\"value\":\"quay.io/coreos/etcd:%s\"}]", - version, - ), - "--namespace", namespace, - ) - _, err = utils.Run(cmd) - ExpectWithOffset(1, err).NotTo(HaveOccurred()) + + By("check etcd cluster is healthy", func() { + Expect(utils.IsEtcdClusterHealthy(ctx, etcdClient)).To(BeTrue()) + }) + By("upgrade etcd cluster to one patch version", func() { + cmd := exec.Command("kubectl", "patch", + "etcdcluster/test", + "--type", "json", + // Strategic Merge Patch is not currently supported by the etcd-operator + // "--patch", + // fmt.Sprintf( + // "{\"spec\":{\"podTemplate\":{\"containers\":[{\"name\":\"etcd\",\"image\":\"quay.io/coreos/etcd:%s\"}]}}}", + // version, + // ), + "--patch", + fmt.Sprintf( + "[{\"op\": \"replace\", \"path\": \"/spec/podTemplate/spec/containers/0/image\", "+ + "\"value\":\"quay.io/coreos/etcd:%s\"}]", + version, + ), + "--namespace", namespace, + ) + _, err := utils.Run(cmd) + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + }) // Give the operator some time to update the sts time.Sleep(2 * time.Second) WaitSTSReady("statefulset/test", namespace) - CheckEtcdClusterHealthy("service/test", namespace) + By("check etcd cluster is healthy", func() { + Expect(utils.IsEtcdClusterHealthy(ctx, etcdClient)).To(BeTrue()) + }) }, Entry( "should upgrade etcd cluster patch version", @@ -279,48 +300,25 @@ func DeleteNamespace(namespace string) error { } func CreateNamespace(namespace string) { - By("create namespace") + By("create namespace", func() { + cmd := exec.Command("kubectl", "create", "namespace", namespace) + _, err := utils.Run(cmd) + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + }) - cmd := exec.Command("kubectl", "create", "namespace", namespace) - _, err := utils.Run(cmd) - ExpectWithOffset(1, err).NotTo(HaveOccurred()) } func WaitSTSReady(stsName, namespace string) { - By("wait for statefulset is ready") - - cmd := exec.Command("kubectl", "wait", - stsName, - "--for", "jsonpath={.status.availableReplicas}=3", - "--namespace", namespace, - "--timeout", "5m", - ) - _, err := utils.Run(cmd) - - ExpectWithOffset(1, err).NotTo(HaveOccurred()) -} - -func CheckEtcdClusterHealthy(serviceName, namespace string) { - var wg sync.WaitGroup - wg.Add(1) - - By("port-forward service to localhost") - port, _ := utils.GetFreePort() - go func() { - defer GinkgoRecover() - defer wg.Done() - - cmd := exec.Command("kubectl", "port-forward", - serviceName, strconv.Itoa(port)+":2379", + By("wait for statefulset is ready", func() { + cmd := exec.Command("kubectl", "wait", + stsName, + "--for", "jsonpath={.status.availableReplicas}=3", "--namespace", namespace, + "--timeout", "5m", ) _, err := utils.Run(cmd) + ExpectWithOffset(1, err).NotTo(HaveOccurred()) - }() + }) - By("check etcd cluster is healthy") - endpoints := []string{"localhost:" + strconv.Itoa(port)} - for i := 0; i < 3; i++ { - Expect(utils.IsEtcdClusterHealthy(endpoints)).To(BeTrue()) - } } From 259fd529e80a75fedfcc24d42bfb992af5d74329 Mon Sep 17 00:00:00 2001 From: Kirill Garbar Date: Tue, 18 Jun 2024 20:40:05 +0000 Subject: [PATCH 4/5] fix failed requests to webhook --- Makefile | 6 +++--- config/manager/manager.yaml | 2 +- test/e2e/testdata/kind.yaml | 4 ++++ 3 files changed, 8 insertions(+), 4 deletions(-) create mode 100644 test/e2e/testdata/kind.yaml diff --git a/Makefile b/Makefile index 6aa29576..5ccfde18 100644 --- a/Makefile +++ b/Makefile @@ -76,7 +76,7 @@ mod-tidy: ## Run go mod tidy against code. test: manifests generate fmt vet envtest ## Run tests. @echo "Check for kubernetes version $(K8S_VERSION_TRIMMED_V) in $(ENVTEST)" @$(ENVTEST) list | grep -q $(K8S_VERSION_TRIMMED_V) - KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(K8S_VERSION_TRIMMED_V) --bin-dir $(LOCALBIN) -p path)" go test $$(go list ./... | grep -v /e2e) -coverprofile cover.out + KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(K8S_VERSION_TRIMMED_V) --bin-dir $(LOCALBIN) -p path)" go test $$(go list ./... | grep -v /e2e) -coverprofile cover.out -v # Utilize Kind or modify the e2e tests to load the image locally, enabling compatibility with other vendors. .PHONY: test-e2e # Run the e2e tests against a Kind k8s instance that is spun up. @@ -203,11 +203,11 @@ kind-load: docker-build kind ## Build and upload docker image to the local Kind .PHONY: kind-create kind-create: kind yq ## Create kubernetes cluster using Kind. @if ! $(KIND) get clusters | grep -q $(KIND_CLUSTER_NAME); then \ - $(KIND) create cluster --name $(KIND_CLUSTER_NAME) --image kindest/node:$(K8S_VERSION); \ + $(KIND) create cluster --name $(KIND_CLUSTER_NAME) --image kindest/node:$(K8S_VERSION) --config=test/e2e/testdata/kind.yaml; \ fi @if ! $(CONTAINER_TOOL) container inspect $$($(KIND) get nodes) | $(YQ) e '.[0].Config.Image' | grep -q $(K8S_VERSION); then \ $(KIND) delete cluster --name $(KIND_CLUSTER_NAME); \ - $(KIND) create cluster --name $(KIND_CLUSTER_NAME) --image kindest/node:$(K8S_VERSION); \ + $(KIND) create cluster --name $(KIND_CLUSTER_NAME) --image kindest/node:$(K8S_VERSION) --config=test/e2e/testdata/kind.yaml; \ fi .PHONY: kind-delete diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml index d657a5e9..4d4f9b29 100644 --- a/config/manager/manager.yaml +++ b/config/manager/manager.yaml @@ -88,7 +88,7 @@ spec: httpGet: path: /readyz port: 8081 - initialDelaySeconds: 5 + initialDelaySeconds: 0 periodSeconds: 10 # TODO(user): Configure the resources accordingly based on the project requirements. # More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ diff --git a/test/e2e/testdata/kind.yaml b/test/e2e/testdata/kind.yaml new file mode 100644 index 00000000..afe81483 --- /dev/null +++ b/test/e2e/testdata/kind.yaml @@ -0,0 +1,4 @@ +kind: Cluster +apiVersion: kind.x-k8s.io/v1alpha4 +networking: + kubeProxyMode: "ipvs" From 57af2c64d60f28ce7bbc0708fdfe9f24bda0129b Mon Sep 17 00:00:00 2001 From: Kirill Garbar Date: Wed, 19 Jun 2024 08:57:20 +0000 Subject: [PATCH 5/5] adjust autotest --- Makefile | 2 +- test/e2e/e2e_test.go | 51 ++++++++++++++++++++------------------------ 2 files changed, 24 insertions(+), 29 deletions(-) diff --git a/Makefile b/Makefile index 5ccfde18..4d052799 100644 --- a/Makefile +++ b/Makefile @@ -270,7 +270,7 @@ HELM_DOCS_VERSION ?= v1.13.1 # renovate: datasource=github-tags depName=mikefarah/yq YQ_VERSION ?= v4.44.1 # renovate: datasource=github-tags depName=onsi/ginkgo -GINKGO_VERSION ?= v2.17.3 +GINKGO_VERSION ?= v2.19.0 ## Tool install scripts KUSTOMIZE_INSTALL_SCRIPT ?= "https://raw.githubusercontent.com/kubernetes-sigs/kustomize/master/hack/install_kustomize.sh" diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.go index 14654fed..3d723d29 100644 --- a/test/e2e/e2e_test.go +++ b/test/e2e/e2e_test.go @@ -20,7 +20,6 @@ import ( "fmt" "os" "os/exec" - "sync" "time" . "github.com/onsi/ginkgo/v2" @@ -76,8 +75,6 @@ var _ = Describe("etcd-operator", Ordered, func() { Context("Simple", func() { It("should deploy etcd cluster", func() { const namespace = "test-simple-etcd-cluster" - var wg sync.WaitGroup - wg.Add(1) CreateNamespace(namespace) DeferCleanup(DeleteNamespaceCB(namespace)) @@ -96,14 +93,14 @@ var _ = Describe("etcd-operator", Ordered, func() { var etcdClient *clientv3.Client - By("get etcd client", func() { - etcdClient, err := utils.GetEtcdClient(ctx, client.ObjectKey{Namespace: namespace, Name: "test"}) + // By("get etcd client", func() { + etcdClient, err := utils.GetEtcdClient(ctx, client.ObjectKey{Namespace: namespace, Name: "test"}) + Expect(err).NotTo(HaveOccurred()) + defer func() { + err := etcdClient.Close() Expect(err).NotTo(HaveOccurred()) - defer func() { - err := etcdClient.Close() - Expect(err).NotTo(HaveOccurred()) - }() - }) + }() + // }) By("check etcd cluster is healthy", func() { Expect(utils.IsEtcdClusterHealthy(ctx, etcdClient)).To(BeTrue()) @@ -116,8 +113,6 @@ var _ = Describe("etcd-operator", Ordered, func() { It("should deploy etcd cluster with auth", func() { var err error const namespace = "test-tls-auth-etcd-cluster" - var wg sync.WaitGroup - wg.Add(1) By("create namespace", func() { cmd := exec.Command("sh", "-c", fmt.Sprintf("kubectl create namespace %s --dry-run=client -o yaml | kubectl apply -f -", namespace)) //nolint:lll @@ -138,7 +133,7 @@ var _ = Describe("etcd-operator", Ordered, func() { By("wait for statefulset is ready", func() { cmd := exec.Command("kubectl", "wait", "statefulset/test", - "--for", "jsonpath={.status.availableReplicas}=3", + "--for", "jsonpath={.status.readyReplicas}=3", "--namespace", namespace, "--timeout", "5m", ) @@ -148,14 +143,14 @@ var _ = Describe("etcd-operator", Ordered, func() { var etcdClient *clientv3.Client - By("get etcd client", func() { - etcdClient, err := utils.GetEtcdClient(ctx, client.ObjectKey{Namespace: namespace, Name: "test"}) + // By("get etcd client", func() { + etcdClient, err = utils.GetEtcdClient(ctx, client.ObjectKey{Namespace: namespace, Name: "test"}) + Expect(err).NotTo(HaveOccurred()) + defer func() { + err := etcdClient.Close() Expect(err).NotTo(HaveOccurred()) - defer func() { - err := etcdClient.Close() - Expect(err).NotTo(HaveOccurred()) - }() - }) + }() + // }) By("check etcd cluster is healthy", func() { Expect(utils.IsEtcdClusterHealthy(ctx, etcdClient)).To(BeTrue()) @@ -200,14 +195,14 @@ var _ = Describe("etcd-operator", Ordered, func() { var etcdClient *clientv3.Client - By("get etcd client", func() { - etcdClient, err := utils.GetEtcdClient(ctx, client.ObjectKey{Namespace: namespace, Name: "test"}) + // By("get etcd client", func() { + etcdClient, err := utils.GetEtcdClient(ctx, client.ObjectKey{Namespace: namespace, Name: "test"}) + Expect(err).NotTo(HaveOccurred()) + defer func() { + err := etcdClient.Close() Expect(err).NotTo(HaveOccurred()) - defer func() { - err := etcdClient.Close() - Expect(err).NotTo(HaveOccurred()) - }() - }) + }() + // }) WaitSTSReady("statefulset/test", namespace) @@ -312,7 +307,7 @@ func WaitSTSReady(stsName, namespace string) { By("wait for statefulset is ready", func() { cmd := exec.Command("kubectl", "wait", stsName, - "--for", "jsonpath={.status.availableReplicas}=3", + "--for", "jsonpath={.status.readyReplicas}=3", "--namespace", namespace, "--timeout", "5m", )