feat(pipeline-gw): health probes (#6728)

* wip

* startup and liveness and readiness probes

* wip: refactor

* tests

* fix typo

* add startup probe for gRPC svc

* lint

* add kafka health checks

* support TLS

* lint

* Update scheduler/cmd/pipelinegateway/main.go

Co-authored-by: Lucian Carata <lc525@users.noreply.github.com>

---------

Co-authored-by: Lucian Carata <lc525@users.noreply.github.com>

Author: domsolutions

k8s/helm-charts/seldon-core-v2-setup/templates/_components-deployments.tpl

@@ -667,6 +667,7 @@ spec:
         - --kafka-config-path=/mnt/kafka/kafka.json
         - --tracing-config-path=/mnt/tracing/tracing.json
         - --log-level=$(LOG_LEVEL)
+        - --health-probe-port=$(HEALTH_PROBE_PORT)
         command:
         - /bin/pipelinegateway
         env:
@@ -753,6 +754,8 @@ spec:
           valueFrom:
             fieldRef:
               fieldPath: metadata.name
+        - name: HEALTH_PROBE_PORT
+          value: "9999"
         - name: POD_IP
           valueFrom:
             fieldRef:
@@ -764,6 +767,12 @@ spec:
         image: '{{ .Values.pipelinegateway.image.registry }}/{{ .Values.pipelinegateway.image.repository
           }}:{{ .Values.pipelinegateway.image.tag }}'
         imagePullPolicy: '{{ .Values.pipelinegateway.image.pullPolicy }}'
+        livenessProbe:
+          failureThreshold: 3
+          httpGet:
+            path: /live
+            port: health
+          periodSeconds: 5
         name: pipelinegateway
         ports:
         - containerPort: 9010
@@ -775,12 +784,28 @@ spec:
         - containerPort: 9006
           name: metrics
           protocol: TCP
+        - containerPort: 9999
+          name: health
+          protocol: TCP
+        readinessProbe:
+          failureThreshold: 3
+          httpGet:
+            path: /ready
+            port: health
+          periodSeconds: 5
         resources:
           limits:
             memory: '{{ .Values.pipelinegateway.resources.memory }}'
           requests:
             cpu: '{{ .Values.pipelinegateway.resources.cpu }}'
             memory: '{{ .Values.pipelinegateway.resources.memory }}'
+        startupProbe:
+          failureThreshold: 3
+          httpGet:
+            path: /startup
+            port: health
+          initialDelaySeconds: 3
+          periodSeconds: 5
         volumeMounts:
         - mountPath: /mnt/kafka
           name: kafka-config-volume

k8s/helm-charts/seldon-core-v2-setup/templates/_components-statefulsets.tpl

@@ -667,6 +667,7 @@ spec:
         - --kafka-config-path=/mnt/kafka/kafka.json
         - --tracing-config-path=/mnt/tracing/tracing.json
         - --log-level=$(LOG_LEVEL)
+        - --health-probe-port=$(HEALTH_PROBE_PORT)
         command:
         - /bin/pipelinegateway
         env:
@@ -753,6 +754,8 @@ spec:
           valueFrom:
             fieldRef:
               fieldPath: metadata.name
+        - name: HEALTH_PROBE_PORT
+          value: "9999"
         - name: POD_IP
           valueFrom:
             fieldRef:
@@ -764,6 +767,12 @@ spec:
         image: '{{ .Values.pipelinegateway.image.registry }}/{{ .Values.pipelinegateway.image.repository
           }}:{{ .Values.pipelinegateway.image.tag }}'
         imagePullPolicy: '{{ .Values.pipelinegateway.image.pullPolicy }}'
+        livenessProbe:
+          failureThreshold: 3
+          httpGet:
+            path: /live
+            port: health
+          periodSeconds: 5
         name: pipelinegateway
         ports:
         - containerPort: 9010
@@ -775,12 +784,28 @@ spec:
         - containerPort: 9006
           name: metrics
           protocol: TCP
+        - containerPort: 9999
+          name: health
+          protocol: TCP
+        readinessProbe:
+          failureThreshold: 3
+          httpGet:
+            path: /ready
+            port: health
+          periodSeconds: 5
         resources:
           limits:
             memory: '{{ .Values.pipelinegateway.resources.memory }}'
           requests:
             cpu: '{{ .Values.pipelinegateway.resources.cpu }}'
             memory: '{{ .Values.pipelinegateway.resources.memory }}'
+        startupProbe:
+          failureThreshold: 3
+          httpGet:
+            path: /startup
+            port: health
+          initialDelaySeconds: 3
+          periodSeconds: 5
         volumeMounts:
         - mountPath: /mnt/kafka
           name: kafka-config-volume

k8s/yaml/components.yaml

@@ -508,6 +508,7 @@ spec:
         - --kafka-config-path=/mnt/kafka/kafka.json
         - --tracing-config-path=/mnt/tracing/tracing.json
         - --log-level=$(LOG_LEVEL)
+        - --health-probe-port=$(HEALTH_PROBE_PORT)
         command:
         - /bin/pipelinegateway
         env:
@@ -589,6 +590,8 @@ spec:
           valueFrom:
             fieldRef:
               fieldPath: metadata.name
+        - name: HEALTH_PROBE_PORT
+          value: "9999"
         - name: POD_IP
           valueFrom:
             fieldRef:
@@ -599,6 +602,12 @@ spec:
               fieldPath: metadata.namespace
         image: 'docker.io/seldonio/seldon-pipelinegateway:latest'
         imagePullPolicy: 'IfNotPresent'
+        livenessProbe:
+          failureThreshold: 3
+          httpGet:
+            path: /live
+            port: health
+          periodSeconds: 5
         name: pipelinegateway
         ports:
         - containerPort: 9010
@@ -610,12 +619,28 @@ spec:
         - containerPort: 9006
           name: metrics
           protocol: TCP
+        - containerPort: 9999
+          name: health
+          protocol: TCP
+        readinessProbe:
+          failureThreshold: 3
+          httpGet:
+            path: /ready
+            port: health
+          periodSeconds: 5
         resources:
           limits:
             memory: '1G'
           requests:
             cpu: '100m'
             memory: '1G'
+        startupProbe:
+          failureThreshold: 3
+          httpGet:
+            path: /startup
+            port: health
+          initialDelaySeconds: 3
+          periodSeconds: 5
         volumeMounts:
         - mountPath: /mnt/kafka
           name: kafka-config-volume

operator/config/seldonconfigs/default.yaml

@@ -172,6 +172,7 @@ spec:
         - --kafka-config-path=/mnt/kafka/kafka.json
         - --tracing-config-path=/mnt/tracing/tracing.json
         - --log-level=$(LOG_LEVEL)
+        - --health-probe-port=$(HEALTH_PROBE_PORT)
         command:
         - /bin/pipelinegateway
         env:
@@ -185,6 +186,8 @@ spec:
           valueFrom:
             fieldRef:
               fieldPath: metadata.name
+        - name: HEALTH_PROBE_PORT
+          value: "9999"
         - name: POD_IP
           valueFrom:
             fieldRef:
@@ -206,12 +209,34 @@ spec:
         - containerPort: 9006
           name: metrics
           protocol: TCP
+        - containerPort: 9999
+          name: health
+          protocol: TCP
         resources:
           limits:
             memory: 1G
           requests:
             cpu: 100m
             memory: 1G
+        startupProbe:
+          httpGet:
+            path: /startup
+            port: health
+          initialDelaySeconds: 3
+          periodSeconds: 5
+          failureThreshold: 3
+        readinessProbe:
+          httpGet:
+            path: /ready
+            port: health
+          periodSeconds: 5
+          failureThreshold: 3
+        livenessProbe:
+          httpGet:
+            path: /live
+            port: health
+          periodSeconds: 5
+          failureThreshold: 3
         volumeMounts:
         - mountPath: /mnt/kafka
           name: kafka-config-volume