Fix the readiness probe using a sidecar container

Author: lobo-ppi

charts/ibm-mq/templates/role-binding.yaml

@@ -0,0 +1,13 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: pod-labeler-binding
+  namespace: ready-check
+roleRef:
+    apiGroup: rbac.authorization.k8s.io
+    kind: Role
+    name: pod-labeler
+subjects:
+- kind: ServiceAccount
+  name: {{ include "ibm-mq.fullname" ( . ) }}
+  namespace: ready-check

charts/ibm-mq/templates/role.yaml

@@ -0,0 +1,9 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: pod-labeler
+  namespace: ready-check
+rules:
+- apiGroups: [""]
+  resources: ["pods"]
+  verbs: ["get", "list","patch"]

charts/ibm-mq/templates/service.yaml

@@ -26,3 +26,4 @@ spec:
     name: qmgr
   selector:
 {{- include "ibm-mq.selectorLabels" . | nindent 4 }}
+    role: master

charts/ibm-mq/templates/stateful-set.yaml

@@ -272,8 +272,46 @@ spec:
           defaultMode: 420
           secretName: {{ .Values.credentials.secret }}
       {{- end }}
+      volumes:
+      - name: state
+        emptyDir: {}
       terminationGracePeriodSeconds: {{.Values.queueManager.terminationGracePeriodSeconds}}
       containers:
+        - name: state-checker
+          command:
+            - sh
+            - '-c'
+            - >
+              sleep 20;
+              while true; do
+                sleep $(($RANDOM % 5 + 5));
+                KUBE_TOKEN=$(cat /var/run/secrets/kubernetes.io/serviceaccount/token)
+                NAMESPACE=$(cat /var/run/secrets/kubernetes.io/serviceaccount/namespace)
+                echo "Checking state of the queue manager"
+                STATE=$(cat /etc/mqm/state/test)
+                if [ $STATE -eq 0 ]; then
+                echo "Queue manager is active"
+                echo "patching the label:"
+                curl -sSk -H "Authorization: Bearer $KUBE_TOKEN" \
+                    --request PATCH \
+                    --header "Content-Type: application/json-patch+json" \
+                    --data '[ { "op": "replace", "path": "/metadata/labels/role", "value": "master" } ]' \
+                    https://$KUBERNETES_SERVICE_HOST:$KUBERNETES_PORT_443_TCP_PORT/api/v1/namespaces/$NAMESPACE/pods/$HOSTNAME
+                else
+                echo "Queue manager is not active"
+                echo "patching the label:"
+                curl -sSk -H "Authorization: Bearer $KUBE_TOKEN" \
+                    --header "Content-Type: application/json-patch+json" \
+                    --request PATCH \
+                    --data '[ { "op": "replace", "path": "/metadata/labels/role", "value": "standby" } ]' \
+                    https://$KUBERNETES_SERVICE_HOST:$KUBERNETES_PORT_443_TCP_PORT/api/v1/namespaces/$NAMESPACE/pods/$HOSTNAME
+                fi
+              done
+          image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
+          imagePullPolicy: IfNotPresent
+          volumeMounts:
+            - mountPath: /etc/mqm/state
+              name: state
         - name: qmgr
           image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
           imagePullPolicy: {{ .Values.image.pullPolicy }}
@@ -371,6 +409,8 @@ spec:
           {{- else if .Values.web.manualConfig.secret.name }}
           volumeMounts:
           {{- end}}
+          - name: state
+            mountPath: /etc/mqm/state
           {{- if .Values.queueManager.nativeha.tls }}
           {{- if .Values.queueManager.nativeha.tls.secretName }}
           - name: ha-tls
@@ -493,7 +533,11 @@ spec:
           readinessProbe:
             exec:
               command:
-              - chkmqready
+              - sh
+              - '-c'
+              - >
+                chkmqready;
+                echo $? > /etc/mqm/state/test
             {{- if or .Values.queueManager.nativeha.enable .Values.queueManager.multiinstance.enable }}
             initialDelaySeconds: {{ .Values.readinessProbe.initialDelaySeconds | default 0 }}
             {{- else }}