ON-15199: Add cplane parameters to Onload CRD

Author: ivatet-amd

api/v1alpha1/onload_types.go

@@ -76,6 +76,18 @@ type OnloadSpec struct {
 	// ImagePullPolicy is the policy used when pulling images.
 	// More info: https://kubernetes.io/docs/concepts/containers/images#updating-images
 	ImagePullPolicy v1.PullPolicy `json:"imagePullPolicy,omitempty"`
+
+	// +optional
+	// ControlPlane allows fine-tuning of the Onload control plane server.
+	ControlPlane *ControlPlaneSpec `json:"controlPlane,omitempty"`
+}
+
+type ControlPlaneSpec struct {
+	// +optional
+	// Parameters is an optional list of parameters passed to the Onload
+	// control plane server when launched by the Onload kernel module.
+	// +kubebuilder:default:={"-K"}
+	Parameters []string `json:"parameters"`
 }
 
 // Currently unimplemented

api/v1alpha1/zz_generated.deepcopy.go

@@ -30,6 +30,9 @@ func main() {
 	// A mount point with the Onload control plane server.
 	onloadCPServerPath := os.Getenv("ONLOAD_CP_SERVER_PATH")
 
+	// Params the Onload kmod should pass to the Onload control plane server.
+	onloadCPServerParams := os.Getenv("ONLOAD_CP_SERVER_PARAMS")
+
 	// There is a race condition when the container is starting, and the
 	// Onload kernel module is loading at the same time, resulting in not
 	// all files being available in the container.
@@ -59,8 +62,8 @@ func main() {
 
 	// Configure Onload kernel module to launch
 	// Onload control plane within a container.
-	err = control_plane.Configure(onloadCPServerPath, containerID,
-		control_plane.NewKernelParametersWriter())
+	err = control_plane.Configure(onloadCPServerPath, onloadCPServerParams,
+		containerID, control_plane.NewKernelParametersWriter())
 	if err != nil {
 		glog.Fatal("Failed to configure Onload control plane: ", err)
 	}

cmd/worker/main.go

@@ -95,6 +95,20 @@ spec:
                 description: Onload is the specification of the version of Onload
                   to be used by this CR
                 properties:
+                  controlPlane:
+                    description: ControlPlane allows fine-tuning of the Onload control
+                      plane server.
+                    properties:
+                      parameters:
+                        default:
+                        - -K
+                        description: Parameters is an optional list of parameters
+                          passed to the Onload control plane server when launched
+                          by the Onload kernel module.
+                        items:
+                          type: string
+                        type: array
+                    type: object
                   imagePullPolicy:
                     description: 'ImagePullPolicy is the policy used when pulling
                       images. More info: https://kubernetes.io/docs/concepts/containers/images#updating-images'

config/crd/bases/onload.amd.com_onloads.yaml

@@ -80,6 +80,15 @@ spec:
     # ImagePullPolicy is the policy used when pulling images. Optional.
     imagePullPolicy: Always
 
+    # ControlPlane allows fine-tuning of the Onload control plane server.
+    # Optional.
+    #controlPlane:
+
+      # Parameters is an optional list of parameters passed to the Onload
+      # control plane server when launched by the Onload kernel module.
+      #parameters:
+      #- -K
+
   # DevicePlugin is further specification for the Onload Device Plugin which
   # uses the device plugin framework to provide an `amd.com/onload` resource.
   # Image location is not configured here; see Onload Operator deployment.

config/samples/onload/base/onload_v1alpha1_onload.yaml

@@ -7,6 +7,7 @@ import (
 	"context"
 	"fmt"
 	"slices"
+	"strings"
 	"time"
 
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -991,6 +992,11 @@ func (r *OnloadReconciler) createDevicePluginDaemonSet(
 
 	workerContainerName := "onload-worker"
 
+	cplaneParams := "-K" // log-to-kmsg
+	if onload.Spec.Onload.ControlPlane != nil && onload.Spec.Onload.ControlPlane.Parameters != nil {
+		cplaneParams = strings.Join(onload.Spec.Onload.ControlPlane.Parameters, " ")
+	}
+
 	workerContainerEnv := []corev1.EnvVar{
 		{
 			Name: "POD_NAME",
@@ -1016,6 +1022,10 @@ func (r *OnloadReconciler) createDevicePluginDaemonSet(
 			Name:  "ONLOAD_CP_SERVER_PATH",
 			Value: "/mnt/onload/sbin/onload_cp_server",
 		},
+		{
+			Name:  "ONLOAD_CP_SERVER_PARAMS",
+			Value: cplaneParams,
+		},
 	}
 
 	workerContainer := corev1.Container{

controllers/onload_controller.go

@@ -4,6 +4,7 @@ package controllers
 
 import (
 	"errors"
+	"slices"
 	"strconv"
 	"time"
 
@@ -871,5 +872,56 @@ var _ = Describe("onload controller", func() {
 				"-libMountPath=qux",
 			),
 		)
+
+		DescribeTable("Testing Onload cplane parameters",
+			func(cplane *onloadv1alpha1.ControlPlaneSpec, expectedParams string) {
+				devicePlugin := appsv1.DaemonSet{}
+				devicePluginName := types.NamespacedName{
+					Name:      onload.Name + "-onload-device-plugin-ds",
+					Namespace: onload.Namespace,
+				}
+
+				if cplane != nil {
+					onload.Spec.Onload.ControlPlane = cplane
+				}
+
+				Expect(k8sClient.Create(ctx, onload)).To(BeNil())
+
+				Eventually(func() bool {
+					err := k8sClient.Get(ctx, devicePluginName, &devicePlugin)
+					return err == nil
+				}, timeout, pollingInterval).Should(BeTrue())
+
+				// Find the worker container.
+				workerIdx := slices.IndexFunc(devicePlugin.Spec.Template.Spec.Containers,
+					func(c corev1.Container) bool { return c.Name == "onload-worker" })
+
+				Expect(workerIdx).NotTo(Equal(-1))
+				workerContainer := devicePlugin.Spec.Template.Spec.Containers[workerIdx]
+
+				// Find the ONLOAD_CP_SERVER_PARAMS env.
+				envIdx := slices.IndexFunc(workerContainer.Env,
+					func(e corev1.EnvVar) bool { return e.Name == "ONLOAD_CP_SERVER_PARAMS" })
+				Expect(envIdx).NotTo(Equal(-1))
+				envValue := workerContainer.Env[envIdx].Value
+
+				// Compare actual vs. expected.
+				Expect(envValue).To(Equal(expectedParams))
+
+			},
+			Entry( /*It*/ "should use the default parameters", nil, "-K"),
+			Entry( /*It*/ "should pass an empty list of parameters",
+				&onloadv1alpha1.ControlPlaneSpec{Parameters: []string{}}, "",
+			),
+			Entry( /*It*/ "should pass a custom list of parameters",
+				&onloadv1alpha1.ControlPlaneSpec{
+					Parameters: []string{
+						"-K",
+						"--llap-max=100",
+					},
+				}, "-K --llap-max=100",
+			),
+		)
+
 	})
 })

controllers/onload_controller_test.go

@@ -12,7 +12,8 @@ import (
 // Configure the loaded Onload kernel module to launch
 // the Onload control plane process within a container.
 func Configure(
-	onloadCPServerPath string, containerID string, kpw KernelParametersWriter,
+	onloadCPServerPath string, onloadCPServerParams string,
+	containerID string, kpw KernelParametersWriter,
 ) error {
 	// Split the container runtime type from identifier.
 	containerID, found := strings.CutPrefix(containerID, "cri-o://")
@@ -31,8 +32,14 @@ func Configure(
 
 	glog.Info("Updated Onload control plane server path to ", crictlPath)
 
+	// Prepend the whitespace to non-empty cplane parameters.
+	if onloadCPServerParams != "" {
+		onloadCPServerParams = " " + onloadCPServerParams
+	}
+
 	// Set the Onload control plane params.
-	params := fmt.Sprintf("exec %s %s -K", containerID, onloadCPServerPath)
+	params := fmt.Sprintf("exec %s %s%s", containerID,
+		onloadCPServerPath, onloadCPServerParams)
 	err = kpw.SetControlPlaneServerParams(params)
 	if err != nil {
 		return err

pkg/control_plane/control_plane.go

@@ -8,39 +8,52 @@ import (
 )
 
 const (
-	mockOnloadCPServerPath = "/mock/sbin/onload_cp_server"
+	mockOnloadCPServerPath   = "/mock/sbin/onload_cp_server"
+	mockOnloadCPServerParams = "-mock -foo -bar"
 
 	mockCRIOContainerID       = "cri-o://0123456789abcdef"
 	mockContainerdContainerID = "containerd://fedcba9876543210"
 )
 
 type mockKernelParametersWriter struct {
+	expectedParams string
 }
 
-func NewMockKernelParametersWriter() KernelParametersWriter {
-	return &mockKernelParametersWriter{}
+func NewMockKernelParametersWriter(expectedParams string) KernelParametersWriter {
+	return &mockKernelParametersWriter{
+		expectedParams: expectedParams,
+	}
 }
 
 func (*mockKernelParametersWriter) SetControlPlaneServerPath(path string) error {
 	Expect(path).To(Equal("/usr/bin/crictl"))
 	return nil
 }
 
-func (*mockKernelParametersWriter) SetControlPlaneServerParams(params string) error {
-	Expect(params).To(Equal("exec 0123456789abcdef /mock/sbin/onload_cp_server -K"))
+func (mkpw *mockKernelParametersWriter) SetControlPlaneServerParams(params string) error {
+	Expect(params).To(Equal(mkpw.expectedParams))
 	return nil
 }
 
 var _ = Describe("Configure Onload to launch the control plane process within a container", func() {
 	It("Should work with the CRI-O runtime", func() {
-		err := Configure(mockOnloadCPServerPath, mockCRIOContainerID,
-			NewMockKernelParametersWriter())
+		expectedParams := "exec 0123456789abcdef /mock/sbin/onload_cp_server -mock -foo -bar"
+		err := Configure(mockOnloadCPServerPath, mockOnloadCPServerParams,
+			mockCRIOContainerID, NewMockKernelParametersWriter(expectedParams))
+		Expect(err).Should(Succeed())
+	})
+
+	It("Should work with the CRI-O runtime and empty parameter list", func() {
+		expectedParams := "exec 0123456789abcdef /mock/sbin/onload_cp_server"
+		err := Configure(mockOnloadCPServerPath, "", mockCRIOContainerID,
+			NewMockKernelParametersWriter(expectedParams))
 		Expect(err).Should(Succeed())
 	})
 
 	It("Should fail with non CRI-O runtime, e.g. containerd", func() {
-		err := Configure(mockOnloadCPServerPath, mockContainerdContainerID,
-			NewMockKernelParametersWriter())
+		notUsed := ""
+		err := Configure(mockOnloadCPServerPath, mockOnloadCPServerParams,
+			mockContainerdContainerID, NewMockKernelParametersWriter(notUsed))
 		Expect(err).Should(HaveOccurred())
 	})
 })