ON-15256: Report topology info from device plugin

The device plugin will now try to read the numa information from sysfs
while finding the nics present in the system. The device plugin will
then add all of the present numa node information to the list of
advertised devices. Because the device plugin is only advertising a
pseudo-device it is unclear how to proceed when multiple nics are
present on different nodes.

Author: tcrawley-xilinx

pkg/deviceplugin/manager.go

@@ -6,6 +6,7 @@ import (
 	"errors"
 	"fmt"
 	"reflect"
+	"slices"
 	"sync"
 
 	"github.com/golang/glog"
@@ -40,12 +41,17 @@ var DefaultConfig = NicManagerConfig{
 	NeedNic:        true,
 }
 
+type nic struct {
+	name string
+	numa int64
+}
+
 // NicManager holds all the state required by the device plugin
 type NicManager struct {
 	// interfaces is used to check the presence of any sfc nics on the node.
 	// Currently it is just used as a check for existence and no additional
 	// logic takes place.
-	interfaces  []string
+	interfaces  []nic
 	deviceFiles []*pluginapi.DeviceSpec
 	mounts      []*pluginapi.Mount
 	devices     []*pluginapi.Device
@@ -56,7 +62,11 @@ type NicManager struct {
 }
 
 func (manager *NicManager) GetInterfaces() []string {
-	return manager.interfaces
+	interfaces := []string{}
+	for _, i := range manager.interfaces {
+		interfaces = append(interfaces, i.name)
+	}
+	return interfaces
 }
 
 func (manager *NicManager) GetDeviceFiles() []*pluginapi.DeviceSpec {
@@ -92,12 +102,32 @@ func NewNicManager(
 
 // Initialises the set of devices to advertise to kubernetes
 func (manager *NicManager) initDevices() {
+
+	// Gets a list of all numa nodes of which there is an associated sfc nic,
+	// this isn't particularly helpful when you have nics on different numa
+	// nodes which are intended for different purposes, but this is basically a
+	// pathological case due to how we only advertise an "onload" device rather
+	// than a "real" one.
+	numaNodes := []*pluginapi.NUMANode{}
+	for _, nic := range manager.interfaces {
+		if nic.numa != -1 {
+			if !slices.ContainsFunc(numaNodes, func(n *pluginapi.NUMANode) bool {
+				return n.ID == nic.numa
+			}) {
+				numaNodes = append(numaNodes, &pluginapi.NUMANode{ID: nic.numa})
+			}
+		}
+	}
+
 	manager.devices = []*pluginapi.Device{}
 	for i := 0; i < manager.config.MaxPodsPerNode; i++ {
 		name := fmt.Sprintf("sfc-%v", i)
 		device := &pluginapi.Device{
 			ID:     name,
 			Health: pluginapi.Healthy,
+			Topology: &pluginapi.TopologyInfo{
+				Nodes: numaNodes,
+			},
 		}
 		manager.devices = append(manager.devices, device)
 	}

pkg/deviceplugin/manager_test.go

@@ -88,3 +88,59 @@ var _ = Describe("Testing command line options", func() {
 		Expect(err).ShouldNot(Succeed())
 	})
 })
+
+var _ = Describe("Testing topology information", func() {
+	var manager *NicManager
+
+	BeforeEach(func() {
+		manager = &NicManager{
+			config: NicManagerConfig{
+				MaxPodsPerNode: 1,
+			},
+		}
+	})
+
+	It("shouldn't provide numa information when none are specified", func() {
+		manager.interfaces = []nic{
+			{name: "A", numa: -1},
+			{name: "B", numa: -1},
+			{name: "C", numa: -1},
+		}
+		manager.initDevices()
+		Expect(len(manager.devices[0].Topology.GetNodes())).To(Equal(0))
+	})
+
+	It("should describe numa information when a single node is present", func() {
+		manager.interfaces = []nic{
+			{name: "A", numa: 1},
+			{name: "B", numa: -1},
+			{name: "C", numa: -1},
+		}
+		manager.initDevices()
+		Expect(len(manager.devices[0].Topology.GetNodes())).To(Equal(1))
+		Expect(manager.devices[0].Topology.GetNodes()[0].ID).To(Equal(int64(1)))
+	})
+
+	It("should describe numa information when multiple nodes are present", func() {
+		manager.interfaces = []nic{
+			{name: "A", numa: 1},
+			{name: "B", numa: 2},
+			{name: "C", numa: -1},
+		}
+		manager.initDevices()
+		Expect(len(manager.devices[0].Topology.GetNodes())).To(Equal(2))
+		Expect(manager.devices[0].Topology.GetNodes()[0].ID).To(Equal(int64(1)))
+		Expect(manager.devices[0].Topology.GetNodes()[1].ID).To(Equal(int64(2)))
+	})
+
+	It("shouldn't provide duplicate numa information", func() {
+		manager.interfaces = []nic{
+			{name: "A", numa: 1},
+			{name: "B", numa: 1},
+			{name: "C", numa: -1},
+		}
+		manager.initDevices()
+		Expect(len(manager.devices[0].Topology.GetNodes())).To(Equal(1))
+		Expect(manager.devices[0].Topology.GetNodes()[0].ID).To(Equal(int64(1)))
+	})
+})

pkg/deviceplugin/nic.go

@@ -6,6 +6,7 @@ import (
 	"errors"
 	"os"
 	"path"
+	"strconv"
 	"strings"
 
 	"github.com/golang/glog"
@@ -43,27 +44,56 @@ func isSFCNic(devicePath string) bool {
 	return vendor == solarflareVendor
 }
 
-func readSysFiles() ([]string, error) {
+// Get numa node from sysfs files.
+// -1 means no specific numa node / unknown
+func getNumaNode(devicePath string) int64 {
+	data, err := os.ReadFile(path.Join(devicePath, "device", "numa_node"))
+	if errors.Is(err, os.ErrNotExist) {
+		// File doesn't exist but that is fine, return -1
+		return -1
+	} else if err != nil {
+		glog.Errorf("Error reading %s (%v)",
+			path.Join(devicePath, "device", "vendor"), err)
+		return -1
+	}
+	numaString := strings.TrimSuffix(string(data), "\n")
+	node, err := strconv.ParseInt(numaString, 10, 64)
+	if err != nil {
+		glog.Errorf("Error parse int from string %s (%v)",
+			numaString, err)
+		return -1
+	}
+	return node
+}
+
+func readSysFiles() ([]nic, error) {
 	infos, err := os.ReadDir(sysClassNetPath)
 	if err != nil {
 		glog.Errorf("Error reading %s (%v)", sysClassNetPath, err)
-		return []string{}, err
+		return []nic{}, err
 	}
-	interfaces := []string{}
+
+	interfaces := []nic{}
 	for _, info := range infos {
-		if isSFCNic(path.Join(sysClassNetPath, info.Name())) {
-			interfaces = append(interfaces, info.Name())
+		devicePath := path.Join(sysClassNetPath, info.Name())
+		if !isSFCNic(devicePath) {
+			continue
+		}
+		nic := nic{
+			name: info.Name(),
+			numa: getNumaNode(devicePath),
 		}
+		interfaces = append(interfaces, nic)
 	}
 	return interfaces, nil
 }
 
 // Returns a list of the Solarflare interfaces present on the node
-func queryNics() ([]string, error) {
+func queryNics() ([]nic, error) {
 	interfaces, err := readSysFiles()
 	if err != nil {
 		glog.Errorf("Failed to list interfaces (%v)", err)
-		return []string{}, err
+		return []nic{}, err
 	}
 	return interfaces, nil
 }