Add rules for IO and network metrics

cmd/ceems_tool/rules.go

@@ -56,6 +56,10 @@ var (
 		amdDevExporterPkgPowerMetric, // AMD metrics device exporter
 		"ceems_compute_unit_gpu_index_flag",
 		"ceems_compute_unit_gpu_sm_count",
+		"ceems_ebpf_read_bytes_total",
+		"ceems_ebpf_write_bytes_total",
+		"ceems_ebpf_ingress_bytes_total",
+		"ceems_ebpf_egress_bytes_total",
 	}
 
 	nvidiaProfSeriesNames = []string{
@@ -115,6 +119,8 @@ type rulesTemplateData struct {
 	HostPowerQuery     string
 	HostPowerSeries    string
 	RAPLAvailable      bool
+	IOAvailable        bool
+	NetAvailable       bool
 	Job                model.LabelValue
 	PUE                float64
 	EmissionFactor     EmissionFactor
@@ -363,6 +369,8 @@ func CreatePromRecordingRules(
 			HostPowerQuery:     hostPowerQuery,
 			HostPowerSeries:    hostPowerSeries,
 			RAPLAvailable:      slices.Contains(jobSeries[job], "ceems_rapl_package_joules_total") && slices.Contains(jobSeries[job], "ceems_rapl_dram_joules_total"),
+			IOAvailable:        slices.Contains(jobSeries[job], "ceems_ebpf_read_bytes_total") || slices.Contains(jobSeries[job], "ceems_ebpf_write_bytes_total"),
+			NetAvailable:       slices.Contains(jobSeries[job], "ceems_ebpf_ingress_bytes_total") || slices.Contains(jobSeries[job], "ceems_ebpf_egress_bytes_total"),
 			Job:                job,
 			PUE:                pueValue,
 			EmissionFactor:     emissionFactor,

cmd/ceems_tool/rules/cpu-cray.rules

@@ -220,3 +220,130 @@ groups:
 {{- end }}
 {{- end }}
 
+{{ if .IOAvailable }}
+
+  - name: compute-unit-io-rules-{{.Job}}
+    interval: {{.EvaluationInterval}}
+    rules:
+      # I/O read bandwidth in bytes/sec.
+      - record: uuid:ceems_io_read_bytes:irate
+        expr: irate(ceems_ebpf_read_bytes_total{job="{{.Job}}"}[{{.RateInterval}}])
+
+      # I/O read requests rate in req/sec.
+      - record: uuid:ceems_io_read_requests:irate
+        expr: irate(ceems_ebpf_read_requests_total{job="{{.Job}}"}[{{.RateInterval}}])
+
+      # I/O read errors rate in err/sec.
+      - record: uuid:ceems_io_read_errors:irate
+        expr: irate(ceems_ebpf_read_errors_total{job="{{.Job}}"}[{{.RateInterval}}])
+
+      # I/O write bandwidth in bytes/sec.
+      - record: uuid:ceems_io_write_bytes:irate
+        expr: irate(ceems_ebpf_write_bytes_total{job="{{.Job}}"}[{{.RateInterval}}])
+
+      # I/O write requests rate in req/sec.
+      - record: uuid:ceems_io_write_requests:irate
+        expr: irate(ceems_ebpf_write_requests_total{job="{{.Job}}"}[{{.RateInterval}}])
+
+      # I/O write errors rate in err/sec.
+      - record: uuid:ceems_io_write_errors:irate
+        expr: irate(ceems_ebpf_write_errors_total{job="{{.Job}}"}[{{.RateInterval}}])
+
+  - name: host-agg-io-rules-{{.Job}}
+    interval: {{.EvaluationInterval}}
+    rules:
+      # Total I/O read bandwidth in bytes/sec for all hosts in a Prometheus job.
+      - record: job:ceems_io_read_bytes:sum_irate
+        expr: |2
+          sum by (job, mountpoint) (
+            sum by (job, mountpoint, instance) (irate(ceems_ebpf_read_bytes_total[1m]))
+          )
+
+      # Total I/O read requests in req/sec for all hosts in a Prometheus job.
+      - record: job:ceems_io_read_requests:sum_irate
+        expr: |2
+          sum by (job, mountpoint) (
+            sum by (job, mountpoint, instance) (irate(ceems_ebpf_read_requests_total[1m]))
+          )
+
+      # Total I/O read errors in err/sec for all hosts in a Prometheus job.
+      - record: job:ceems_io_read_errors:sum_irate
+        expr: |2
+          sum by (job, mountpoint) (
+            sum by (job, mountpoint, instance) (irate(ceems_ebpf_read_errors_total[1m]))
+          )
+
+      # Total I/O write bandwidth in bytes/sec for all hosts in a Prometheus job.
+      - record: job:ceems_io_write_bytes:sum_irate
+        expr: |2
+          sum by (job, mountpoint) (
+            sum by (job, mountpoint, instance) (irate(ceems_ebpf_write_bytes_total[1m]))
+          )
+
+      # Total I/O write requests in req/sec for all hosts in a Prometheus job.
+      - record: job:ceems_io_write_requests:sum_irate
+        expr: |2
+          sum by (job, mountpoint) (
+            sum by (job, mountpoint, instance) (irate(ceems_ebpf_write_requests_total[1m]))
+          )
+
+      # Total I/O write errors in err/sec for all hosts in a Prometheus job.
+      - record: job:ceems_io_write_errors:sum_irate
+        expr: |2
+          sum by (job, mountpoint) (
+            sum by (job, mountpoint, instance) (irate(ceems_ebpf_write_errors_total[1m]))
+          )
+{{- end }}
+
+{{- if .NetAvailable }}
+
+  - name: compute-unit-network-rules-{{.Job}}
+    interval: {{.EvaluationInterval}}
+    rules:
+      # Netwok ingress bandwidth in bytes/sec.
+      - record: uuid:ceems_net_ingress_bytes:irate
+        expr: irate(ceems_ebpf_ingress_bytes_total{job="{{.Job}}"}[{{.RateInterval}}])
+
+      # Netwok ingress packets rate in pkts/sec.
+      - record: uuid:ceems_net_ingress_packets:irate
+        expr: irate(ceems_ebpf_ingress_packets_total{job="{{.Job}}"}[{{.RateInterval}}])
+
+      # Netwok egress bandwidth in bytes/sec.
+      - record: uuid:ceems_net_egress_bytes:irate
+        expr: irate(ceems_ebpf_egress_bytes_total{job="{{.Job}}"}[{{.RateInterval}}])
+
+      # Netwok egress packets rate in pkts/sec.
+      - record: uuid:ceems_net_egress_packets:irate
+        expr: irate(ceems_ebpf_egress_packets_total{job="{{.Job}}"}[{{.RateInterval}}])
+
+  - name: host-agg-network-rules-{{.Job}}
+    interval: {{.EvaluationInterval}}
+    rules:
+      # Total network ingress bandwidth in bytes/sec for all hosts in a Prometheus job.
+      - record: job:ceems_net_ingress_bytes:sum_irate
+        expr: |2
+          sum by (job, family, proto) (
+            sum by (job, family, proto, instance) (irate(ceems_ebpf_ingress_bytes_total[1m]))
+          )
+
+      # Total network ingress packets rate in pkts/sec for all hosts in a Prometheus job.
+      - record: job:ceems_net_ingress_packets:sum_irate
+        expr: |2
+          sum by (job, family, proto) (
+            sum by (job, family, proto, instance) (irate(ceems_ebpf_ingress_packets_total[1m]))
+          )
+
+      # Total network egress bandwidth in bytes/sec for all hosts in a Prometheus job.
+      - record: job:ceems_net_egress_bytes:sum_irate
+        expr: |2
+          sum by (job, family, proto) (
+            sum by (job, family, proto, instance) (irate(ceems_ebpf_egress_bytes_total[1m]))
+          )
+
+      # Total network egress packets rate in pkts/sec for all hosts in a Prometheus job.
+      - record: job:ceems_net_egress_packets:sum_irate
+        expr: |2
+          sum by (job, family, proto) (
+            sum by (job, family, proto, instance) (irate(ceems_ebpf_egress_packets_total[1m]))
+          )
+{{- end }}

cmd/ceems_tool/rules/cpu-ipmi-redfish-hwmon.rules

@@ -245,3 +245,131 @@ groups:
           )
 {{- end }}
 {{- end }}
+
+{{- if .IOAvailable }}
+
+  - name: compute-unit-io-rules-{{.Job}}
+    interval: {{.EvaluationInterval}}
+    rules:
+      # I/O read bandwidth in bytes/sec.
+      - record: uuid:ceems_io_read_bytes:irate
+        expr: irate(ceems_ebpf_read_bytes_total{job="{{.Job}}"}[{{.RateInterval}}])
+
+      # I/O read requests rate in req/sec.
+      - record: uuid:ceems_io_read_requests:irate
+        expr: irate(ceems_ebpf_read_requests_total{job="{{.Job}}"}[{{.RateInterval}}])
+
+      # I/O read errors rate in err/sec.
+      - record: uuid:ceems_io_read_errors:irate
+        expr: irate(ceems_ebpf_read_errors_total{job="{{.Job}}"}[{{.RateInterval}}])
+
+      # I/O write bandwidth in bytes/sec.
+      - record: uuid:ceems_io_write_bytes:irate
+        expr: irate(ceems_ebpf_write_bytes_total{job="{{.Job}}"}[{{.RateInterval}}])
+
+      # I/O write requests rate in req/sec.
+      - record: uuid:ceems_io_write_requests:irate
+        expr: irate(ceems_ebpf_write_requests_total{job="{{.Job}}"}[{{.RateInterval}}])
+
+      # I/O write errors rate in err/sec.
+      - record: uuid:ceems_io_write_errors:irate
+        expr: irate(ceems_ebpf_write_errors_total{job="{{.Job}}"}[{{.RateInterval}}])
+
+  - name: host-agg-io-rules-{{.Job}}
+    interval: {{.EvaluationInterval}}
+    rules:
+      # Total I/O read bandwidth in bytes/sec for all hosts in a Prometheus job.
+      - record: job:ceems_io_read_bytes:sum_irate
+        expr: |2
+          sum by (job, mountpoint) (
+            sum by (job, mountpoint, instance) (irate(ceems_ebpf_read_bytes_total[1m]))
+          )
+
+      # Total I/O read requests in req/sec for all hosts in a Prometheus job.
+      - record: job:ceems_io_read_requests:sum_irate
+        expr: |2
+          sum by (job, mountpoint) (
+            sum by (job, mountpoint, instance) (irate(ceems_ebpf_read_requests_total[1m]))
+          )
+
+      # Total I/O read errors in err/sec for all hosts in a Prometheus job.
+      - record: job:ceems_io_read_errors:sum_irate
+        expr: |2
+          sum by (job, mountpoint) (
+            sum by (job, mountpoint, instance) (irate(ceems_ebpf_read_errors_total[1m]))
+          )
+
+      # Total I/O write bandwidth in bytes/sec for all hosts in a Prometheus job.
+      - record: job:ceems_io_write_bytes:sum_irate
+        expr: |2
+          sum by (job, mountpoint) (
+            sum by (job, mountpoint, instance) (irate(ceems_ebpf_write_bytes_total[1m]))
+          )
+
+      # Total I/O write requests in req/sec for all hosts in a Prometheus job.
+      - record: job:ceems_io_write_requests:sum_irate
+        expr: |2
+          sum by (job, mountpoint) (
+            sum by (job, mountpoint, instance) (irate(ceems_ebpf_write_requests_total[1m]))
+          )
+
+      # Total I/O write errors in err/sec for all hosts in a Prometheus job.
+      - record: job:ceems_io_write_errors:sum_irate
+        expr: |2
+          sum by (job, mountpoint) (
+            sum by (job, mountpoint, instance) (irate(ceems_ebpf_write_errors_total[1m]))
+          )
+{{- end }}
+
+{{- if .NetAvailable }}
+
+  - name: compute-unit-network-rules-{{.Job}}
+    interval: {{.EvaluationInterval}}
+    rules:
+      # Netwok ingress bandwidth in bytes/sec.
+      - record: uuid:ceems_net_ingress_bytes:irate
+        expr: irate(ceems_ebpf_ingress_bytes_total{job="{{.Job}}"}[{{.RateInterval}}])
+
+      # Netwok ingress packets rate in pkts/sec.
+      - record: uuid:ceems_net_ingress_packets:irate
+        expr: irate(ceems_ebpf_ingress_packets_total{job="{{.Job}}"}[{{.RateInterval}}])
+
+      # Netwok egress bandwidth in bytes/sec.
+      - record: uuid:ceems_net_egress_bytes:irate
+        expr: irate(ceems_ebpf_egress_bytes_total{job="{{.Job}}"}[{{.RateInterval}}])
+
+      # Netwok egress packets rate in pkts/sec.
+      - record: uuid:ceems_net_egress_packets:irate
+        expr: irate(ceems_ebpf_egress_packets_total{job="{{.Job}}"}[{{.RateInterval}}])
+
+  - name: host-agg-network-rules-{{.Job}}
+    interval: {{.EvaluationInterval}}
+    rules:
+      # Total network ingress bandwidth in bytes/sec for all hosts in a Prometheus job.
+      - record: job:ceems_net_ingress_bytes:sum_irate
+        expr: |2
+          sum by (job, family, proto) (
+            sum by (job, family, proto, instance) (irate(ceems_ebpf_ingress_bytes_total[1m]))
+          )
+
+      # Total network ingress packets rate in pkts/sec for all hosts in a Prometheus job.
+      - record: job:ceems_net_ingress_packets:sum_irate
+        expr: |2
+          sum by (job, family, proto) (
+            sum by (job, family, proto, instance) (irate(ceems_ebpf_ingress_packets_total[1m]))
+          )
+
+      # Total network egress bandwidth in bytes/sec for all hosts in a Prometheus job.
+      - record: job:ceems_net_egress_bytes:sum_irate
+        expr: |2
+          sum by (job, family, proto) (
+            sum by (job, family, proto, instance) (irate(ceems_ebpf_egress_bytes_total[1m]))
+          )
+
+      # Total network egress packets rate in pkts/sec for all hosts in a Prometheus job.
+      - record: job:ceems_net_egress_packets:sum_irate
+        expr: |2
+          sum by (job, family, proto) (
+            sum by (job, family, proto, instance) (irate(ceems_ebpf_egress_packets_total[1m]))
+          )
+{{- end }}