SAGE-1578: added upload metrics by node

Author: seanshahkarami

docker-compose.yml

@@ -2,6 +2,7 @@
 services:
   loader:
     build: .
+    restart: always
     ports:
       - "127.0.0.1:8080:8080"
     environment:

file_uploader.go

@@ -90,8 +90,13 @@ func (up *s3FileUploader) UploadFile(src, dst string, meta *MetaData) error {
 	// update metrics
 	// TODO(sean) make these non-global variables
 	// TODO(sean) think about splitting data vs meta files
+	// TODO(sean) figure out how to get VSN metadata for metrics
+	size := float64(stat.Size())
+	node := meta.Meta["node"]
 	uploadsProcessedTotal.Inc()
-	uploadsProcessedBytes.Add(float64(stat.Size()))
+	uploadsProcessedBytes.Add(size)
+	uploadsProcessedTotalByNode.WithLabelValues(node).Inc()
+	uploadsProcessedBytesByNode.WithLabelValues(node).Add(size)
 
 	return nil
 }

go.mod

@@ -3,8 +3,8 @@ module github.com/sagecontinuum/sage-storage-loader
 go 1.19
 
 require (
-	github.com/aws/aws-sdk-go v1.44.131
-	github.com/prometheus/client_golang v1.13.1
+	github.com/aws/aws-sdk-go v1.44.151
+	github.com/prometheus/client_golang v1.14.0
 )
 
 require (
@@ -17,6 +17,6 @@ require (
 	github.com/prometheus/client_model v0.3.0 // indirect
 	github.com/prometheus/common v0.37.0 // indirect
 	github.com/prometheus/procfs v0.8.0 // indirect
-	golang.org/x/sys v0.1.0 // indirect
+	golang.org/x/sys v0.2.0 // indirect
 	google.golang.org/protobuf v1.28.1 // indirect
 )

go.sum

@@ -42,6 +42,8 @@ github.com/aws/aws-sdk-go v1.44.3 h1:GA9bJsWeJdwPtcKK9Uq3jfMaPko0ROWzVuqwFM7BBP0
 github.com/aws/aws-sdk-go v1.44.3/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo=
 github.com/aws/aws-sdk-go v1.44.131 h1:kd61x79ax0vyiC/SZ9X1hKh8E0pt1BUOOcVBJEFhxkg=
 github.com/aws/aws-sdk-go v1.44.131/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI=
+github.com/aws/aws-sdk-go v1.44.151 h1:2FrJZm3kTcyTtfpE7LEQT9XW+jkoi4KEvBhFWqHEZmo=
+github.com/aws/aws-sdk-go v1.44.151/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
 github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
@@ -173,6 +175,8 @@ github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqr
 github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY=
 github.com/prometheus/client_golang v1.13.1 h1:3gMjIY2+/hzmqhtUC/aQNYldJA6DtH3CgQvwS+02K1c=
 github.com/prometheus/client_golang v1.13.1/go.mod h1:vTeo+zgvILHsnnj/39Ou/1fPN5nJFOEMgftOUOmlvYQ=
+github.com/prometheus/client_golang v1.14.0 h1:nJdhIvne2eSX/XRAFV9PcvFFRbrjbcTUj0VP62TMhnw=
+github.com/prometheus/client_golang v1.14.0/go.mod h1:8vpkKitgIVNcqrRBWh1C4TIUQgYNtG/XQE4E/Zae36Y=
 github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
 github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
 github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
@@ -344,6 +348,8 @@ golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.1.0 h1:kunALQeHf1/185U1i0GOB/fy1IPRDDpuoOOqRReG57U=
 golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.2.0 h1:ljd4t30dBnAvMZaQCevtY0xLLD0A+bRZXbgLMLU1F/A=
+golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=

main.go

@@ -20,14 +20,32 @@ import (
 
 // TODO(sean) make this part of the service
 var (
-	uploadsProcessedTotal = promauto.NewCounter(prometheus.CounterOpts{
-		Name: "storageloader_uploads_total",
-		Help: "Total number of uploads processed.",
-	})
-	uploadsProcessedBytes = promauto.NewCounter(prometheus.CounterOpts{
-		Name: "storageloader_uploads_bytes_total",
-		Help: "Total upload bytes processed.",
-	})
+	uploadsProcessedTotal = promauto.NewCounter(
+		prometheus.CounterOpts{
+			Name: "storageloader_uploads_total",
+			Help: "Total number of uploads processed.",
+		},
+	)
+	uploadsProcessedBytes = promauto.NewCounter(
+		prometheus.CounterOpts{
+			Name: "storageloader_upload_bytes_total",
+			Help: "Total upload bytes processed.",
+		},
+	)
+	uploadsProcessedTotalByNode = promauto.NewCounterVec(
+		prometheus.CounterOpts{
+			Name: "storageloader_uploads_total_by_node_total",
+			Help: "Total number of uploads processed by node.",
+		},
+		[]string{"node"},
+	)
+	uploadsProcessedBytesByNode = promauto.NewCounterVec(
+		prometheus.CounterOpts{
+			Name: "storageloader_upload_bytes_by_node_total",
+			Help: "Total number of uploads processed by node.",
+		},
+		[]string{"node"},
+	)
 )
 
 // TODO(sean) consider updating the design to decoupling and splitting the "scan filesystem" step into its own

worker.go

@@ -108,7 +108,7 @@ func (w *Worker) Process(job Job) error {
 		return fmt.Errorf("error uploading data file: %s", err.Error())
 	}
 
-	if err := w.Uploader.UploadFile(metaPath, filepath.Join(s3path, targetNameMeta), nil); err != nil {
+	if err := w.Uploader.UploadFile(metaPath, filepath.Join(s3path, targetNameMeta), &meta); err != nil {
 		return fmt.Errorf("error uploading meta file: %s", err.Error())
 	}