added total uploads and total bytes processed metrics (#3)

Author: seanshahkarami

docker-compose.yml

@@ -2,6 +2,8 @@
 services:
   loader:
     build: .
+    ports:
+      - "127.0.0.1:8080:8080"
     environment:
       - LOADER_NUM_WORKERS=3
       - LOADER_DATA_DIR=/data
@@ -16,8 +18,8 @@ services:
     image: minio/minio
     command: [ "server", "/data", "--console-address", ":9001" ]
     ports:
-      - "9000:9000"
-      - "9001:9001"
+      - "127.0.0.1:9000:9000"
+      - "127.0.0.1:9001:9001"
     environment:
       - MINIO_ROOT_USER=admin
       - MINIO_ROOT_PASSWORD=password

file_uploader.go

@@ -64,6 +64,11 @@ func (up *s3FileUploader) UploadFile(src, dst string, meta *MetaData) error {
 		return fmt.Errorf("could not create a new uploader")
 	}
 
+	stat, err := os.Stat(src)
+	if err != nil {
+		return err
+	}
+
 	f, err := os.Open(src)
 	if err != nil {
 		return err
@@ -82,6 +87,12 @@ func (up *s3FileUploader) UploadFile(src, dst string, meta *MetaData) error {
 		return fmt.Errorf("s3 uploader failed: %s", err.Error())
 	}
 
+	// update metrics
+	// TODO(sean) make these non-global variables
+	// TODO(sean) think about splitting data vs meta files
+	uploadsProcessedTotal.Inc()
+	uploadsProcessedBytes.Add(float64(stat.Size()))
+
 	return nil
 }
 

go.mod

@@ -1,10 +1,22 @@
 module github.com/sagecontinuum/sage-storage-loader
 
-go 1.18
+go 1.19
 
-require github.com/aws/aws-sdk-go v1.44.3
+require (
+	github.com/aws/aws-sdk-go v1.44.131
+	github.com/prometheus/client_golang v1.13.1
+)
 
 require (
+	github.com/beorn7/perks v1.0.1 // indirect
+	github.com/cespare/xxhash/v2 v2.1.2 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/golang/protobuf v1.5.2 // indirect
 	github.com/jmespath/go-jmespath v0.4.0 // indirect
+	github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
+	github.com/prometheus/client_model v0.3.0 // indirect
+	github.com/prometheus/common v0.37.0 // indirect
+	github.com/prometheus/procfs v0.8.0 // indirect
+	golang.org/x/sys v0.1.0 // indirect
+	google.golang.org/protobuf v1.28.1 // indirect
 )