feat: structured logging in digger (backend) (#1927)

* feat: add structured logging in backend

Author: aldoborrero

backend/bootstrap/main.go

@@ -6,6 +6,7 @@ import (
 	"html/template"
 	"io/fs"
 	"log"
+	"log/slog"
 	"net/http"
 	"os"
 	"path/filepath"
@@ -59,12 +60,12 @@ func periodicProfiling() {
 			memProfilePath := filepath.Join("/tmp/profiles", fmt.Sprintf("memory-%s.pprof", timestamp))
 			f, err := os.Create(memProfilePath)
 			if err != nil {
-				log.Printf("Failed to create memory profile: %v", err)
+				slog.Error("Failed to create memory profile", "error", err)
 				continue
 			}
 
 			if err := pprof.WriteHeapProfile(f); err != nil {
-				log.Printf("Failed to write memory profile: %v", err)
+				slog.Error("Failed to write memory profile", "error", err)
 			}
 			f.Close()
 
@@ -77,7 +78,7 @@ func periodicProfiling() {
 func cleanupOldProfiles(dir string, keep int) {
 	files, err := filepath.Glob(filepath.Join(dir, "memory-*.pprof"))
 	if err != nil {
-		log.Printf("Failed to list profile files: %v", err)
+		slog.Error("Failed to list profile files", "error", err)
 		return
 	}
 
@@ -88,7 +89,7 @@ func cleanupOldProfiles(dir string, keep int) {
 	// Sort files by name (which includes timestamp)
 	for i := 0; i < len(files)-keep; i++ {
 		if err := os.Remove(files[i]); err != nil {
-			log.Printf("Failed to remove old profile %s: %v", files[i], err)
+			slog.Error("Failed to remove old profile", "file", files[i], "error", err)
 		}
 	}
 }
@@ -108,7 +109,7 @@ func Bootstrap(templates embed.FS, diggerController controllers.DiggerController
 		Release:          "api@" + Version,
 		Debug:            true,
 	}); err != nil {
-		log.Printf("Sentry initialization failed: %v\n", err)
+		slog.Error("Sentry initialization failed", "error", err)
 	}
 
 	//database migrations
@@ -241,7 +242,9 @@ func Bootstrap(templates embed.FS, diggerController controllers.DiggerController
 }
 
 func initLogging() {
-	log.SetOutput(os.Stdout)
-	log.SetFlags(log.Ldate | log.Ltime | log.Lshortfile)
-	log.Println("Initialized the logger successfully")
+	handler := slog.NewTextHandler(os.Stdout, &slog.HandlerOptions{
+		Level: slog.LevelInfo,
+	})
+	logger := slog.New(handler)
+	slog.SetDefault(logger)
 }

backend/ci_backends/github_actions.go

@@ -4,19 +4,20 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
+	"log/slog"
+
 	"github.com/diggerhq/digger/backend/utils"
 	orchestrator_scheduler "github.com/diggerhq/digger/libs/scheduler"
 	"github.com/diggerhq/digger/libs/spec"
 	"github.com/google/go-github/v61/github"
-	"log"
 )
 
 type GithubActionCi struct {
 	Client *github.Client
 }
 
 func (g GithubActionCi) TriggerWorkflow(spec spec.Spec, runName string, vcsToken string) error {
-	log.Printf("TriggerGithubWorkflow: repoOwner: %v, repoName: %v, commentId: %v", spec.VCS.RepoOwner, spec.VCS.RepoName, spec.CommentId)
+	slog.Info("TriggerGithubWorkflow", "repoOwner", spec.VCS.RepoOwner, "repoName", spec.VCS.RepoName, "commentId", spec.CommentId)
 	client := g.Client
 	specBytes, err := json.Marshal(spec)
 
@@ -35,13 +36,13 @@ func (g GithubActionCi) TriggerWorkflow(spec spec.Spec, runName string, vcsToken
 
 func (g GithubActionCi) GetWorkflowUrl(spec spec.Spec) (string, error) {
 	if spec.JobId == "" {
-		log.Printf("Cannot get workflow URL: JobId is empty")
+		slog.Error("Cannot get workflow URL: JobId is empty")
 		return "", fmt.Errorf("job ID is required to fetch workflow URL")
 	}
 
 	_, workflowRunUrl, err := utils.GetWorkflowIdAndUrlFromDiggerJobId(g.Client, spec.VCS.RepoOwner, spec.VCS.RepoName, spec.JobId)
 	if err != nil {
-		log.Printf("Error getting workflow ID from job: %v", err)
+		slog.Error("Error getting workflow ID from job", "error", err)
 		return "", err
 	} else {
 		return workflowRunUrl, nil

backend/ci_backends/provider.go

@@ -2,8 +2,9 @@ package ci_backends
 
 import (
 	"fmt"
+	"log/slog"
+
 	"github.com/diggerhq/digger/backend/utils"
-	"log"
 )
 
 type CiBackendProvider interface {
@@ -15,7 +16,7 @@ type DefaultBackendProvider struct{}
 func (d DefaultBackendProvider) GetCiBackend(options CiBackendOptions) (CiBackend, error) {
 	client, _, err := utils.GetGithubClientFromAppId(options.GithubClientProvider, options.GithubInstallationId, options.GithubAppId, options.RepoFullName)
 	if err != nil {
-		log.Printf("GetCiBackend: could not get github client: %v", err)
+		slog.Error("GetCiBackend: could not get github client", "error", err)
 		return nil, fmt.Errorf("could not get github client: %v", err)
 	}
 	backend := &GithubActionCi{

backend/controllers/cache.go

@@ -2,15 +2,16 @@ package controllers
 
 import (
 	"fmt"
-	"github.com/diggerhq/digger/backend/models"
-	"github.com/diggerhq/digger/backend/utils"
-	dg_configuration "github.com/diggerhq/digger/libs/digger_config"
-	"github.com/gin-gonic/gin"
-	"log"
+	"log/slog"
 	"net/http"
 	"os"
 	"path"
 	"strings"
+
+	"github.com/diggerhq/digger/backend/models"
+	"github.com/diggerhq/digger/backend/utils"
+	dg_configuration "github.com/diggerhq/digger/libs/digger_config"
+	"github.com/gin-gonic/gin"
 )
 
 func (d DiggerController) UpdateRepoCache(c *gin.Context) {
@@ -24,7 +25,7 @@ func (d DiggerController) UpdateRepoCache(c *gin.Context) {
 	var request UpdateCacheRequest
 	err := c.BindJSON(&request)
 	if err != nil {
-		log.Printf("Error binding JSON: %v", err)
+		slog.Error("Error binding JSON", "error", err)
 		c.JSON(http.StatusInternalServerError, gin.H{"error": "Error binding JSON"})
 		return
 	}
@@ -33,22 +34,21 @@ func (d DiggerController) UpdateRepoCache(c *gin.Context) {
 	installationId := request.InstallationId
 	link, err := models.DB.GetGithubAppInstallationLink(installationId)
 	if err != nil {
-		log.Printf("could not installation link: %v", err)
-		c.String(500, fmt.Sprintf("coulnt not find installation link %v %v", repoFullName, installationId))
+		slog.Error("Could not get installation link", "error", err, "repoFullName", repoFullName, "installationId", installationId)
+		c.String(http.StatusInternalServerError, fmt.Sprintf("could not find installation link %v %v", repoFullName, installationId))
 		return
-
 	}
 	orgId := link.OrganisationId
 
-	log.Printf("the org id is %v", orgId)
+	slog.Info("Processing repo cache update", "orgId", orgId)
 
 	repoOwner, repoName, _ := strings.Cut(repoFullName, "/")
 	repoDiggerName := strings.ReplaceAll(repoFullName, "/", "-")
 
 	repo, err := models.DB.GetRepo(orgId, repoDiggerName)
 	if err != nil {
-		log.Printf("could not get repo: %v", err)
-		c.String(500, fmt.Sprintf("coulnt not get repository %v %v", repoFullName, orgId))
+		slog.Error("Could not get repo", "error", err, "repoFullName", repoFullName, "orgId", orgId)
+		c.String(http.StatusInternalServerError, fmt.Sprintf("could not get repository %v %v", repoFullName, orgId))
 		return
 	}
 
@@ -57,8 +57,8 @@ func (d DiggerController) UpdateRepoCache(c *gin.Context) {
 
 	_, token, err := utils.GetGithubService(d.GithubClientProvider, installationId, repoFullName, repoOwner, repoName)
 	if err != nil {
-		log.Printf("could not get github service :%v", err)
-		c.String(500, fmt.Sprintf("could not get github service %v %v", repoFullName, orgId))
+		slog.Error("Could not get GitHub service", "error", err, "repoFullName", repoFullName, "orgId", orgId)
+		c.String(http.StatusInternalServerError, fmt.Sprintf("could not get github service %v %v", repoFullName, orgId))
 		return
 	}
 
@@ -72,23 +72,23 @@ func (d DiggerController) UpdateRepoCache(c *gin.Context) {
 			diggerYmlStr = string(diggerYmlBytes)
 			config, _, _, err = dg_configuration.LoadDiggerConfig(dir, true, nil)
 			if err != nil {
-				log.Printf("Error loading digger config: %v", err)
+				slog.Error("Error loading digger config", "error", err)
 				return err
 			}
 			return nil
 		})
 
 		if err != nil {
-			log.Printf("could not load digger config :%v", err)
+			slog.Error("Could not load digger config", "error", err)
 			return
 		}
 		_, err = models.DB.UpsertRepoCache(orgId, repoFullName, diggerYmlStr, *config)
 		if err != nil {
-			log.Printf("could upadate repo cache :%v", err)
+			slog.Error("Could not update repo cache", "error", err)
 			return
-
 		}
+		slog.Info("Successfully updated repo cache", "repoFullName", repoFullName, "orgId", orgId)
 	}()
 
-	c.String(200, "successfully submitted cache for processing, check backend logs for progress")
+	c.String(http.StatusOK, "successfully submitted cache for processing, check backend logs for progress")
 }

backend/controllers/connections.go

@@ -2,11 +2,12 @@ package controllers
 
 import (
 	"errors"
-	"github.com/samber/lo"
-	"log"
+	"log/slog"
 	"net/http"
 	"os"
 
+	"github.com/samber/lo"
+
 	"github.com/diggerhq/digger/backend/utils"
 	"gorm.io/gorm"
 
@@ -22,15 +23,15 @@ func ListVCSConnectionsApi(c *gin.Context) {
 	var org models.Organisation
 	err := models.DB.GormDB.Where("external_id = ? AND external_source = ?", organisationId, organisationSource).First(&org).Error
 	if err != nil {
-		log.Printf("could not fetch organisation: %v err: %v", organisationId, err)
+		slog.Error("Could not fetch organisation", "organisationId", organisationId, "error", err)
 		c.JSON(http.StatusNotFound, gin.H{"error": "Could not fetch organisation"})
 		return
 	}
 
 	var connections []models.VCSConnection
 	err = models.DB.GormDB.Where("organisation_id = ?", org.ID).Find(&connections).Error
 	if err != nil {
-		log.Printf("could not fetch VCS connections: %v", err)
+		slog.Error("Could not fetch VCS connections", "organisationId", organisationId, "error", err)
 		c.JSON(http.StatusInternalServerError, gin.H{"error": "Could not fetch VCS connections"})
 		return
 	}
@@ -54,7 +55,7 @@ func CreateVCSConnectionApi(c *gin.Context) {
 	var org models.Organisation
 	err := models.DB.GormDB.Where("external_id = ? AND external_source = ?", organisationId, organisationSource).First(&org).Error
 	if err != nil {
-		log.Printf("could not fetch organisation: %v err: %v", organisationId, err)
+		slog.Error("Could not fetch organisation", "organisationId", organisationId, "error", err)
 		c.JSON(http.StatusNotFound, gin.H{"error": "Could not fetch organisation"})
 		return
 	}
@@ -68,33 +69,34 @@ func CreateVCSConnectionApi(c *gin.Context) {
 
 	var request CreateVCSConnectionRequest
 	if err := c.BindJSON(&request); err != nil {
+		slog.Error("Invalid request body", "error", err)
 		c.JSON(http.StatusBadRequest, gin.H{"error": "Invalid request body"})
 		return
 	}
 
 	if request.VCS != "bitbucket" {
-		log.Printf("VCS type not supported: %v", request.VCS)
+		slog.Error("VCS type not supported", "type", request.VCS)
 		c.JSON(http.StatusBadRequest, gin.H{"error": "VCS type not supported"})
 		return
 	}
 
 	secret := os.Getenv("DIGGER_ENCRYPTION_SECRET")
 	if secret == "" {
-		log.Printf("ERROR: no encryption secret specified")
+		slog.Error("No encryption secret specified")
 		c.JSON(http.StatusInternalServerError, gin.H{"error": "Could not encrypt access token"})
 		return
 	}
 
 	bitbucketAccessTokenEncrypted, err := utils.AESEncrypt([]byte(secret), request.BitbucketAccessToken)
 	if err != nil {
-		log.Printf("could not encrypt access token: %v", err)
+		slog.Error("Could not encrypt access token", "error", err)
 		c.JSON(http.StatusInternalServerError, gin.H{"error": "Could not encrypt access token"})
 		return
 	}
 
 	bitbucketWebhookSecretEncrypted, err := utils.AESEncrypt([]byte(secret), request.BitbucketWebhookSecret)
 	if err != nil {
-		log.Printf("could not encrypt webhook secret: %v", err)
+		slog.Error("Could not encrypt webhook secret", "error", err)
 		c.JSON(http.StatusInternalServerError, gin.H{"error": "Could not encrypt webhook secret"})
 		return
 	}
@@ -114,9 +116,12 @@ func CreateVCSConnectionApi(c *gin.Context) {
 		org.ID,
 	)
 	if err != nil {
-		log.Printf("")
+		slog.Error("Could not create VCS connection", "error", err)
+		c.JSON(http.StatusInternalServerError, gin.H{"error": "Could not create VCS connection"})
+		return
 	}
 
+	slog.Info("Created VCS connection", "connectionId", connection.ID, "organisationId", org.ID)
 	c.JSON(http.StatusCreated, gin.H{
 		"connection": connection.ID,
 	})
@@ -131,9 +136,10 @@ func GetVCSConnection(c *gin.Context) {
 	err := models.DB.GormDB.Where("external_id = ? AND external_source = ?", organisationId, organisationSource).First(&org).Error
 	if err != nil {
 		if errors.Is(err, gorm.ErrRecordNotFound) {
+			slog.Info("Organisation not found", "organisationId", organisationId)
 			c.String(http.StatusNotFound, "Could not find organisation: "+organisationId)
 		} else {
-			log.Printf("could not fetch organisation: %v err: %v", organisationId, err)
+			slog.Error("Could not fetch organisation", "organisationId", organisationId, "error", err)
 			c.String(http.StatusNotFound, "Could not fetch organisation: "+organisationId)
 		}
 		return
@@ -143,9 +149,10 @@ func GetVCSConnection(c *gin.Context) {
 	err = models.DB.GormDB.Where("id = ? AND organisation_id = ?", connectionId, org.ID).First(&connection).Error
 	if err != nil {
 		if errors.Is(err, gorm.ErrRecordNotFound) {
+			slog.Info("Connection not found", "connectionId", connectionId, "organisationId", org.ID)
 			c.String(http.StatusNotFound, "Could not find connection: "+connectionId)
 		} else {
-			log.Printf("could not fetch connection: %v err: %v", connectionId, err)
+			slog.Error("Could not fetch connection", "connectionId", connectionId, "error", err)
 			c.String(http.StatusInternalServerError, "Could not fetch connection")
 		}
 		return
@@ -166,9 +173,10 @@ func DeleteVCSConnection(c *gin.Context) {
 	err := models.DB.GormDB.Where("external_id = ? AND external_source = ?", organisationId, organisationSource).First(&org).Error
 	if err != nil {
 		if errors.Is(err, gorm.ErrRecordNotFound) {
+			slog.Info("Organisation not found", "organisationId", organisationId)
 			c.String(http.StatusNotFound, "Could not find organisation: "+organisationId)
 		} else {
-			log.Printf("could not fetch organisation: %v err: %v", organisationId, err)
+			slog.Error("Could not fetch organisation", "organisationId", organisationId, "error", err)
 			c.String(http.StatusNotFound, "Could not fetch organisation: "+organisationId)
 		}
 		return
@@ -178,21 +186,23 @@ func DeleteVCSConnection(c *gin.Context) {
 	err = models.DB.GormDB.Where("id = ? AND organisation_id = ?", connectionId, org.ID).First(&connection).Error
 	if err != nil {
 		if errors.Is(err, gorm.ErrRecordNotFound) {
+			slog.Info("Connection not found", "connectionId", connectionId, "organisationId", org.ID)
 			c.String(http.StatusNotFound, "Could not find connection: "+connectionId)
 		} else {
-			log.Printf("could not fetch connection: %v err: %v", connectionId, err)
+			slog.Error("Could not fetch connection", "connectionId", connectionId, "error", err)
 			c.String(http.StatusInternalServerError, "Could not fetch connection")
 		}
 		return
 	}
 
 	err = models.DB.GormDB.Delete(&connection).Error
 	if err != nil {
-		log.Printf("could not delete connection: %v err: %v", connectionId, err)
+		slog.Error("Could not delete connection", "connectionId", connectionId, "error", err)
 		c.String(http.StatusInternalServerError, "Could not delete connection")
 		return
 	}
 
+	slog.Info("Successfully deleted VCS connection", "connectionId", connectionId, "organisationId", org.ID)
 	c.JSON(http.StatusOK, gin.H{
 		"status": "success",
 	})