feat/support layers (#2061)

* support layers in digger.yml

Author: motatoes

backend/controllers/github.go

@@ -720,17 +720,6 @@ func handlePullRequestEvent(gh utils.GithubClientProvider, payload *github.PullR
 		return fmt.Errorf("error initializing comment reporter")
 	}
 
-	err = utils.ReportInitialJobsStatus(commentReporter, jobsForImpactedProjects)
-	if err != nil {
-		slog.Error("Failed to comment initial status for jobs",
-			"prNumber", prNumber,
-			"jobCount", len(jobsForImpactedProjects),
-			"error", err,
-		)
-		commentReporterManager.UpdateComment(fmt.Sprintf(":x: Failed to comment initial status for jobs: %v", err))
-		return fmt.Errorf("failed to comment initial status for jobs")
-	}
-
 	err = utils.SetPRStatusForJobs(ghService, prNumber, jobsForImpactedProjects)
 	if err != nil {
 		slog.Error("Error setting status for PR",
@@ -741,6 +730,40 @@ func handlePullRequestEvent(gh utils.GithubClientProvider, payload *github.PullR
 		return fmt.Errorf("error setting status for PR: %v", err)
 	}
 
+	nLayers, _ := orchestrator_scheduler.CountUniqueLayers(jobsForImpactedProjects)
+	slog.Debug("Number of layers",
+		"prNumber", prNumber,
+		"nLayers", nLayers,
+		"respectLayers", config.RespectLayers,
+	)
+	if config.RespectLayers && nLayers > 1 {
+		slog.Debug("Respecting layers",
+			"prNumber", prNumber)
+		err = utils.ReportLayersTableForJobs(commentReporter, jobsForImpactedProjects)
+		if err != nil {
+			slog.Error("Failed to comment initial status for jobs",
+				"prNumber", prNumber,
+				"jobCount", len(jobsForImpactedProjects),
+				"error", err,
+			)
+			commentReporterManager.UpdateComment(fmt.Sprintf(":x: Failed to comment initial status for jobs: %v", err))
+			return fmt.Errorf("failed to comment initial status for jobs")
+		}
+		slog.Debug("not performing plan since there are multiple layers and respect_layers is enabled")
+		return nil
+	} else {
+		err = utils.ReportInitialJobsStatus(commentReporter, jobsForImpactedProjects)
+		if err != nil {
+			slog.Error("Failed to comment initial status for jobs",
+				"prNumber", prNumber,
+				"jobCount", len(jobsForImpactedProjects),
+				"error", err,
+			)
+			commentReporterManager.UpdateComment(fmt.Sprintf(":x: Failed to comment initial status for jobs: %v", err))
+			return fmt.Errorf("failed to comment initial status for jobs")
+		}
+	}
+
 	slog.Debug("Preparing job and project maps",
 		"prNumber", prNumber,
 		"projectCount", len(impactedProjects),
@@ -788,6 +811,9 @@ func handlePullRequestEvent(gh utils.GithubClientProvider, payload *github.PullR
 		"jobCount", len(impactedJobsMap),
 	)
 
+	if config.RespectLayers {
+
+	}
 	batchId, _, err := utils.ConvertJobsToDiggerJobs(
 		*diggerCommand,
 		models.DiggerVCSGithub,
@@ -1427,7 +1453,7 @@ func handleIssueCommentEvent(gh utils.GithubClientProvider, payload *github.Issu
 		"branchName", prBranchName,
 	)
 
-	impactedProjects, impactedProjectsSourceMapping, requestedProject, _, err := generic.ProcessIssueCommentEvent(issueNumber, commentBody, config, projectsGraph, ghService)
+	processEventResult, err := generic.ProcessIssueCommentEvent(issueNumber, commentBody, config, projectsGraph, ghService)
 	if err != nil {
 		slog.Error("Error processing issue comment event",
 			"issueNumber", issueNumber,
@@ -1436,45 +1462,49 @@ func handleIssueCommentEvent(gh utils.GithubClientProvider, payload *github.Issu
 		commentReporterManager.UpdateComment(fmt.Sprintf(":x: Error processing event: %v", err))
 		return fmt.Errorf("error processing event")
 	}
+	impactedProjectsForComment := processEventResult.ImpactedProjectsForComment
+	impactedProjectsSourceMapping := processEventResult.ImpactedProjectsSourceMapping
+	allImpactedProjects := processEventResult.AllImpactedProjects
 
 	slog.Info("Issue comment event processed successfully",
 		"issueNumber", issueNumber,
-		"impactedProjectCount", len(impactedProjects),
-		"requestedProject", requestedProject,
+		"impactedProjectCount", len(impactedProjectsForComment),
+		"allImpactedProjectsCount", len(allImpactedProjects),
+
 	)
 
-	jobs, coverAllImpactedProjects, err := generic.ConvertIssueCommentEventToJobs(repoFullName, actor, issueNumber, commentBody, impactedProjects, requestedProject, config.Workflows, prBranchName, defaultBranch)
+	jobs, coverAllImpactedProjects, err := generic.ConvertIssueCommentEventToJobs(repoFullName, actor, issueNumber, commentBody, impactedProjectsForComment, allImpactedProjects, config.Workflows, prBranchName, defaultBranch)
 	if err != nil {
 		slog.Error("Error converting event to jobs",
 			"issueNumber", issueNumber,
-			"impactedProjectCount", len(impactedProjects),
+			"impactedProjectCount", len(impactedProjectsForComment),
 			"error", err,
 		)
 		commentReporterManager.UpdateComment(fmt.Sprintf(":x: Error converting event to jobs: %v", err))
 		return fmt.Errorf("error converting event to jobs")
 	}
-
+	
 	slog.Info("Issue comment event converted to jobs successfully",
 		"issueNumber", issueNumber,
 		"jobCount", len(jobs),
 	)
 
 	// if flag set we dont allow more projects impacted than the number of changed files in PR (safety check)
 	if config2.LimitByNumOfFilesChanged() {
-		if len(impactedProjects) > len(changedFiles) {
+		if len(impactedProjectsForComment) > len(changedFiles) {
 			slog.Error("Number of impacted projects exceeds number of changed files",
 				"issueNumber", issueNumber,
-				"impactedProjectCount", len(impactedProjects),
+				"impactedProjectCount", len(impactedProjectsForComment),
 				"changedFileCount", len(changedFiles),
 			)
 
-			commentReporterManager.UpdateComment(fmt.Sprintf(":x: Error the number impacted projects %v exceeds number of changed files: %v", len(impactedProjects), len(changedFiles)))
+			commentReporterManager.UpdateComment(fmt.Sprintf(":x: Error the number impacted projects %v exceeds number of changed files: %v", len(impactedProjectsForComment), len(changedFiles)))
 
 			slog.Debug("Detailed event information",
 				slog.Group("details",
 					slog.Any("changedFiles", changedFiles),
 					slog.Int("configLength", len(diggerYmlStr)),
-					slog.Int("impactedProjectCount", len(impactedProjects)),
+					slog.Int("impactedProjectCount", len(impactedProjectsForComment)),
 				),
 			)
 
@@ -1483,20 +1513,20 @@ func handleIssueCommentEvent(gh utils.GithubClientProvider, payload *github.Issu
 	}
 
 	maxImpactedProjectsPerChange := config2.MaxImpactedProjectsPerChange()
-	if len(impactedProjects) > maxImpactedProjectsPerChange {
+	if len(impactedProjectsForComment) > maxImpactedProjectsPerChange {
 		slog.Error("Number of impacted projects exceeds number of changed files",
 			"prNumber", issueNumber,
-			"impactedProjectCount", len(impactedProjects),
+			"impactedProjectCount", len(impactedProjectsForComment),
 			"changedFileCount", len(changedFiles),
 		)
 
-		commentReporterManager.UpdateComment(fmt.Sprintf(":x: Error the number impacted projects %v exceeds Max allowed ImpactedProjectsPerChange: %v, we set this limit to protect against hitting github API limits", len(impactedProjects), maxImpactedProjectsPerChange))
+		commentReporterManager.UpdateComment(fmt.Sprintf(":x: Error the number impacted projects %v exceeds Max allowed ImpactedProjectsPerChange: %v, we set this limit to protect against hitting github API limits", len(impactedProjectsForComment), maxImpactedProjectsPerChange))
 
 		slog.Debug("Detailed event information",
 			slog.Group("details",
 				slog.Any("changedFiles", changedFiles),
 				slog.Int("configLength", len(diggerYmlStr)),
-				slog.Int("impactedProjectCount", len(impactedProjects)),
+				slog.Int("impactedProjectCount", len(impactedProjectsForComment)),
 			),
 		)
 		return fmt.Errorf("error processing event")
@@ -1506,11 +1536,11 @@ func handleIssueCommentEvent(gh utils.GithubClientProvider, payload *github.Issu
 	if config.PrLocks {
 		slog.Info("Processing PR locks for impacted projects",
 			"issueNumber", issueNumber,
-			"projectCount", len(impactedProjects),
+			"projectCount", len(impactedProjectsForComment),
 			"command", *diggerCommand,
 		)
 
-		for _, project := range impactedProjects {
+		for _, project := range impactedProjectsForComment {
 			prLock := dg_locking.PullRequestLock{
 				InternalLock: locking.BackendDBLock{
 					OrgId: orgId,
@@ -1580,12 +1610,12 @@ func handleIssueCommentEvent(gh utils.GithubClientProvider, payload *github.Issu
 
 	slog.Debug("Preparing job and project maps",
 		"issueNumber", issueNumber,
-		"projectCount", len(impactedProjects),
+		"projectCount", len(impactedProjectsForComment),
 		"jobCount", len(jobs),
 	)
 
 	impactedProjectsMap := make(map[string]dg_configuration.Project)
-	for _, p := range impactedProjects {
+	for _, p := range impactedProjectsForComment {
 		impactedProjectsMap[p.Name] = p
 	}
 

backend/migrations/20250730002811.sql

@@ -0,0 +1,2 @@
+-- Modify "digger_batches" table
+ALTER TABLE "public"."digger_batches" ADD COLUMN "layer" bigint NULL;

backend/migrations/atlas.sum

@@ -1,4 +1,4 @@
-h1:DO/UG0tYCnZA69yRLMd2ZBKUORlJtZJLkVCiCogKp+k=
+h1:g7vBQEDCfExWRVQBcLZ9Jnxb1y/szNvlaeZjw1ZfRSY=
 20231227132525.sql h1:43xn7XC0GoJsCnXIMczGXWis9d504FAWi4F1gViTIcw=
 20240115170600.sql h1:IW8fF/8vc40+eWqP/xDK+R4K9jHJ9QBSGO6rN9LtfSA=
 20240116123649.sql h1:R1JlUIgxxF6Cyob9HdtMqiKmx/BfnsctTl5rvOqssQw=
@@ -61,3 +61,4 @@ h1:DO/UG0tYCnZA69yRLMd2ZBKUORlJtZJLkVCiCogKp+k=
 20250716222603.sql h1:a66teR/6BQwNwhJ/zgNA/NjUVTz5DcoQl7cDZvzVS8c=
 20250717032021.sql h1:HaIhNsz3C+c87CDmFjgFlc9zGqoE5BU4m0dofDpeDYk=
 20250725041417.sql h1:Dds6fqS415FD1jlsoVEahcEEm1px3EHV5435moL+Vp8=
+20250730002811.sql h1:VqNDMnhVinkBXlDJN4BCq5JzZr4NHVu6zuHEGP9dMvk=

backend/models/scheduler.go

@@ -26,6 +26,7 @@ const DiggerVCSBitbucket DiggerVCSType = "bitbucket"
 type DiggerBatch struct {
 	gorm.Model
 	ID                       uuid.UUID `gorm:"primary_key"`
+	Layer                    uint
 	VCS                      DiggerVCSType
 	PrNumber                 int
 	CommentId                *int64

backend/utils/pr_comment.go

@@ -3,6 +3,7 @@ package utils
 import (
 	"fmt"
 	"log/slog"
+	"sort"
 
 	"github.com/diggerhq/digger/libs/ci"
 	"github.com/diggerhq/digger/libs/scheduler"
@@ -135,6 +136,28 @@ func UpdateCRComment(cr *CommentReporter, comment string) error {
 	return nil
 }
 
+func trimMessageIfExceedsMaxLength(message string) string {
+
+	const GithubCommentMaxLength = 65536
+
+	if len(message) > GithubCommentMaxLength {
+		slog.Warn("Comment message is too long, trimming",
+			"originalLength", len(message),
+			"maxLength", GithubCommentMaxLength,
+		)
+
+		const footer = "[trimmed]"
+		trimLength := len(message) - GithubCommentMaxLength + len(footer)
+		message = message[:len(message)-trimLength] + footer
+
+		slog.Debug("Trimmed comment message",
+			"newLength", len(message),
+			"trimmedBytes", trimLength,
+		)
+	}
+	return message
+}
+
 func ReportInitialJobsStatus(cr *CommentReporter, jobs []scheduler.Job) error {
 	prNumber := cr.PrNumber
 	prService := cr.PrService
@@ -158,24 +181,7 @@ func ReportInitialJobsStatus(cr *CommentReporter, jobs []scheduler.Job) error {
 		}
 	}
 
-	const GithubCommentMaxLength = 65536
-
-	if len(message) > GithubCommentMaxLength {
-		slog.Warn("Comment message is too long, trimming",
-			"originalLength", len(message),
-			"maxLength", GithubCommentMaxLength,
-		)
-
-		const footer = "[trimmed]"
-		trimLength := len(message) - GithubCommentMaxLength + len(footer)
-		message = message[:len(message)-trimLength] + footer
-
-		slog.Debug("Trimmed comment message",
-			"newLength", len(message),
-			"trimmedBytes", trimLength,
-		)
-	}
-
+	message = trimMessageIfExceedsMaxLength(message)
 	err := prService.EditComment(prNumber, commentId, message)
 	if err != nil {
 		slog.Error("Failed to update comment with initial jobs status",
@@ -190,39 +196,61 @@ func ReportInitialJobsStatus(cr *CommentReporter, jobs []scheduler.Job) error {
 	return nil
 }
 
-func ReportNoProjectsImpacted(cr *CommentReporter, jobs []scheduler.Job) error {
+func ReportLayersTableForJobs(cr *CommentReporter, jobs []scheduler.Job) error {
 	prNumber := cr.PrNumber
 	prService := cr.PrService
 	commentId := cr.CommentId
 
-	slog.Info("Reporting no projects impacted",
+	// sort jobs by layer for better display (sort by name too)
+	sort.Slice(jobs, func(i, j int) bool {
+		if jobs[i].Layer == jobs[j].Layer {
+			return jobs[i].ProjectName < jobs[j].ProjectName
+		}
+		return jobs[i].Layer < jobs[j].Layer
+	})
+
+	slog.Info("Reporting initial jobs status",
 		"prNumber", prNumber,
 		"commentId", commentId,
 		"jobCount", len(jobs),
 	)
 
-	message := "" +
-		":construction_worker: The following projects are impacted\n\n"
-	for _, job := range jobs {
-		message = message + fmt.Sprintf(""+
-			"<!-- PROJECTHOLDER %v -->\n"+
-			":clock11: **%v**: pending...\n"+
-			"<!-- PROJECTHOLDEREND %v -->\n"+
-			"", job.ProjectName, job.ProjectName, job.ProjectName)
-
-		slog.Debug("Added project placeholder to message", "projectName", job.ProjectName)
+	message := ""
+	if len(jobs) == 0 {
+		message = message + ":construction_worker: No projects impacted"
+	} else {
+		message = message + fmt.Sprintf("| Project | Layer |\n")
+		message = message + fmt.Sprintf("|---------|--------|\n")
+		for _, job := range jobs {
+			message = message + fmt.Sprintf(""+
+				"|:clock11: **%v**|%v|\n", job.ProjectName, job.Layer)
+		}
 	}
 
+	message += "----------------\n\n"
+	message += `
+<details>
+  <summary>Instructions</summary>
+
+Since you enabled layers in your configuration, you can proceed to perform a layer-by-layer deployment.
+To start planning the first layer you can run "digger plan --layer 0". To apply the first layer, run "digger apply --layer 0".
+
+To deploy the next layer, run "digger plan --layer 1". To apply the next layer, run "digger apply --layer 1".
+
+And so on. A new commit on the branch will restart this deployment process.
+</details>
+`
+	message = trimMessageIfExceedsMaxLength(message)
 	err := prService.EditComment(prNumber, commentId, message)
 	if err != nil {
-		slog.Error("Failed to update comment with no projects impacted message",
+		slog.Error("Failed to update comment with initial jobs status",
 			"prNumber", prNumber,
 			"commentId", commentId,
 			"error", err,
 		)
 		return err
 	}
 
-	slog.Debug("Successfully reported no projects impacted", "prNumber", prNumber, "commentId", commentId)
+	slog.Debug("Successfully reported initial jobs status", "prNumber", prNumber, "commentId", commentId)
 	return nil
 }

cli/cmd/digger/main_test.go

@@ -921,7 +921,7 @@ func TestGitHubNewCommentContext(t *testing.T) {
 	lock := &locking.MockLock{}
 	prManager := ci.MockPullRequestManager{ChangedFiles: []string{"dev/test.tf"}}
 	planStorage := storage.MockPlanStorage{}
-	impactedProjects, requestedProject, prNumber, err := dggithub.ProcessGitHubEvent(ghEvent, &diggerConfig, &prManager)
+	impactedProjects, _, prNumber, err := dggithub.ProcessGitHubEvent(ghEvent, &diggerConfig, &prManager)
 	reporter := &reporting.CiReporter{
 		CiService: &prManager,
 		PrNumber:  prNumber,
@@ -930,7 +930,7 @@ func TestGitHubNewCommentContext(t *testing.T) {
 	policyChecker := policy.MockPolicyChecker{}
 	backendApi := backendapi.MockBackendApi{}
 
-	jobs, _, err := generic.ConvertIssueCommentEventToJobs("", "", 0, "", impactedProjects, requestedProject, map[string]configuration.Workflow{}, "prbranch", "")
+	jobs, _, err := generic.ConvertIssueCommentEventToJobs("", "", 0, "", impactedProjects, impactedProjects, map[string]configuration.Workflow{}, "prbranch", "")
 	_, _, err = digger.RunJobs(jobs, &prManager, prManager, lock, reporter, &planStorage, policyChecker, comment_updater.NoopCommentUpdater{}, backendApi, "123", false, false, "1", "")
 	assert.NoError(t, err)
 	if err != nil {
@@ -1020,10 +1020,9 @@ func TestGitHubTestPRCommandCaseInsensitivity(t *testing.T) {
 	var impactedProjects []configuration.Project
 	impactedProjects = make([]configuration.Project, 1)
 	impactedProjects[0] = project
-	var requestedProject = project
 	workflows := make(map[string]configuration.Workflow, 1)
 	workflows["default"] = configuration.Workflow{}
-	jobs, _, err := generic.ConvertIssueCommentEventToJobs("", "", 0, "digger plan", impactedProjects, &requestedProject, workflows, "prbranch", "main")
+	jobs, _, err := generic.ConvertIssueCommentEventToJobs("", "", 0, "digger plan", impactedProjects, impactedProjects, workflows, "prbranch", "main")
 
 	assert.Equal(t, 1, len(jobs))
 	assert.Equal(t, "digger plan", jobs[0].Commands[0])

cli/pkg/github/github.go

@@ -302,7 +302,14 @@ func GitHubCI(lock core_locking.Lock, policyCheckerProvider core_policy.PolicyCh
 			repoFullName := *commentEvent.Repo.FullName
 			requestedBy := *commentEvent.Sender.Login
 			commentBody := *commentEvent.Comment.Body
-			jobs, coversAllImpactedProjects, err = generic.ConvertIssueCommentEventToJobs(repoFullName, requestedBy, prNumber, commentBody, impactedProjects, requestedProject, diggerConfig.Workflows, prBranchName, defaultBranch)
+
+			var impactedProjectsForEvent []digger_config.Project
+			if requestedProject != nil {
+				impactedProjectsForEvent = []digger_config.Project{*requestedProject}
+			} else {
+				impactedProjectsForEvent = impactedProjects
+			}
+			jobs, coversAllImpactedProjects, err = generic.ConvertIssueCommentEventToJobs(repoFullName, requestedBy, prNumber, commentBody, impactedProjectsForEvent, impactedProjects, diggerConfig.Workflows, prBranchName, defaultBranch)
 		} else {
 			usage.ReportErrorAndExit(githubActor, fmt.Sprintf("Unsupported GitHub event type. %s", err), 6)
 		}