From 87521d5e48a889ab6fcad1e7f61c0f426f4ec21d Mon Sep 17 00:00:00 2001 From: Prakash Kumar Date: Mon, 21 Jul 2025 14:37:56 +0530 Subject: [PATCH 01/31] Add workflow status latest service with CI/CD status optimization --- .../WorkflowStatusLatestRepository.go | 197 +++++++++++ .../status/WorkflowStatusLatestService.go | 320 ++++++++++++++++++ .../status/WorkflowStatusUpdateService.go | 160 +++++++++ .../status/wire_workflow_status_latest.go | 31 ++ pkg/workflow/wire_workflow.go | 1 + ...900_workflow_status_latest_tables.down.sql | 15 + ...03900_workflow_status_latest_tables.up.sql | 41 +++ 7 files changed, 765 insertions(+) create mode 100644 internal/sql/repository/pipelineConfig/WorkflowStatusLatestRepository.go create mode 100644 pkg/workflow/status/WorkflowStatusLatestService.go create mode 100644 pkg/workflow/status/WorkflowStatusUpdateService.go create mode 100644 pkg/workflow/status/wire_workflow_status_latest.go create mode 100644 scripts/sql/34203900_workflow_status_latest_tables.down.sql create mode 100644 scripts/sql/34203900_workflow_status_latest_tables.up.sql diff --git a/internal/sql/repository/pipelineConfig/WorkflowStatusLatestRepository.go b/internal/sql/repository/pipelineConfig/WorkflowStatusLatestRepository.go new file mode 100644 index 0000000000..7f36cb27f8 --- /dev/null +++ b/internal/sql/repository/pipelineConfig/WorkflowStatusLatestRepository.go @@ -0,0 +1,197 @@ +/* + * Copyright (c) 2024. Devtron Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package pipelineConfig + +import ( + "github.com/devtron-labs/devtron/pkg/sql" + "github.com/go-pg/pg" + "go.uber.org/zap" +) + +type WorkflowStatusLatestRepository interface { + // CI Workflow Status Latest methods + SaveCiWorkflowStatusLatest(model *CiWorkflowStatusLatest) error + UpdateCiWorkflowStatusLatest(model *CiWorkflowStatusLatest) error + GetCiWorkflowStatusLatestByPipelineId(pipelineId int) (*CiWorkflowStatusLatest, error) + GetCiWorkflowStatusLatestByAppId(appId int) ([]*CiWorkflowStatusLatest, error) + DeleteCiWorkflowStatusLatestByPipelineId(pipelineId int) error + + // CD Workflow Status Latest methods + SaveCdWorkflowStatusLatest(model *CdWorkflowStatusLatest) error + UpdateCdWorkflowStatusLatest(model *CdWorkflowStatusLatest) error + GetCdWorkflowStatusLatestByPipelineIdAndWorkflowType(pipelineId int, workflowType string) (*CdWorkflowStatusLatest, error) + GetCdWorkflowStatusLatestByAppId(appId int) ([]*CdWorkflowStatusLatest, error) + GetCdWorkflowStatusLatestByPipelineId(pipelineId int) ([]*CdWorkflowStatusLatest, error) + DeleteCdWorkflowStatusLatestByPipelineId(pipelineId int) error +} + +type WorkflowStatusLatestRepositoryImpl struct { + dbConnection *pg.DB + logger *zap.SugaredLogger +} + +func NewWorkflowStatusLatestRepositoryImpl(dbConnection *pg.DB, logger *zap.SugaredLogger) *WorkflowStatusLatestRepositoryImpl { + return &WorkflowStatusLatestRepositoryImpl{ + dbConnection: dbConnection, + logger: logger, + } +} + +// CI Workflow Status Latest model +type CiWorkflowStatusLatest struct { + tableName struct{} `sql:"ci_workflow_status_latest" pg:",discard_unknown_columns"` + Id int `sql:"id,pk"` + PipelineId int `sql:"pipeline_id"` + AppId int `sql:"app_id"` + CiWorkflowId int `sql:"ci_workflow_id"` + Status string `sql:"status"` + sql.AuditLog +} + +// CD Workflow Status Latest model +type CdWorkflowStatusLatest struct { + tableName struct{} `sql:"cd_workflow_status_latest" pg:",discard_unknown_columns"` + Id int `sql:"id,pk"` + PipelineId int `sql:"pipeline_id"` + AppId int `sql:"app_id"` + EnvironmentId int `sql:"environment_id"` + WorkflowType string `sql:"workflow_type"` + WorkflowRunnerId int `sql:"workflow_runner_id"` + Status string `sql:"status"` + sql.AuditLog +} + +// CI Workflow Status Latest methods implementation +func (impl *WorkflowStatusLatestRepositoryImpl) SaveCiWorkflowStatusLatest(model *CiWorkflowStatusLatest) error { + err := impl.dbConnection.Insert(model) + if err != nil { + impl.logger.Errorw("error in saving ci workflow status latest", "err", err, "model", model) + return err + } + return nil +} + +func (impl *WorkflowStatusLatestRepositoryImpl) UpdateCiWorkflowStatusLatest(model *CiWorkflowStatusLatest) error { + _, err := impl.dbConnection.Model(model).WherePK().UpdateNotNull() + if err != nil { + impl.logger.Errorw("error in updating ci workflow status latest", "err", err, "model", model) + return err + } + return nil +} + +func (impl *WorkflowStatusLatestRepositoryImpl) GetCiWorkflowStatusLatestByPipelineId(pipelineId int) (*CiWorkflowStatusLatest, error) { + model := &CiWorkflowStatusLatest{} + err := impl.dbConnection.Model(model). + Where("pipeline_id = ?", pipelineId). + Select() + if err != nil { + impl.logger.Errorw("error in getting ci workflow status latest by pipeline id", "err", err, "pipelineId", pipelineId) + return nil, err + } + return model, nil +} + +func (impl *WorkflowStatusLatestRepositoryImpl) GetCiWorkflowStatusLatestByAppId(appId int) ([]*CiWorkflowStatusLatest, error) { + var models []*CiWorkflowStatusLatest + err := impl.dbConnection.Model(&models). + Where("app_id = ?", appId). + Select() + if err != nil { + impl.logger.Errorw("error in getting ci workflow status latest by app id", "err", err, "appId", appId) + return nil, err + } + return models, nil +} + +func (impl *WorkflowStatusLatestRepositoryImpl) DeleteCiWorkflowStatusLatestByPipelineId(pipelineId int) error { + _, err := impl.dbConnection.Model(&CiWorkflowStatusLatest{}). + Where("pipeline_id = ?", pipelineId). + Delete() + if err != nil { + impl.logger.Errorw("error in deleting ci workflow status latest by pipeline id", "err", err, "pipelineId", pipelineId) + return err + } + return nil +} + +// CD Workflow Status Latest methods implementation +func (impl *WorkflowStatusLatestRepositoryImpl) SaveCdWorkflowStatusLatest(model *CdWorkflowStatusLatest) error { + err := impl.dbConnection.Insert(model) + if err != nil { + impl.logger.Errorw("error in saving cd workflow status latest", "err", err, "model", model) + return err + } + return nil +} + +func (impl *WorkflowStatusLatestRepositoryImpl) UpdateCdWorkflowStatusLatest(model *CdWorkflowStatusLatest) error { + _, err := impl.dbConnection.Model(model).WherePK().UpdateNotNull() + if err != nil { + impl.logger.Errorw("error in updating cd workflow status latest", "err", err, "model", model) + return err + } + return nil +} + +func (impl *WorkflowStatusLatestRepositoryImpl) GetCdWorkflowStatusLatestByPipelineIdAndWorkflowType(pipelineId int, workflowType string) (*CdWorkflowStatusLatest, error) { + model := &CdWorkflowStatusLatest{} + err := impl.dbConnection.Model(model). + Where("pipeline_id = ?", pipelineId). + Where("workflow_type = ?", workflowType). + Select() + if err != nil { + impl.logger.Errorw("error in getting cd workflow status latest by pipeline id and workflow type", "err", err, "pipelineId", pipelineId, "workflowType", workflowType) + return nil, err + } + return model, nil +} + +func (impl *WorkflowStatusLatestRepositoryImpl) GetCdWorkflowStatusLatestByAppId(appId int) ([]*CdWorkflowStatusLatest, error) { + var models []*CdWorkflowStatusLatest + err := impl.dbConnection.Model(&models). + Where("app_id = ?", appId). + Select() + if err != nil { + impl.logger.Errorw("error in getting cd workflow status latest by app id", "err", err, "appId", appId) + return nil, err + } + return models, nil +} + +func (impl *WorkflowStatusLatestRepositoryImpl) GetCdWorkflowStatusLatestByPipelineId(pipelineId int) ([]*CdWorkflowStatusLatest, error) { + var models []*CdWorkflowStatusLatest + err := impl.dbConnection.Model(&models). + Where("pipeline_id = ?", pipelineId). + Select() + if err != nil { + impl.logger.Errorw("error in getting cd workflow status latest by pipeline id", "err", err, "pipelineId", pipelineId) + return nil, err + } + return models, nil +} + +func (impl *WorkflowStatusLatestRepositoryImpl) DeleteCdWorkflowStatusLatestByPipelineId(pipelineId int) error { + _, err := impl.dbConnection.Model(&CdWorkflowStatusLatest{}). + Where("pipeline_id = ?", pipelineId). + Delete() + if err != nil { + impl.logger.Errorw("error in deleting cd workflow status latest by pipeline id", "err", err, "pipelineId", pipelineId) + return err + } + return nil +} diff --git a/pkg/workflow/status/WorkflowStatusLatestService.go b/pkg/workflow/status/WorkflowStatusLatestService.go new file mode 100644 index 0000000000..13894154dd --- /dev/null +++ b/pkg/workflow/status/WorkflowStatusLatestService.go @@ -0,0 +1,320 @@ +/* + * Copyright (c) 2024. Devtron Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package status + +import ( + util2 "github.com/devtron-labs/devtron/internal/util" + "time" + + "github.com/devtron-labs/devtron/api/bean" + "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" + "github.com/go-pg/pg" + "go.uber.org/zap" +) + +type WorkflowStatusLatestService interface { + // CI Workflow Status Latest methods + SaveOrUpdateCiWorkflowStatusLatest(pipelineId, appId, ciWorkflowId int, status string, userId int32) error + GetCiWorkflowStatusLatestByPipelineId(pipelineId int) (*CiWorkflowStatusLatest, error) + GetCiWorkflowStatusLatestByAppId(appId int) ([]*CiWorkflowStatusLatest, error) + + // CD Workflow Status Latest methods + SaveOrUpdateCdWorkflowStatusLatest(pipelineId, appId, environmentId, workflowRunnerId int, workflowType, status string, userId int32) error + GetCdWorkflowStatusLatestByPipelineIdAndWorkflowType(pipelineId int, workflowType string) (*CdWorkflowStatusLatest, error) + GetCdWorkflowStatusLatestByAppId(appId int) ([]*CdWorkflowStatusLatest, error) + GetCdWorkflowStatusLatestByPipelineId(pipelineId int) ([]*CdWorkflowStatusLatest, error) +} + +type WorkflowStatusLatestServiceImpl struct { + logger *zap.SugaredLogger + workflowStatusLatestRepository pipelineConfig.WorkflowStatusLatestRepository + ciWorkflowRepository pipelineConfig.CiWorkflowRepository + cdWorkflowRepository pipelineConfig.CdWorkflowRepository +} + +func NewWorkflowStatusLatestServiceImpl( + logger *zap.SugaredLogger, + workflowStatusLatestRepository pipelineConfig.WorkflowStatusLatestRepository, + ciWorkflowRepository pipelineConfig.CiWorkflowRepository, + cdWorkflowRepository pipelineConfig.CdWorkflowRepository, +) *WorkflowStatusLatestServiceImpl { + return &WorkflowStatusLatestServiceImpl{ + logger: logger, + workflowStatusLatestRepository: workflowStatusLatestRepository, + ciWorkflowRepository: ciWorkflowRepository, + cdWorkflowRepository: cdWorkflowRepository, + } +} + +// DTOs for service layer +type CiWorkflowStatusLatest struct { + PipelineId int `json:"pipelineId"` + AppId int `json:"appId"` + CiWorkflowId int `json:"ciWorkflowId"` + Status string `json:"status"` +} + +type CdWorkflowStatusLatest struct { + PipelineId int `json:"pipelineId"` + AppId int `json:"appId"` + EnvironmentId int `json:"environmentId"` + WorkflowType string `json:"workflowType"` + WorkflowRunnerId int `json:"workflowRunnerId"` + Status string `json:"status"` +} + +// CI Workflow Status Latest methods implementation +func (impl *WorkflowStatusLatestServiceImpl) SaveOrUpdateCiWorkflowStatusLatest(pipelineId, appId, ciWorkflowId int, status string, userId int32) error { + // Check if entry exists + existingEntry, err := impl.workflowStatusLatestRepository.GetCiWorkflowStatusLatestByPipelineId(pipelineId) + if err != nil && err != pg.ErrNoRows { + impl.logger.Errorw("error in getting ci workflow status latest", "err", err, "pipelineId", pipelineId) + return err + } + + now := time.Now() + if err == pg.ErrNoRows { + // Create new entry + model := &pipelineConfig.CiWorkflowStatusLatest{ + PipelineId: pipelineId, + AppId: appId, + CiWorkflowId: ciWorkflowId, + Status: status, + } + model.CreatedBy = userId + model.CreatedOn = now + model.UpdatedBy = userId + model.UpdatedOn = now + + return impl.workflowStatusLatestRepository.SaveCiWorkflowStatusLatest(model) + } else { + // Update existing entry + existingEntry.CiWorkflowId = ciWorkflowId + existingEntry.Status = status + existingEntry.UpdatedBy = userId + existingEntry.UpdatedOn = now + + return impl.workflowStatusLatestRepository.UpdateCiWorkflowStatusLatest(existingEntry) + } +} + +func (impl *WorkflowStatusLatestServiceImpl) GetCiWorkflowStatusLatestByPipelineId(pipelineId int) (*CiWorkflowStatusLatest, error) { + model, err := impl.workflowStatusLatestRepository.GetCiWorkflowStatusLatestByPipelineId(pipelineId) + if err != nil { + if err == pg.ErrNoRows { + // Fallback to old method + return impl.getCiWorkflowStatusFromOldMethod(pipelineId) + } + impl.logger.Errorw("error in getting ci workflow status latest", "err", err, "pipelineId", pipelineId) + return nil, err + } + + return &CiWorkflowStatusLatest{ + PipelineId: model.PipelineId, + AppId: model.AppId, + CiWorkflowId: model.CiWorkflowId, + Status: model.Status, + }, nil +} + +func (impl *WorkflowStatusLatestServiceImpl) GetCiWorkflowStatusLatestByAppId(appId int) ([]*CiWorkflowStatusLatest, error) { + models, err := impl.workflowStatusLatestRepository.GetCiWorkflowStatusLatestByAppId(appId) + if err != nil { + impl.logger.Errorw("error in getting ci workflow status latest by app id", "err", err, "appId", appId) + return nil, err + } + + var result []*CiWorkflowStatusLatest + for _, model := range models { + result = append(result, &CiWorkflowStatusLatest{ + PipelineId: model.PipelineId, + AppId: model.AppId, + CiWorkflowId: model.CiWorkflowId, + Status: model.Status, + }) + } + + return result, nil +} + +// CD Workflow Status Latest methods implementation +func (impl *WorkflowStatusLatestServiceImpl) SaveOrUpdateCdWorkflowStatusLatest(pipelineId, appId, environmentId, workflowRunnerId int, workflowType, status string, userId int32) error { + // Check if entry exists + existingEntry, err := impl.workflowStatusLatestRepository.GetCdWorkflowStatusLatestByPipelineIdAndWorkflowType(pipelineId, workflowType) + if err != nil && err != pg.ErrNoRows { + impl.logger.Errorw("error in getting cd workflow status latest", "err", err, "pipelineId", pipelineId, "workflowType", workflowType) + return err + } + + now := time.Now() + if err == pg.ErrNoRows { + // Create new entry + model := &pipelineConfig.CdWorkflowStatusLatest{ + PipelineId: pipelineId, + AppId: appId, + EnvironmentId: environmentId, + WorkflowType: workflowType, + WorkflowRunnerId: workflowRunnerId, + Status: status, + } + model.CreatedBy = userId + model.CreatedOn = now + model.UpdatedBy = userId + model.UpdatedOn = now + + return impl.workflowStatusLatestRepository.SaveCdWorkflowStatusLatest(model) + } else { + // Update existing entry + existingEntry.WorkflowRunnerId = workflowRunnerId + existingEntry.Status = status + existingEntry.UpdatedBy = userId + existingEntry.UpdatedOn = now + + return impl.workflowStatusLatestRepository.UpdateCdWorkflowStatusLatest(existingEntry) + } +} + +func (impl *WorkflowStatusLatestServiceImpl) GetCdWorkflowStatusLatestByPipelineIdAndWorkflowType(pipelineId int, workflowType string) (*CdWorkflowStatusLatest, error) { + model, err := impl.workflowStatusLatestRepository.GetCdWorkflowStatusLatestByPipelineIdAndWorkflowType(pipelineId, workflowType) + if err != nil { + if err == pg.ErrNoRows { + // Fallback to old method + return impl.getCdWorkflowStatusFromOldMethod(pipelineId, workflowType) + } + impl.logger.Errorw("error in getting cd workflow status latest", "err", err, "pipelineId", pipelineId, "workflowType", workflowType) + return nil, err + } + + return &CdWorkflowStatusLatest{ + PipelineId: model.PipelineId, + AppId: model.AppId, + EnvironmentId: model.EnvironmentId, + WorkflowType: model.WorkflowType, + WorkflowRunnerId: model.WorkflowRunnerId, + Status: model.Status, + }, nil +} + +func (impl *WorkflowStatusLatestServiceImpl) GetCdWorkflowStatusLatestByAppId(appId int) ([]*CdWorkflowStatusLatest, error) { + models, err := impl.workflowStatusLatestRepository.GetCdWorkflowStatusLatestByAppId(appId) + if err != nil { + impl.logger.Errorw("error in getting cd workflow status latest by app id", "err", err, "appId", appId) + return nil, err + } + + var result []*CdWorkflowStatusLatest + for _, model := range models { + result = append(result, &CdWorkflowStatusLatest{ + PipelineId: model.PipelineId, + AppId: model.AppId, + EnvironmentId: model.EnvironmentId, + WorkflowType: model.WorkflowType, + WorkflowRunnerId: model.WorkflowRunnerId, + Status: model.Status, + }) + } + + return result, nil +} + +func (impl *WorkflowStatusLatestServiceImpl) GetCdWorkflowStatusLatestByPipelineId(pipelineId int) ([]*CdWorkflowStatusLatest, error) { + models, err := impl.workflowStatusLatestRepository.GetCdWorkflowStatusLatestByPipelineId(pipelineId) + if err != nil { + impl.logger.Errorw("error in getting cd workflow status latest by pipeline id", "err", err, "pipelineId", pipelineId) + return nil, err + } + + var result []*CdWorkflowStatusLatest + for _, model := range models { + result = append(result, &CdWorkflowStatusLatest{ + PipelineId: model.PipelineId, + AppId: model.AppId, + EnvironmentId: model.EnvironmentId, + WorkflowType: model.WorkflowType, + WorkflowRunnerId: model.WorkflowRunnerId, + Status: model.Status, + }) + } + + return result, nil +} + +// Fallback methods to old implementation when no entry found in latest status tables +func (impl *WorkflowStatusLatestServiceImpl) getCiWorkflowStatusFromOldMethod(pipelineId int) (*CiWorkflowStatusLatest, error) { + // Get the latest CI workflow for this pipeline using the old method + workflow, err := impl.ciWorkflowRepository.FindLastTriggeredWorkflow(pipelineId) + if err != nil { + if util2.IsErrNoRows(err) { + return &CiWorkflowStatusLatest{ + PipelineId: pipelineId, + AppId: 0, // Will need to be populated from pipeline info + CiWorkflowId: 0, + Status: "Not Triggered", + }, nil + } + impl.logger.Errorw("error in getting last triggered workflow", "err", err, "pipelineId", pipelineId) + return nil, err + } + + return &CiWorkflowStatusLatest{ + PipelineId: pipelineId, + AppId: workflow.CiPipeline.AppId, + CiWorkflowId: workflow.Id, + Status: workflow.Status, + }, nil +} + +func (impl *WorkflowStatusLatestServiceImpl) getCdWorkflowStatusFromOldMethod(pipelineId int, workflowType string) (*CdWorkflowStatusLatest, error) { + // Convert workflowType to the appropriate enum + var runnerType bean.WorkflowType + switch workflowType { + case "PRE": + runnerType = bean.CD_WORKFLOW_TYPE_PRE + case "DEPLOY": + runnerType = bean.CD_WORKFLOW_TYPE_DEPLOY + case "POST": + runnerType = bean.CD_WORKFLOW_TYPE_POST + default: + runnerType = bean.WorkflowType(workflowType) + } + + // Get the latest CD workflow runner for this pipeline and type using the old method + wfr, err := impl.cdWorkflowRepository.FindLatestByPipelineIdAndRunnerType(pipelineId, runnerType) + if err != nil { + if err == pg.ErrNoRows { + return &CdWorkflowStatusLatest{ + PipelineId: pipelineId, + AppId: 0, // Will need to be populated from pipeline info + EnvironmentId: 0, + WorkflowType: workflowType, + WorkflowRunnerId: 0, + Status: "Not Triggered", + }, nil + } + impl.logger.Errorw("error in getting latest cd workflow runner", "err", err, "pipelineId", pipelineId, "runnerType", runnerType) + return nil, err + } + + return &CdWorkflowStatusLatest{ + PipelineId: pipelineId, + AppId: wfr.CdWorkflow.Pipeline.AppId, + EnvironmentId: wfr.CdWorkflow.Pipeline.EnvironmentId, + WorkflowType: workflowType, + WorkflowRunnerId: wfr.Id, + Status: wfr.Status, + }, nil +} diff --git a/pkg/workflow/status/WorkflowStatusUpdateService.go b/pkg/workflow/status/WorkflowStatusUpdateService.go new file mode 100644 index 0000000000..1cd0f2a641 --- /dev/null +++ b/pkg/workflow/status/WorkflowStatusUpdateService.go @@ -0,0 +1,160 @@ +/* + * Copyright (c) 2024. Devtron Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package status + +import ( + "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" + "go.uber.org/zap" +) + +type WorkflowStatusUpdateService interface { + // Methods to update latest status tables when workflow status changes + UpdateCiWorkflowStatusLatest(pipelineId, appId, ciWorkflowId int, status string, userId int32) error + UpdateCdWorkflowStatusLatest(pipelineId, appId, environmentId, workflowRunnerId int, workflowType, status string, userId int32) error + + // Methods to fetch optimized status for trigger view + FetchCiStatusForTriggerViewOptimized(appId int) ([]*pipelineConfig.CiWorkflowStatus, error) + FetchCdStatusForTriggerViewOptimized(appId int) ([]*pipelineConfig.CdWorkflowStatus, error) +} + +type WorkflowStatusUpdateServiceImpl struct { + logger *zap.SugaredLogger + workflowStatusLatestService WorkflowStatusLatestService + ciWorkflowRepository pipelineConfig.CiWorkflowRepository + cdWorkflowRepository pipelineConfig.CdWorkflowRepository + ciPipelineRepository pipelineConfig.CiPipelineRepository + pipelineRepository pipelineConfig.PipelineRepository +} + +func NewWorkflowStatusUpdateServiceImpl( + logger *zap.SugaredLogger, + workflowStatusLatestService WorkflowStatusLatestService, + ciWorkflowRepository pipelineConfig.CiWorkflowRepository, + cdWorkflowRepository pipelineConfig.CdWorkflowRepository, + ciPipelineRepository pipelineConfig.CiPipelineRepository, + pipelineRepository pipelineConfig.PipelineRepository, +) *WorkflowStatusUpdateServiceImpl { + return &WorkflowStatusUpdateServiceImpl{ + logger: logger, + workflowStatusLatestService: workflowStatusLatestService, + ciWorkflowRepository: ciWorkflowRepository, + cdWorkflowRepository: cdWorkflowRepository, + ciPipelineRepository: ciPipelineRepository, + pipelineRepository: pipelineRepository, + } +} + +func (impl *WorkflowStatusUpdateServiceImpl) UpdateCiWorkflowStatusLatest(pipelineId, appId, ciWorkflowId int, status string, userId int32) error { + return impl.workflowStatusLatestService.SaveOrUpdateCiWorkflowStatusLatest(pipelineId, appId, ciWorkflowId, status, userId) +} + +func (impl *WorkflowStatusUpdateServiceImpl) UpdateCdWorkflowStatusLatest(pipelineId, appId, environmentId, workflowRunnerId int, workflowType, status string, userId int32) error { + return impl.workflowStatusLatestService.SaveOrUpdateCdWorkflowStatusLatest(pipelineId, appId, environmentId, workflowRunnerId, workflowType, status, userId) +} + +func (impl *WorkflowStatusUpdateServiceImpl) FetchCiStatusForTriggerViewOptimized(appId int) ([]*pipelineConfig.CiWorkflowStatus, error) { + // First try to get from the optimized latest status table + latestStatuses, err := impl.workflowStatusLatestService.GetCiWorkflowStatusLatestByAppId(appId) + if err != nil { + impl.logger.Errorw("error in getting ci workflow status latest by app id", "err", err, "appId", appId) + // Fallback to old method + return impl.ciWorkflowRepository.FIndCiWorkflowStatusesByAppId(appId) + } + + // Convert to the expected format + var ciWorkflowStatuses []*pipelineConfig.CiWorkflowStatus + for _, latestStatus := range latestStatuses { + // Get pipeline name from CI pipeline repository + ciPipeline, err := impl.ciPipelineRepository.FindById(latestStatus.PipelineId) + if err != nil { + impl.logger.Errorw("error in getting ci pipeline", "err", err, "pipelineId", latestStatus.PipelineId) + continue + } + + ciWorkflowStatus := &pipelineConfig.CiWorkflowStatus{ + CiPipelineId: latestStatus.PipelineId, + CiPipelineName: ciPipeline.Name, + CiStatus: latestStatus.Status, + CiWorkflowId: latestStatus.CiWorkflowId, + StorageConfigured: true, // Default value, can be enhanced later + } + ciWorkflowStatuses = append(ciWorkflowStatuses, ciWorkflowStatus) + } + + // If no entries found in latest status table, fallback to old method + if len(ciWorkflowStatuses) == 0 { + impl.logger.Infow("no entries found in ci workflow status latest table, falling back to old method", "appId", appId) + return impl.ciWorkflowRepository.FIndCiWorkflowStatusesByAppId(appId) + } + + return ciWorkflowStatuses, nil +} + +func (impl *WorkflowStatusUpdateServiceImpl) FetchCdStatusForTriggerViewOptimized(appId int) ([]*pipelineConfig.CdWorkflowStatus, error) { + // First try to get from the optimized latest status table + latestStatuses, err := impl.workflowStatusLatestService.GetCdWorkflowStatusLatestByAppId(appId) + if err != nil { + impl.logger.Errorw("error in getting cd workflow status latest by app id", "err", err, "appId", appId) + // Fallback to old method - would need to implement this based on existing CD status fetching logic + return nil, err + } + + // Convert to the expected format + var cdWorkflowStatuses []*pipelineConfig.CdWorkflowStatus + for _, latestStatus := range latestStatuses { + // Get pipeline info + pipeline, err := impl.pipelineRepository.FindById(latestStatus.PipelineId) + if err != nil { + impl.logger.Errorw("error in getting pipeline", "err", err, "pipelineId", latestStatus.PipelineId) + continue + } + + var status string + switch latestStatus.WorkflowType { + case "PRE": + status = latestStatus.Status + case "DEPLOY": + status = latestStatus.Status + case "POST": + status = latestStatus.Status + default: + status = latestStatus.Status + } + + cdWorkflowStatus := &pipelineConfig.CdWorkflowStatus{ + CiPipelineId: pipeline.CiPipelineId, + PipelineId: latestStatus.PipelineId, + PipelineName: pipeline.Name, + WorkflowType: latestStatus.WorkflowType, + WfrId: latestStatus.WorkflowRunnerId, + } + + // Set the appropriate status field based on workflow type + switch latestStatus.WorkflowType { + case "PRE": + cdWorkflowStatus.PreStatus = status + case "DEPLOY": + cdWorkflowStatus.DeployStatus = status + case "POST": + cdWorkflowStatus.PostStatus = status + } + + cdWorkflowStatuses = append(cdWorkflowStatuses, cdWorkflowStatus) + } + + return cdWorkflowStatuses, nil +} diff --git a/pkg/workflow/status/wire_workflow_status_latest.go b/pkg/workflow/status/wire_workflow_status_latest.go new file mode 100644 index 0000000000..b6b1f971f5 --- /dev/null +++ b/pkg/workflow/status/wire_workflow_status_latest.go @@ -0,0 +1,31 @@ +/* + * Copyright (c) 2024. Devtron Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package status + +import ( + "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" + "github.com/google/wire" +) + +var WorkflowStatusLatestWireSet = wire.NewSet( + pipelineConfig.NewWorkflowStatusLatestRepositoryImpl, + wire.Bind(new(pipelineConfig.WorkflowStatusLatestRepository), new(*pipelineConfig.WorkflowStatusLatestRepositoryImpl)), + NewWorkflowStatusLatestServiceImpl, + wire.Bind(new(WorkflowStatusLatestService), new(*WorkflowStatusLatestServiceImpl)), + NewWorkflowStatusUpdateServiceImpl, + wire.Bind(new(WorkflowStatusUpdateService), new(*WorkflowStatusUpdateServiceImpl)), +) diff --git a/pkg/workflow/wire_workflow.go b/pkg/workflow/wire_workflow.go index 3b48d0ae5e..e01a39efa1 100644 --- a/pkg/workflow/wire_workflow.go +++ b/pkg/workflow/wire_workflow.go @@ -28,6 +28,7 @@ import ( var WorkflowWireSet = wire.NewSet( cd.CdWorkflowWireSet, status.WorkflowStatusWireSet, + status.WorkflowStatusLatestWireSet, hook.NewTriggerAuditHookImpl, wire.Bind(new(hook.TriggerAuditHook), new(*hook.TriggerAuditHookImpl)), service.NewWorkflowTriggerAuditServiceImpl, diff --git a/scripts/sql/34203900_workflow_status_latest_tables.down.sql b/scripts/sql/34203900_workflow_status_latest_tables.down.sql new file mode 100644 index 0000000000..26ce9546a2 --- /dev/null +++ b/scripts/sql/34203900_workflow_status_latest_tables.down.sql @@ -0,0 +1,15 @@ +BEGIN; + +-- Drop cd_workflow_status_latest table +DROP TABLE IF EXISTS "public"."cd_workflow_status_latest"; + +-- Drop sequence for cd_workflow_status_latest +DROP SEQUENCE IF EXISTS id_seq_cd_workflow_status_latest; + +-- Drop ci_workflow_status_latest table +DROP TABLE IF EXISTS "public"."ci_workflow_status_latest"; + +-- Drop sequence for ci_workflow_status_latest +DROP SEQUENCE IF EXISTS id_seq_ci_workflow_status_latest; + +COMMIT; diff --git a/scripts/sql/34203900_workflow_status_latest_tables.up.sql b/scripts/sql/34203900_workflow_status_latest_tables.up.sql new file mode 100644 index 0000000000..1c8a24f76f --- /dev/null +++ b/scripts/sql/34203900_workflow_status_latest_tables.up.sql @@ -0,0 +1,41 @@ +BEGIN; + +-- Create Sequence for ci_workflow_status_latest +CREATE SEQUENCE IF NOT EXISTS id_seq_ci_workflow_status_latest; + +-- Create ci_workflow_status_latest table +CREATE TABLE IF NOT EXISTS "public"."ci_workflow_status_latest" ( + "id" int4 NOT NULL DEFAULT nextval('id_seq_ci_workflow_status_latest'::regclass), + "pipeline_id" int4 NOT NULL, + "app_id" int4 NOT NULL, + "ci_workflow_id" int4 NOT NULL, + "status" varchar(50) NOT NULL, + "created_on" timestamptz NOT NULL, + "created_by" int4 NOT NULL, + "updated_on" timestamptz NOT NULL, + "updated_by" int4 NOT NULL, + PRIMARY KEY ("id"), + UNIQUE ("pipeline_id") +); + +-- Create Sequence for cd_workflow_status_latest +CREATE SEQUENCE IF NOT EXISTS id_seq_cd_workflow_status_latest; + +-- Create cd_workflow_status_latest table +CREATE TABLE IF NOT EXISTS "public"."cd_workflow_status_latest" ( + "id" int4 NOT NULL DEFAULT nextval('id_seq_cd_workflow_status_latest'::regclass), + "pipeline_id" int4 NOT NULL, + "app_id" int4 NOT NULL, + "environment_id" int4 NOT NULL, + "workflow_type" varchar(20) NOT NULL, -- PRE, DEPLOY, POST + "workflow_runner_id" int4 NOT NULL, + "status" varchar(50) NOT NULL, + "created_on" timestamptz NOT NULL, + "created_by" int4 NOT NULL, + "updated_on" timestamptz NOT NULL, + "updated_by" int4 NOT NULL, + PRIMARY KEY ("id"), + UNIQUE ("pipeline_id", "workflow_type") +); + +COMMIT; From b97e7e4b8b4052f8415134b379267abea3cd33ba Mon Sep 17 00:00:00 2001 From: Prakash Kumar Date: Mon, 21 Jul 2025 15:04:39 +0530 Subject: [PATCH 02/31] Decouple workflow status management by deriving `status` from respective workflow tables. --- .../WorkflowStatusLatestRepository.go | 2 -- .../status/WorkflowStatusLatestService.go | 30 +++++++++++++------ ...03900_workflow_status_latest_tables.up.sql | 2 -- 3 files changed, 21 insertions(+), 13 deletions(-) diff --git a/internal/sql/repository/pipelineConfig/WorkflowStatusLatestRepository.go b/internal/sql/repository/pipelineConfig/WorkflowStatusLatestRepository.go index 7f36cb27f8..4a11673e58 100644 --- a/internal/sql/repository/pipelineConfig/WorkflowStatusLatestRepository.go +++ b/internal/sql/repository/pipelineConfig/WorkflowStatusLatestRepository.go @@ -58,7 +58,6 @@ type CiWorkflowStatusLatest struct { PipelineId int `sql:"pipeline_id"` AppId int `sql:"app_id"` CiWorkflowId int `sql:"ci_workflow_id"` - Status string `sql:"status"` sql.AuditLog } @@ -71,7 +70,6 @@ type CdWorkflowStatusLatest struct { EnvironmentId int `sql:"environment_id"` WorkflowType string `sql:"workflow_type"` WorkflowRunnerId int `sql:"workflow_runner_id"` - Status string `sql:"status"` sql.AuditLog } diff --git a/pkg/workflow/status/WorkflowStatusLatestService.go b/pkg/workflow/status/WorkflowStatusLatestService.go index 13894154dd..536871fd84 100644 --- a/pkg/workflow/status/WorkflowStatusLatestService.go +++ b/pkg/workflow/status/WorkflowStatusLatestService.go @@ -28,12 +28,12 @@ import ( type WorkflowStatusLatestService interface { // CI Workflow Status Latest methods - SaveOrUpdateCiWorkflowStatusLatest(pipelineId, appId, ciWorkflowId int, status string, userId int32) error + SaveOrUpdateCiWorkflowStatusLatest(pipelineId, appId, ciWorkflowId int, userId int32) error GetCiWorkflowStatusLatestByPipelineId(pipelineId int) (*CiWorkflowStatusLatest, error) GetCiWorkflowStatusLatestByAppId(appId int) ([]*CiWorkflowStatusLatest, error) // CD Workflow Status Latest methods - SaveOrUpdateCdWorkflowStatusLatest(pipelineId, appId, environmentId, workflowRunnerId int, workflowType, status string, userId int32) error + SaveOrUpdateCdWorkflowStatusLatest(pipelineId, appId, environmentId, workflowRunnerId int, workflowType string, userId int32) error GetCdWorkflowStatusLatestByPipelineIdAndWorkflowType(pipelineId int, workflowType string) (*CdWorkflowStatusLatest, error) GetCdWorkflowStatusLatestByAppId(appId int) ([]*CdWorkflowStatusLatest, error) GetCdWorkflowStatusLatestByPipelineId(pipelineId int) ([]*CdWorkflowStatusLatest, error) @@ -65,7 +65,7 @@ type CiWorkflowStatusLatest struct { PipelineId int `json:"pipelineId"` AppId int `json:"appId"` CiWorkflowId int `json:"ciWorkflowId"` - Status string `json:"status"` + Status string `json:"status"` // Derived from ci_workflow table } type CdWorkflowStatusLatest struct { @@ -74,11 +74,11 @@ type CdWorkflowStatusLatest struct { EnvironmentId int `json:"environmentId"` WorkflowType string `json:"workflowType"` WorkflowRunnerId int `json:"workflowRunnerId"` - Status string `json:"status"` + Status string `json:"status"` // Derived from cd_workflow_runner table } // CI Workflow Status Latest methods implementation -func (impl *WorkflowStatusLatestServiceImpl) SaveOrUpdateCiWorkflowStatusLatest(pipelineId, appId, ciWorkflowId int, status string, userId int32) error { +func (impl *WorkflowStatusLatestServiceImpl) SaveOrUpdateCiWorkflowStatusLatest(pipelineId, appId, ciWorkflowId int, userId int32) error { // Check if entry exists existingEntry, err := impl.workflowStatusLatestRepository.GetCiWorkflowStatusLatestByPipelineId(pipelineId) if err != nil && err != pg.ErrNoRows { @@ -93,7 +93,6 @@ func (impl *WorkflowStatusLatestServiceImpl) SaveOrUpdateCiWorkflowStatusLatest( PipelineId: pipelineId, AppId: appId, CiWorkflowId: ciWorkflowId, - Status: status, } model.CreatedBy = userId model.CreatedOn = now @@ -104,7 +103,6 @@ func (impl *WorkflowStatusLatestServiceImpl) SaveOrUpdateCiWorkflowStatusLatest( } else { // Update existing entry existingEntry.CiWorkflowId = ciWorkflowId - existingEntry.Status = status existingEntry.UpdatedBy = userId existingEntry.UpdatedOn = now @@ -123,11 +121,18 @@ func (impl *WorkflowStatusLatestServiceImpl) GetCiWorkflowStatusLatestByPipeline return nil, err } + // Get status from ci_workflow table + ciWorkflow, err := impl.ciWorkflowRepository.FindById(model.CiWorkflowId) + if err != nil { + impl.logger.Errorw("error in getting ci workflow", "err", err, "ciWorkflowId", model.CiWorkflowId) + return nil, err + } + return &CiWorkflowStatusLatest{ PipelineId: model.PipelineId, AppId: model.AppId, CiWorkflowId: model.CiWorkflowId, - Status: model.Status, + Status: ciWorkflow.Status, }, nil } @@ -140,11 +145,18 @@ func (impl *WorkflowStatusLatestServiceImpl) GetCiWorkflowStatusLatestByAppId(ap var result []*CiWorkflowStatusLatest for _, model := range models { + // Get status from ci_workflow table + ciWorkflow, err := impl.ciWorkflowRepository.FindById(model.CiWorkflowId) + if err != nil { + impl.logger.Errorw("error in getting ci workflow", "err", err, "ciWorkflowId", model.CiWorkflowId) + continue // Skip this entry if we can't get the workflow + } + result = append(result, &CiWorkflowStatusLatest{ PipelineId: model.PipelineId, AppId: model.AppId, CiWorkflowId: model.CiWorkflowId, - Status: model.Status, + Status: ciWorkflow.Status, }) } diff --git a/scripts/sql/34203900_workflow_status_latest_tables.up.sql b/scripts/sql/34203900_workflow_status_latest_tables.up.sql index 1c8a24f76f..6c87a7ec1d 100644 --- a/scripts/sql/34203900_workflow_status_latest_tables.up.sql +++ b/scripts/sql/34203900_workflow_status_latest_tables.up.sql @@ -9,7 +9,6 @@ CREATE TABLE IF NOT EXISTS "public"."ci_workflow_status_latest" ( "pipeline_id" int4 NOT NULL, "app_id" int4 NOT NULL, "ci_workflow_id" int4 NOT NULL, - "status" varchar(50) NOT NULL, "created_on" timestamptz NOT NULL, "created_by" int4 NOT NULL, "updated_on" timestamptz NOT NULL, @@ -29,7 +28,6 @@ CREATE TABLE IF NOT EXISTS "public"."cd_workflow_status_latest" ( "environment_id" int4 NOT NULL, "workflow_type" varchar(20) NOT NULL, -- PRE, DEPLOY, POST "workflow_runner_id" int4 NOT NULL, - "status" varchar(50) NOT NULL, "created_on" timestamptz NOT NULL, "created_by" int4 NOT NULL, "updated_on" timestamptz NOT NULL, From bc7e274d1023392fc6a7da42182ce7d1830313ba Mon Sep 17 00:00:00 2001 From: Prakash Kumar Date: Mon, 21 Jul 2025 15:42:34 +0530 Subject: [PATCH 03/31] Refactor `WorkflowStatusLatestService` to derive `status` from workflow runner tables and remove redundant `status` parameters. Integrate CI status optimization fallback in `CiHandlerImpl`. --- pkg/pipeline/CiHandler.go | 18 ++++++++--- .../status/WorkflowStatusLatestService.go | 31 +++++++++++++++---- .../status/WorkflowStatusUpdateService.go | 12 +++---- 3 files changed, 45 insertions(+), 16 deletions(-) diff --git a/pkg/pipeline/CiHandler.go b/pkg/pipeline/CiHandler.go index e636f67d44..70eb9a2246 100644 --- a/pkg/pipeline/CiHandler.go +++ b/pkg/pipeline/CiHandler.go @@ -49,6 +49,7 @@ import ( "github.com/devtron-labs/devtron/pkg/pipeline/executors" "github.com/devtron-labs/devtron/pkg/pipeline/types" "github.com/devtron-labs/devtron/pkg/resourceGroup" + workflowStatusLatest "github.com/devtron-labs/devtron/pkg/workflow/status" "github.com/devtron-labs/devtron/util/rbac" "github.com/go-pg/pg" "go.uber.org/zap" @@ -97,6 +98,7 @@ type CiHandlerImpl struct { config *types.CiConfig k8sCommonService k8sPkg.K8sCommonService workFlowStageStatusService workflowStatus.WorkFlowStageStatusService + workflowStatusUpdateService workflowStatusLatest.WorkflowStatusUpdateService } func NewCiHandlerImpl(Logger *zap.SugaredLogger, ciService CiService, ciPipelineMaterialRepository pipelineConfig.CiPipelineMaterialRepository, gitSensorClient gitSensor.Client, ciWorkflowRepository pipelineConfig.CiWorkflowRepository, @@ -104,6 +106,7 @@ func NewCiHandlerImpl(Logger *zap.SugaredLogger, ciService CiService, ciPipeline appListingRepository repository.AppListingRepository, cdPipelineRepository pipelineConfig.PipelineRepository, enforcerUtil rbac.EnforcerUtil, resourceGroupService resourceGroup.ResourceGroupService, envRepository repository2.EnvironmentRepository, imageTaggingService imageTagging.ImageTaggingService, k8sCommonService k8sPkg.K8sCommonService, appWorkflowRepository appWorkflow.AppWorkflowRepository, customTagService CustomTagService, workFlowStageStatusService workflowStatus.WorkFlowStageStatusService, + workflowStatusUpdateService workflowStatusLatest.WorkflowStatusUpdateService, ) *CiHandlerImpl { cih := &CiHandlerImpl{ Logger: Logger, @@ -126,6 +129,7 @@ func NewCiHandlerImpl(Logger *zap.SugaredLogger, ciService CiService, ciPipeline appWorkflowRepository: appWorkflowRepository, k8sCommonService: k8sCommonService, workFlowStageStatusService: workFlowStageStatusService, + workflowStatusUpdateService: workflowStatusUpdateService, } config, err := types.GetCiConfig() if err != nil { @@ -644,10 +648,16 @@ func (impl *CiHandlerImpl) stateChanged(status string, podStatus string, msg str } func (impl *CiHandlerImpl) FetchCiStatusForTriggerViewV1(appId int) ([]*pipelineConfig.CiWorkflowStatus, error) { - ciWorkflowStatuses, err := impl.ciWorkflowRepository.FIndCiWorkflowStatusesByAppId(appId) - if err != nil && !util.IsErrNoRows(err) { - impl.Logger.Errorw("err in fetching ciWorkflowStatuses from ciWorkflowRepository", "appId", appId, "err", err) - return ciWorkflowStatuses, err + // Try to use the optimized workflow status latest service first + ciWorkflowStatuses, err := impl.workflowStatusUpdateService.FetchCiStatusForTriggerViewOptimized(appId) + if err != nil { + impl.Logger.Errorw("error in fetching ci status from optimized service, falling back to old method", "appId", appId, "err", err) + // Fallback to old method if optimized service fails + ciWorkflowStatuses, err = impl.ciWorkflowRepository.FIndCiWorkflowStatusesByAppId(appId) + if err != nil && !util.IsErrNoRows(err) { + impl.Logger.Errorw("err in fetching ciWorkflowStatuses from ciWorkflowRepository", "appId", appId, "err", err) + return ciWorkflowStatuses, err + } } return ciWorkflowStatuses, err diff --git a/pkg/workflow/status/WorkflowStatusLatestService.go b/pkg/workflow/status/WorkflowStatusLatestService.go index 536871fd84..3d7f204394 100644 --- a/pkg/workflow/status/WorkflowStatusLatestService.go +++ b/pkg/workflow/status/WorkflowStatusLatestService.go @@ -164,7 +164,7 @@ func (impl *WorkflowStatusLatestServiceImpl) GetCiWorkflowStatusLatestByAppId(ap } // CD Workflow Status Latest methods implementation -func (impl *WorkflowStatusLatestServiceImpl) SaveOrUpdateCdWorkflowStatusLatest(pipelineId, appId, environmentId, workflowRunnerId int, workflowType, status string, userId int32) error { +func (impl *WorkflowStatusLatestServiceImpl) SaveOrUpdateCdWorkflowStatusLatest(pipelineId, appId, environmentId, workflowRunnerId int, workflowType string, userId int32) error { // Check if entry exists existingEntry, err := impl.workflowStatusLatestRepository.GetCdWorkflowStatusLatestByPipelineIdAndWorkflowType(pipelineId, workflowType) if err != nil && err != pg.ErrNoRows { @@ -181,7 +181,6 @@ func (impl *WorkflowStatusLatestServiceImpl) SaveOrUpdateCdWorkflowStatusLatest( EnvironmentId: environmentId, WorkflowType: workflowType, WorkflowRunnerId: workflowRunnerId, - Status: status, } model.CreatedBy = userId model.CreatedOn = now @@ -192,7 +191,6 @@ func (impl *WorkflowStatusLatestServiceImpl) SaveOrUpdateCdWorkflowStatusLatest( } else { // Update existing entry existingEntry.WorkflowRunnerId = workflowRunnerId - existingEntry.Status = status existingEntry.UpdatedBy = userId existingEntry.UpdatedOn = now @@ -211,13 +209,20 @@ func (impl *WorkflowStatusLatestServiceImpl) GetCdWorkflowStatusLatestByPipeline return nil, err } + // Get status from cd_workflow_runner table + cdWorkflowRunner, err := impl.cdWorkflowRepository.FindBasicWorkflowRunnerById(model.WorkflowRunnerId) + if err != nil { + impl.logger.Errorw("error in getting cd workflow runner", "err", err, "workflowRunnerId", model.WorkflowRunnerId) + return nil, err + } + return &CdWorkflowStatusLatest{ PipelineId: model.PipelineId, AppId: model.AppId, EnvironmentId: model.EnvironmentId, WorkflowType: model.WorkflowType, WorkflowRunnerId: model.WorkflowRunnerId, - Status: model.Status, + Status: cdWorkflowRunner.Status, }, nil } @@ -230,13 +235,20 @@ func (impl *WorkflowStatusLatestServiceImpl) GetCdWorkflowStatusLatestByAppId(ap var result []*CdWorkflowStatusLatest for _, model := range models { + // Get status from cd_workflow_runner table + cdWorkflowRunner, err := impl.cdWorkflowRepository.FindBasicWorkflowRunnerById(model.WorkflowRunnerId) + if err != nil { + impl.logger.Errorw("error in getting cd workflow runner", "err", err, "workflowRunnerId", model.WorkflowRunnerId) + continue // Skip this entry if we can't get the workflow runner + } + result = append(result, &CdWorkflowStatusLatest{ PipelineId: model.PipelineId, AppId: model.AppId, EnvironmentId: model.EnvironmentId, WorkflowType: model.WorkflowType, WorkflowRunnerId: model.WorkflowRunnerId, - Status: model.Status, + Status: cdWorkflowRunner.Status, }) } @@ -252,13 +264,20 @@ func (impl *WorkflowStatusLatestServiceImpl) GetCdWorkflowStatusLatestByPipeline var result []*CdWorkflowStatusLatest for _, model := range models { + // Get status from cd_workflow_runner table + cdWorkflowRunner, err := impl.cdWorkflowRepository.FindBasicWorkflowRunnerById(model.WorkflowRunnerId) + if err != nil { + impl.logger.Errorw("error in getting cd workflow runner", "err", err, "workflowRunnerId", model.WorkflowRunnerId) + continue // Skip this entry if we can't get the workflow runner + } + result = append(result, &CdWorkflowStatusLatest{ PipelineId: model.PipelineId, AppId: model.AppId, EnvironmentId: model.EnvironmentId, WorkflowType: model.WorkflowType, WorkflowRunnerId: model.WorkflowRunnerId, - Status: model.Status, + Status: cdWorkflowRunner.Status, }) } diff --git a/pkg/workflow/status/WorkflowStatusUpdateService.go b/pkg/workflow/status/WorkflowStatusUpdateService.go index 1cd0f2a641..6c56f6c17d 100644 --- a/pkg/workflow/status/WorkflowStatusUpdateService.go +++ b/pkg/workflow/status/WorkflowStatusUpdateService.go @@ -23,8 +23,8 @@ import ( type WorkflowStatusUpdateService interface { // Methods to update latest status tables when workflow status changes - UpdateCiWorkflowStatusLatest(pipelineId, appId, ciWorkflowId int, status string, userId int32) error - UpdateCdWorkflowStatusLatest(pipelineId, appId, environmentId, workflowRunnerId int, workflowType, status string, userId int32) error + UpdateCiWorkflowStatusLatest(pipelineId, appId, ciWorkflowId int, userId int32) error + UpdateCdWorkflowStatusLatest(pipelineId, appId, environmentId, workflowRunnerId int, workflowType string, userId int32) error // Methods to fetch optimized status for trigger view FetchCiStatusForTriggerViewOptimized(appId int) ([]*pipelineConfig.CiWorkflowStatus, error) @@ -58,12 +58,12 @@ func NewWorkflowStatusUpdateServiceImpl( } } -func (impl *WorkflowStatusUpdateServiceImpl) UpdateCiWorkflowStatusLatest(pipelineId, appId, ciWorkflowId int, status string, userId int32) error { - return impl.workflowStatusLatestService.SaveOrUpdateCiWorkflowStatusLatest(pipelineId, appId, ciWorkflowId, status, userId) +func (impl *WorkflowStatusUpdateServiceImpl) UpdateCiWorkflowStatusLatest(pipelineId, appId, ciWorkflowId int, userId int32) error { + return impl.workflowStatusLatestService.SaveOrUpdateCiWorkflowStatusLatest(pipelineId, appId, ciWorkflowId, userId) } -func (impl *WorkflowStatusUpdateServiceImpl) UpdateCdWorkflowStatusLatest(pipelineId, appId, environmentId, workflowRunnerId int, workflowType, status string, userId int32) error { - return impl.workflowStatusLatestService.SaveOrUpdateCdWorkflowStatusLatest(pipelineId, appId, environmentId, workflowRunnerId, workflowType, status, userId) +func (impl *WorkflowStatusUpdateServiceImpl) UpdateCdWorkflowStatusLatest(pipelineId, appId, environmentId, workflowRunnerId int, workflowType string, userId int32) error { + return impl.workflowStatusLatestService.SaveOrUpdateCdWorkflowStatusLatest(pipelineId, appId, environmentId, workflowRunnerId, workflowType, userId) } func (impl *WorkflowStatusUpdateServiceImpl) FetchCiStatusForTriggerViewOptimized(appId int) ([]*pipelineConfig.CiWorkflowStatus, error) { From 604baa88b3fd03301c35443b292d8640677d121a Mon Sep 17 00:00:00 2001 From: Prakash Kumar Date: Mon, 21 Jul 2025 16:41:06 +0530 Subject: [PATCH 04/31] Remove `WorkflowStatusLatestService` usage and fallback logic from `CiHandlerImpl`. Streamline CI status fetching through existing repository methods. --- pkg/pipeline/CiHandler.go | 18 ++++-------------- 1 file changed, 4 insertions(+), 14 deletions(-) diff --git a/pkg/pipeline/CiHandler.go b/pkg/pipeline/CiHandler.go index 70eb9a2246..e636f67d44 100644 --- a/pkg/pipeline/CiHandler.go +++ b/pkg/pipeline/CiHandler.go @@ -49,7 +49,6 @@ import ( "github.com/devtron-labs/devtron/pkg/pipeline/executors" "github.com/devtron-labs/devtron/pkg/pipeline/types" "github.com/devtron-labs/devtron/pkg/resourceGroup" - workflowStatusLatest "github.com/devtron-labs/devtron/pkg/workflow/status" "github.com/devtron-labs/devtron/util/rbac" "github.com/go-pg/pg" "go.uber.org/zap" @@ -98,7 +97,6 @@ type CiHandlerImpl struct { config *types.CiConfig k8sCommonService k8sPkg.K8sCommonService workFlowStageStatusService workflowStatus.WorkFlowStageStatusService - workflowStatusUpdateService workflowStatusLatest.WorkflowStatusUpdateService } func NewCiHandlerImpl(Logger *zap.SugaredLogger, ciService CiService, ciPipelineMaterialRepository pipelineConfig.CiPipelineMaterialRepository, gitSensorClient gitSensor.Client, ciWorkflowRepository pipelineConfig.CiWorkflowRepository, @@ -106,7 +104,6 @@ func NewCiHandlerImpl(Logger *zap.SugaredLogger, ciService CiService, ciPipeline appListingRepository repository.AppListingRepository, cdPipelineRepository pipelineConfig.PipelineRepository, enforcerUtil rbac.EnforcerUtil, resourceGroupService resourceGroup.ResourceGroupService, envRepository repository2.EnvironmentRepository, imageTaggingService imageTagging.ImageTaggingService, k8sCommonService k8sPkg.K8sCommonService, appWorkflowRepository appWorkflow.AppWorkflowRepository, customTagService CustomTagService, workFlowStageStatusService workflowStatus.WorkFlowStageStatusService, - workflowStatusUpdateService workflowStatusLatest.WorkflowStatusUpdateService, ) *CiHandlerImpl { cih := &CiHandlerImpl{ Logger: Logger, @@ -129,7 +126,6 @@ func NewCiHandlerImpl(Logger *zap.SugaredLogger, ciService CiService, ciPipeline appWorkflowRepository: appWorkflowRepository, k8sCommonService: k8sCommonService, workFlowStageStatusService: workFlowStageStatusService, - workflowStatusUpdateService: workflowStatusUpdateService, } config, err := types.GetCiConfig() if err != nil { @@ -648,16 +644,10 @@ func (impl *CiHandlerImpl) stateChanged(status string, podStatus string, msg str } func (impl *CiHandlerImpl) FetchCiStatusForTriggerViewV1(appId int) ([]*pipelineConfig.CiWorkflowStatus, error) { - // Try to use the optimized workflow status latest service first - ciWorkflowStatuses, err := impl.workflowStatusUpdateService.FetchCiStatusForTriggerViewOptimized(appId) - if err != nil { - impl.Logger.Errorw("error in fetching ci status from optimized service, falling back to old method", "appId", appId, "err", err) - // Fallback to old method if optimized service fails - ciWorkflowStatuses, err = impl.ciWorkflowRepository.FIndCiWorkflowStatusesByAppId(appId) - if err != nil && !util.IsErrNoRows(err) { - impl.Logger.Errorw("err in fetching ciWorkflowStatuses from ciWorkflowRepository", "appId", appId, "err", err) - return ciWorkflowStatuses, err - } + ciWorkflowStatuses, err := impl.ciWorkflowRepository.FIndCiWorkflowStatusesByAppId(appId) + if err != nil && !util.IsErrNoRows(err) { + impl.Logger.Errorw("err in fetching ciWorkflowStatuses from ciWorkflowRepository", "appId", appId, "err", err) + return ciWorkflowStatuses, err } return ciWorkflowStatuses, err From 17c1dae8b7fc95ec8afb95e9b74ec2b9dcbb5c17 Mon Sep 17 00:00:00 2001 From: Prakash Kumar Date: Mon, 21 Jul 2025 17:23:59 +0530 Subject: [PATCH 05/31] Refactor workflow status handling: rename `WorkflowStatusLatestService` package, integrate optimized CI status fetching and fallback logic in `CiHandlerImpl`. --- pkg/pipeline/CiHandler.go | 17 +++++++++++++---- .../WorkflowStatusLatestService.go | 2 +- .../WorkflowStatusUpdateService.go | 2 +- .../wire_workflow_status_latest.go | 2 +- pkg/workflow/wire_workflow.go | 3 ++- wire_gen.go | 8 ++++++-- 6 files changed, 24 insertions(+), 10 deletions(-) rename pkg/workflow/status/{ => workflowStatusLatest}/WorkflowStatusLatestService.go (99%) rename pkg/workflow/status/{ => workflowStatusLatest}/WorkflowStatusUpdateService.go (99%) rename pkg/workflow/status/{ => workflowStatusLatest}/wire_workflow_status_latest.go (97%) diff --git a/pkg/pipeline/CiHandler.go b/pkg/pipeline/CiHandler.go index e636f67d44..126e84e04f 100644 --- a/pkg/pipeline/CiHandler.go +++ b/pkg/pipeline/CiHandler.go @@ -29,6 +29,7 @@ import ( eventProcessorBean "github.com/devtron-labs/devtron/pkg/eventProcessor/bean" "github.com/devtron-labs/devtron/pkg/pipeline/constants" "github.com/devtron-labs/devtron/pkg/pipeline/workflowStatus" + "github.com/devtron-labs/devtron/pkg/workflow/status/workflowStatusLatest" "regexp" "slices" "strconv" @@ -97,6 +98,7 @@ type CiHandlerImpl struct { config *types.CiConfig k8sCommonService k8sPkg.K8sCommonService workFlowStageStatusService workflowStatus.WorkFlowStageStatusService + workflowStatusUpdateService workflowStatusLatest.WorkflowStatusUpdateService } func NewCiHandlerImpl(Logger *zap.SugaredLogger, ciService CiService, ciPipelineMaterialRepository pipelineConfig.CiPipelineMaterialRepository, gitSensorClient gitSensor.Client, ciWorkflowRepository pipelineConfig.CiWorkflowRepository, @@ -104,6 +106,7 @@ func NewCiHandlerImpl(Logger *zap.SugaredLogger, ciService CiService, ciPipeline appListingRepository repository.AppListingRepository, cdPipelineRepository pipelineConfig.PipelineRepository, enforcerUtil rbac.EnforcerUtil, resourceGroupService resourceGroup.ResourceGroupService, envRepository repository2.EnvironmentRepository, imageTaggingService imageTagging.ImageTaggingService, k8sCommonService k8sPkg.K8sCommonService, appWorkflowRepository appWorkflow.AppWorkflowRepository, customTagService CustomTagService, workFlowStageStatusService workflowStatus.WorkFlowStageStatusService, + workflowStatusUpdateService workflowStatusLatest.WorkflowStatusUpdateService, ) *CiHandlerImpl { cih := &CiHandlerImpl{ Logger: Logger, @@ -126,6 +129,7 @@ func NewCiHandlerImpl(Logger *zap.SugaredLogger, ciService CiService, ciPipeline appWorkflowRepository: appWorkflowRepository, k8sCommonService: k8sCommonService, workFlowStageStatusService: workFlowStageStatusService, + workflowStatusUpdateService: workflowStatusUpdateService, } config, err := types.GetCiConfig() if err != nil { @@ -644,10 +648,15 @@ func (impl *CiHandlerImpl) stateChanged(status string, podStatus string, msg str } func (impl *CiHandlerImpl) FetchCiStatusForTriggerViewV1(appId int) ([]*pipelineConfig.CiWorkflowStatus, error) { - ciWorkflowStatuses, err := impl.ciWorkflowRepository.FIndCiWorkflowStatusesByAppId(appId) - if err != nil && !util.IsErrNoRows(err) { - impl.Logger.Errorw("err in fetching ciWorkflowStatuses from ciWorkflowRepository", "appId", appId, "err", err) - return ciWorkflowStatuses, err + ciWorkflowStatuses, err := impl.workflowStatusUpdateService.FetchCiStatusForTriggerViewOptimized(appId) + if err != nil { + impl.Logger.Errorw("error in fetching ci status from optimized service, falling back to old method", "appId", appId, "err", err) + // Fallback to old method if optimized service fails + ciWorkflowStatuses, err = impl.ciWorkflowRepository.FIndCiWorkflowStatusesByAppId(appId) + if err != nil && !util.IsErrNoRows(err) { + impl.Logger.Errorw("err in fetching ciWorkflowStatuses from ciWorkflowRepository", "appId", appId, "err", err) + return ciWorkflowStatuses, err + } } return ciWorkflowStatuses, err diff --git a/pkg/workflow/status/WorkflowStatusLatestService.go b/pkg/workflow/status/workflowStatusLatest/WorkflowStatusLatestService.go similarity index 99% rename from pkg/workflow/status/WorkflowStatusLatestService.go rename to pkg/workflow/status/workflowStatusLatest/WorkflowStatusLatestService.go index 3d7f204394..cf86a1421b 100644 --- a/pkg/workflow/status/WorkflowStatusLatestService.go +++ b/pkg/workflow/status/workflowStatusLatest/WorkflowStatusLatestService.go @@ -14,7 +14,7 @@ * limitations under the License. */ -package status +package workflowStatusLatest import ( util2 "github.com/devtron-labs/devtron/internal/util" diff --git a/pkg/workflow/status/WorkflowStatusUpdateService.go b/pkg/workflow/status/workflowStatusLatest/WorkflowStatusUpdateService.go similarity index 99% rename from pkg/workflow/status/WorkflowStatusUpdateService.go rename to pkg/workflow/status/workflowStatusLatest/WorkflowStatusUpdateService.go index 6c56f6c17d..e52ef1c44d 100644 --- a/pkg/workflow/status/WorkflowStatusUpdateService.go +++ b/pkg/workflow/status/workflowStatusLatest/WorkflowStatusUpdateService.go @@ -14,7 +14,7 @@ * limitations under the License. */ -package status +package workflowStatusLatest import ( "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" diff --git a/pkg/workflow/status/wire_workflow_status_latest.go b/pkg/workflow/status/workflowStatusLatest/wire_workflow_status_latest.go similarity index 97% rename from pkg/workflow/status/wire_workflow_status_latest.go rename to pkg/workflow/status/workflowStatusLatest/wire_workflow_status_latest.go index b6b1f971f5..a0b18557f7 100644 --- a/pkg/workflow/status/wire_workflow_status_latest.go +++ b/pkg/workflow/status/workflowStatusLatest/wire_workflow_status_latest.go @@ -14,7 +14,7 @@ * limitations under the License. */ -package status +package workflowStatusLatest import ( "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" diff --git a/pkg/workflow/wire_workflow.go b/pkg/workflow/wire_workflow.go index e01a39efa1..d2850fc073 100644 --- a/pkg/workflow/wire_workflow.go +++ b/pkg/workflow/wire_workflow.go @@ -19,6 +19,7 @@ package workflow import ( "github.com/devtron-labs/devtron/pkg/workflow/cd" "github.com/devtron-labs/devtron/pkg/workflow/status" + "github.com/devtron-labs/devtron/pkg/workflow/status/workflowStatusLatest" "github.com/devtron-labs/devtron/pkg/workflow/trigger/audit/hook" "github.com/devtron-labs/devtron/pkg/workflow/trigger/audit/repository" "github.com/devtron-labs/devtron/pkg/workflow/trigger/audit/service" @@ -28,7 +29,7 @@ import ( var WorkflowWireSet = wire.NewSet( cd.CdWorkflowWireSet, status.WorkflowStatusWireSet, - status.WorkflowStatusLatestWireSet, + workflowStatusLatest.WorkflowStatusLatestWireSet, hook.NewTriggerAuditHookImpl, wire.Bind(new(hook.TriggerAuditHook), new(*hook.TriggerAuditHookImpl)), service.NewWorkflowTriggerAuditServiceImpl, diff --git a/wire_gen.go b/wire_gen.go index 9a899a2eab..40615a5be0 100644 --- a/wire_gen.go +++ b/wire_gen.go @@ -1,6 +1,6 @@ // Code generated by Wire. DO NOT EDIT. -//go:generate go run github.com/google/wire/cmd/wire +//go:generate go run -mod=mod github.com/google/wire/cmd/wire //go:build !wireinject // +build !wireinject @@ -273,6 +273,7 @@ import ( read20 "github.com/devtron-labs/devtron/pkg/workflow/cd/read" "github.com/devtron-labs/devtron/pkg/workflow/dag" status2 "github.com/devtron-labs/devtron/pkg/workflow/status" + "github.com/devtron-labs/devtron/pkg/workflow/status/workflowStatusLatest" "github.com/devtron-labs/devtron/pkg/workflow/trigger/audit/hook" repository19 "github.com/devtron-labs/devtron/pkg/workflow/trigger/audit/repository" service3 "github.com/devtron-labs/devtron/pkg/workflow/trigger/audit/service" @@ -748,7 +749,10 @@ func InitializeApp() (*App, error) { deploymentTemplateValidationServiceEntImpl := validator.NewDeploymentTemplateValidationServiceEntImpl() deploymentTemplateValidationServiceImpl := validator.NewDeploymentTemplateValidationServiceImpl(sugaredLogger, chartRefServiceImpl, scopedVariableManagerImpl, deployedAppMetricsServiceImpl, deploymentTemplateValidationServiceEntImpl) devtronAppGitOpConfigServiceImpl := gitOpsConfig.NewDevtronAppGitOpConfigServiceImpl(sugaredLogger, chartRepositoryImpl, chartServiceImpl, gitOpsConfigReadServiceImpl, gitOpsValidationServiceImpl, argoClientWrapperServiceImpl, deploymentConfigServiceImpl, chartReadServiceImpl) - ciHandlerImpl := pipeline.NewCiHandlerImpl(sugaredLogger, ciServiceImpl, ciPipelineMaterialRepositoryImpl, clientImpl, ciWorkflowRepositoryImpl, ciArtifactRepositoryImpl, userServiceImpl, eventRESTClientImpl, eventSimpleFactoryImpl, ciPipelineRepositoryImpl, appListingRepositoryImpl, pipelineRepositoryImpl, enforcerUtilImpl, resourceGroupServiceImpl, environmentRepositoryImpl, imageTaggingServiceImpl, k8sCommonServiceImpl, appWorkflowRepositoryImpl, customTagServiceImpl, workFlowStageStatusServiceImpl) + workflowStatusLatestRepositoryImpl := pipelineConfig.NewWorkflowStatusLatestRepositoryImpl(db, sugaredLogger) + workflowStatusLatestServiceImpl := workflowStatusLatest.NewWorkflowStatusLatestServiceImpl(sugaredLogger, workflowStatusLatestRepositoryImpl, ciWorkflowRepositoryImpl, cdWorkflowRepositoryImpl) + workflowStatusUpdateServiceImpl := workflowStatusLatest.NewWorkflowStatusUpdateServiceImpl(sugaredLogger, workflowStatusLatestServiceImpl, ciWorkflowRepositoryImpl, cdWorkflowRepositoryImpl, ciPipelineRepositoryImpl, pipelineRepositoryImpl) + ciHandlerImpl := pipeline.NewCiHandlerImpl(sugaredLogger, ciServiceImpl, ciPipelineMaterialRepositoryImpl, clientImpl, ciWorkflowRepositoryImpl, ciArtifactRepositoryImpl, userServiceImpl, eventRESTClientImpl, eventSimpleFactoryImpl, ciPipelineRepositoryImpl, appListingRepositoryImpl, pipelineRepositoryImpl, enforcerUtilImpl, resourceGroupServiceImpl, environmentRepositoryImpl, imageTaggingServiceImpl, k8sCommonServiceImpl, appWorkflowRepositoryImpl, customTagServiceImpl, workFlowStageStatusServiceImpl, workflowStatusUpdateServiceImpl) cdHandlerImpl := pipeline.NewCdHandlerImpl(sugaredLogger, userServiceImpl, cdWorkflowRepositoryImpl, ciArtifactRepositoryImpl, ciPipelineMaterialRepositoryImpl, pipelineRepositoryImpl, environmentRepositoryImpl, ciWorkflowRepositoryImpl, enforcerUtilImpl, resourceGroupServiceImpl, imageTaggingServiceImpl, k8sServiceImpl, customTagServiceImpl, deploymentConfigServiceImpl, workFlowStageStatusServiceImpl, cdWorkflowRunnerServiceImpl) appWorkflowServiceImpl := appWorkflow2.NewAppWorkflowServiceImpl(sugaredLogger, appWorkflowRepositoryImpl, ciCdPipelineOrchestratorImpl, ciPipelineRepositoryImpl, pipelineRepositoryImpl, enforcerUtilImpl, resourceGroupServiceImpl, appRepositoryImpl, userAuthServiceImpl, chartServiceImpl, deploymentConfigServiceImpl, pipelineBuilderImpl) appCloneServiceImpl := appClone.NewAppCloneServiceImpl(sugaredLogger, pipelineBuilderImpl, attributesServiceImpl, chartServiceImpl, configMapServiceImpl, appWorkflowServiceImpl, appListingServiceImpl, propertiesConfigServiceImpl, pipelineStageServiceImpl, ciTemplateReadServiceImpl, appRepositoryImpl, ciPipelineRepositoryImpl, pipelineRepositoryImpl, ciPipelineConfigServiceImpl, gitOpsConfigReadServiceImpl, chartReadServiceImpl) From cb6ce4930d0da414e4f3ad2c58cc4eb02c8abf8f Mon Sep 17 00:00:00 2001 From: Prakash Kumar Date: Mon, 21 Jul 2025 18:03:22 +0530 Subject: [PATCH 06/31] Integrate optimized CI workflow fetching in `CiHandlerImpl` using `FindLastTriggeredWorkflowByCiIdsOptimized`. Add fallback to legacy method for error handling. Update repository with the optimized query method. --- .../pipelineConfig/CiWorkflowRepository.go | 18 ++++++++++++++++++ pkg/pipeline/CiHandler.go | 4 ++-- 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/internal/sql/repository/pipelineConfig/CiWorkflowRepository.go b/internal/sql/repository/pipelineConfig/CiWorkflowRepository.go index 411662fcc6..76d10c9770 100644 --- a/internal/sql/repository/pipelineConfig/CiWorkflowRepository.go +++ b/internal/sql/repository/pipelineConfig/CiWorkflowRepository.go @@ -40,6 +40,7 @@ type CiWorkflowRepository interface { FindByName(name string) (*CiWorkflow, error) FindLastTriggeredWorkflowByCiIds(pipelineId []int) (ciWorkflow []*CiWorkflow, err error) + FindLastTriggeredWorkflowByCiIdsOptimized(pipelineId []int) (ciWorkflow []*CiWorkflow, err error) FindLastTriggeredWorkflowByArtifactId(ciArtifactId int) (ciWorkflow *CiWorkflow, err error) FindAllLastTriggeredWorkflowByArtifactId(ciArtifactId []int) (ciWorkflow []*CiWorkflow, err error) FindAllTriggeredWorkflowCountInLast24Hour() (ciWorkflowCount int, err error) @@ -290,6 +291,23 @@ func (impl *CiWorkflowRepositoryImpl) FindLastTriggeredWorkflowByCiIds(pipelineI return ciWorkflow, err } +// FindLastTriggeredWorkflowByCiIdsOptimized uses the ci_workflow_status_latest table for better performance +func (impl *CiWorkflowRepositoryImpl) FindLastTriggeredWorkflowByCiIdsOptimized(pipelineId []int) (ciWorkflow []*CiWorkflow, err error) { + err = impl.dbConnection.Model(&ciWorkflow). + Column("ci_workflow.*", "CiPipeline"). + Join("INNER JOIN ci_workflow_status_latest cwsl ON cwsl.ci_workflow_id = ci_workflow.id"). + Where("cwsl.pipeline_id IN (?)", pg.In(pipelineId)). + Select() + + if err != nil { + impl.logger.Errorw("error in optimized query, falling back to old method", "err", err, "pipelineIds", pipelineId) + // Fallback to the old method if optimized query fails + return impl.FindLastTriggeredWorkflowByCiIds(pipelineId) + } + + return ciWorkflow, err +} + func (impl *CiWorkflowRepositoryImpl) FindLastTriggeredWorkflowByArtifactId(ciArtifactId int) (ciWorkflow *CiWorkflow, err error) { workflow := &CiWorkflow{} err = impl.dbConnection.Model(workflow). diff --git a/pkg/pipeline/CiHandler.go b/pkg/pipeline/CiHandler.go index 126e84e04f..ba9f6cceca 100644 --- a/pkg/pipeline/CiHandler.go +++ b/pkg/pipeline/CiHandler.go @@ -870,9 +870,9 @@ func (impl *CiHandlerImpl) FetchCiStatusForTriggerViewForEnvironment(request res if len(ciPipelineIds) == 0 { return ciWorkflowStatuses, nil } - latestCiWorkflows, err := impl.ciWorkflowRepository.FindLastTriggeredWorkflowByCiIds(ciPipelineIds) + latestCiWorkflows, err := impl.ciWorkflowRepository.FindLastTriggeredWorkflowByCiIdsOptimized(ciPipelineIds) if err != nil && !util.IsErrNoRows(err) { - impl.Logger.Errorw("err", "ciPipelineIds", ciPipelineIds, "err", err) + impl.Logger.Errorw("err in optimized ci workflow fetch", "ciPipelineIds", ciPipelineIds, "err", err) return ciWorkflowStatuses, err } From 96815d372070c3d6eddd543ed31ee33c6235ced9 Mon Sep 17 00:00:00 2001 From: Prakash Kumar Date: Tue, 22 Jul 2025 13:14:55 +0530 Subject: [PATCH 07/31] Add `StorageConfigured` field to `CiWorkflowStatusLatest` and update services to handle it. Integrate `BlobStorageEnabled` from the data model and replace default hardcoded value in `WorkflowStatusUpdateServiceImpl`. --- .../WorkflowStatusLatestService.go | 19 ++++++++++--------- .../WorkflowStatusUpdateService.go | 4 +--- 2 files changed, 11 insertions(+), 12 deletions(-) diff --git a/pkg/workflow/status/workflowStatusLatest/WorkflowStatusLatestService.go b/pkg/workflow/status/workflowStatusLatest/WorkflowStatusLatestService.go index cf86a1421b..b338185ef1 100644 --- a/pkg/workflow/status/workflowStatusLatest/WorkflowStatusLatestService.go +++ b/pkg/workflow/status/workflowStatusLatest/WorkflowStatusLatestService.go @@ -60,12 +60,12 @@ func NewWorkflowStatusLatestServiceImpl( } } -// DTOs for service layer type CiWorkflowStatusLatest struct { - PipelineId int `json:"pipelineId"` - AppId int `json:"appId"` - CiWorkflowId int `json:"ciWorkflowId"` - Status string `json:"status"` // Derived from ci_workflow table + PipelineId int `json:"pipelineId"` + AppId int `json:"appId"` + CiWorkflowId int `json:"ciWorkflowId"` + Status string `json:"status"` // Derived from ci_workflow table + StorageConfigured bool `json:"storageConfigured"` } type CdWorkflowStatusLatest struct { @@ -153,10 +153,11 @@ func (impl *WorkflowStatusLatestServiceImpl) GetCiWorkflowStatusLatestByAppId(ap } result = append(result, &CiWorkflowStatusLatest{ - PipelineId: model.PipelineId, - AppId: model.AppId, - CiWorkflowId: model.CiWorkflowId, - Status: ciWorkflow.Status, + PipelineId: model.PipelineId, + AppId: model.AppId, + CiWorkflowId: model.CiWorkflowId, + Status: ciWorkflow.Status, + StorageConfigured: ciWorkflow.BlobStorageEnabled, }) } diff --git a/pkg/workflow/status/workflowStatusLatest/WorkflowStatusUpdateService.go b/pkg/workflow/status/workflowStatusLatest/WorkflowStatusUpdateService.go index e52ef1c44d..45d4437f80 100644 --- a/pkg/workflow/status/workflowStatusLatest/WorkflowStatusUpdateService.go +++ b/pkg/workflow/status/workflowStatusLatest/WorkflowStatusUpdateService.go @@ -67,7 +67,6 @@ func (impl *WorkflowStatusUpdateServiceImpl) UpdateCdWorkflowStatusLatest(pipeli } func (impl *WorkflowStatusUpdateServiceImpl) FetchCiStatusForTriggerViewOptimized(appId int) ([]*pipelineConfig.CiWorkflowStatus, error) { - // First try to get from the optimized latest status table latestStatuses, err := impl.workflowStatusLatestService.GetCiWorkflowStatusLatestByAppId(appId) if err != nil { impl.logger.Errorw("error in getting ci workflow status latest by app id", "err", err, "appId", appId) @@ -78,7 +77,6 @@ func (impl *WorkflowStatusUpdateServiceImpl) FetchCiStatusForTriggerViewOptimize // Convert to the expected format var ciWorkflowStatuses []*pipelineConfig.CiWorkflowStatus for _, latestStatus := range latestStatuses { - // Get pipeline name from CI pipeline repository ciPipeline, err := impl.ciPipelineRepository.FindById(latestStatus.PipelineId) if err != nil { impl.logger.Errorw("error in getting ci pipeline", "err", err, "pipelineId", latestStatus.PipelineId) @@ -90,7 +88,7 @@ func (impl *WorkflowStatusUpdateServiceImpl) FetchCiStatusForTriggerViewOptimize CiPipelineName: ciPipeline.Name, CiStatus: latestStatus.Status, CiWorkflowId: latestStatus.CiWorkflowId, - StorageConfigured: true, // Default value, can be enhanced later + StorageConfigured: latestStatus.StorageConfigured, } ciWorkflowStatuses = append(ciWorkflowStatuses, ciWorkflowStatus) } From 4a0bc3a47ed4a71371528c5747316578615747b8 Mon Sep 17 00:00:00 2001 From: ayushmaheshwari Date: Tue, 22 Jul 2025 18:03:21 +0530 Subject: [PATCH 08/31] wip: adding code for cd --- .../pipelineConfig/CdWorfkflowRepository.go | 18 ++- .../WorkflowStatusLatestRepository.go | 50 +++++++-- .../trigger/devtronApps/HandlerService.go | 10 +- .../devtronApps/deployStageHandlerCode.go | 63 ++++++++++- .../devtronApps/preStageHandlerCode.go | 2 +- pkg/pipeline/CdHandler.go | 104 +++++++++++++++++- pkg/workflow/cd/CdWorkflowRunnerService.go | 34 ++++-- .../status/WorkflowStatusLatestService.go | 32 +++++- .../status/WorkflowStatusUpdateService.go | 7 +- 9 files changed, 275 insertions(+), 45 deletions(-) diff --git a/internal/sql/repository/pipelineConfig/CdWorfkflowRepository.go b/internal/sql/repository/pipelineConfig/CdWorfkflowRepository.go index 6b6001a8dd..d07411be4e 100644 --- a/internal/sql/repository/pipelineConfig/CdWorfkflowRepository.go +++ b/internal/sql/repository/pipelineConfig/CdWorfkflowRepository.go @@ -19,6 +19,7 @@ package pipelineConfig import ( "context" "errors" + "fmt" apiBean "github.com/devtron-labs/devtron/api/bean" "github.com/devtron-labs/devtron/internal/sql/repository" "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig/bean/workflow" @@ -28,6 +29,7 @@ import ( "github.com/go-pg/pg" "go.opentelemetry.io/otel" "go.uber.org/zap" + "strings" "time" ) @@ -69,7 +71,7 @@ type CdWorkflowRepository interface { IsLatestWf(pipelineId int, wfId int) (bool, error) FindLatestCdWorkflowByPipelineId(pipelineIds []int) (*CdWorkflow, error) FindLatestCdWorkflowByPipelineIdV2(pipelineIds []int) ([]*CdWorkflow, error) - FetchAllCdStagesLatestEntity(pipelineIds []int) ([]*CdWorkflowStatus, error) + FetchAllCdStagesLatestEntity(pipelineWorkflowPairs map[int]apiBean.WorkflowType) ([]*CdWorkflowStatus, error) FetchAllCdStagesLatestEntityStatus(wfrIds []int) ([]*CdWorkflowRunner, error) ExistsByStatus(status string) (bool, error) FetchEnvAllCdStagesLatestEntityStatus(wfrIds []int, envID int) ([]*CdWorkflowRunner, error) @@ -578,15 +580,15 @@ func (impl *CdWorkflowRepositoryImpl) IsLatestWf(pipelineId int, wfId int) (bool return !exists, err } -func (impl *CdWorkflowRepositoryImpl) FetchAllCdStagesLatestEntity(pipelineIds []int) ([]*CdWorkflowStatus, error) { +func (impl *CdWorkflowRepositoryImpl) FetchAllCdStagesLatestEntity(pipelineWorkflowPairs map[int]apiBean.WorkflowType) ([]*CdWorkflowStatus, error) { var cdWorkflowStatus []*CdWorkflowStatus - if len(pipelineIds) == 0 { + if len(pipelineWorkflowPairs) == 0 { return cdWorkflowStatus, nil } query := "select p.ci_pipeline_id, wf.pipeline_id, wfr.workflow_type, max(wfr.id) as wfr_id from cd_workflow_runner wfr" + " inner join cd_workflow wf on wf.id=wfr.cd_workflow_id" + " inner join pipeline p on p.id = wf.pipeline_id" + - " where wf.pipeline_id in (" + sqlIntSeq(pipelineIds) + ")" + + " where (wf.pipeline_id, wfr.workflow_type) in (" + buildPipelineTypeValuesList(pipelineWorkflowPairs) + ")" + " group by p.ci_pipeline_id, wf.pipeline_id, wfr.workflow_type order by wfr_id desc;" _, err := impl.dbConnection.Query(&cdWorkflowStatus, query) if err != nil { @@ -596,6 +598,14 @@ func (impl *CdWorkflowRepositoryImpl) FetchAllCdStagesLatestEntity(pipelineIds [ return cdWorkflowStatus, nil } +func buildPipelineTypeValuesList(pairs map[int]apiBean.WorkflowType) string { + var values []string + for pipelineId, workflowType := range pairs { + values = append(values, fmt.Sprintf("(%d,'%s')", pipelineId, workflowType)) + } + return strings.Join(values, ",") +} + func (impl *CdWorkflowRepositoryImpl) FetchAllCdStagesLatestEntityStatus(wfrIds []int) ([]*CdWorkflowRunner, error) { var wfrList []*CdWorkflowRunner err := impl.dbConnection.Model(&wfrList). diff --git a/internal/sql/repository/pipelineConfig/WorkflowStatusLatestRepository.go b/internal/sql/repository/pipelineConfig/WorkflowStatusLatestRepository.go index 4a11673e58..233753b17c 100644 --- a/internal/sql/repository/pipelineConfig/WorkflowStatusLatestRepository.go +++ b/internal/sql/repository/pipelineConfig/WorkflowStatusLatestRepository.go @@ -19,6 +19,7 @@ package pipelineConfig import ( "github.com/devtron-labs/devtron/pkg/sql" "github.com/go-pg/pg" + "github.com/go-pg/pg/orm" "go.uber.org/zap" ) @@ -31,12 +32,13 @@ type WorkflowStatusLatestRepository interface { DeleteCiWorkflowStatusLatestByPipelineId(pipelineId int) error // CD Workflow Status Latest methods - SaveCdWorkflowStatusLatest(model *CdWorkflowStatusLatest) error - UpdateCdWorkflowStatusLatest(model *CdWorkflowStatusLatest) error - GetCdWorkflowStatusLatestByPipelineIdAndWorkflowType(pipelineId int, workflowType string) (*CdWorkflowStatusLatest, error) + SaveCdWorkflowStatusLatest(tx *pg.Tx, model *CdWorkflowStatusLatest) error + UpdateCdWorkflowStatusLatest(tx *pg.Tx, model *CdWorkflowStatusLatest) error + GetCdWorkflowStatusLatestByPipelineIdAndWorkflowType(tx *pg.Tx, pipelineId int, workflowType string) (*CdWorkflowStatusLatest, error) GetCdWorkflowStatusLatestByAppId(appId int) ([]*CdWorkflowStatusLatest, error) GetCdWorkflowStatusLatestByPipelineId(pipelineId int) ([]*CdWorkflowStatusLatest, error) DeleteCdWorkflowStatusLatestByPipelineId(pipelineId int) error + GetCdWorkflowStatusLatestByPipelineIds(pipelineIds []int) ([]*CdWorkflowStatusLatest, error) } type WorkflowStatusLatestRepositoryImpl struct { @@ -128,8 +130,14 @@ func (impl *WorkflowStatusLatestRepositoryImpl) DeleteCiWorkflowStatusLatestByPi } // CD Workflow Status Latest methods implementation -func (impl *WorkflowStatusLatestRepositoryImpl) SaveCdWorkflowStatusLatest(model *CdWorkflowStatusLatest) error { - err := impl.dbConnection.Insert(model) +func (impl *WorkflowStatusLatestRepositoryImpl) SaveCdWorkflowStatusLatest(tx *pg.Tx, model *CdWorkflowStatusLatest) error { + var connection orm.DB + if tx != nil { + connection = tx + } else { + connection = impl.dbConnection + } + err := connection.Insert(model) if err != nil { impl.logger.Errorw("error in saving cd workflow status latest", "err", err, "model", model) return err @@ -137,8 +145,14 @@ func (impl *WorkflowStatusLatestRepositoryImpl) SaveCdWorkflowStatusLatest(model return nil } -func (impl *WorkflowStatusLatestRepositoryImpl) UpdateCdWorkflowStatusLatest(model *CdWorkflowStatusLatest) error { - _, err := impl.dbConnection.Model(model).WherePK().UpdateNotNull() +func (impl *WorkflowStatusLatestRepositoryImpl) UpdateCdWorkflowStatusLatest(tx *pg.Tx, model *CdWorkflowStatusLatest) error { + var connection orm.DB + if tx != nil { + connection = tx + } else { + connection = impl.dbConnection + } + _, err := connection.Model(model).WherePK().UpdateNotNull() if err != nil { impl.logger.Errorw("error in updating cd workflow status latest", "err", err, "model", model) return err @@ -146,9 +160,15 @@ func (impl *WorkflowStatusLatestRepositoryImpl) UpdateCdWorkflowStatusLatest(mod return nil } -func (impl *WorkflowStatusLatestRepositoryImpl) GetCdWorkflowStatusLatestByPipelineIdAndWorkflowType(pipelineId int, workflowType string) (*CdWorkflowStatusLatest, error) { +func (impl *WorkflowStatusLatestRepositoryImpl) GetCdWorkflowStatusLatestByPipelineIdAndWorkflowType(tx *pg.Tx, pipelineId int, workflowType string) (*CdWorkflowStatusLatest, error) { model := &CdWorkflowStatusLatest{} - err := impl.dbConnection.Model(model). + var connection orm.DB + if tx != nil { + connection = tx + } else { + connection = impl.dbConnection + } + err := connection.Model(model). Where("pipeline_id = ?", pipelineId). Where("workflow_type = ?", workflowType). Select() @@ -193,3 +213,15 @@ func (impl *WorkflowStatusLatestRepositoryImpl) DeleteCdWorkflowStatusLatestByPi } return nil } + +func (impl *WorkflowStatusLatestRepositoryImpl) GetCdWorkflowStatusLatestByPipelineIds(pipelineIds []int) ([]*CdWorkflowStatusLatest, error) { + var models []*CdWorkflowStatusLatest + err := impl.dbConnection.Model(&models). + Where("pipeline_id IN (?)", pg.In(pipelineIds)). + Select() + if err != nil { + impl.logger.Errorw("error in getting cd workflow status latest by pipeline ids", "err", err, "pipelineIds", pipelineIds) + return nil, err + } + return models, nil +} diff --git a/pkg/deployment/trigger/devtronApps/HandlerService.go b/pkg/deployment/trigger/devtronApps/HandlerService.go index 28ff4ca715..fd174071eb 100644 --- a/pkg/deployment/trigger/devtronApps/HandlerService.go +++ b/pkg/deployment/trigger/devtronApps/HandlerService.go @@ -20,8 +20,9 @@ import ( "bufio" "context" "github.com/devtron-labs/common-lib/async" - service2 "github.com/devtron-labs/devtron/pkg/workflow/trigger/audit/service" "github.com/devtron-labs/devtron/client/fluxcd" + status2 "github.com/devtron-labs/devtron/pkg/workflow/status" + service2 "github.com/devtron-labs/devtron/pkg/workflow/trigger/audit/service" "os" "time" @@ -174,6 +175,7 @@ type HandlerServiceImpl struct { asyncRunnable *async.Runnable workflowTriggerAuditService service2.WorkflowTriggerAuditService fluxCdDeploymentService fluxcd.DeploymentService + workflowStatusUpdateService status2.WorkflowStatusUpdateService } func NewHandlerServiceImpl(logger *zap.SugaredLogger, @@ -238,7 +240,8 @@ func NewHandlerServiceImpl(logger *zap.SugaredLogger, deploymentEventHandler app.DeploymentEventHandler, asyncRunnable *async.Runnable, workflowTriggerAuditService service2.WorkflowTriggerAuditService, - fluxCdDeploymentService fluxcd.DeploymentService) (*HandlerServiceImpl, error) { + fluxCdDeploymentService fluxcd.DeploymentService, + workflowStatusUpdateService status2.WorkflowStatusUpdateService) (*HandlerServiceImpl, error) { impl := &HandlerServiceImpl{ logger: logger, cdWorkflowCommonService: cdWorkflowCommonService, @@ -307,7 +310,8 @@ func NewHandlerServiceImpl(logger *zap.SugaredLogger, deploymentEventHandler: deploymentEventHandler, asyncRunnable: asyncRunnable, workflowTriggerAuditService: workflowTriggerAuditService, - fluxCdDeploymentService: fluxCdDeploymentService, + fluxCdDeploymentService: fluxCdDeploymentService, + workflowStatusUpdateService: workflowStatusUpdateService, } config, err := types.GetCdConfig() if err != nil { diff --git a/pkg/deployment/trigger/devtronApps/deployStageHandlerCode.go b/pkg/deployment/trigger/devtronApps/deployStageHandlerCode.go index 0b082f38c8..18dccdeed4 100644 --- a/pkg/deployment/trigger/devtronApps/deployStageHandlerCode.go +++ b/pkg/deployment/trigger/devtronApps/deployStageHandlerCode.go @@ -269,7 +269,20 @@ func (impl *HandlerServiceImpl) ManualCdTrigger(triggerContext bean.TriggerConte } cdWorkflowId = cdWf.Id } - + tx, err := impl.cdWorkflowRepository.GetConnection().Begin() + if err != nil { + impl.logger.Errorw("error in getting connection for cdWorkflowRepository, ManualCdTrigger", "err", err) + return 0, "", nil, err + } + isRollbackNeeded := true + defer func() { + if isRollbackNeeded { + err = tx.Rollback() + if err != nil { + impl.logger.Errorw("error in rolling back transaction for cdWorkflowRunner, ManualCdTrigger", "cdWorkflowId", cdWorkflowId, "err", err) + } + } + }() runner := &pipelineConfig.CdWorkflowRunner{ Name: cdPipeline.Name, WorkflowType: bean3.CD_WORKFLOW_TYPE_DEPLOY, @@ -282,11 +295,26 @@ func (impl *HandlerServiceImpl) ManualCdTrigger(triggerContext bean.TriggerConte AuditLog: sql.AuditLog{CreatedOn: triggeredAt, CreatedBy: overrideRequest.UserId, UpdatedOn: triggeredAt, UpdatedBy: overrideRequest.UserId}, ReferenceId: triggerContext.ReferenceId, } - err = impl.cdWorkflowRunnerService.SaveWfr(nil, runner) + + err = impl.cdWorkflowRunnerService.SaveWfr(tx, runner) if err != nil { impl.logger.Errorw("err in creating cdWorkflowRunner, ManualCdTrigger", "cdWorkflowId", cdWorkflowId, "err", err) return 0, "", nil, err } + + err = impl.workflowStatusUpdateService.UpdateCdWorkflowStatusLatest(tx, overrideRequest.PipelineId, overrideRequest.AppId, overrideRequest.EnvId, runner.Id, runner.WorkflowType.String(), overrideRequest.UserId) + if err != nil { + impl.logger.Errorw("error in updating workflow status latest, ManualCdTrigger", "runnerId", overrideRequest.WfrId, "err", err) + return 0, "", nil, err + } + + isRollbackNeeded = false + err = tx.Commit() + if err != nil { + impl.logger.Errorw("error in committing transaction for cdWorkflowRunner, ManualCdTrigger", "cdWorkflowId", cdWorkflowId, "err", err) + return 0, "", nil, err + } + runner.CdWorkflow = &pipelineConfig.CdWorkflow{ Pipeline: cdPipeline, } @@ -407,6 +435,22 @@ func (impl *HandlerServiceImpl) TriggerAutomaticDeployment(request bean.CdTrigge } } + tx, err := impl.cdWorkflowRepository.GetConnection().Begin() + if err != nil { + impl.logger.Errorw("error in getting connection for cdWorkflowRepository, ManualCdTrigger", "err", err) + return err + } + + isRollbackNeeded := true + defer func() { + if isRollbackNeeded { + err = tx.Rollback() + if err != nil { + impl.logger.Errorw("error in rolling back transaction for cdWorkflowRunner, ManualCdTrigger", "cdWorkflowId", cdWf.Id, "err", err) + } + } + }() + runner := &pipelineConfig.CdWorkflowRunner{ Name: pipeline.Name, WorkflowType: bean3.CD_WORKFLOW_TYPE_DEPLOY, @@ -419,10 +463,23 @@ func (impl *HandlerServiceImpl) TriggerAutomaticDeployment(request bean.CdTrigge AuditLog: sql.AuditLog{CreatedOn: triggeredAt, CreatedBy: triggeredBy, UpdatedOn: triggeredAt, UpdatedBy: triggeredBy}, ReferenceId: request.TriggerContext.ReferenceId, } - err := impl.cdWorkflowRunnerService.SaveWfr(nil, runner) + err = impl.cdWorkflowRunnerService.SaveWfr(tx, runner) if err != nil { return err } + + err = impl.workflowStatusUpdateService.UpdateCdWorkflowStatusLatest(tx, pipeline.Id, pipeline.AppId, pipeline.EnvironmentId, runner.Id, runner.WorkflowType.String(), request.TriggeredBy) + if err != nil { + impl.logger.Errorw("error in updating workflow status latest, ManualCdTrigger", "runnerId", runner.Id, "err", err) + return err + } + + err = tx.Commit() + if err != nil { + impl.logger.Errorw("error in committing transaction for cdWorkflowRunner, ManualCdTrigger", "cdWorkflowId", cdWf.Id, "err", err) + return err + } + runner.CdWorkflow = &pipelineConfig.CdWorkflow{ Pipeline: pipeline, } diff --git a/pkg/deployment/trigger/devtronApps/preStageHandlerCode.go b/pkg/deployment/trigger/devtronApps/preStageHandlerCode.go index ae8594cdc0..e91f9e67b9 100644 --- a/pkg/deployment/trigger/devtronApps/preStageHandlerCode.go +++ b/pkg/deployment/trigger/devtronApps/preStageHandlerCode.go @@ -312,7 +312,7 @@ func (impl *HandlerServiceImpl) createStartingWfAndRunner(request bean.CdTrigger ReferenceId: request.TriggerContext.ReferenceId, } _, span := otel.Tracer("orchestrator").Start(ctx, "cdWorkflowRepository.SaveWorkFlowRunner") - _, err = impl.cdWorkflowRunnerService.SaveCDWorkflowRunnerWithStage(runner) + _, err = impl.cdWorkflowRunnerService.SaveCDWorkflowRunnerWithStage(runner, cdWf, pipeline) span.End() if err != nil { return nil, nil, err diff --git a/pkg/pipeline/CdHandler.go b/pkg/pipeline/CdHandler.go index c0080b0dd0..5683db2269 100644 --- a/pkg/pipeline/CdHandler.go +++ b/pkg/pipeline/CdHandler.go @@ -28,9 +28,11 @@ import ( repository3 "github.com/devtron-labs/devtron/pkg/cluster/environment/repository" common2 "github.com/devtron-labs/devtron/pkg/deployment/common" eventProcessorBean "github.com/devtron-labs/devtron/pkg/eventProcessor/bean" + repository2 "github.com/devtron-labs/devtron/pkg/pipeline/repository" "github.com/devtron-labs/devtron/pkg/pipeline/workflowStatus" bean5 "github.com/devtron-labs/devtron/pkg/pipeline/workflowStatus/bean" "github.com/devtron-labs/devtron/pkg/workflow/cd" + "github.com/devtron-labs/devtron/pkg/workflow/status" "slices" "strconv" "strings" @@ -89,6 +91,8 @@ type CdHandlerImpl struct { deploymentConfigService common2.DeploymentConfigService workflowStageStatusService workflowStatus.WorkFlowStageStatusService cdWorkflowRunnerService cd.CdWorkflowRunnerService + WorkflowStatusLatestService status.WorkflowStatusLatestService + pipelineStageRepository repository2.PipelineStageRepository } func NewCdHandlerImpl(Logger *zap.SugaredLogger, userService user.UserService, @@ -103,6 +107,8 @@ func NewCdHandlerImpl(Logger *zap.SugaredLogger, userService user.UserService, deploymentConfigService common2.DeploymentConfigService, workflowStageStatusService workflowStatus.WorkFlowStageStatusService, cdWorkflowRunnerService cd.CdWorkflowRunnerService, + WorkflowStatusLatestService status.WorkflowStatusLatestService, + pipelineStageRepository repository2.PipelineStageRepository, ) *CdHandlerImpl { cdh := &CdHandlerImpl{ Logger: Logger, @@ -121,6 +127,8 @@ func NewCdHandlerImpl(Logger *zap.SugaredLogger, userService user.UserService, deploymentConfigService: deploymentConfigService, workflowStageStatusService: workflowStageStatusService, cdWorkflowRunnerService: cdWorkflowRunnerService, + WorkflowStatusLatestService: WorkflowStatusLatestService, + pipelineStageRepository: pipelineStageRepository, } config, err := types.GetCdConfig() if err != nil { @@ -594,16 +602,18 @@ func (impl *CdHandlerImpl) FetchAppWorkflowStatusForTriggerView(appId int) ([]*p return cdWorkflowStatus, nil } - cdMap := make(map[int]*pipelineConfig.CdWorkflowStatus) - result, err := impl.cdWorkflowRepository.FetchAllCdStagesLatestEntity(pipelineIds) + result, err := impl.getWfrStatusForLatestRunners(pipelineIds, pipelines) if err != nil { + impl.Logger.Errorw("error in fetching wfrIds", "pipelineIds", pipelineIds, "err", err) return cdWorkflowStatus, err } + var wfrIds []int for _, item := range result { wfrIds = append(wfrIds, item.WfrId) } + var cdMap = make(map[int]*pipelineConfig.CdWorkflowStatus) statusMap := make(map[int]string) if len(wfrIds) > 0 { wfrList, err := impl.cdWorkflowRepository.FetchAllCdStagesLatestEntityStatus(wfrIds) @@ -682,6 +692,88 @@ func (impl *CdHandlerImpl) FetchAppWorkflowStatusForTriggerView(appId int) ([]*p return cdWorkflowStatus, err } +func (impl *CdHandlerImpl) getWfrStatusForLatestRunners(pipelineIds []int, pipelines []*pipelineConfig.Pipeline) ([]*pipelineConfig.CdWorkflowStatus, error) { + // fetching the latest pipeline from the index table - cdWorkflowLatest + var result []*pipelineConfig.CdWorkflowStatus + cdWorkflowLatest, err := impl.WorkflowStatusLatestService.GetCdWorkflowLatestByPipelineIds(pipelineIds) + if err != nil { + impl.Logger.Errorw("error in getting latest by pipelineId", "pipelineId", pipelineIds, "err", err) + return nil, err + } + + var pipelineIdToCiPipelineIdMap map[int]int + for _, item := range pipelines { + pipelineIdToCiPipelineIdMap[item.Id] = item.CiPipelineId + } + + for _, item := range cdWorkflowLatest { + result = append(result, &pipelineConfig.CdWorkflowStatus{ + CiPipelineId: pipelineIdToCiPipelineIdMap[item.PipelineId], + PipelineId: item.PipelineId, + WorkflowType: item.WorkflowType, + WfrId: item.WorkflowRunnerId, + }) + } + + var cdWorfklowLatestMap map[int]map[bean.WorkflowType]bool + for _, item := range cdWorkflowLatest { + if _, ok := cdWorfklowLatestMap[item.PipelineId]; !ok { + cdWorfklowLatestMap[item.PipelineId] = make(map[bean.WorkflowType]bool) + } + cdWorfklowLatestMap[item.PipelineId][bean.WorkflowType(item.WorkflowType)] = true + } + + pipelineStage, err := impl.pipelineStageRepository.GetAllCdStagesByCdPipelineIds(pipelineIds) + if err != nil { + impl.Logger.Errorw("error in fetching pipeline stages", "pipelineId", pipelineIds, "err", err) + return nil, err + } + pipelineStageMap := make(map[int]map[bean.WorkflowType]bool) + for _, item := range pipelineStage { + if _, ok := pipelineStageMap[item.CdPipelineId]; !ok { + pipelineStageMap[item.CdPipelineId] = make(map[bean.WorkflowType]bool) + } + if item.Type == repository2.PIPELINE_STAGE_TYPE_PRE_CD { + pipelineStageMap[item.CdPipelineId][bean.CD_WORKFLOW_TYPE_PRE] = true + } else if item.Type == repository2.PIPELINE_STAGE_TYPE_POST_CD { + pipelineStageMap[item.CdPipelineId][bean.CD_WORKFLOW_TYPE_POST] = true + } + } + + // calculating all the pipelines not present in the index table cdWorkflowLatest + var pipelinesAbsentInCache map[int]bean.WorkflowType + for _, item := range pipelines { + if _, ok := cdWorfklowLatestMap[item.Id]; !ok { + pipelinesAbsentInCache[item.Id] = bean.CD_WORKFLOW_TYPE_PRE + pipelinesAbsentInCache[item.Id] = bean.CD_WORKFLOW_TYPE_DEPLOY + pipelinesAbsentInCache[item.Id] = bean.CD_WORKFLOW_TYPE_POST + } else { + if _, ok := pipelineStageMap[item.Id][bean.CD_WORKFLOW_TYPE_PRE]; ok { + if val, ok := cdWorfklowLatestMap[item.Id][bean.CD_WORKFLOW_TYPE_PRE]; !ok || !val { + pipelinesAbsentInCache[item.Id] = bean.CD_WORKFLOW_TYPE_PRE + } + } + if _, ok := pipelineStageMap[item.Id][bean.CD_WORKFLOW_TYPE_POST]; ok { + if val, ok := cdWorfklowLatestMap[item.Id][bean.CD_WORKFLOW_TYPE_POST]; !ok || !val { + pipelinesAbsentInCache[item.Id] = bean.CD_WORKFLOW_TYPE_POST + } + } + if val, ok := cdWorfklowLatestMap[item.Id][bean.CD_WORKFLOW_TYPE_DEPLOY]; !ok || !val { + pipelinesAbsentInCache[item.Id] = bean.CD_WORKFLOW_TYPE_POST + } + } + } + if len(pipelinesAbsentInCache) > 0 { + remainingRunners, err := impl.cdWorkflowRepository.FetchAllCdStagesLatestEntity(pipelinesAbsentInCache) + if err != nil { + impl.Logger.Errorw("error in fetching all cd stages latest entity", "pipelinesAbsentInCache", pipelinesAbsentInCache, "err", err) + return nil, err + } + result = append(result, remainingRunners...) + } + return result, nil +} + func (impl *CdHandlerImpl) FetchAppWorkflowStatusForTriggerViewForEnvironment(request resourceGroup2.ResourceGroupingRequest, token string) ([]*pipelineConfig.CdWorkflowStatus, error) { cdWorkflowStatus := make([]*pipelineConfig.CdWorkflowStatus, 0) var pipelines []*pipelineConfig.Pipeline @@ -753,11 +845,15 @@ func (impl *CdHandlerImpl) FetchAppWorkflowStatusForTriggerViewForEnvironment(re if len(pipelineIds) == 0 { return cdWorkflowStatus, nil } + cdMap := make(map[int]*pipelineConfig.CdWorkflowStatus) - wfrStatus, err := impl.cdWorkflowRepository.FetchAllCdStagesLatestEntity(pipelineIds) + + wfrStatus, err := impl.getWfrStatusForLatestRunners(pipelineIds, pipelines) if err != nil { + impl.Logger.Errorw("error in fetching wfrIds", "pipelineIds", pipelineIds, "err", err) return cdWorkflowStatus, err } + var wfrIds []int for _, item := range wfrStatus { wfrIds = append(wfrIds, item.WfrId) @@ -904,7 +1000,7 @@ func (impl *CdHandlerImpl) FetchAppDeploymentStatusForEnvironments(request resou return deploymentStatuses, nil } _, span = otel.Tracer("orchestrator").Start(request.Ctx, "pipelineBuilder.FetchAllCdStagesLatestEntity") - result, err := impl.cdWorkflowRepository.FetchAllCdStagesLatestEntity(pipelineIds) + result, err := impl.getWfrStatusForLatestRunners(pipelineIds, cdPipelines) span.End() if err != nil { return deploymentStatuses, err diff --git a/pkg/workflow/cd/CdWorkflowRunnerService.go b/pkg/workflow/cd/CdWorkflowRunnerService.go index b27fd7a8cc..522c18c13a 100644 --- a/pkg/workflow/cd/CdWorkflowRunnerService.go +++ b/pkg/workflow/cd/CdWorkflowRunnerService.go @@ -28,6 +28,7 @@ import ( "github.com/devtron-labs/devtron/pkg/sql" "github.com/devtron-labs/devtron/pkg/workflow/cd/adapter" "github.com/devtron-labs/devtron/pkg/workflow/cd/bean" + "github.com/devtron-labs/devtron/pkg/workflow/status" "github.com/devtron-labs/devtron/util" "github.com/go-pg/pg" "go.uber.org/zap" @@ -37,28 +38,31 @@ type CdWorkflowRunnerService interface { UpdateWfr(dto *bean.CdWorkflowRunnerDto, updatedBy int) error SaveWfr(tx *pg.Tx, wfr *pipelineConfig.CdWorkflowRunner) error UpdateIsArtifactUploaded(wfrId int, isArtifactUploaded bool) error - SaveCDWorkflowRunnerWithStage(wfr *pipelineConfig.CdWorkflowRunner) (*pipelineConfig.CdWorkflowRunner, error) + SaveCDWorkflowRunnerWithStage(wfr *pipelineConfig.CdWorkflowRunner, cdWf *pipelineConfig.CdWorkflow, pipeline *pipelineConfig.Pipeline) (*pipelineConfig.CdWorkflowRunner, error) UpdateCdWorkflowRunnerWithStage(wfr *pipelineConfig.CdWorkflowRunner) error GetPrePostWorkflowStagesByWorkflowRunnerIdsList(wfIdWfTypeMap map[int]bean4.CdWorkflowWithArtifact) (map[int]map[string][]*bean3.WorkflowStageDto, error) } type CdWorkflowRunnerServiceImpl struct { - logger *zap.SugaredLogger - cdWorkflowRepository pipelineConfig.CdWorkflowRepository - workflowStageService workflowStatus.WorkFlowStageStatusService - transactionManager sql.TransactionWrapper - config *types.CiConfig + logger *zap.SugaredLogger + cdWorkflowRepository pipelineConfig.CdWorkflowRepository + workflowStageService workflowStatus.WorkFlowStageStatusService + transactionManager sql.TransactionWrapper + config *types.CiConfig + workflowStatusUpdateService status.WorkflowStatusUpdateService } func NewCdWorkflowRunnerServiceImpl(logger *zap.SugaredLogger, cdWorkflowRepository pipelineConfig.CdWorkflowRepository, workflowStageService workflowStatus.WorkFlowStageStatusService, - transactionManager sql.TransactionWrapper) *CdWorkflowRunnerServiceImpl { + transactionManager sql.TransactionWrapper, + workflowStatusUpdateService status.WorkflowStatusUpdateService) *CdWorkflowRunnerServiceImpl { impl := &CdWorkflowRunnerServiceImpl{ - logger: logger, - cdWorkflowRepository: cdWorkflowRepository, - workflowStageService: workflowStageService, - transactionManager: transactionManager, + logger: logger, + cdWorkflowRepository: cdWorkflowRepository, + workflowStageService: workflowStageService, + transactionManager: transactionManager, + workflowStatusUpdateService: workflowStatusUpdateService, } ciConfig, err := types.GetCiConfig() if err != nil { @@ -88,7 +92,7 @@ func (impl *CdWorkflowRunnerServiceImpl) UpdateIsArtifactUploaded(wfrId int, isA return nil } -func (impl *CdWorkflowRunnerServiceImpl) SaveCDWorkflowRunnerWithStage(wfr *pipelineConfig.CdWorkflowRunner) (*pipelineConfig.CdWorkflowRunner, error) { +func (impl *CdWorkflowRunnerServiceImpl) SaveCDWorkflowRunnerWithStage(wfr *pipelineConfig.CdWorkflowRunner, cdWf *pipelineConfig.CdWorkflow, pipeline *pipelineConfig.Pipeline) (*pipelineConfig.CdWorkflowRunner, error) { // implementation tx, err := impl.transactionManager.StartTx() if err != nil { @@ -117,6 +121,12 @@ func (impl *CdWorkflowRunnerServiceImpl) SaveCDWorkflowRunnerWithStage(wfr *pipe return wfr, err } + err = impl.workflowStatusUpdateService.UpdateCdWorkflowStatusLatest(tx, cdWf.PipelineId, pipeline.AppId, pipeline.EnvironmentId, wfr.Id, wfr.WorkflowType.String(), wfr.CreatedBy) + if err != nil { + impl.logger.Errorw("error in updating workflow status latest, ManualCdTrigger", "runnerId", wfr.CreatedBy, "err", err) + return wfr, err + } + err = impl.transactionManager.CommitTx(tx) if err != nil { impl.logger.Errorw("error in committing transaction", "workflowName", wfr.Name, "error", err) diff --git a/pkg/workflow/status/WorkflowStatusLatestService.go b/pkg/workflow/status/WorkflowStatusLatestService.go index 3d7f204394..12a8cc6b80 100644 --- a/pkg/workflow/status/WorkflowStatusLatestService.go +++ b/pkg/workflow/status/WorkflowStatusLatestService.go @@ -33,10 +33,11 @@ type WorkflowStatusLatestService interface { GetCiWorkflowStatusLatestByAppId(appId int) ([]*CiWorkflowStatusLatest, error) // CD Workflow Status Latest methods - SaveOrUpdateCdWorkflowStatusLatest(pipelineId, appId, environmentId, workflowRunnerId int, workflowType string, userId int32) error + SaveOrUpdateCdWorkflowStatusLatest(tx *pg.Tx, pipelineId, appId, environmentId, workflowRunnerId int, workflowType string, userId int32) error GetCdWorkflowStatusLatestByPipelineIdAndWorkflowType(pipelineId int, workflowType string) (*CdWorkflowStatusLatest, error) GetCdWorkflowStatusLatestByAppId(appId int) ([]*CdWorkflowStatusLatest, error) GetCdWorkflowStatusLatestByPipelineId(pipelineId int) ([]*CdWorkflowStatusLatest, error) + GetCdWorkflowLatestByPipelineIds(pipelineIds []int) ([]*CdWorkflowStatusLatest, error) } type WorkflowStatusLatestServiceImpl struct { @@ -164,9 +165,9 @@ func (impl *WorkflowStatusLatestServiceImpl) GetCiWorkflowStatusLatestByAppId(ap } // CD Workflow Status Latest methods implementation -func (impl *WorkflowStatusLatestServiceImpl) SaveOrUpdateCdWorkflowStatusLatest(pipelineId, appId, environmentId, workflowRunnerId int, workflowType string, userId int32) error { +func (impl *WorkflowStatusLatestServiceImpl) SaveOrUpdateCdWorkflowStatusLatest(tx *pg.Tx, pipelineId, appId, environmentId, workflowRunnerId int, workflowType string, userId int32) error { // Check if entry exists - existingEntry, err := impl.workflowStatusLatestRepository.GetCdWorkflowStatusLatestByPipelineIdAndWorkflowType(pipelineId, workflowType) + existingEntry, err := impl.workflowStatusLatestRepository.GetCdWorkflowStatusLatestByPipelineIdAndWorkflowType(tx, pipelineId, workflowType) if err != nil && err != pg.ErrNoRows { impl.logger.Errorw("error in getting cd workflow status latest", "err", err, "pipelineId", pipelineId, "workflowType", workflowType) return err @@ -187,19 +188,19 @@ func (impl *WorkflowStatusLatestServiceImpl) SaveOrUpdateCdWorkflowStatusLatest( model.UpdatedBy = userId model.UpdatedOn = now - return impl.workflowStatusLatestRepository.SaveCdWorkflowStatusLatest(model) + return impl.workflowStatusLatestRepository.SaveCdWorkflowStatusLatest(tx, model) } else { // Update existing entry existingEntry.WorkflowRunnerId = workflowRunnerId existingEntry.UpdatedBy = userId existingEntry.UpdatedOn = now - return impl.workflowStatusLatestRepository.UpdateCdWorkflowStatusLatest(existingEntry) + return impl.workflowStatusLatestRepository.UpdateCdWorkflowStatusLatest(tx, existingEntry) } } func (impl *WorkflowStatusLatestServiceImpl) GetCdWorkflowStatusLatestByPipelineIdAndWorkflowType(pipelineId int, workflowType string) (*CdWorkflowStatusLatest, error) { - model, err := impl.workflowStatusLatestRepository.GetCdWorkflowStatusLatestByPipelineIdAndWorkflowType(pipelineId, workflowType) + model, err := impl.workflowStatusLatestRepository.GetCdWorkflowStatusLatestByPipelineIdAndWorkflowType(nil, pipelineId, workflowType) if err != nil { if err == pg.ErrNoRows { // Fallback to old method @@ -349,3 +350,22 @@ func (impl *WorkflowStatusLatestServiceImpl) getCdWorkflowStatusFromOldMethod(pi Status: wfr.Status, }, nil } + +func (impl *WorkflowStatusLatestServiceImpl) GetCdWorkflowLatestByPipelineIds(pipelineIds []int) ([]*CdWorkflowStatusLatest, error) { + cdWorkflowStatusLatest, err := impl.workflowStatusLatestRepository.GetCdWorkflowStatusLatestByPipelineIds(pipelineIds) + if err != nil { + impl.logger.Errorw("error in getting cd workflow status latest by pipeline ids", "pipelineIds", pipelineIds, "err", err) + return nil, err + } + var result []*CdWorkflowStatusLatest + for _, model := range cdWorkflowStatusLatest { + result = append(result, &CdWorkflowStatusLatest{ + PipelineId: model.PipelineId, + AppId: model.AppId, + EnvironmentId: model.EnvironmentId, + WorkflowType: model.WorkflowType, + WorkflowRunnerId: model.WorkflowRunnerId, + }) + } + return result, nil +} diff --git a/pkg/workflow/status/WorkflowStatusUpdateService.go b/pkg/workflow/status/WorkflowStatusUpdateService.go index 6c56f6c17d..9c1bfab40d 100644 --- a/pkg/workflow/status/WorkflowStatusUpdateService.go +++ b/pkg/workflow/status/WorkflowStatusUpdateService.go @@ -18,13 +18,14 @@ package status import ( "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" + "github.com/go-pg/pg" "go.uber.org/zap" ) type WorkflowStatusUpdateService interface { // Methods to update latest status tables when workflow status changes UpdateCiWorkflowStatusLatest(pipelineId, appId, ciWorkflowId int, userId int32) error - UpdateCdWorkflowStatusLatest(pipelineId, appId, environmentId, workflowRunnerId int, workflowType string, userId int32) error + UpdateCdWorkflowStatusLatest(tx *pg.Tx, pipelineId, appId, environmentId, workflowRunnerId int, workflowType string, userId int32) error // Methods to fetch optimized status for trigger view FetchCiStatusForTriggerViewOptimized(appId int) ([]*pipelineConfig.CiWorkflowStatus, error) @@ -62,8 +63,8 @@ func (impl *WorkflowStatusUpdateServiceImpl) UpdateCiWorkflowStatusLatest(pipeli return impl.workflowStatusLatestService.SaveOrUpdateCiWorkflowStatusLatest(pipelineId, appId, ciWorkflowId, userId) } -func (impl *WorkflowStatusUpdateServiceImpl) UpdateCdWorkflowStatusLatest(pipelineId, appId, environmentId, workflowRunnerId int, workflowType string, userId int32) error { - return impl.workflowStatusLatestService.SaveOrUpdateCdWorkflowStatusLatest(pipelineId, appId, environmentId, workflowRunnerId, workflowType, userId) +func (impl *WorkflowStatusUpdateServiceImpl) UpdateCdWorkflowStatusLatest(tx *pg.Tx, pipelineId, appId, environmentId, workflowRunnerId int, workflowType string, userId int32) error { + return impl.workflowStatusLatestService.SaveOrUpdateCdWorkflowStatusLatest(tx, pipelineId, appId, environmentId, workflowRunnerId, workflowType, userId) } func (impl *WorkflowStatusUpdateServiceImpl) FetchCiStatusForTriggerViewOptimized(appId int) ([]*pipelineConfig.CiWorkflowStatus, error) { From fc4c6aae98c34ba7470d0e686f9faa388618396c Mon Sep 17 00:00:00 2001 From: Prakash Kumar Date: Tue, 22 Jul 2025 18:30:27 +0530 Subject: [PATCH 09/31] Implement hybrid CI workflow status fetching in `CiHandlerImpl` using `ci_workflow_status_latest` table with fallback to legacy query methods. Update repositories with new helper methods to support this approach. --- .../pipelineConfig/CiWorkflowRepository.go | 31 +- .../WorkflowStatusLatestRepository.go | 35 +++ pkg/pipeline/CiHandler.go | 294 ++++++++++++++---- wire_gen.go | 5 +- 4 files changed, 294 insertions(+), 71 deletions(-) diff --git a/internal/sql/repository/pipelineConfig/CiWorkflowRepository.go b/internal/sql/repository/pipelineConfig/CiWorkflowRepository.go index 76d10c9770..ef43684d21 100644 --- a/internal/sql/repository/pipelineConfig/CiWorkflowRepository.go +++ b/internal/sql/repository/pipelineConfig/CiWorkflowRepository.go @@ -40,7 +40,7 @@ type CiWorkflowRepository interface { FindByName(name string) (*CiWorkflow, error) FindLastTriggeredWorkflowByCiIds(pipelineId []int) (ciWorkflow []*CiWorkflow, err error) - FindLastTriggeredWorkflowByCiIdsOptimized(pipelineId []int) (ciWorkflow []*CiWorkflow, err error) + FindWorkflowsByCiWorkflowIds(ciWorkflowIds []int) (ciWorkflow []*CiWorkflow, err error) FindLastTriggeredWorkflowByArtifactId(ciArtifactId int) (ciWorkflow *CiWorkflow, err error) FindAllLastTriggeredWorkflowByArtifactId(ciArtifactId []int) (ciWorkflow []*CiWorkflow, err error) FindAllTriggeredWorkflowCountInLast24Hour() (ciWorkflowCount int, err error) @@ -49,6 +49,7 @@ type CiWorkflowRepository interface { ExistsByStatus(status string) (bool, error) FindBuildTypeAndStatusDataOfLast1Day() ([]*BuildTypeCount, error) FIndCiWorkflowStatusesByAppId(appId int) ([]*CiWorkflowStatus, error) + FindCiPipelineIdsByAppId(appId int) ([]int, error) MigrateIsArtifactUploaded(wfId int, isArtifactUploaded bool) MigrateCiArtifactLocation(wfId int, artifactLocation string) @@ -291,20 +292,16 @@ func (impl *CiWorkflowRepositoryImpl) FindLastTriggeredWorkflowByCiIds(pipelineI return ciWorkflow, err } -// FindLastTriggeredWorkflowByCiIdsOptimized uses the ci_workflow_status_latest table for better performance -func (impl *CiWorkflowRepositoryImpl) FindLastTriggeredWorkflowByCiIdsOptimized(pipelineId []int) (ciWorkflow []*CiWorkflow, err error) { +// FindWorkflowsByCiWorkflowIds fetches workflows by their workflow IDs (simple query) +func (impl *CiWorkflowRepositoryImpl) FindWorkflowsByCiWorkflowIds(ciWorkflowIds []int) (ciWorkflow []*CiWorkflow, err error) { + if len(ciWorkflowIds) == 0 { + return ciWorkflow, nil + } + err = impl.dbConnection.Model(&ciWorkflow). Column("ci_workflow.*", "CiPipeline"). - Join("INNER JOIN ci_workflow_status_latest cwsl ON cwsl.ci_workflow_id = ci_workflow.id"). - Where("cwsl.pipeline_id IN (?)", pg.In(pipelineId)). + Where("ci_workflow.id IN (?)", pg.In(ciWorkflowIds)). Select() - - if err != nil { - impl.logger.Errorw("error in optimized query, falling back to old method", "err", err, "pipelineIds", pipelineId) - // Fallback to the old method if optimized query fails - return impl.FindLastTriggeredWorkflowByCiIds(pipelineId) - } - return ciWorkflow, err } @@ -397,6 +394,16 @@ func (impl *CiWorkflowRepositoryImpl) FIndCiWorkflowStatusesByAppId(appId int) ( return ciworkflowStatuses, err } +// FindCiPipelineIdsByAppId gets all CI pipeline IDs for an app (simple query) +func (impl *CiWorkflowRepositoryImpl) FindCiPipelineIdsByAppId(appId int) ([]int, error) { + var ciPipelineIds []int + err := impl.dbConnection.Model((*CiPipeline)(nil)). + Column("id"). + Where("app_id = ? AND deleted = false", appId). + Select(&ciPipelineIds) + return ciPipelineIds, err +} + func (impl *CiWorkflowRepositoryImpl) MigrateIsArtifactUploaded(wfId int, isArtifactUploaded bool) { _, err := impl.dbConnection.Model((*CiWorkflow)(nil)). Set("is_artifact_uploaded = ?", workflow.GetArtifactUploadedType(isArtifactUploaded)). diff --git a/internal/sql/repository/pipelineConfig/WorkflowStatusLatestRepository.go b/internal/sql/repository/pipelineConfig/WorkflowStatusLatestRepository.go index 4a11673e58..901272eb05 100644 --- a/internal/sql/repository/pipelineConfig/WorkflowStatusLatestRepository.go +++ b/internal/sql/repository/pipelineConfig/WorkflowStatusLatestRepository.go @@ -28,6 +28,8 @@ type WorkflowStatusLatestRepository interface { UpdateCiWorkflowStatusLatest(model *CiWorkflowStatusLatest) error GetCiWorkflowStatusLatestByPipelineId(pipelineId int) (*CiWorkflowStatusLatest, error) GetCiWorkflowStatusLatestByAppId(appId int) ([]*CiWorkflowStatusLatest, error) + GetCachedPipelineIds(pipelineIds []int) ([]int, error) + GetCiWorkflowStatusLatestByPipelineIds(pipelineIds []int) ([]*CiWorkflowStatusLatest, error) DeleteCiWorkflowStatusLatestByPipelineId(pipelineId int) error // CD Workflow Status Latest methods @@ -127,6 +129,39 @@ func (impl *WorkflowStatusLatestRepositoryImpl) DeleteCiWorkflowStatusLatestByPi return nil } +func (impl *WorkflowStatusLatestRepositoryImpl) GetCachedPipelineIds(pipelineIds []int) ([]int, error) { + if len(pipelineIds) == 0 { + return []int{}, nil + } + + var cachedPipelineIds []int + err := impl.dbConnection.Model(&CiWorkflowStatusLatest{}). + Column("pipeline_id"). + Where("pipeline_id IN (?)", pg.In(pipelineIds)). + Select(&cachedPipelineIds) + if err != nil { + impl.logger.Errorw("error in getting cached pipeline ids", "err", err, "pipelineIds", pipelineIds) + return nil, err + } + return cachedPipelineIds, nil +} + +func (impl *WorkflowStatusLatestRepositoryImpl) GetCiWorkflowStatusLatestByPipelineIds(pipelineIds []int) ([]*CiWorkflowStatusLatest, error) { + if len(pipelineIds) == 0 { + return []*CiWorkflowStatusLatest{}, nil + } + + var models []*CiWorkflowStatusLatest + err := impl.dbConnection.Model(&models). + Where("pipeline_id IN (?)", pg.In(pipelineIds)). + Select() + if err != nil { + impl.logger.Errorw("error in getting ci workflow status latest by pipeline ids", "err", err, "pipelineIds", pipelineIds) + return nil, err + } + return models, nil +} + // CD Workflow Status Latest methods implementation func (impl *WorkflowStatusLatestRepositoryImpl) SaveCdWorkflowStatusLatest(model *CdWorkflowStatusLatest) error { err := impl.dbConnection.Insert(model) diff --git a/pkg/pipeline/CiHandler.go b/pkg/pipeline/CiHandler.go index ba9f6cceca..711dd82120 100644 --- a/pkg/pipeline/CiHandler.go +++ b/pkg/pipeline/CiHandler.go @@ -29,7 +29,6 @@ import ( eventProcessorBean "github.com/devtron-labs/devtron/pkg/eventProcessor/bean" "github.com/devtron-labs/devtron/pkg/pipeline/constants" "github.com/devtron-labs/devtron/pkg/pipeline/workflowStatus" - "github.com/devtron-labs/devtron/pkg/workflow/status/workflowStatusLatest" "regexp" "slices" "strconv" @@ -77,28 +76,28 @@ type CiHandler interface { } type CiHandlerImpl struct { - Logger *zap.SugaredLogger - ciPipelineMaterialRepository pipelineConfig.CiPipelineMaterialRepository - ciService CiService - gitSensorClient gitSensor.Client - ciWorkflowRepository pipelineConfig.CiWorkflowRepository - ciArtifactRepository repository.CiArtifactRepository - userService user.UserService - eventClient client.EventClient - eventFactory client.EventFactory - ciPipelineRepository pipelineConfig.CiPipelineRepository - appListingRepository repository.AppListingRepository - cdPipelineRepository pipelineConfig.PipelineRepository - enforcerUtil rbac.EnforcerUtil - resourceGroupService resourceGroup.ResourceGroupService - envRepository repository2.EnvironmentRepository - imageTaggingService imageTagging.ImageTaggingService - customTagService CustomTagService - appWorkflowRepository appWorkflow.AppWorkflowRepository - config *types.CiConfig - k8sCommonService k8sPkg.K8sCommonService - workFlowStageStatusService workflowStatus.WorkFlowStageStatusService - workflowStatusUpdateService workflowStatusLatest.WorkflowStatusUpdateService + Logger *zap.SugaredLogger + ciPipelineMaterialRepository pipelineConfig.CiPipelineMaterialRepository + ciService CiService + gitSensorClient gitSensor.Client + ciWorkflowRepository pipelineConfig.CiWorkflowRepository + ciArtifactRepository repository.CiArtifactRepository + userService user.UserService + eventClient client.EventClient + eventFactory client.EventFactory + ciPipelineRepository pipelineConfig.CiPipelineRepository + appListingRepository repository.AppListingRepository + cdPipelineRepository pipelineConfig.PipelineRepository + enforcerUtil rbac.EnforcerUtil + resourceGroupService resourceGroup.ResourceGroupService + envRepository repository2.EnvironmentRepository + imageTaggingService imageTagging.ImageTaggingService + customTagService CustomTagService + appWorkflowRepository appWorkflow.AppWorkflowRepository + config *types.CiConfig + k8sCommonService k8sPkg.K8sCommonService + workFlowStageStatusService workflowStatus.WorkFlowStageStatusService + workflowStatusLatestRepository pipelineConfig.WorkflowStatusLatestRepository } func NewCiHandlerImpl(Logger *zap.SugaredLogger, ciService CiService, ciPipelineMaterialRepository pipelineConfig.CiPipelineMaterialRepository, gitSensorClient gitSensor.Client, ciWorkflowRepository pipelineConfig.CiWorkflowRepository, @@ -106,30 +105,30 @@ func NewCiHandlerImpl(Logger *zap.SugaredLogger, ciService CiService, ciPipeline appListingRepository repository.AppListingRepository, cdPipelineRepository pipelineConfig.PipelineRepository, enforcerUtil rbac.EnforcerUtil, resourceGroupService resourceGroup.ResourceGroupService, envRepository repository2.EnvironmentRepository, imageTaggingService imageTagging.ImageTaggingService, k8sCommonService k8sPkg.K8sCommonService, appWorkflowRepository appWorkflow.AppWorkflowRepository, customTagService CustomTagService, workFlowStageStatusService workflowStatus.WorkFlowStageStatusService, - workflowStatusUpdateService workflowStatusLatest.WorkflowStatusUpdateService, + workflowStatusLatestRepository pipelineConfig.WorkflowStatusLatestRepository, ) *CiHandlerImpl { cih := &CiHandlerImpl{ - Logger: Logger, - ciService: ciService, - ciPipelineMaterialRepository: ciPipelineMaterialRepository, - gitSensorClient: gitSensorClient, - ciWorkflowRepository: ciWorkflowRepository, - ciArtifactRepository: ciArtifactRepository, - userService: userService, - eventClient: eventClient, - eventFactory: eventFactory, - ciPipelineRepository: ciPipelineRepository, - appListingRepository: appListingRepository, - cdPipelineRepository: cdPipelineRepository, - enforcerUtil: enforcerUtil, - resourceGroupService: resourceGroupService, - envRepository: envRepository, - imageTaggingService: imageTaggingService, - customTagService: customTagService, - appWorkflowRepository: appWorkflowRepository, - k8sCommonService: k8sCommonService, - workFlowStageStatusService: workFlowStageStatusService, - workflowStatusUpdateService: workflowStatusUpdateService, + Logger: Logger, + ciService: ciService, + ciPipelineMaterialRepository: ciPipelineMaterialRepository, + gitSensorClient: gitSensorClient, + ciWorkflowRepository: ciWorkflowRepository, + ciArtifactRepository: ciArtifactRepository, + userService: userService, + eventClient: eventClient, + eventFactory: eventFactory, + ciPipelineRepository: ciPipelineRepository, + appListingRepository: appListingRepository, + cdPipelineRepository: cdPipelineRepository, + enforcerUtil: enforcerUtil, + resourceGroupService: resourceGroupService, + envRepository: envRepository, + imageTaggingService: imageTaggingService, + customTagService: customTagService, + appWorkflowRepository: appWorkflowRepository, + k8sCommonService: k8sCommonService, + workFlowStageStatusService: workFlowStageStatusService, + workflowStatusLatestRepository: workflowStatusLatestRepository, } config, err := types.GetCiConfig() if err != nil { @@ -648,18 +647,202 @@ func (impl *CiHandlerImpl) stateChanged(status string, podStatus string, msg str } func (impl *CiHandlerImpl) FetchCiStatusForTriggerViewV1(appId int) ([]*pipelineConfig.CiWorkflowStatus, error) { - ciWorkflowStatuses, err := impl.workflowStatusUpdateService.FetchCiStatusForTriggerViewOptimized(appId) + // Get all CI pipeline IDs for this app + allPipelineIds, err := impl.ciWorkflowRepository.FindCiPipelineIdsByAppId(appId) if err != nil { - impl.Logger.Errorw("error in fetching ci status from optimized service, falling back to old method", "appId", appId, "err", err) - // Fallback to old method if optimized service fails - ciWorkflowStatuses, err = impl.ciWorkflowRepository.FIndCiWorkflowStatusesByAppId(appId) - if err != nil && !util.IsErrNoRows(err) { - impl.Logger.Errorw("err in fetching ciWorkflowStatuses from ciWorkflowRepository", "appId", appId, "err", err) - return ciWorkflowStatuses, err + impl.Logger.Errorw("error in getting ci pipeline ids for app, falling back to old method", "err", err, "appId", appId) + return impl.ciWorkflowRepository.FIndCiWorkflowStatusesByAppId(appId) + } + + if len(allPipelineIds) == 0 { + return []*pipelineConfig.CiWorkflowStatus{}, nil + } + + // Find which pipeline IDs have entries in latest status table + pipelinesInLatestTable, err := impl.workflowStatusLatestRepository.GetCachedPipelineIds(allPipelineIds) + if err != nil { + impl.Logger.Errorw("error in checking latest status table, falling back to old method", "err", err, "appId", appId) + return impl.ciWorkflowRepository.FIndCiWorkflowStatusesByAppId(appId) + } + + var allStatuses []*pipelineConfig.CiWorkflowStatus + + // Fetch from latest status table for available pipelines + if len(pipelinesInLatestTable) > 0 { + statusesFromLatestTable, err := impl.fetchCiStatusFromLatestTable(pipelinesInLatestTable) + if err != nil { + impl.Logger.Errorw("error in fetching from latest status table", "err", err, "pipelineIds", pipelinesInLatestTable) + return nil, err + } else { + allStatuses = append(allStatuses, statusesFromLatestTable...) + } + } + + // Find pipeline IDs that are NOT in latest status table + pipelinesNotInLatestTable := impl.findMissingPipelineIds(allPipelineIds, pipelinesInLatestTable) + + // Fetch using complex query for missing pipeline IDs + if len(pipelinesNotInLatestTable) > 0 { + statusesFromComplexQuery, err := impl.fetchCiStatusUsingComplexQuery(pipelinesNotInLatestTable) + if err != nil { + impl.Logger.Errorw("error in fetching using complex query", "err", err, "pipelineIds", pipelinesNotInLatestTable) + return nil, err + } else { + allStatuses = append(allStatuses, statusesFromComplexQuery...) + } + } + + impl.Logger.Debugw("hybrid ci status fetch completed", + "appId", appId, + "totalPipelines", len(allPipelineIds), + "pipelinesFromLatestTable", len(pipelinesInLatestTable), + "pipelinesFromComplexQuery", len(pipelinesNotInLatestTable), + "totalResults", len(allStatuses)) + + return allStatuses, nil +} + +// fetchCiStatusFromLatestTable fetches CI status from ci_workflow_status_latest table +func (impl *CiHandlerImpl) fetchCiStatusFromLatestTable(pipelineIds []int) ([]*pipelineConfig.CiWorkflowStatus, error) { + // Get entries from latest status table + latestStatusEntries, err := impl.workflowStatusLatestRepository.GetCiWorkflowStatusLatestByPipelineIds(pipelineIds) + if err != nil { + return nil, err + } + + // Extract workflow IDs + var workflowIds []int + for _, entry := range latestStatusEntries { + workflowIds = append(workflowIds, entry.CiWorkflowId) + } + + // Get workflows by IDs + workflows, err := impl.ciWorkflowRepository.FindWorkflowsByCiWorkflowIds(workflowIds) + if err != nil { + return nil, err + } + + // Convert to CiWorkflowStatus format + var statuses []*pipelineConfig.CiWorkflowStatus + for _, workflow := range workflows { + status := &pipelineConfig.CiWorkflowStatus{ + CiWorkflowId: workflow.Id, + CiPipelineName: workflow.CiPipeline.Name, + CiPipelineId: workflow.CiPipelineId, + CiStatus: workflow.Status, + StorageConfigured: workflow.BlobStorageEnabled, } + statuses = append(statuses, status) } - return ciWorkflowStatuses, err + return statuses, nil +} + +// fetchCiStatusUsingComplexQuery fetches CI status using complex query for specific pipeline IDs +func (impl *CiHandlerImpl) fetchCiStatusUsingComplexQuery(pipelineIds []int) ([]*pipelineConfig.CiWorkflowStatus, error) { + // Get latest workflows for these pipelines + workflows, err := impl.ciWorkflowRepository.FindLastTriggeredWorkflowByCiIds(pipelineIds) + if err != nil { + return nil, err + } + + // Convert to CiWorkflowStatus format + var statuses []*pipelineConfig.CiWorkflowStatus + for _, workflow := range workflows { + status := &pipelineConfig.CiWorkflowStatus{ + CiWorkflowId: workflow.Id, + CiPipelineName: workflow.CiPipeline.Name, + CiPipelineId: workflow.CiPipelineId, + CiStatus: workflow.Status, + StorageConfigured: workflow.BlobStorageEnabled, + } + statuses = append(statuses, status) + } + + return statuses, nil +} + +// fetchWorkflowsFromLatestTable fetches workflows from ci_workflow_status_latest table +func (impl *CiHandlerImpl) fetchWorkflowsFromLatestTable(pipelineIds []int) ([]*pipelineConfig.CiWorkflow, error) { + // Get entries from latest status table + latestStatusEntries, err := impl.workflowStatusLatestRepository.GetCiWorkflowStatusLatestByPipelineIds(pipelineIds) + if err != nil { + return nil, err + } + + // Extract workflow IDs + var workflowIds []int + for _, entry := range latestStatusEntries { + workflowIds = append(workflowIds, entry.CiWorkflowId) + } + + // Get workflows by IDs + return impl.ciWorkflowRepository.FindWorkflowsByCiWorkflowIds(workflowIds) +} + +// fetchLastTriggeredWorkflowsHybrid implements hybrid approach for workflow fetching +// Uses latest status table for available pipelines, fallback to complex query for missing pipelines +func (impl *CiHandlerImpl) fetchLastTriggeredWorkflowsHybrid(pipelineIds []int) ([]*pipelineConfig.CiWorkflow, error) { + if len(pipelineIds) == 0 { + return []*pipelineConfig.CiWorkflow{}, nil + } + + // Find which pipeline IDs have entries in latest status table + pipelinesInLatestTable, err := impl.workflowStatusLatestRepository.GetCachedPipelineIds(pipelineIds) + if err != nil { + impl.Logger.Errorw("error in checking latest status table, falling back to complex query", "err", err, "pipelineIds", pipelineIds) + return impl.ciWorkflowRepository.FindLastTriggeredWorkflowByCiIds(pipelineIds) + } + + var allWorkflows []*pipelineConfig.CiWorkflow + + // Fetch from latest status table for available pipelines + if len(pipelinesInLatestTable) > 0 { + workflowsFromLatestTable, err := impl.fetchWorkflowsFromLatestTable(pipelinesInLatestTable) + if err != nil { + impl.Logger.Errorw("error in fetching from latest status table", "err", err, "pipelineIds", pipelinesInLatestTable) + return nil, err + } else { + allWorkflows = append(allWorkflows, workflowsFromLatestTable...) + } + } + + // Find pipeline IDs that are NOT in latest status table + pipelinesNotInLatestTable := impl.findMissingPipelineIds(pipelineIds, pipelinesInLatestTable) + + // Fetch using complex query for missing pipeline IDs + if len(pipelinesNotInLatestTable) > 0 { + workflowsFromComplexQuery, err := impl.ciWorkflowRepository.FindLastTriggeredWorkflowByCiIds(pipelinesNotInLatestTable) + if err != nil { + impl.Logger.Errorw("error in fetching using complex query", "err", err, "pipelineIds", pipelinesNotInLatestTable) + return nil, err + } else { + allWorkflows = append(allWorkflows, workflowsFromComplexQuery...) + } + } + + impl.Logger.Debugw("hybrid workflow fetch completed", + "totalPipelines", len(pipelineIds), + "pipelinesFromLatestTable", len(pipelinesInLatestTable), + "pipelinesFromOldQuery", len(pipelinesNotInLatestTable)) + + return allWorkflows, nil +} + +// findMissingPipelineIds finds pipeline IDs that are NOT in the latest status table +func (impl *CiHandlerImpl) findMissingPipelineIds(allPipelineIds, pipelinesInLatestTable []int) []int { + pipelineIdMap := make(map[int]bool) + for _, id := range pipelinesInLatestTable { + pipelineIdMap[id] = true + } + + var missingPipelineIds []int + for _, id := range allPipelineIds { + if !pipelineIdMap[id] { + missingPipelineIds = append(missingPipelineIds, id) + } + } + return missingPipelineIds } func (impl *CiHandlerImpl) FetchCiStatusForTriggerView(appId int) ([]*pipelineConfig.CiWorkflowStatus, error) { @@ -870,9 +1053,10 @@ func (impl *CiHandlerImpl) FetchCiStatusForTriggerViewForEnvironment(request res if len(ciPipelineIds) == 0 { return ciWorkflowStatuses, nil } - latestCiWorkflows, err := impl.ciWorkflowRepository.FindLastTriggeredWorkflowByCiIdsOptimized(ciPipelineIds) + // Hybrid approach: Use latest status table for available pipelines, fallback to complex old query for others + latestCiWorkflows, err := impl.fetchLastTriggeredWorkflowsHybrid(ciPipelineIds) if err != nil && !util.IsErrNoRows(err) { - impl.Logger.Errorw("err in optimized ci workflow fetch", "ciPipelineIds", ciPipelineIds, "err", err) + impl.Logger.Errorw("err in hybrid ci workflow fetch", "ciPipelineIds", ciPipelineIds, "err", err) return ciWorkflowStatuses, err } diff --git a/wire_gen.go b/wire_gen.go index 40615a5be0..1623a1cb5c 100644 --- a/wire_gen.go +++ b/wire_gen.go @@ -273,7 +273,6 @@ import ( read20 "github.com/devtron-labs/devtron/pkg/workflow/cd/read" "github.com/devtron-labs/devtron/pkg/workflow/dag" status2 "github.com/devtron-labs/devtron/pkg/workflow/status" - "github.com/devtron-labs/devtron/pkg/workflow/status/workflowStatusLatest" "github.com/devtron-labs/devtron/pkg/workflow/trigger/audit/hook" repository19 "github.com/devtron-labs/devtron/pkg/workflow/trigger/audit/repository" service3 "github.com/devtron-labs/devtron/pkg/workflow/trigger/audit/service" @@ -750,9 +749,7 @@ func InitializeApp() (*App, error) { deploymentTemplateValidationServiceImpl := validator.NewDeploymentTemplateValidationServiceImpl(sugaredLogger, chartRefServiceImpl, scopedVariableManagerImpl, deployedAppMetricsServiceImpl, deploymentTemplateValidationServiceEntImpl) devtronAppGitOpConfigServiceImpl := gitOpsConfig.NewDevtronAppGitOpConfigServiceImpl(sugaredLogger, chartRepositoryImpl, chartServiceImpl, gitOpsConfigReadServiceImpl, gitOpsValidationServiceImpl, argoClientWrapperServiceImpl, deploymentConfigServiceImpl, chartReadServiceImpl) workflowStatusLatestRepositoryImpl := pipelineConfig.NewWorkflowStatusLatestRepositoryImpl(db, sugaredLogger) - workflowStatusLatestServiceImpl := workflowStatusLatest.NewWorkflowStatusLatestServiceImpl(sugaredLogger, workflowStatusLatestRepositoryImpl, ciWorkflowRepositoryImpl, cdWorkflowRepositoryImpl) - workflowStatusUpdateServiceImpl := workflowStatusLatest.NewWorkflowStatusUpdateServiceImpl(sugaredLogger, workflowStatusLatestServiceImpl, ciWorkflowRepositoryImpl, cdWorkflowRepositoryImpl, ciPipelineRepositoryImpl, pipelineRepositoryImpl) - ciHandlerImpl := pipeline.NewCiHandlerImpl(sugaredLogger, ciServiceImpl, ciPipelineMaterialRepositoryImpl, clientImpl, ciWorkflowRepositoryImpl, ciArtifactRepositoryImpl, userServiceImpl, eventRESTClientImpl, eventSimpleFactoryImpl, ciPipelineRepositoryImpl, appListingRepositoryImpl, pipelineRepositoryImpl, enforcerUtilImpl, resourceGroupServiceImpl, environmentRepositoryImpl, imageTaggingServiceImpl, k8sCommonServiceImpl, appWorkflowRepositoryImpl, customTagServiceImpl, workFlowStageStatusServiceImpl, workflowStatusUpdateServiceImpl) + ciHandlerImpl := pipeline.NewCiHandlerImpl(sugaredLogger, ciServiceImpl, ciPipelineMaterialRepositoryImpl, clientImpl, ciWorkflowRepositoryImpl, ciArtifactRepositoryImpl, userServiceImpl, eventRESTClientImpl, eventSimpleFactoryImpl, ciPipelineRepositoryImpl, appListingRepositoryImpl, pipelineRepositoryImpl, enforcerUtilImpl, resourceGroupServiceImpl, environmentRepositoryImpl, imageTaggingServiceImpl, k8sCommonServiceImpl, appWorkflowRepositoryImpl, customTagServiceImpl, workFlowStageStatusServiceImpl, workflowStatusLatestRepositoryImpl) cdHandlerImpl := pipeline.NewCdHandlerImpl(sugaredLogger, userServiceImpl, cdWorkflowRepositoryImpl, ciArtifactRepositoryImpl, ciPipelineMaterialRepositoryImpl, pipelineRepositoryImpl, environmentRepositoryImpl, ciWorkflowRepositoryImpl, enforcerUtilImpl, resourceGroupServiceImpl, imageTaggingServiceImpl, k8sServiceImpl, customTagServiceImpl, deploymentConfigServiceImpl, workFlowStageStatusServiceImpl, cdWorkflowRunnerServiceImpl) appWorkflowServiceImpl := appWorkflow2.NewAppWorkflowServiceImpl(sugaredLogger, appWorkflowRepositoryImpl, ciCdPipelineOrchestratorImpl, ciPipelineRepositoryImpl, pipelineRepositoryImpl, enforcerUtilImpl, resourceGroupServiceImpl, appRepositoryImpl, userAuthServiceImpl, chartServiceImpl, deploymentConfigServiceImpl, pipelineBuilderImpl) appCloneServiceImpl := appClone.NewAppCloneServiceImpl(sugaredLogger, pipelineBuilderImpl, attributesServiceImpl, chartServiceImpl, configMapServiceImpl, appWorkflowServiceImpl, appListingServiceImpl, propertiesConfigServiceImpl, pipelineStageServiceImpl, ciTemplateReadServiceImpl, appRepositoryImpl, ciPipelineRepositoryImpl, pipelineRepositoryImpl, ciPipelineConfigServiceImpl, gitOpsConfigReadServiceImpl, chartReadServiceImpl) From 7f327f2c79f53d3e79448b42117b489b6f85f033 Mon Sep 17 00:00:00 2001 From: ayushmaheshwari Date: Wed, 23 Jul 2025 11:16:42 +0530 Subject: [PATCH 10/31] wip --- pkg/deployment/trigger/devtronApps/deployStageHandlerCode.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/deployment/trigger/devtronApps/deployStageHandlerCode.go b/pkg/deployment/trigger/devtronApps/deployStageHandlerCode.go index 18dccdeed4..b99fa4265a 100644 --- a/pkg/deployment/trigger/devtronApps/deployStageHandlerCode.go +++ b/pkg/deployment/trigger/devtronApps/deployStageHandlerCode.go @@ -479,7 +479,7 @@ func (impl *HandlerServiceImpl) TriggerAutomaticDeployment(request bean.CdTrigge impl.logger.Errorw("error in committing transaction for cdWorkflowRunner, ManualCdTrigger", "cdWorkflowId", cdWf.Id, "err", err) return err } - + isRollbackNeeded = false runner.CdWorkflow = &pipelineConfig.CdWorkflow{ Pipeline: pipeline, } From 19f6886072203eeb6a7c21bb0fb311486ba41e1c Mon Sep 17 00:00:00 2001 From: Prakash Kumar Date: Wed, 23 Jul 2025 11:54:47 +0530 Subject: [PATCH 11/31] Rename `GetCachedPipelineIds` to `GetByPipelineIds` in `WorkflowStatusLatestRepository` and update references in `CiHandlerImpl` for clarity. --- .../pipelineConfig/WorkflowStatusLatestRepository.go | 4 ++-- pkg/pipeline/CiHandler.go | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/internal/sql/repository/pipelineConfig/WorkflowStatusLatestRepository.go b/internal/sql/repository/pipelineConfig/WorkflowStatusLatestRepository.go index 901272eb05..cd6fc2d9eb 100644 --- a/internal/sql/repository/pipelineConfig/WorkflowStatusLatestRepository.go +++ b/internal/sql/repository/pipelineConfig/WorkflowStatusLatestRepository.go @@ -28,7 +28,7 @@ type WorkflowStatusLatestRepository interface { UpdateCiWorkflowStatusLatest(model *CiWorkflowStatusLatest) error GetCiWorkflowStatusLatestByPipelineId(pipelineId int) (*CiWorkflowStatusLatest, error) GetCiWorkflowStatusLatestByAppId(appId int) ([]*CiWorkflowStatusLatest, error) - GetCachedPipelineIds(pipelineIds []int) ([]int, error) + GetByPipelineIds(pipelineIds []int) ([]int, error) GetCiWorkflowStatusLatestByPipelineIds(pipelineIds []int) ([]*CiWorkflowStatusLatest, error) DeleteCiWorkflowStatusLatestByPipelineId(pipelineId int) error @@ -129,7 +129,7 @@ func (impl *WorkflowStatusLatestRepositoryImpl) DeleteCiWorkflowStatusLatestByPi return nil } -func (impl *WorkflowStatusLatestRepositoryImpl) GetCachedPipelineIds(pipelineIds []int) ([]int, error) { +func (impl *WorkflowStatusLatestRepositoryImpl) GetByPipelineIds(pipelineIds []int) ([]int, error) { if len(pipelineIds) == 0 { return []int{}, nil } diff --git a/pkg/pipeline/CiHandler.go b/pkg/pipeline/CiHandler.go index 711dd82120..c94c58b025 100644 --- a/pkg/pipeline/CiHandler.go +++ b/pkg/pipeline/CiHandler.go @@ -659,7 +659,7 @@ func (impl *CiHandlerImpl) FetchCiStatusForTriggerViewV1(appId int) ([]*pipeline } // Find which pipeline IDs have entries in latest status table - pipelinesInLatestTable, err := impl.workflowStatusLatestRepository.GetCachedPipelineIds(allPipelineIds) + pipelinesInLatestTable, err := impl.workflowStatusLatestRepository.GetByPipelineIds(allPipelineIds) if err != nil { impl.Logger.Errorw("error in checking latest status table, falling back to old method", "err", err, "appId", appId) return impl.ciWorkflowRepository.FIndCiWorkflowStatusesByAppId(appId) @@ -788,7 +788,7 @@ func (impl *CiHandlerImpl) fetchLastTriggeredWorkflowsHybrid(pipelineIds []int) } // Find which pipeline IDs have entries in latest status table - pipelinesInLatestTable, err := impl.workflowStatusLatestRepository.GetCachedPipelineIds(pipelineIds) + pipelinesInLatestTable, err := impl.workflowStatusLatestRepository.GetByPipelineIds(pipelineIds) if err != nil { impl.Logger.Errorw("error in checking latest status table, falling back to complex query", "err", err, "pipelineIds", pipelineIds) return impl.ciWorkflowRepository.FindLastTriggeredWorkflowByCiIds(pipelineIds) From 01b8bc6ba959e60f34af2f75d7c2471cc84deb76 Mon Sep 17 00:00:00 2001 From: Prakash Kumar Date: Wed, 23 Jul 2025 12:34:22 +0530 Subject: [PATCH 12/31] Refactor CI status fetching in `CiHandlerImpl` by renaming methods for clarity, introducing `GetCiWorkflowStatusFromCiWorkflow` in `adapter`, and improving log consistency. --- pkg/pipeline/CiHandler.go | 72 +++++++++------------------------ pkg/pipeline/adapter/adapter.go | 10 +++++ 2 files changed, 29 insertions(+), 53 deletions(-) diff --git a/pkg/pipeline/CiHandler.go b/pkg/pipeline/CiHandler.go index c94c58b025..837e59ea67 100644 --- a/pkg/pipeline/CiHandler.go +++ b/pkg/pipeline/CiHandler.go @@ -27,6 +27,7 @@ import ( buildBean "github.com/devtron-labs/devtron/pkg/build/pipeline/bean" repository2 "github.com/devtron-labs/devtron/pkg/cluster/environment/repository" eventProcessorBean "github.com/devtron-labs/devtron/pkg/eventProcessor/bean" + "github.com/devtron-labs/devtron/pkg/pipeline/adapter" "github.com/devtron-labs/devtron/pkg/pipeline/constants" "github.com/devtron-labs/devtron/pkg/pipeline/workflowStatus" "regexp" @@ -647,10 +648,9 @@ func (impl *CiHandlerImpl) stateChanged(status string, podStatus string, msg str } func (impl *CiHandlerImpl) FetchCiStatusForTriggerViewV1(appId int) ([]*pipelineConfig.CiWorkflowStatus, error) { - // Get all CI pipeline IDs for this app allPipelineIds, err := impl.ciWorkflowRepository.FindCiPipelineIdsByAppId(appId) if err != nil { - impl.Logger.Errorw("error in getting ci pipeline ids for app, falling back to old method", "err", err, "appId", appId) + impl.Logger.Errorw("error in getting ci pipeline ids for app, falling back to old method", "appId", appId, "err", err) return impl.ciWorkflowRepository.FIndCiWorkflowStatusesByAppId(appId) } @@ -658,89 +658,69 @@ func (impl *CiHandlerImpl) FetchCiStatusForTriggerViewV1(appId int) ([]*pipeline return []*pipelineConfig.CiWorkflowStatus{}, nil } - // Find which pipeline IDs have entries in latest status table pipelinesInLatestTable, err := impl.workflowStatusLatestRepository.GetByPipelineIds(allPipelineIds) if err != nil { - impl.Logger.Errorw("error in checking latest status table, falling back to old method", "err", err, "appId", appId) + impl.Logger.Errorw("error in checking latest status table, falling back to old method", "appId", appId, "err", err) return impl.ciWorkflowRepository.FIndCiWorkflowStatusesByAppId(appId) } var allStatuses []*pipelineConfig.CiWorkflowStatus - // Fetch from latest status table for available pipelines if len(pipelinesInLatestTable) > 0 { statusesFromLatestTable, err := impl.fetchCiStatusFromLatestTable(pipelinesInLatestTable) if err != nil { - impl.Logger.Errorw("error in fetching from latest status table", "err", err, "pipelineIds", pipelinesInLatestTable) + impl.Logger.Errorw("error in fetching from latest status table", "pipelineIds", pipelinesInLatestTable, "err", err) return nil, err } else { allStatuses = append(allStatuses, statusesFromLatestTable...) } } - // Find pipeline IDs that are NOT in latest status table - pipelinesNotInLatestTable := impl.findMissingPipelineIds(allPipelineIds, pipelinesInLatestTable) + pipelinesNotInLatestTable := impl.getPipelineIdsNotInLatestTable(allPipelineIds, pipelinesInLatestTable) - // Fetch using complex query for missing pipeline IDs if len(pipelinesNotInLatestTable) > 0 { - statusesFromComplexQuery, err := impl.fetchCiStatusUsingComplexQuery(pipelinesNotInLatestTable) + statusesFromComplexQuery, err := impl.fetchCiStatusUsingFallbackMethod(pipelinesNotInLatestTable) if err != nil { - impl.Logger.Errorw("error in fetching using complex query", "err", err, "pipelineIds", pipelinesNotInLatestTable) + impl.Logger.Errorw("error in fetching using complex query", "pipelineIds", pipelinesNotInLatestTable, "err", err) return nil, err } else { allStatuses = append(allStatuses, statusesFromComplexQuery...) } } - impl.Logger.Debugw("hybrid ci status fetch completed", - "appId", appId, - "totalPipelines", len(allPipelineIds), - "pipelinesFromLatestTable", len(pipelinesInLatestTable), - "pipelinesFromComplexQuery", len(pipelinesNotInLatestTable), - "totalResults", len(allStatuses)) + impl.Logger.Debugw("hybrid ci status fetch completed", "appId", appId, "totalPipelines", len(allPipelineIds), "pipelinesFromLatestTable", len(pipelinesInLatestTable), "pipelinesFromOldQuery", len(pipelinesNotInLatestTable)) return allStatuses, nil } // fetchCiStatusFromLatestTable fetches CI status from ci_workflow_status_latest table func (impl *CiHandlerImpl) fetchCiStatusFromLatestTable(pipelineIds []int) ([]*pipelineConfig.CiWorkflowStatus, error) { - // Get entries from latest status table latestStatusEntries, err := impl.workflowStatusLatestRepository.GetCiWorkflowStatusLatestByPipelineIds(pipelineIds) if err != nil { return nil, err } - // Extract workflow IDs var workflowIds []int for _, entry := range latestStatusEntries { workflowIds = append(workflowIds, entry.CiWorkflowId) } - // Get workflows by IDs workflows, err := impl.ciWorkflowRepository.FindWorkflowsByCiWorkflowIds(workflowIds) if err != nil { return nil, err } - // Convert to CiWorkflowStatus format var statuses []*pipelineConfig.CiWorkflowStatus for _, workflow := range workflows { - status := &pipelineConfig.CiWorkflowStatus{ - CiWorkflowId: workflow.Id, - CiPipelineName: workflow.CiPipeline.Name, - CiPipelineId: workflow.CiPipelineId, - CiStatus: workflow.Status, - StorageConfigured: workflow.BlobStorageEnabled, - } + status := adapter.GetCiWorkflowStatusFromCiWorkflow(workflow) statuses = append(statuses, status) } return statuses, nil } -// fetchCiStatusUsingComplexQuery fetches CI status using complex query for specific pipeline IDs -func (impl *CiHandlerImpl) fetchCiStatusUsingComplexQuery(pipelineIds []int) ([]*pipelineConfig.CiWorkflowStatus, error) { - // Get latest workflows for these pipelines +// fetchCiStatusUsingFallbackMethod fetches CI status directly from workflow table having multiple joins +func (impl *CiHandlerImpl) fetchCiStatusUsingFallbackMethod(pipelineIds []int) ([]*pipelineConfig.CiWorkflowStatus, error) { workflows, err := impl.ciWorkflowRepository.FindLastTriggeredWorkflowByCiIds(pipelineIds) if err != nil { return nil, err @@ -749,13 +729,7 @@ func (impl *CiHandlerImpl) fetchCiStatusUsingComplexQuery(pipelineIds []int) ([] // Convert to CiWorkflowStatus format var statuses []*pipelineConfig.CiWorkflowStatus for _, workflow := range workflows { - status := &pipelineConfig.CiWorkflowStatus{ - CiWorkflowId: workflow.Id, - CiPipelineName: workflow.CiPipeline.Name, - CiPipelineId: workflow.CiPipelineId, - CiStatus: workflow.Status, - StorageConfigured: workflow.BlobStorageEnabled, - } + status := adapter.GetCiWorkflowStatusFromCiWorkflow(workflow) statuses = append(statuses, status) } @@ -787,50 +761,43 @@ func (impl *CiHandlerImpl) fetchLastTriggeredWorkflowsHybrid(pipelineIds []int) return []*pipelineConfig.CiWorkflow{}, nil } - // Find which pipeline IDs have entries in latest status table pipelinesInLatestTable, err := impl.workflowStatusLatestRepository.GetByPipelineIds(pipelineIds) if err != nil { - impl.Logger.Errorw("error in checking latest status table, falling back to complex query", "err", err, "pipelineIds", pipelineIds) + impl.Logger.Errorw("error in checking latest status table, falling back to complex query", "pipelineIds", pipelineIds, "err", err) return impl.ciWorkflowRepository.FindLastTriggeredWorkflowByCiIds(pipelineIds) } var allWorkflows []*pipelineConfig.CiWorkflow - // Fetch from latest status table for available pipelines if len(pipelinesInLatestTable) > 0 { workflowsFromLatestTable, err := impl.fetchWorkflowsFromLatestTable(pipelinesInLatestTable) if err != nil { - impl.Logger.Errorw("error in fetching from latest status table", "err", err, "pipelineIds", pipelinesInLatestTable) + impl.Logger.Errorw("error in fetching from latest status table", "pipelineIds", pipelinesInLatestTable, "err", err) return nil, err } else { allWorkflows = append(allWorkflows, workflowsFromLatestTable...) } } - // Find pipeline IDs that are NOT in latest status table - pipelinesNotInLatestTable := impl.findMissingPipelineIds(pipelineIds, pipelinesInLatestTable) + pipelinesNotInLatestTable := impl.getPipelineIdsNotInLatestTable(pipelineIds, pipelinesInLatestTable) - // Fetch using complex query for missing pipeline IDs if len(pipelinesNotInLatestTable) > 0 { workflowsFromComplexQuery, err := impl.ciWorkflowRepository.FindLastTriggeredWorkflowByCiIds(pipelinesNotInLatestTable) if err != nil { - impl.Logger.Errorw("error in fetching using complex query", "err", err, "pipelineIds", pipelinesNotInLatestTable) + impl.Logger.Errorw("error in fetching using complex query", "pipelineIds", pipelinesNotInLatestTable, "err", err) return nil, err } else { allWorkflows = append(allWorkflows, workflowsFromComplexQuery...) } } - impl.Logger.Debugw("hybrid workflow fetch completed", - "totalPipelines", len(pipelineIds), - "pipelinesFromLatestTable", len(pipelinesInLatestTable), - "pipelinesFromOldQuery", len(pipelinesNotInLatestTable)) + impl.Logger.Debugw("hybrid workflow fetch completed", "totalPipelines", len(pipelineIds), "pipelinesFromLatestTable", len(pipelinesInLatestTable), "pipelinesFromOldQuery", len(pipelinesNotInLatestTable)) return allWorkflows, nil } -// findMissingPipelineIds finds pipeline IDs that are NOT in the latest status table -func (impl *CiHandlerImpl) findMissingPipelineIds(allPipelineIds, pipelinesInLatestTable []int) []int { +// getPipelineIdsNotInLatestTable finds pipeline IDs that are NOT in the latest status table +func (impl *CiHandlerImpl) getPipelineIdsNotInLatestTable(allPipelineIds, pipelinesInLatestTable []int) []int { pipelineIdMap := make(map[int]bool) for _, id := range pipelinesInLatestTable { pipelineIdMap[id] = true @@ -1053,7 +1020,6 @@ func (impl *CiHandlerImpl) FetchCiStatusForTriggerViewForEnvironment(request res if len(ciPipelineIds) == 0 { return ciWorkflowStatuses, nil } - // Hybrid approach: Use latest status table for available pipelines, fallback to complex old query for others latestCiWorkflows, err := impl.fetchLastTriggeredWorkflowsHybrid(ciPipelineIds) if err != nil && !util.IsErrNoRows(err) { impl.Logger.Errorw("err in hybrid ci workflow fetch", "ciPipelineIds", ciPipelineIds, "err", err) diff --git a/pkg/pipeline/adapter/adapter.go b/pkg/pipeline/adapter/adapter.go index 3ac8b35f7a..c4a8ddd5e4 100644 --- a/pkg/pipeline/adapter/adapter.go +++ b/pkg/pipeline/adapter/adapter.go @@ -406,3 +406,13 @@ func NewMigrateExternalAppValidationRequest(pipeline *bean.CDPipelineConfigObjec } return request } + +func GetCiWorkflowStatusFromCiWorkflow(ciWorkflow *pipelineConfig.CiWorkflow) *pipelineConfig.CiWorkflowStatus { + return &pipelineConfig.CiWorkflowStatus{ + CiPipelineId: ciWorkflow.CiPipelineId, + CiPipelineName: ciWorkflow.CiPipeline.Name, + CiStatus: ciWorkflow.Status, + StorageConfigured: ciWorkflow.BlobStorageEnabled, + CiWorkflowId: ciWorkflow.Id, + } +} From 3bc8e3d01ae838364ff5d613558f774120023ff1 Mon Sep 17 00:00:00 2001 From: Prakash Kumar Date: Wed, 23 Jul 2025 13:01:03 +0530 Subject: [PATCH 13/31] Refactor method names and variable references in `CiHandlerImpl` for improved clarity and consistency. Update logging and fallback logic in CI status and workflow fetching. --- pkg/pipeline/CiHandler.go | 64 +++++++++++++++------------------------ 1 file changed, 25 insertions(+), 39 deletions(-) diff --git a/pkg/pipeline/CiHandler.go b/pkg/pipeline/CiHandler.go index 837e59ea67..0d53e48938 100644 --- a/pkg/pipeline/CiHandler.go +++ b/pkg/pipeline/CiHandler.go @@ -658,7 +658,7 @@ func (impl *CiHandlerImpl) FetchCiStatusForTriggerViewV1(appId int) ([]*pipeline return []*pipelineConfig.CiWorkflowStatus{}, nil } - pipelinesInLatestTable, err := impl.workflowStatusLatestRepository.GetByPipelineIds(allPipelineIds) + latestStatusEntries, err := impl.workflowStatusLatestRepository.GetCiWorkflowStatusLatestByPipelineIds(allPipelineIds) if err != nil { impl.Logger.Errorw("error in checking latest status table, falling back to old method", "appId", appId, "err", err) return impl.ciWorkflowRepository.FIndCiWorkflowStatusesByAppId(appId) @@ -666,42 +666,35 @@ func (impl *CiHandlerImpl) FetchCiStatusForTriggerViewV1(appId int) ([]*pipeline var allStatuses []*pipelineConfig.CiWorkflowStatus - if len(pipelinesInLatestTable) > 0 { - statusesFromLatestTable, err := impl.fetchCiStatusFromLatestTable(pipelinesInLatestTable) + if len(latestStatusEntries) > 0 { + statusesFromLatestTable, err := impl.fetchCiWorkflowStatusFromLatestEntries(latestStatusEntries) if err != nil { - impl.Logger.Errorw("error in fetching from latest status table", "pipelineIds", pipelinesInLatestTable, "err", err) + impl.Logger.Errorw("error in fetching ci workflow status from latest ci workflow entries ", "latestStatusEntries", latestStatusEntries, "err", err) return nil, err } else { allStatuses = append(allStatuses, statusesFromLatestTable...) } } - pipelinesNotInLatestTable := impl.getPipelineIdsNotInLatestTable(allPipelineIds, pipelinesInLatestTable) + pipelinesNotInLatestTable := impl.getPipelineIdsNotInLatestTable(allPipelineIds, latestStatusEntries) if len(pipelinesNotInLatestTable) > 0 { - statusesFromComplexQuery, err := impl.fetchCiStatusUsingFallbackMethod(pipelinesNotInLatestTable) + statusesFromOldQuery, err := impl.fetchCiStatusUsingFallbackMethod(pipelinesNotInLatestTable) if err != nil { - impl.Logger.Errorw("error in fetching using complex query", "pipelineIds", pipelinesNotInLatestTable, "err", err) + impl.Logger.Errorw("error in fetching using fallback method by pipelineIds", "pipelineIds", pipelinesNotInLatestTable, "err", err) return nil, err } else { - allStatuses = append(allStatuses, statusesFromComplexQuery...) + allStatuses = append(allStatuses, statusesFromOldQuery...) } } - impl.Logger.Debugw("hybrid ci status fetch completed", "appId", appId, "totalPipelines", len(allPipelineIds), "pipelinesFromLatestTable", len(pipelinesInLatestTable), "pipelinesFromOldQuery", len(pipelinesNotInLatestTable)) - return allStatuses, nil } -// fetchCiStatusFromLatestTable fetches CI status from ci_workflow_status_latest table -func (impl *CiHandlerImpl) fetchCiStatusFromLatestTable(pipelineIds []int) ([]*pipelineConfig.CiWorkflowStatus, error) { - latestStatusEntries, err := impl.workflowStatusLatestRepository.GetCiWorkflowStatusLatestByPipelineIds(pipelineIds) - if err != nil { - return nil, err - } - +// fetchCiWorkflowStatusFromLatestEntries fetches CI status from ci_workflow_status_latest table +func (impl *CiHandlerImpl) fetchCiWorkflowStatusFromLatestEntries(latestCiWorkflowStatusEntries []*pipelineConfig.CiWorkflowStatusLatest) ([]*pipelineConfig.CiWorkflowStatus, error) { var workflowIds []int - for _, entry := range latestStatusEntries { + for _, entry := range latestCiWorkflowStatusEntries { workflowIds = append(workflowIds, entry.CiWorkflowId) } @@ -736,21 +729,12 @@ func (impl *CiHandlerImpl) fetchCiStatusUsingFallbackMethod(pipelineIds []int) ( return statuses, nil } -// fetchWorkflowsFromLatestTable fetches workflows from ci_workflow_status_latest table -func (impl *CiHandlerImpl) fetchWorkflowsFromLatestTable(pipelineIds []int) ([]*pipelineConfig.CiWorkflow, error) { - // Get entries from latest status table - latestStatusEntries, err := impl.workflowStatusLatestRepository.GetCiWorkflowStatusLatestByPipelineIds(pipelineIds) - if err != nil { - return nil, err - } - - // Extract workflow IDs +func (impl *CiHandlerImpl) fetchWorkflowsFromLatestTable(latestStatusEntries []*pipelineConfig.CiWorkflowStatusLatest) ([]*pipelineConfig.CiWorkflow, error) { var workflowIds []int for _, entry := range latestStatusEntries { workflowIds = append(workflowIds, entry.CiWorkflowId) } - // Get workflows by IDs return impl.ciWorkflowRepository.FindWorkflowsByCiWorkflowIds(workflowIds) } @@ -761,7 +745,7 @@ func (impl *CiHandlerImpl) fetchLastTriggeredWorkflowsHybrid(pipelineIds []int) return []*pipelineConfig.CiWorkflow{}, nil } - pipelinesInLatestTable, err := impl.workflowStatusLatestRepository.GetByPipelineIds(pipelineIds) + latestStatusEntries, err := impl.workflowStatusLatestRepository.GetCiWorkflowStatusLatestByPipelineIds(pipelineIds) if err != nil { impl.Logger.Errorw("error in checking latest status table, falling back to complex query", "pipelineIds", pipelineIds, "err", err) return impl.ciWorkflowRepository.FindLastTriggeredWorkflowByCiIds(pipelineIds) @@ -769,35 +753,37 @@ func (impl *CiHandlerImpl) fetchLastTriggeredWorkflowsHybrid(pipelineIds []int) var allWorkflows []*pipelineConfig.CiWorkflow - if len(pipelinesInLatestTable) > 0 { - workflowsFromLatestTable, err := impl.fetchWorkflowsFromLatestTable(pipelinesInLatestTable) + if len(latestStatusEntries) > 0 { + workflowsFromLatestTable, err := impl.fetchWorkflowsFromLatestTable(latestStatusEntries) if err != nil { - impl.Logger.Errorw("error in fetching from latest status table", "pipelineIds", pipelinesInLatestTable, "err", err) + impl.Logger.Errorw("error in fetching from latest status table", "latestStatusEntries", latestStatusEntries, "err", err) return nil, err } else { allWorkflows = append(allWorkflows, workflowsFromLatestTable...) } } - pipelinesNotInLatestTable := impl.getPipelineIdsNotInLatestTable(pipelineIds, pipelinesInLatestTable) + pipelinesNotInLatestTable := impl.getPipelineIdsNotInLatestTable(pipelineIds, latestStatusEntries) if len(pipelinesNotInLatestTable) > 0 { - workflowsFromComplexQuery, err := impl.ciWorkflowRepository.FindLastTriggeredWorkflowByCiIds(pipelinesNotInLatestTable) + workflowsFromOldQuery, err := impl.ciWorkflowRepository.FindLastTriggeredWorkflowByCiIds(pipelinesNotInLatestTable) if err != nil { - impl.Logger.Errorw("error in fetching using complex query", "pipelineIds", pipelinesNotInLatestTable, "err", err) + impl.Logger.Errorw("error in fetching using old query by pipeline ids", "pipelineIds", pipelinesNotInLatestTable, "err", err) return nil, err } else { - allWorkflows = append(allWorkflows, workflowsFromComplexQuery...) + allWorkflows = append(allWorkflows, workflowsFromOldQuery...) } } - impl.Logger.Debugw("hybrid workflow fetch completed", "totalPipelines", len(pipelineIds), "pipelinesFromLatestTable", len(pipelinesInLatestTable), "pipelinesFromOldQuery", len(pipelinesNotInLatestTable)) - return allWorkflows, nil } // getPipelineIdsNotInLatestTable finds pipeline IDs that are NOT in the latest status table -func (impl *CiHandlerImpl) getPipelineIdsNotInLatestTable(allPipelineIds, pipelinesInLatestTable []int) []int { +func (impl *CiHandlerImpl) getPipelineIdsNotInLatestTable(allPipelineIds []int, latestStatusEntries []*pipelineConfig.CiWorkflowStatusLatest) []int { + var pipelinesInLatestTable []int + for _, entry := range latestStatusEntries { + pipelinesInLatestTable = append(pipelinesInLatestTable, entry.PipelineId) + } pipelineIdMap := make(map[int]bool) for _, id := range pipelinesInLatestTable { pipelineIdMap[id] = true From 02fd2b4e35113c0787c16199f073fa8bf39fdb16 Mon Sep 17 00:00:00 2001 From: Prakash Kumar Date: Wed, 23 Jul 2025 13:02:40 +0530 Subject: [PATCH 14/31] Remove unused `GetByPipelineIds` method from `WorkflowStatusLatestRepository` to streamline repository interface. --- .../WorkflowStatusLatestRepository.go | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/internal/sql/repository/pipelineConfig/WorkflowStatusLatestRepository.go b/internal/sql/repository/pipelineConfig/WorkflowStatusLatestRepository.go index cd6fc2d9eb..6616111ad2 100644 --- a/internal/sql/repository/pipelineConfig/WorkflowStatusLatestRepository.go +++ b/internal/sql/repository/pipelineConfig/WorkflowStatusLatestRepository.go @@ -28,7 +28,6 @@ type WorkflowStatusLatestRepository interface { UpdateCiWorkflowStatusLatest(model *CiWorkflowStatusLatest) error GetCiWorkflowStatusLatestByPipelineId(pipelineId int) (*CiWorkflowStatusLatest, error) GetCiWorkflowStatusLatestByAppId(appId int) ([]*CiWorkflowStatusLatest, error) - GetByPipelineIds(pipelineIds []int) ([]int, error) GetCiWorkflowStatusLatestByPipelineIds(pipelineIds []int) ([]*CiWorkflowStatusLatest, error) DeleteCiWorkflowStatusLatestByPipelineId(pipelineId int) error @@ -129,23 +128,6 @@ func (impl *WorkflowStatusLatestRepositoryImpl) DeleteCiWorkflowStatusLatestByPi return nil } -func (impl *WorkflowStatusLatestRepositoryImpl) GetByPipelineIds(pipelineIds []int) ([]int, error) { - if len(pipelineIds) == 0 { - return []int{}, nil - } - - var cachedPipelineIds []int - err := impl.dbConnection.Model(&CiWorkflowStatusLatest{}). - Column("pipeline_id"). - Where("pipeline_id IN (?)", pg.In(pipelineIds)). - Select(&cachedPipelineIds) - if err != nil { - impl.logger.Errorw("error in getting cached pipeline ids", "err", err, "pipelineIds", pipelineIds) - return nil, err - } - return cachedPipelineIds, nil -} - func (impl *WorkflowStatusLatestRepositoryImpl) GetCiWorkflowStatusLatestByPipelineIds(pipelineIds []int) ([]*CiWorkflowStatusLatest, error) { if len(pipelineIds) == 0 { return []*CiWorkflowStatusLatest{}, nil From b3e96fc139d549bba1f4df021eba92898fb74094 Mon Sep 17 00:00:00 2001 From: Prakash Kumar Date: Wed, 23 Jul 2025 13:52:28 +0530 Subject: [PATCH 15/31] Refactor `WorkflowStatusLatestService` package imports and CI status fetching in `CiHandlerImpl` to improve logging and error handling consistency. --- pkg/pipeline/CiHandler.go | 5 +++-- pkg/workflow/wire_workflow.go | 2 +- .../workflowStatusLatest/WorkflowStatusLatestService.go | 0 .../workflowStatusLatest/WorkflowStatusUpdateService.go | 0 .../workflowStatusLatest/wire_workflow_status_latest.go | 0 5 files changed, 4 insertions(+), 3 deletions(-) rename pkg/workflow/{status => }/workflowStatusLatest/WorkflowStatusLatestService.go (100%) rename pkg/workflow/{status => }/workflowStatusLatest/WorkflowStatusUpdateService.go (100%) rename pkg/workflow/{status => }/workflowStatusLatest/wire_workflow_status_latest.go (100%) diff --git a/pkg/pipeline/CiHandler.go b/pkg/pipeline/CiHandler.go index 0d53e48938..5c56eb82d5 100644 --- a/pkg/pipeline/CiHandler.go +++ b/pkg/pipeline/CiHandler.go @@ -700,6 +700,7 @@ func (impl *CiHandlerImpl) fetchCiWorkflowStatusFromLatestEntries(latestCiWorkfl workflows, err := impl.ciWorkflowRepository.FindWorkflowsByCiWorkflowIds(workflowIds) if err != nil { + impl.Logger.Errorw("error in fetching ci workflows by ci workflow ids", "workflowIds", workflowIds, "err", err) return nil, err } @@ -712,14 +713,14 @@ func (impl *CiHandlerImpl) fetchCiWorkflowStatusFromLatestEntries(latestCiWorkfl return statuses, nil } -// fetchCiStatusUsingFallbackMethod fetches CI status directly from workflow table having multiple joins +// fetchCiStatusUsingFallbackMethod fetches CI status directly from ci_workflow table func (impl *CiHandlerImpl) fetchCiStatusUsingFallbackMethod(pipelineIds []int) ([]*pipelineConfig.CiWorkflowStatus, error) { workflows, err := impl.ciWorkflowRepository.FindLastTriggeredWorkflowByCiIds(pipelineIds) if err != nil { + impl.Logger.Errorw("error in fetching ci workflows by ci ids", "pipelineIds", pipelineIds, "err", err) return nil, err } - // Convert to CiWorkflowStatus format var statuses []*pipelineConfig.CiWorkflowStatus for _, workflow := range workflows { status := adapter.GetCiWorkflowStatusFromCiWorkflow(workflow) diff --git a/pkg/workflow/wire_workflow.go b/pkg/workflow/wire_workflow.go index d2850fc073..54ca23bdad 100644 --- a/pkg/workflow/wire_workflow.go +++ b/pkg/workflow/wire_workflow.go @@ -19,10 +19,10 @@ package workflow import ( "github.com/devtron-labs/devtron/pkg/workflow/cd" "github.com/devtron-labs/devtron/pkg/workflow/status" - "github.com/devtron-labs/devtron/pkg/workflow/status/workflowStatusLatest" "github.com/devtron-labs/devtron/pkg/workflow/trigger/audit/hook" "github.com/devtron-labs/devtron/pkg/workflow/trigger/audit/repository" "github.com/devtron-labs/devtron/pkg/workflow/trigger/audit/service" + "github.com/devtron-labs/devtron/pkg/workflow/workflowStatusLatest" "github.com/google/wire" ) diff --git a/pkg/workflow/status/workflowStatusLatest/WorkflowStatusLatestService.go b/pkg/workflow/workflowStatusLatest/WorkflowStatusLatestService.go similarity index 100% rename from pkg/workflow/status/workflowStatusLatest/WorkflowStatusLatestService.go rename to pkg/workflow/workflowStatusLatest/WorkflowStatusLatestService.go diff --git a/pkg/workflow/status/workflowStatusLatest/WorkflowStatusUpdateService.go b/pkg/workflow/workflowStatusLatest/WorkflowStatusUpdateService.go similarity index 100% rename from pkg/workflow/status/workflowStatusLatest/WorkflowStatusUpdateService.go rename to pkg/workflow/workflowStatusLatest/WorkflowStatusUpdateService.go diff --git a/pkg/workflow/status/workflowStatusLatest/wire_workflow_status_latest.go b/pkg/workflow/workflowStatusLatest/wire_workflow_status_latest.go similarity index 100% rename from pkg/workflow/status/workflowStatusLatest/wire_workflow_status_latest.go rename to pkg/workflow/workflowStatusLatest/wire_workflow_status_latest.go From 03f78d448bc8fac1e8ac90bbc09a0738b5350d00 Mon Sep 17 00:00:00 2001 From: ayushmaheshwari Date: Wed, 23 Jul 2025 15:57:06 +0530 Subject: [PATCH 16/31] circular import fix --- pkg/deployment/trigger/devtronApps/HandlerService.go | 6 +++--- pkg/pipeline/CdHandler.go | 6 +++--- pkg/workflow/cd/CdWorkflowRunnerService.go | 6 +++--- wire_gen.go | 11 +++++++---- 4 files changed, 16 insertions(+), 13 deletions(-) diff --git a/pkg/deployment/trigger/devtronApps/HandlerService.go b/pkg/deployment/trigger/devtronApps/HandlerService.go index fd174071eb..dcc8fc3b0e 100644 --- a/pkg/deployment/trigger/devtronApps/HandlerService.go +++ b/pkg/deployment/trigger/devtronApps/HandlerService.go @@ -21,8 +21,8 @@ import ( "context" "github.com/devtron-labs/common-lib/async" "github.com/devtron-labs/devtron/client/fluxcd" - status2 "github.com/devtron-labs/devtron/pkg/workflow/status" service2 "github.com/devtron-labs/devtron/pkg/workflow/trigger/audit/service" + "github.com/devtron-labs/devtron/pkg/workflow/workflowStatusLatest" "os" "time" @@ -175,7 +175,7 @@ type HandlerServiceImpl struct { asyncRunnable *async.Runnable workflowTriggerAuditService service2.WorkflowTriggerAuditService fluxCdDeploymentService fluxcd.DeploymentService - workflowStatusUpdateService status2.WorkflowStatusUpdateService + workflowStatusUpdateService workflowStatusLatest.WorkflowStatusUpdateService } func NewHandlerServiceImpl(logger *zap.SugaredLogger, @@ -241,7 +241,7 @@ func NewHandlerServiceImpl(logger *zap.SugaredLogger, asyncRunnable *async.Runnable, workflowTriggerAuditService service2.WorkflowTriggerAuditService, fluxCdDeploymentService fluxcd.DeploymentService, - workflowStatusUpdateService status2.WorkflowStatusUpdateService) (*HandlerServiceImpl, error) { + workflowStatusUpdateService workflowStatusLatest.WorkflowStatusUpdateService) (*HandlerServiceImpl, error) { impl := &HandlerServiceImpl{ logger: logger, cdWorkflowCommonService: cdWorkflowCommonService, diff --git a/pkg/pipeline/CdHandler.go b/pkg/pipeline/CdHandler.go index 5683db2269..0eac520103 100644 --- a/pkg/pipeline/CdHandler.go +++ b/pkg/pipeline/CdHandler.go @@ -32,7 +32,7 @@ import ( "github.com/devtron-labs/devtron/pkg/pipeline/workflowStatus" bean5 "github.com/devtron-labs/devtron/pkg/pipeline/workflowStatus/bean" "github.com/devtron-labs/devtron/pkg/workflow/cd" - "github.com/devtron-labs/devtron/pkg/workflow/status" + "github.com/devtron-labs/devtron/pkg/workflow/workflowStatusLatest" "slices" "strconv" "strings" @@ -91,7 +91,7 @@ type CdHandlerImpl struct { deploymentConfigService common2.DeploymentConfigService workflowStageStatusService workflowStatus.WorkFlowStageStatusService cdWorkflowRunnerService cd.CdWorkflowRunnerService - WorkflowStatusLatestService status.WorkflowStatusLatestService + WorkflowStatusLatestService workflowStatusLatest.WorkflowStatusLatestService pipelineStageRepository repository2.PipelineStageRepository } @@ -107,7 +107,7 @@ func NewCdHandlerImpl(Logger *zap.SugaredLogger, userService user.UserService, deploymentConfigService common2.DeploymentConfigService, workflowStageStatusService workflowStatus.WorkFlowStageStatusService, cdWorkflowRunnerService cd.CdWorkflowRunnerService, - WorkflowStatusLatestService status.WorkflowStatusLatestService, + WorkflowStatusLatestService workflowStatusLatest.WorkflowStatusLatestService, pipelineStageRepository repository2.PipelineStageRepository, ) *CdHandlerImpl { cdh := &CdHandlerImpl{ diff --git a/pkg/workflow/cd/CdWorkflowRunnerService.go b/pkg/workflow/cd/CdWorkflowRunnerService.go index 522c18c13a..49ea1f8928 100644 --- a/pkg/workflow/cd/CdWorkflowRunnerService.go +++ b/pkg/workflow/cd/CdWorkflowRunnerService.go @@ -28,7 +28,7 @@ import ( "github.com/devtron-labs/devtron/pkg/sql" "github.com/devtron-labs/devtron/pkg/workflow/cd/adapter" "github.com/devtron-labs/devtron/pkg/workflow/cd/bean" - "github.com/devtron-labs/devtron/pkg/workflow/status" + "github.com/devtron-labs/devtron/pkg/workflow/workflowStatusLatest" "github.com/devtron-labs/devtron/util" "github.com/go-pg/pg" "go.uber.org/zap" @@ -49,14 +49,14 @@ type CdWorkflowRunnerServiceImpl struct { workflowStageService workflowStatus.WorkFlowStageStatusService transactionManager sql.TransactionWrapper config *types.CiConfig - workflowStatusUpdateService status.WorkflowStatusUpdateService + workflowStatusUpdateService workflowStatusLatest.WorkflowStatusUpdateService } func NewCdWorkflowRunnerServiceImpl(logger *zap.SugaredLogger, cdWorkflowRepository pipelineConfig.CdWorkflowRepository, workflowStageService workflowStatus.WorkFlowStageStatusService, transactionManager sql.TransactionWrapper, - workflowStatusUpdateService status.WorkflowStatusUpdateService) *CdWorkflowRunnerServiceImpl { + workflowStatusUpdateService workflowStatusLatest.WorkflowStatusUpdateService) *CdWorkflowRunnerServiceImpl { impl := &CdWorkflowRunnerServiceImpl{ logger: logger, cdWorkflowRepository: cdWorkflowRepository, diff --git a/wire_gen.go b/wire_gen.go index 1623a1cb5c..3f5df6e799 100644 --- a/wire_gen.go +++ b/wire_gen.go @@ -276,6 +276,7 @@ import ( "github.com/devtron-labs/devtron/pkg/workflow/trigger/audit/hook" repository19 "github.com/devtron-labs/devtron/pkg/workflow/trigger/audit/repository" service3 "github.com/devtron-labs/devtron/pkg/workflow/trigger/audit/service" + "github.com/devtron-labs/devtron/pkg/workflow/workflowStatusLatest" util2 "github.com/devtron-labs/devtron/util" "github.com/devtron-labs/devtron/util/commonEnforcementFunctionsUtil" "github.com/devtron-labs/devtron/util/cron" @@ -623,7 +624,10 @@ func InitializeApp() (*App, error) { appListingServiceImpl := app2.NewAppListingServiceImpl(sugaredLogger, appListingRepositoryImpl, appDetailsReadServiceImpl, appRepositoryImpl, appListingViewBuilderImpl, pipelineRepositoryImpl, linkoutsRepositoryImpl, cdWorkflowRepositoryImpl, pipelineOverrideRepositoryImpl, environmentRepositoryImpl, chartRepositoryImpl, ciPipelineRepositoryImpl, dockerRegistryIpsConfigServiceImpl, userRepositoryImpl, deployedAppMetricsServiceImpl, ciArtifactRepositoryImpl, envConfigOverrideReadServiceImpl, ciPipelineConfigReadServiceImpl) workflowStageRepositoryImpl := repository18.NewWorkflowStageRepositoryImpl(sugaredLogger, db) workFlowStageStatusServiceImpl := workflowStatus.NewWorkflowStageFlowStatusServiceImpl(sugaredLogger, workflowStageRepositoryImpl, ciWorkflowRepositoryImpl, cdWorkflowRepositoryImpl, transactionUtilImpl) - cdWorkflowRunnerServiceImpl := cd.NewCdWorkflowRunnerServiceImpl(sugaredLogger, cdWorkflowRepositoryImpl, workFlowStageStatusServiceImpl, transactionUtilImpl) + workflowStatusLatestRepositoryImpl := pipelineConfig.NewWorkflowStatusLatestRepositoryImpl(db, sugaredLogger) + workflowStatusLatestServiceImpl := workflowStatusLatest.NewWorkflowStatusLatestServiceImpl(sugaredLogger, workflowStatusLatestRepositoryImpl, ciWorkflowRepositoryImpl, cdWorkflowRepositoryImpl) + workflowStatusUpdateServiceImpl := workflowStatusLatest.NewWorkflowStatusUpdateServiceImpl(sugaredLogger, workflowStatusLatestServiceImpl, ciWorkflowRepositoryImpl, cdWorkflowRepositoryImpl, ciPipelineRepositoryImpl, pipelineRepositoryImpl) + cdWorkflowRunnerServiceImpl := cd.NewCdWorkflowRunnerServiceImpl(sugaredLogger, cdWorkflowRepositoryImpl, workFlowStageStatusServiceImpl, transactionUtilImpl, workflowStatusUpdateServiceImpl) deploymentEventHandlerImpl := app2.NewDeploymentEventHandlerImpl(sugaredLogger, eventRESTClientImpl, eventSimpleFactoryImpl, runnable) appServiceImpl := app2.NewAppService(pipelineOverrideRepositoryImpl, utilMergeUtil, sugaredLogger, pipelineRepositoryImpl, eventRESTClientImpl, eventSimpleFactoryImpl, appRepositoryImpl, configMapRepositoryImpl, chartRepositoryImpl, cdWorkflowRepositoryImpl, commonServiceImpl, chartTemplateServiceImpl, pipelineStatusTimelineRepositoryImpl, pipelineStatusTimelineResourcesServiceImpl, pipelineStatusSyncDetailServiceImpl, pipelineStatusTimelineServiceImpl, appServiceConfig, appStatusServiceImpl, installedAppReadServiceImpl, installedAppVersionHistoryRepositoryImpl, scopedVariableCMCSManagerImpl, acdConfig, gitOpsConfigReadServiceImpl, gitOperationServiceImpl, deploymentTemplateServiceImpl, appListingServiceImpl, deploymentConfigServiceImpl, envConfigOverrideReadServiceImpl, cdWorkflowRunnerServiceImpl, deploymentEventHandlerImpl) scopedVariableManagerImpl, err := variables.NewScopedVariableManagerImpl(sugaredLogger, scopedVariableServiceImpl, variableEntityMappingServiceImpl, variableSnapshotHistoryServiceImpl, variableTemplateParserImpl) @@ -748,9 +752,8 @@ func InitializeApp() (*App, error) { deploymentTemplateValidationServiceEntImpl := validator.NewDeploymentTemplateValidationServiceEntImpl() deploymentTemplateValidationServiceImpl := validator.NewDeploymentTemplateValidationServiceImpl(sugaredLogger, chartRefServiceImpl, scopedVariableManagerImpl, deployedAppMetricsServiceImpl, deploymentTemplateValidationServiceEntImpl) devtronAppGitOpConfigServiceImpl := gitOpsConfig.NewDevtronAppGitOpConfigServiceImpl(sugaredLogger, chartRepositoryImpl, chartServiceImpl, gitOpsConfigReadServiceImpl, gitOpsValidationServiceImpl, argoClientWrapperServiceImpl, deploymentConfigServiceImpl, chartReadServiceImpl) - workflowStatusLatestRepositoryImpl := pipelineConfig.NewWorkflowStatusLatestRepositoryImpl(db, sugaredLogger) ciHandlerImpl := pipeline.NewCiHandlerImpl(sugaredLogger, ciServiceImpl, ciPipelineMaterialRepositoryImpl, clientImpl, ciWorkflowRepositoryImpl, ciArtifactRepositoryImpl, userServiceImpl, eventRESTClientImpl, eventSimpleFactoryImpl, ciPipelineRepositoryImpl, appListingRepositoryImpl, pipelineRepositoryImpl, enforcerUtilImpl, resourceGroupServiceImpl, environmentRepositoryImpl, imageTaggingServiceImpl, k8sCommonServiceImpl, appWorkflowRepositoryImpl, customTagServiceImpl, workFlowStageStatusServiceImpl, workflowStatusLatestRepositoryImpl) - cdHandlerImpl := pipeline.NewCdHandlerImpl(sugaredLogger, userServiceImpl, cdWorkflowRepositoryImpl, ciArtifactRepositoryImpl, ciPipelineMaterialRepositoryImpl, pipelineRepositoryImpl, environmentRepositoryImpl, ciWorkflowRepositoryImpl, enforcerUtilImpl, resourceGroupServiceImpl, imageTaggingServiceImpl, k8sServiceImpl, customTagServiceImpl, deploymentConfigServiceImpl, workFlowStageStatusServiceImpl, cdWorkflowRunnerServiceImpl) + cdHandlerImpl := pipeline.NewCdHandlerImpl(sugaredLogger, userServiceImpl, cdWorkflowRepositoryImpl, ciArtifactRepositoryImpl, ciPipelineMaterialRepositoryImpl, pipelineRepositoryImpl, environmentRepositoryImpl, ciWorkflowRepositoryImpl, enforcerUtilImpl, resourceGroupServiceImpl, imageTaggingServiceImpl, k8sServiceImpl, customTagServiceImpl, deploymentConfigServiceImpl, workFlowStageStatusServiceImpl, cdWorkflowRunnerServiceImpl, workflowStatusLatestServiceImpl, pipelineStageRepositoryImpl) appWorkflowServiceImpl := appWorkflow2.NewAppWorkflowServiceImpl(sugaredLogger, appWorkflowRepositoryImpl, ciCdPipelineOrchestratorImpl, ciPipelineRepositoryImpl, pipelineRepositoryImpl, enforcerUtilImpl, resourceGroupServiceImpl, appRepositoryImpl, userAuthServiceImpl, chartServiceImpl, deploymentConfigServiceImpl, pipelineBuilderImpl) appCloneServiceImpl := appClone.NewAppCloneServiceImpl(sugaredLogger, pipelineBuilderImpl, attributesServiceImpl, chartServiceImpl, configMapServiceImpl, appWorkflowServiceImpl, appListingServiceImpl, propertiesConfigServiceImpl, pipelineStageServiceImpl, ciTemplateReadServiceImpl, appRepositoryImpl, ciPipelineRepositoryImpl, pipelineRepositoryImpl, ciPipelineConfigServiceImpl, gitOpsConfigReadServiceImpl, chartReadServiceImpl) deploymentTemplateRepositoryImpl := repository2.NewDeploymentTemplateRepositoryImpl(db, sugaredLogger) @@ -781,7 +784,7 @@ func InitializeApp() (*App, error) { scanToolExecutionHistoryMappingRepositoryImpl := repository25.NewScanToolExecutionHistoryMappingRepositoryImpl(db, sugaredLogger) cdWorkflowReadServiceImpl := read20.NewCdWorkflowReadServiceImpl(sugaredLogger, cdWorkflowRepositoryImpl) imageScanServiceImpl := imageScanning.NewImageScanServiceImpl(sugaredLogger, imageScanHistoryRepositoryImpl, imageScanResultRepositoryImpl, imageScanObjectMetaRepositoryImpl, cveStoreRepositoryImpl, imageScanDeployInfoRepositoryImpl, userServiceImpl, appRepositoryImpl, environmentServiceImpl, ciArtifactRepositoryImpl, policyServiceImpl, pipelineRepositoryImpl, ciPipelineRepositoryImpl, scanToolMetadataRepositoryImpl, scanToolExecutionHistoryMappingRepositoryImpl, cvePolicyRepositoryImpl, cdWorkflowReadServiceImpl) - devtronAppsHandlerServiceImpl, err := devtronApps.NewHandlerServiceImpl(sugaredLogger, cdWorkflowCommonServiceImpl, gitOpsManifestPushServiceImpl, gitOpsConfigReadServiceImpl, argoK8sClientImpl, acdConfig, argoClientWrapperServiceImpl, pipelineStatusTimelineServiceImpl, chartTemplateServiceImpl, workflowEventPublishServiceImpl, manifestCreationServiceImpl, deployedConfigurationHistoryServiceImpl, pipelineStageServiceImpl, globalPluginServiceImpl, customTagServiceImpl, pluginInputVariableParserImpl, prePostCdScriptHistoryServiceImpl, scopedVariableCMCSManagerImpl, imageDigestPolicyServiceImpl, userServiceImpl, helmAppServiceImpl, enforcerUtilImpl, userDeploymentRequestServiceImpl, helmAppClientImpl, eventSimpleFactoryImpl, eventRESTClientImpl, environmentVariables, appRepositoryImpl, ciPipelineMaterialRepositoryImpl, imageScanHistoryReadServiceImpl, imageScanDeployInfoReadServiceImpl, imageScanDeployInfoServiceImpl, pipelineRepositoryImpl, pipelineOverrideRepositoryImpl, manifestPushConfigRepositoryImpl, chartRepositoryImpl, environmentRepositoryImpl, cdWorkflowRepositoryImpl, ciWorkflowRepositoryImpl, ciArtifactRepositoryImpl, ciTemplateReadServiceImpl, gitMaterialReadServiceImpl, appLabelRepositoryImpl, ciPipelineRepositoryImpl, appWorkflowRepositoryImpl, dockerArtifactStoreRepositoryImpl, imageScanServiceImpl, k8sServiceImpl, transactionUtilImpl, deploymentConfigServiceImpl, ciCdPipelineOrchestratorImpl, gitOperationServiceImpl, attributesServiceImpl, clusterRepositoryImpl, cdWorkflowRunnerServiceImpl, clusterServiceImplExtended, ciLogServiceImpl, workflowServiceImpl, blobStorageConfigServiceImpl, deploymentEventHandlerImpl, runnable, workflowTriggerAuditServiceImpl, deploymentServiceImpl) + devtronAppsHandlerServiceImpl, err := devtronApps.NewHandlerServiceImpl(sugaredLogger, cdWorkflowCommonServiceImpl, gitOpsManifestPushServiceImpl, gitOpsConfigReadServiceImpl, argoK8sClientImpl, acdConfig, argoClientWrapperServiceImpl, pipelineStatusTimelineServiceImpl, chartTemplateServiceImpl, workflowEventPublishServiceImpl, manifestCreationServiceImpl, deployedConfigurationHistoryServiceImpl, pipelineStageServiceImpl, globalPluginServiceImpl, customTagServiceImpl, pluginInputVariableParserImpl, prePostCdScriptHistoryServiceImpl, scopedVariableCMCSManagerImpl, imageDigestPolicyServiceImpl, userServiceImpl, helmAppServiceImpl, enforcerUtilImpl, userDeploymentRequestServiceImpl, helmAppClientImpl, eventSimpleFactoryImpl, eventRESTClientImpl, environmentVariables, appRepositoryImpl, ciPipelineMaterialRepositoryImpl, imageScanHistoryReadServiceImpl, imageScanDeployInfoReadServiceImpl, imageScanDeployInfoServiceImpl, pipelineRepositoryImpl, pipelineOverrideRepositoryImpl, manifestPushConfigRepositoryImpl, chartRepositoryImpl, environmentRepositoryImpl, cdWorkflowRepositoryImpl, ciWorkflowRepositoryImpl, ciArtifactRepositoryImpl, ciTemplateReadServiceImpl, gitMaterialReadServiceImpl, appLabelRepositoryImpl, ciPipelineRepositoryImpl, appWorkflowRepositoryImpl, dockerArtifactStoreRepositoryImpl, imageScanServiceImpl, k8sServiceImpl, transactionUtilImpl, deploymentConfigServiceImpl, ciCdPipelineOrchestratorImpl, gitOperationServiceImpl, attributesServiceImpl, clusterRepositoryImpl, cdWorkflowRunnerServiceImpl, clusterServiceImplExtended, ciLogServiceImpl, workflowServiceImpl, blobStorageConfigServiceImpl, deploymentEventHandlerImpl, runnable, workflowTriggerAuditServiceImpl, deploymentServiceImpl, workflowStatusUpdateServiceImpl) if err != nil { return nil, err } From 48468ce41623d603b0f4f7664340ff32c7804c87 Mon Sep 17 00:00:00 2001 From: ayushmaheshwari Date: Wed, 23 Jul 2025 16:51:16 +0530 Subject: [PATCH 17/31] refactoring --- pkg/pipeline/CdHandler.go | 92 ++-------------- .../cd/read/CdWorkflowRunnerReadService.go | 102 +++++++++++++++++- wire_gen.go | 20 ++-- 3 files changed, 114 insertions(+), 100 deletions(-) diff --git a/pkg/pipeline/CdHandler.go b/pkg/pipeline/CdHandler.go index 0eac520103..ec4089c2eb 100644 --- a/pkg/pipeline/CdHandler.go +++ b/pkg/pipeline/CdHandler.go @@ -32,6 +32,7 @@ import ( "github.com/devtron-labs/devtron/pkg/pipeline/workflowStatus" bean5 "github.com/devtron-labs/devtron/pkg/pipeline/workflowStatus/bean" "github.com/devtron-labs/devtron/pkg/workflow/cd" + "github.com/devtron-labs/devtron/pkg/workflow/cd/read" "github.com/devtron-labs/devtron/pkg/workflow/workflowStatusLatest" "slices" "strconv" @@ -93,6 +94,7 @@ type CdHandlerImpl struct { cdWorkflowRunnerService cd.CdWorkflowRunnerService WorkflowStatusLatestService workflowStatusLatest.WorkflowStatusLatestService pipelineStageRepository repository2.PipelineStageRepository + cdWorkflowRunnerReadService read.CdWorkflowRunnerReadService } func NewCdHandlerImpl(Logger *zap.SugaredLogger, userService user.UserService, @@ -109,6 +111,7 @@ func NewCdHandlerImpl(Logger *zap.SugaredLogger, userService user.UserService, cdWorkflowRunnerService cd.CdWorkflowRunnerService, WorkflowStatusLatestService workflowStatusLatest.WorkflowStatusLatestService, pipelineStageRepository repository2.PipelineStageRepository, + cdWorkflowRunnerReadService read.CdWorkflowRunnerReadService, ) *CdHandlerImpl { cdh := &CdHandlerImpl{ Logger: Logger, @@ -129,6 +132,7 @@ func NewCdHandlerImpl(Logger *zap.SugaredLogger, userService user.UserService, cdWorkflowRunnerService: cdWorkflowRunnerService, WorkflowStatusLatestService: WorkflowStatusLatestService, pipelineStageRepository: pipelineStageRepository, + cdWorkflowRunnerReadService: cdWorkflowRunnerReadService, } config, err := types.GetCdConfig() if err != nil { @@ -602,7 +606,7 @@ func (impl *CdHandlerImpl) FetchAppWorkflowStatusForTriggerView(appId int) ([]*p return cdWorkflowStatus, nil } - result, err := impl.getWfrStatusForLatestRunners(pipelineIds, pipelines) + result, err := impl.cdWorkflowRunnerReadService.GetWfrStatusForLatestRunners(pipelineIds, pipelines) if err != nil { impl.Logger.Errorw("error in fetching wfrIds", "pipelineIds", pipelineIds, "err", err) return cdWorkflowStatus, err @@ -692,88 +696,6 @@ func (impl *CdHandlerImpl) FetchAppWorkflowStatusForTriggerView(appId int) ([]*p return cdWorkflowStatus, err } -func (impl *CdHandlerImpl) getWfrStatusForLatestRunners(pipelineIds []int, pipelines []*pipelineConfig.Pipeline) ([]*pipelineConfig.CdWorkflowStatus, error) { - // fetching the latest pipeline from the index table - cdWorkflowLatest - var result []*pipelineConfig.CdWorkflowStatus - cdWorkflowLatest, err := impl.WorkflowStatusLatestService.GetCdWorkflowLatestByPipelineIds(pipelineIds) - if err != nil { - impl.Logger.Errorw("error in getting latest by pipelineId", "pipelineId", pipelineIds, "err", err) - return nil, err - } - - var pipelineIdToCiPipelineIdMap map[int]int - for _, item := range pipelines { - pipelineIdToCiPipelineIdMap[item.Id] = item.CiPipelineId - } - - for _, item := range cdWorkflowLatest { - result = append(result, &pipelineConfig.CdWorkflowStatus{ - CiPipelineId: pipelineIdToCiPipelineIdMap[item.PipelineId], - PipelineId: item.PipelineId, - WorkflowType: item.WorkflowType, - WfrId: item.WorkflowRunnerId, - }) - } - - var cdWorfklowLatestMap map[int]map[bean.WorkflowType]bool - for _, item := range cdWorkflowLatest { - if _, ok := cdWorfklowLatestMap[item.PipelineId]; !ok { - cdWorfklowLatestMap[item.PipelineId] = make(map[bean.WorkflowType]bool) - } - cdWorfklowLatestMap[item.PipelineId][bean.WorkflowType(item.WorkflowType)] = true - } - - pipelineStage, err := impl.pipelineStageRepository.GetAllCdStagesByCdPipelineIds(pipelineIds) - if err != nil { - impl.Logger.Errorw("error in fetching pipeline stages", "pipelineId", pipelineIds, "err", err) - return nil, err - } - pipelineStageMap := make(map[int]map[bean.WorkflowType]bool) - for _, item := range pipelineStage { - if _, ok := pipelineStageMap[item.CdPipelineId]; !ok { - pipelineStageMap[item.CdPipelineId] = make(map[bean.WorkflowType]bool) - } - if item.Type == repository2.PIPELINE_STAGE_TYPE_PRE_CD { - pipelineStageMap[item.CdPipelineId][bean.CD_WORKFLOW_TYPE_PRE] = true - } else if item.Type == repository2.PIPELINE_STAGE_TYPE_POST_CD { - pipelineStageMap[item.CdPipelineId][bean.CD_WORKFLOW_TYPE_POST] = true - } - } - - // calculating all the pipelines not present in the index table cdWorkflowLatest - var pipelinesAbsentInCache map[int]bean.WorkflowType - for _, item := range pipelines { - if _, ok := cdWorfklowLatestMap[item.Id]; !ok { - pipelinesAbsentInCache[item.Id] = bean.CD_WORKFLOW_TYPE_PRE - pipelinesAbsentInCache[item.Id] = bean.CD_WORKFLOW_TYPE_DEPLOY - pipelinesAbsentInCache[item.Id] = bean.CD_WORKFLOW_TYPE_POST - } else { - if _, ok := pipelineStageMap[item.Id][bean.CD_WORKFLOW_TYPE_PRE]; ok { - if val, ok := cdWorfklowLatestMap[item.Id][bean.CD_WORKFLOW_TYPE_PRE]; !ok || !val { - pipelinesAbsentInCache[item.Id] = bean.CD_WORKFLOW_TYPE_PRE - } - } - if _, ok := pipelineStageMap[item.Id][bean.CD_WORKFLOW_TYPE_POST]; ok { - if val, ok := cdWorfklowLatestMap[item.Id][bean.CD_WORKFLOW_TYPE_POST]; !ok || !val { - pipelinesAbsentInCache[item.Id] = bean.CD_WORKFLOW_TYPE_POST - } - } - if val, ok := cdWorfklowLatestMap[item.Id][bean.CD_WORKFLOW_TYPE_DEPLOY]; !ok || !val { - pipelinesAbsentInCache[item.Id] = bean.CD_WORKFLOW_TYPE_POST - } - } - } - if len(pipelinesAbsentInCache) > 0 { - remainingRunners, err := impl.cdWorkflowRepository.FetchAllCdStagesLatestEntity(pipelinesAbsentInCache) - if err != nil { - impl.Logger.Errorw("error in fetching all cd stages latest entity", "pipelinesAbsentInCache", pipelinesAbsentInCache, "err", err) - return nil, err - } - result = append(result, remainingRunners...) - } - return result, nil -} - func (impl *CdHandlerImpl) FetchAppWorkflowStatusForTriggerViewForEnvironment(request resourceGroup2.ResourceGroupingRequest, token string) ([]*pipelineConfig.CdWorkflowStatus, error) { cdWorkflowStatus := make([]*pipelineConfig.CdWorkflowStatus, 0) var pipelines []*pipelineConfig.Pipeline @@ -848,7 +770,7 @@ func (impl *CdHandlerImpl) FetchAppWorkflowStatusForTriggerViewForEnvironment(re cdMap := make(map[int]*pipelineConfig.CdWorkflowStatus) - wfrStatus, err := impl.getWfrStatusForLatestRunners(pipelineIds, pipelines) + wfrStatus, err := impl.cdWorkflowRunnerReadService.GetWfrStatusForLatestRunners(pipelineIds, pipelines) if err != nil { impl.Logger.Errorw("error in fetching wfrIds", "pipelineIds", pipelineIds, "err", err) return cdWorkflowStatus, err @@ -1000,7 +922,7 @@ func (impl *CdHandlerImpl) FetchAppDeploymentStatusForEnvironments(request resou return deploymentStatuses, nil } _, span = otel.Tracer("orchestrator").Start(request.Ctx, "pipelineBuilder.FetchAllCdStagesLatestEntity") - result, err := impl.getWfrStatusForLatestRunners(pipelineIds, cdPipelines) + result, err := impl.cdWorkflowRunnerReadService.GetWfrStatusForLatestRunners(pipelineIds, cdPipelines) span.End() if err != nil { return deploymentStatuses, err diff --git a/pkg/workflow/cd/read/CdWorkflowRunnerReadService.go b/pkg/workflow/cd/read/CdWorkflowRunnerReadService.go index e9b1fefb4c..331077c090 100644 --- a/pkg/workflow/cd/read/CdWorkflowRunnerReadService.go +++ b/pkg/workflow/cd/read/CdWorkflowRunnerReadService.go @@ -1,9 +1,12 @@ package read import ( + bean2 "github.com/devtron-labs/devtron/api/bean" "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" + repository2 "github.com/devtron-labs/devtron/pkg/pipeline/repository" "github.com/devtron-labs/devtron/pkg/workflow/cd/adapter" "github.com/devtron-labs/devtron/pkg/workflow/cd/bean" + "github.com/devtron-labs/devtron/pkg/workflow/workflowStatusLatest" "github.com/go-pg/pg" "go.uber.org/zap" ) @@ -11,18 +14,25 @@ import ( type CdWorkflowRunnerReadService interface { FindWorkflowRunnerById(wfrId int) (*bean.CdWorkflowRunnerDto, error) CheckIfWfrLatest(wfrId, pipelineId int) (isLatest bool, err error) + GetWfrStatusForLatestRunners(pipelineIds []int, pipelines []*pipelineConfig.Pipeline) ([]*pipelineConfig.CdWorkflowStatus, error) } type CdWorkflowRunnerReadServiceImpl struct { - logger *zap.SugaredLogger - cdWorkflowRepository pipelineConfig.CdWorkflowRepository + logger *zap.SugaredLogger + cdWorkflowRepository pipelineConfig.CdWorkflowRepository + WorkflowStatusLatestService workflowStatusLatest.WorkflowStatusLatestService + pipelineStageRepository repository2.PipelineStageRepository } func NewCdWorkflowRunnerReadServiceImpl(logger *zap.SugaredLogger, - cdWorkflowRepository pipelineConfig.CdWorkflowRepository) *CdWorkflowRunnerReadServiceImpl { + cdWorkflowRepository pipelineConfig.CdWorkflowRepository, + WorkflowStatusLatestService workflowStatusLatest.WorkflowStatusLatestService, + pipelineStageRepository repository2.PipelineStageRepository) *CdWorkflowRunnerReadServiceImpl { return &CdWorkflowRunnerReadServiceImpl{ - logger: logger, - cdWorkflowRepository: cdWorkflowRepository, + logger: logger, + cdWorkflowRepository: cdWorkflowRepository, + WorkflowStatusLatestService: WorkflowStatusLatestService, + pipelineStageRepository: pipelineStageRepository, } } @@ -44,3 +54,85 @@ func (impl *CdWorkflowRunnerReadServiceImpl) CheckIfWfrLatest(wfrId, pipelineId } return isLatest, nil } + +func (impl *CdWorkflowRunnerReadServiceImpl) GetWfrStatusForLatestRunners(pipelineIds []int, pipelines []*pipelineConfig.Pipeline) ([]*pipelineConfig.CdWorkflowStatus, error) { + // fetching the latest pipeline from the index table - cdWorkflowLatest + var result []*pipelineConfig.CdWorkflowStatus + cdWorkflowLatest, err := impl.WorkflowStatusLatestService.GetCdWorkflowLatestByPipelineIds(pipelineIds) + if err != nil { + impl.logger.Errorw("error in getting latest by pipelineId", "pipelineId", pipelineIds, "err", err) + return nil, err + } + + var pipelineIdToCiPipelineIdMap map[int]int + for _, item := range pipelines { + pipelineIdToCiPipelineIdMap[item.Id] = item.CiPipelineId + } + + for _, item := range cdWorkflowLatest { + result = append(result, &pipelineConfig.CdWorkflowStatus{ + CiPipelineId: pipelineIdToCiPipelineIdMap[item.PipelineId], + PipelineId: item.PipelineId, + WorkflowType: item.WorkflowType, + WfrId: item.WorkflowRunnerId, + }) + } + + var cdWorfklowLatestMap map[int]map[bean2.WorkflowType]bool + for _, item := range cdWorkflowLatest { + if _, ok := cdWorfklowLatestMap[item.PipelineId]; !ok { + cdWorfklowLatestMap[item.PipelineId] = make(map[bean2.WorkflowType]bool) + } + cdWorfklowLatestMap[item.PipelineId][bean2.WorkflowType(item.WorkflowType)] = true + } + + pipelineStage, err := impl.pipelineStageRepository.GetAllCdStagesByCdPipelineIds(pipelineIds) + if err != nil { + impl.logger.Errorw("error in fetching pipeline stages", "pipelineId", pipelineIds, "err", err) + return nil, err + } + pipelineStageMap := make(map[int]map[bean2.WorkflowType]bool) + for _, item := range pipelineStage { + if _, ok := pipelineStageMap[item.CdPipelineId]; !ok { + pipelineStageMap[item.CdPipelineId] = make(map[bean2.WorkflowType]bool) + } + if item.Type == repository2.PIPELINE_STAGE_TYPE_PRE_CD { + pipelineStageMap[item.CdPipelineId][bean2.CD_WORKFLOW_TYPE_PRE] = true + } else if item.Type == repository2.PIPELINE_STAGE_TYPE_POST_CD { + pipelineStageMap[item.CdPipelineId][bean2.CD_WORKFLOW_TYPE_POST] = true + } + } + + // calculating all the pipelines not present in the index table cdWorkflowLatest + var pipelinesAbsentInCache map[int]bean2.WorkflowType + for _, item := range pipelines { + if _, ok := cdWorfklowLatestMap[item.Id]; !ok { + pipelinesAbsentInCache[item.Id] = bean2.CD_WORKFLOW_TYPE_PRE + pipelinesAbsentInCache[item.Id] = bean2.CD_WORKFLOW_TYPE_DEPLOY + pipelinesAbsentInCache[item.Id] = bean2.CD_WORKFLOW_TYPE_POST + } else { + if _, ok := pipelineStageMap[item.Id][bean2.CD_WORKFLOW_TYPE_PRE]; ok { + if val, ok := cdWorfklowLatestMap[item.Id][bean2.CD_WORKFLOW_TYPE_PRE]; !ok || !val { + pipelinesAbsentInCache[item.Id] = bean2.CD_WORKFLOW_TYPE_PRE + } + } + if _, ok := pipelineStageMap[item.Id][bean2.CD_WORKFLOW_TYPE_POST]; ok { + if val, ok := cdWorfklowLatestMap[item.Id][bean2.CD_WORKFLOW_TYPE_POST]; !ok || !val { + pipelinesAbsentInCache[item.Id] = bean2.CD_WORKFLOW_TYPE_POST + } + } + if val, ok := cdWorfklowLatestMap[item.Id][bean2.CD_WORKFLOW_TYPE_DEPLOY]; !ok || !val { + pipelinesAbsentInCache[item.Id] = bean2.CD_WORKFLOW_TYPE_POST + } + } + } + if len(pipelinesAbsentInCache) > 0 { + remainingRunners, err := impl.cdWorkflowRepository.FetchAllCdStagesLatestEntity(pipelinesAbsentInCache) + if err != nil { + impl.logger.Errorw("error in fetching all cd stages latest entity", "pipelinesAbsentInCache", pipelinesAbsentInCache, "err", err) + return nil, err + } + result = append(result, remainingRunners...) + } + return result, nil +} diff --git a/wire_gen.go b/wire_gen.go index 3f5df6e799..1ba1dd8001 100644 --- a/wire_gen.go +++ b/wire_gen.go @@ -185,7 +185,7 @@ import ( "github.com/devtron-labs/devtron/pkg/deployment/gitOps/validation" "github.com/devtron-labs/devtron/pkg/deployment/manifest" "github.com/devtron-labs/devtron/pkg/deployment/manifest/configMapAndSecret" - read19 "github.com/devtron-labs/devtron/pkg/deployment/manifest/configMapAndSecret/read" + read20 "github.com/devtron-labs/devtron/pkg/deployment/manifest/configMapAndSecret/read" "github.com/devtron-labs/devtron/pkg/deployment/manifest/deployedAppMetrics" repository17 "github.com/devtron-labs/devtron/pkg/deployment/manifest/deployedAppMetrics/repository" "github.com/devtron-labs/devtron/pkg/deployment/manifest/deploymentTemplate" @@ -248,7 +248,7 @@ import ( "github.com/devtron-labs/devtron/pkg/plugin" repository21 "github.com/devtron-labs/devtron/pkg/plugin/repository" "github.com/devtron-labs/devtron/pkg/policyGovernance/security/imageScanning" - read18 "github.com/devtron-labs/devtron/pkg/policyGovernance/security/imageScanning/read" + read19 "github.com/devtron-labs/devtron/pkg/policyGovernance/security/imageScanning/read" repository25 "github.com/devtron-labs/devtron/pkg/policyGovernance/security/imageScanning/repository" "github.com/devtron-labs/devtron/pkg/policyGovernance/security/scanTool" repository16 "github.com/devtron-labs/devtron/pkg/policyGovernance/security/scanTool/repository" @@ -270,7 +270,7 @@ import ( repository13 "github.com/devtron-labs/devtron/pkg/variables/repository" "github.com/devtron-labs/devtron/pkg/webhook/helm" "github.com/devtron-labs/devtron/pkg/workflow/cd" - read20 "github.com/devtron-labs/devtron/pkg/workflow/cd/read" + read18 "github.com/devtron-labs/devtron/pkg/workflow/cd/read" "github.com/devtron-labs/devtron/pkg/workflow/dag" status2 "github.com/devtron-labs/devtron/pkg/workflow/status" "github.com/devtron-labs/devtron/pkg/workflow/trigger/audit/hook" @@ -753,7 +753,8 @@ func InitializeApp() (*App, error) { deploymentTemplateValidationServiceImpl := validator.NewDeploymentTemplateValidationServiceImpl(sugaredLogger, chartRefServiceImpl, scopedVariableManagerImpl, deployedAppMetricsServiceImpl, deploymentTemplateValidationServiceEntImpl) devtronAppGitOpConfigServiceImpl := gitOpsConfig.NewDevtronAppGitOpConfigServiceImpl(sugaredLogger, chartRepositoryImpl, chartServiceImpl, gitOpsConfigReadServiceImpl, gitOpsValidationServiceImpl, argoClientWrapperServiceImpl, deploymentConfigServiceImpl, chartReadServiceImpl) ciHandlerImpl := pipeline.NewCiHandlerImpl(sugaredLogger, ciServiceImpl, ciPipelineMaterialRepositoryImpl, clientImpl, ciWorkflowRepositoryImpl, ciArtifactRepositoryImpl, userServiceImpl, eventRESTClientImpl, eventSimpleFactoryImpl, ciPipelineRepositoryImpl, appListingRepositoryImpl, pipelineRepositoryImpl, enforcerUtilImpl, resourceGroupServiceImpl, environmentRepositoryImpl, imageTaggingServiceImpl, k8sCommonServiceImpl, appWorkflowRepositoryImpl, customTagServiceImpl, workFlowStageStatusServiceImpl, workflowStatusLatestRepositoryImpl) - cdHandlerImpl := pipeline.NewCdHandlerImpl(sugaredLogger, userServiceImpl, cdWorkflowRepositoryImpl, ciArtifactRepositoryImpl, ciPipelineMaterialRepositoryImpl, pipelineRepositoryImpl, environmentRepositoryImpl, ciWorkflowRepositoryImpl, enforcerUtilImpl, resourceGroupServiceImpl, imageTaggingServiceImpl, k8sServiceImpl, customTagServiceImpl, deploymentConfigServiceImpl, workFlowStageStatusServiceImpl, cdWorkflowRunnerServiceImpl, workflowStatusLatestServiceImpl, pipelineStageRepositoryImpl) + cdWorkflowRunnerReadServiceImpl := read18.NewCdWorkflowRunnerReadServiceImpl(sugaredLogger, cdWorkflowRepositoryImpl, workflowStatusLatestServiceImpl, pipelineStageRepositoryImpl) + cdHandlerImpl := pipeline.NewCdHandlerImpl(sugaredLogger, userServiceImpl, cdWorkflowRepositoryImpl, ciArtifactRepositoryImpl, ciPipelineMaterialRepositoryImpl, pipelineRepositoryImpl, environmentRepositoryImpl, ciWorkflowRepositoryImpl, enforcerUtilImpl, resourceGroupServiceImpl, imageTaggingServiceImpl, k8sServiceImpl, customTagServiceImpl, deploymentConfigServiceImpl, workFlowStageStatusServiceImpl, cdWorkflowRunnerServiceImpl, workflowStatusLatestServiceImpl, pipelineStageRepositoryImpl, cdWorkflowRunnerReadServiceImpl) appWorkflowServiceImpl := appWorkflow2.NewAppWorkflowServiceImpl(sugaredLogger, appWorkflowRepositoryImpl, ciCdPipelineOrchestratorImpl, ciPipelineRepositoryImpl, pipelineRepositoryImpl, enforcerUtilImpl, resourceGroupServiceImpl, appRepositoryImpl, userAuthServiceImpl, chartServiceImpl, deploymentConfigServiceImpl, pipelineBuilderImpl) appCloneServiceImpl := appClone.NewAppCloneServiceImpl(sugaredLogger, pipelineBuilderImpl, attributesServiceImpl, chartServiceImpl, configMapServiceImpl, appWorkflowServiceImpl, appListingServiceImpl, propertiesConfigServiceImpl, pipelineStageServiceImpl, ciTemplateReadServiceImpl, appRepositoryImpl, ciPipelineRepositoryImpl, pipelineRepositoryImpl, ciPipelineConfigServiceImpl, gitOpsConfigReadServiceImpl, chartReadServiceImpl) deploymentTemplateRepositoryImpl := repository2.NewDeploymentTemplateRepositoryImpl(db, sugaredLogger) @@ -767,22 +768,22 @@ func InitializeApp() (*App, error) { imageScanDeployInfoRepositoryImpl := repository25.NewImageScanDeployInfoRepositoryImpl(db, sugaredLogger) imageScanObjectMetaRepositoryImpl := repository25.NewImageScanObjectMetaRepositoryImpl(db, sugaredLogger) imageScanHistoryRepositoryImpl := repository25.NewImageScanHistoryRepositoryImpl(db, sugaredLogger) - imageScanHistoryReadServiceImpl := read18.NewImageScanHistoryReadService(sugaredLogger, imageScanHistoryRepositoryImpl) + imageScanHistoryReadServiceImpl := read19.NewImageScanHistoryReadService(sugaredLogger, imageScanHistoryRepositoryImpl) cveStoreRepositoryImpl := repository25.NewCveStoreRepositoryImpl(db, sugaredLogger) policyServiceImpl := imageScanning.NewPolicyServiceImpl(environmentServiceImpl, sugaredLogger, appRepositoryImpl, pipelineOverrideRepositoryImpl, cvePolicyRepositoryImpl, clusterServiceImplExtended, pipelineRepositoryImpl, imageScanResultRepositoryImpl, imageScanDeployInfoRepositoryImpl, imageScanObjectMetaRepositoryImpl, httpClient, ciArtifactRepositoryImpl, ciCdConfig, imageScanHistoryReadServiceImpl, cveStoreRepositoryImpl, ciTemplateRepositoryImpl, clusterReadServiceImpl, transactionUtilImpl) - imageScanResultReadServiceImpl := read18.NewImageScanResultReadServiceImpl(sugaredLogger, imageScanResultRepositoryImpl) + imageScanResultReadServiceImpl := read19.NewImageScanResultReadServiceImpl(sugaredLogger, imageScanResultRepositoryImpl) draftAwareConfigServiceImpl := draftAwareConfigService.NewDraftAwareResourceServiceImpl(sugaredLogger, configMapServiceImpl, chartServiceImpl, propertiesConfigServiceImpl) gitOpsManifestPushServiceImpl := publish.NewGitOpsManifestPushServiceImpl(sugaredLogger, pipelineStatusTimelineServiceImpl, pipelineOverrideRepositoryImpl, acdConfig, chartRefServiceImpl, gitOpsConfigReadServiceImpl, chartServiceImpl, gitOperationServiceImpl, argoClientWrapperServiceImpl, transactionUtilImpl, deploymentConfigServiceImpl, chartTemplateServiceImpl) manifestCreationServiceImpl := manifest.NewManifestCreationServiceImpl(sugaredLogger, dockerRegistryIpsConfigServiceImpl, chartRefServiceImpl, scopedVariableCMCSManagerImpl, k8sCommonServiceImpl, deployedAppMetricsServiceImpl, imageDigestPolicyServiceImpl, utilMergeUtil, appCrudOperationServiceImpl, deploymentTemplateServiceImpl, argoClientWrapperServiceImpl, configMapHistoryRepositoryImpl, configMapRepositoryImpl, chartRepositoryImpl, envConfigOverrideRepositoryImpl, environmentRepositoryImpl, pipelineRepositoryImpl, ciArtifactRepositoryImpl, pipelineOverrideRepositoryImpl, pipelineStrategyHistoryRepositoryImpl, pipelineConfigRepositoryImpl, deploymentTemplateHistoryRepositoryImpl, deploymentConfigServiceImpl, envConfigOverrideReadServiceImpl) - configMapHistoryReadServiceImpl := read19.NewConfigMapHistoryReadService(sugaredLogger, configMapHistoryRepositoryImpl, scopedVariableCMCSManagerImpl) + configMapHistoryReadServiceImpl := read20.NewConfigMapHistoryReadService(sugaredLogger, configMapHistoryRepositoryImpl, scopedVariableCMCSManagerImpl) deployedConfigurationHistoryServiceImpl := history.NewDeployedConfigurationHistoryServiceImpl(sugaredLogger, userServiceImpl, deploymentTemplateHistoryServiceImpl, pipelineStrategyHistoryServiceImpl, configMapHistoryServiceImpl, cdWorkflowRepositoryImpl, scopedVariableCMCSManagerImpl, deploymentTemplateHistoryReadServiceImpl, configMapHistoryReadServiceImpl) userDeploymentRequestRepositoryImpl := repository26.NewUserDeploymentRequestRepositoryImpl(db, transactionUtilImpl) userDeploymentRequestServiceImpl := service4.NewUserDeploymentRequestServiceImpl(sugaredLogger, userDeploymentRequestRepositoryImpl) - imageScanDeployInfoReadServiceImpl := read18.NewImageScanDeployInfoReadService(sugaredLogger, imageScanDeployInfoRepositoryImpl) + imageScanDeployInfoReadServiceImpl := read19.NewImageScanDeployInfoReadService(sugaredLogger, imageScanDeployInfoRepositoryImpl) imageScanDeployInfoServiceImpl := imageScanning.NewImageScanDeployInfoService(sugaredLogger, imageScanDeployInfoRepositoryImpl) manifestPushConfigRepositoryImpl := repository20.NewManifestPushConfigRepository(sugaredLogger, db) scanToolExecutionHistoryMappingRepositoryImpl := repository25.NewScanToolExecutionHistoryMappingRepositoryImpl(db, sugaredLogger) - cdWorkflowReadServiceImpl := read20.NewCdWorkflowReadServiceImpl(sugaredLogger, cdWorkflowRepositoryImpl) + cdWorkflowReadServiceImpl := read18.NewCdWorkflowReadServiceImpl(sugaredLogger, cdWorkflowRepositoryImpl) imageScanServiceImpl := imageScanning.NewImageScanServiceImpl(sugaredLogger, imageScanHistoryRepositoryImpl, imageScanResultRepositoryImpl, imageScanObjectMetaRepositoryImpl, cveStoreRepositoryImpl, imageScanDeployInfoRepositoryImpl, userServiceImpl, appRepositoryImpl, environmentServiceImpl, ciArtifactRepositoryImpl, policyServiceImpl, pipelineRepositoryImpl, ciPipelineRepositoryImpl, scanToolMetadataRepositoryImpl, scanToolExecutionHistoryMappingRepositoryImpl, cvePolicyRepositoryImpl, cdWorkflowReadServiceImpl) devtronAppsHandlerServiceImpl, err := devtronApps.NewHandlerServiceImpl(sugaredLogger, cdWorkflowCommonServiceImpl, gitOpsManifestPushServiceImpl, gitOpsConfigReadServiceImpl, argoK8sClientImpl, acdConfig, argoClientWrapperServiceImpl, pipelineStatusTimelineServiceImpl, chartTemplateServiceImpl, workflowEventPublishServiceImpl, manifestCreationServiceImpl, deployedConfigurationHistoryServiceImpl, pipelineStageServiceImpl, globalPluginServiceImpl, customTagServiceImpl, pluginInputVariableParserImpl, prePostCdScriptHistoryServiceImpl, scopedVariableCMCSManagerImpl, imageDigestPolicyServiceImpl, userServiceImpl, helmAppServiceImpl, enforcerUtilImpl, userDeploymentRequestServiceImpl, helmAppClientImpl, eventSimpleFactoryImpl, eventRESTClientImpl, environmentVariables, appRepositoryImpl, ciPipelineMaterialRepositoryImpl, imageScanHistoryReadServiceImpl, imageScanDeployInfoReadServiceImpl, imageScanDeployInfoServiceImpl, pipelineRepositoryImpl, pipelineOverrideRepositoryImpl, manifestPushConfigRepositoryImpl, chartRepositoryImpl, environmentRepositoryImpl, cdWorkflowRepositoryImpl, ciWorkflowRepositoryImpl, ciArtifactRepositoryImpl, ciTemplateReadServiceImpl, gitMaterialReadServiceImpl, appLabelRepositoryImpl, ciPipelineRepositoryImpl, appWorkflowRepositoryImpl, dockerArtifactStoreRepositoryImpl, imageScanServiceImpl, k8sServiceImpl, transactionUtilImpl, deploymentConfigServiceImpl, ciCdPipelineOrchestratorImpl, gitOperationServiceImpl, attributesServiceImpl, clusterRepositoryImpl, cdWorkflowRunnerServiceImpl, clusterServiceImplExtended, ciLogServiceImpl, workflowServiceImpl, blobStorageConfigServiceImpl, deploymentEventHandlerImpl, runnable, workflowTriggerAuditServiceImpl, deploymentServiceImpl, workflowStatusUpdateServiceImpl) if err != nil { @@ -1103,7 +1104,6 @@ func InitializeApp() (*App, error) { muxRouter := router.NewMuxRouter(sugaredLogger, environmentRouterImpl, clusterRouterImpl, webhookRouterImpl, userAuthRouterImpl, gitProviderRouterImpl, gitHostRouterImpl, dockerRegRouterImpl, notificationRouterImpl, teamRouterImpl, userRouterImpl, chartRefRouterImpl, configMapRouterImpl, appStoreRouterImpl, chartRepositoryRouterImpl, releaseMetricsRouterImpl, deploymentGroupRouterImpl, batchOperationRouterImpl, chartGroupRouterImpl, imageScanRouterImpl, policyRouterImpl, gitOpsConfigRouterImpl, dashboardRouterImpl, attributesRouterImpl, userAttributesRouterImpl, commonRouterImpl, grafanaRouterImpl, ssoLoginRouterImpl, telemetryRouterImpl, telemetryEventClientImplExtended, bulkUpdateRouterImpl, webhookListenerRouterImpl, appRouterImpl, coreAppRouterImpl, helmAppRouterImpl, k8sApplicationRouterImpl, pProfRouterImpl, deploymentConfigRouterImpl, dashboardTelemetryRouterImpl, commonDeploymentRouterImpl, externalLinkRouterImpl, globalPluginRouterImpl, moduleRouterImpl, serverRouterImpl, apiTokenRouterImpl, cdApplicationStatusUpdateHandlerImpl, k8sCapacityRouterImpl, webhookHelmRouterImpl, globalCMCSRouterImpl, userTerminalAccessRouterImpl, jobRouterImpl, ciStatusUpdateCronImpl, resourceGroupingRouterImpl, rbacRoleRouterImpl, scopedVariableRouterImpl, ciTriggerCronImpl, proxyRouterImpl, deploymentConfigurationRouterImpl, infraConfigRouterImpl, argoApplicationRouterImpl, devtronResourceRouterImpl, fluxApplicationRouterImpl, scanningResultRouterImpl, routerImpl) loggingMiddlewareImpl := util4.NewLoggingMiddlewareImpl(userServiceImpl) cdWorkflowServiceImpl := cd.NewCdWorkflowServiceImpl(sugaredLogger, cdWorkflowRepositoryImpl) - cdWorkflowRunnerReadServiceImpl := read20.NewCdWorkflowRunnerReadServiceImpl(sugaredLogger, cdWorkflowRepositoryImpl) webhookServiceImpl := pipeline.NewWebhookServiceImpl(ciArtifactRepositoryImpl, sugaredLogger, ciPipelineRepositoryImpl, ciWorkflowRepositoryImpl, cdWorkflowCommonServiceImpl, workFlowStageStatusServiceImpl, ciServiceImpl) workflowEventProcessorImpl, err := in.NewWorkflowEventProcessorImpl(sugaredLogger, pubSubClientServiceImpl, cdWorkflowServiceImpl, cdWorkflowReadServiceImpl, cdWorkflowRunnerServiceImpl, cdWorkflowRunnerReadServiceImpl, workflowDagExecutorImpl, ciHandlerImpl, cdHandlerImpl, eventSimpleFactoryImpl, eventRESTClientImpl, devtronAppsHandlerServiceImpl, deployedAppServiceImpl, webhookServiceImpl, validate, environmentVariables, cdWorkflowCommonServiceImpl, cdPipelineConfigServiceImpl, userDeploymentRequestServiceImpl, serviceImpl, pipelineRepositoryImpl, ciArtifactRepositoryImpl, cdWorkflowRepositoryImpl, deploymentConfigServiceImpl, handlerServiceImpl, runnable) if err != nil { From 61a1c74101c642d283e3d628b1a105707d9f401c Mon Sep 17 00:00:00 2001 From: prakhar katiyar <39842461+prkhrkat@users.noreply.github.com> Date: Wed, 23 Jul 2025 17:27:29 +0530 Subject: [PATCH 18/31] opt (#6747) * opt * nil handling * removed * removed --- go.sum | 1 + .../WorkflowStatusLatestRepository.go | 24 +++- pkg/pipeline/CiHandler.go | 17 +++ pkg/pipeline/CiService.go | 56 ++++++--- pkg/workflow/cd/CdWorkflowRunnerService.go | 16 +++ pkg/workflow/dag/WorkflowDagExecutor.go | 16 ++- .../WorkflowStatusLatestService.go | 106 +++++++++++++++++- .../WorkflowStatusUpdateService.go | 6 +- wire_gen.go | 9 +- 9 files changed, 213 insertions(+), 38 deletions(-) diff --git a/go.sum b/go.sum index adfcbe0b44..038454d052 100644 --- a/go.sum +++ b/go.sum @@ -469,6 +469,7 @@ github.com/google/s2a-go v0.1.9 h1:LGD7gtMgezd8a/Xak7mEWL0PjoTQFvpRudN895yqKW0= github.com/google/s2a-go v0.1.9/go.mod h1:YA0Ei2ZQL3acow2O62kdp9UlnvMmU7kA6Eutn0dXayM= github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4= github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ= +github.com/google/subcommands v1.2.0 h1:vWQspBTo2nEqTUFita5/KeEWlUL8kQObDFbub/EN9oE= github.com/google/subcommands v1.2.0/go.mod h1:ZjhPrFU+Olkh9WazFPsl27BQ4UPiG37m3yTrtFlrHVk= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= diff --git a/internal/sql/repository/pipelineConfig/WorkflowStatusLatestRepository.go b/internal/sql/repository/pipelineConfig/WorkflowStatusLatestRepository.go index d88cea9752..d0ffdac7c3 100644 --- a/internal/sql/repository/pipelineConfig/WorkflowStatusLatestRepository.go +++ b/internal/sql/repository/pipelineConfig/WorkflowStatusLatestRepository.go @@ -25,8 +25,8 @@ import ( type WorkflowStatusLatestRepository interface { // CI Workflow Status Latest methods - SaveCiWorkflowStatusLatest(model *CiWorkflowStatusLatest) error - UpdateCiWorkflowStatusLatest(model *CiWorkflowStatusLatest) error + SaveCiWorkflowStatusLatest(tx *pg.Tx, model *CiWorkflowStatusLatest) error + UpdateCiWorkflowStatusLatest(tx *pg.Tx, model *CiWorkflowStatusLatest) error GetCiWorkflowStatusLatestByPipelineId(pipelineId int) (*CiWorkflowStatusLatest, error) GetCiWorkflowStatusLatestByAppId(appId int) ([]*CiWorkflowStatusLatest, error) GetCiWorkflowStatusLatestByPipelineIds(pipelineIds []int) ([]*CiWorkflowStatusLatest, error) @@ -77,8 +77,14 @@ type CdWorkflowStatusLatest struct { } // CI Workflow Status Latest methods implementation -func (impl *WorkflowStatusLatestRepositoryImpl) SaveCiWorkflowStatusLatest(model *CiWorkflowStatusLatest) error { - err := impl.dbConnection.Insert(model) +func (impl *WorkflowStatusLatestRepositoryImpl) SaveCiWorkflowStatusLatest(tx *pg.Tx, model *CiWorkflowStatusLatest) error { + var connection orm.DB + if tx != nil { + connection = tx + } else { + connection = impl.dbConnection + } + err := connection.Insert(model) if err != nil { impl.logger.Errorw("error in saving ci workflow status latest", "err", err, "model", model) return err @@ -86,8 +92,14 @@ func (impl *WorkflowStatusLatestRepositoryImpl) SaveCiWorkflowStatusLatest(model return nil } -func (impl *WorkflowStatusLatestRepositoryImpl) UpdateCiWorkflowStatusLatest(model *CiWorkflowStatusLatest) error { - _, err := impl.dbConnection.Model(model).WherePK().UpdateNotNull() +func (impl *WorkflowStatusLatestRepositoryImpl) UpdateCiWorkflowStatusLatest(tx *pg.Tx, model *CiWorkflowStatusLatest) error { + var connection orm.DB + if tx != nil { + connection = tx + } else { + connection = impl.dbConnection + } + _, err := connection.Model(model).WherePK().UpdateNotNull() if err != nil { impl.logger.Errorw("error in updating ci workflow status latest", "err", err, "model", model) return err diff --git a/pkg/pipeline/CiHandler.go b/pkg/pipeline/CiHandler.go index 5c56eb82d5..f222c2d894 100644 --- a/pkg/pipeline/CiHandler.go +++ b/pkg/pipeline/CiHandler.go @@ -30,6 +30,7 @@ import ( "github.com/devtron-labs/devtron/pkg/pipeline/adapter" "github.com/devtron-labs/devtron/pkg/pipeline/constants" "github.com/devtron-labs/devtron/pkg/pipeline/workflowStatus" + "github.com/devtron-labs/devtron/pkg/workflow/workflowStatusLatest" "regexp" "slices" "strconv" @@ -99,6 +100,7 @@ type CiHandlerImpl struct { k8sCommonService k8sPkg.K8sCommonService workFlowStageStatusService workflowStatus.WorkFlowStageStatusService workflowStatusLatestRepository pipelineConfig.WorkflowStatusLatestRepository + workflowStatusUpdateService workflowStatusLatest.WorkflowStatusUpdateService } func NewCiHandlerImpl(Logger *zap.SugaredLogger, ciService CiService, ciPipelineMaterialRepository pipelineConfig.CiPipelineMaterialRepository, gitSensorClient gitSensor.Client, ciWorkflowRepository pipelineConfig.CiWorkflowRepository, @@ -107,6 +109,7 @@ func NewCiHandlerImpl(Logger *zap.SugaredLogger, ciService CiService, ciPipeline imageTaggingService imageTagging.ImageTaggingService, k8sCommonService k8sPkg.K8sCommonService, appWorkflowRepository appWorkflow.AppWorkflowRepository, customTagService CustomTagService, workFlowStageStatusService workflowStatus.WorkFlowStageStatusService, workflowStatusLatestRepository pipelineConfig.WorkflowStatusLatestRepository, + workflowStatusUpdateService workflowStatusLatest.WorkflowStatusUpdateService, ) *CiHandlerImpl { cih := &CiHandlerImpl{ Logger: Logger, @@ -130,6 +133,7 @@ func NewCiHandlerImpl(Logger *zap.SugaredLogger, ciService CiService, ciPipeline k8sCommonService: k8sCommonService, workFlowStageStatusService: workFlowStageStatusService, workflowStatusLatestRepository: workflowStatusLatestRepository, + workflowStatusUpdateService: workflowStatusUpdateService, } config, err := types.GetCiConfig() if err != nil { @@ -601,6 +605,19 @@ func (impl *CiHandlerImpl) UpdateWorkflow(workflowStatus eventProcessorBean.CiCd impl.Logger.Error("update wf failed for id " + strconv.Itoa(savedWorkflow.Id)) return savedWorkflow.Id, true, err } + + // Update latest status table for CI workflow + // Check if CiPipeline is loaded, if not pass 0 as appId to let the function fetch it + appId := 0 + if savedWorkflow.CiPipeline != nil { + appId = savedWorkflow.CiPipeline.AppId + } + err = impl.workflowStatusUpdateService.UpdateCiWorkflowStatusLatest(nil, savedWorkflow.CiPipelineId, appId, savedWorkflow.Id, savedWorkflow.TriggeredBy) + if err != nil { + impl.Logger.Errorw("error in updating ci workflow status latest", "err", err, "pipelineId", savedWorkflow.CiPipelineId, "workflowId", savedWorkflow.Id) + // Don't return error here as the main workflow update was successful + } + impl.sendCIFailEvent(savedWorkflow, status, message) return savedWorkflow.Id, true, nil } diff --git a/pkg/pipeline/CiService.go b/pkg/pipeline/CiService.go index 55fb2e1c20..08b82aa9fc 100644 --- a/pkg/pipeline/CiService.go +++ b/pkg/pipeline/CiService.go @@ -26,6 +26,7 @@ import ( "github.com/devtron-labs/devtron/pkg/pipeline/types" "github.com/devtron-labs/devtron/pkg/pipeline/workflowStatus" "github.com/devtron-labs/devtron/pkg/sql" + "github.com/devtron-labs/devtron/pkg/workflow/workflowStatusLatest" util3 "github.com/devtron-labs/devtron/util" util2 "github.com/devtron-labs/devtron/util/event" "go.uber.org/zap" @@ -39,13 +40,14 @@ type CiService interface { } type CiServiceImpl struct { - Logger *zap.SugaredLogger - workflowStageStatusService workflowStatus.WorkFlowStageStatusService - eventClient client.EventClient - eventFactory client.EventFactory - config *types.CiConfig - ciWorkflowRepository pipelineConfig.CiWorkflowRepository - transactionManager sql.TransactionWrapper + Logger *zap.SugaredLogger + workflowStageStatusService workflowStatus.WorkFlowStageStatusService + eventClient client.EventClient + eventFactory client.EventFactory + config *types.CiConfig + ciWorkflowRepository pipelineConfig.CiWorkflowRepository + transactionManager sql.TransactionWrapper + workflowStatusUpdateService workflowStatusLatest.WorkflowStatusUpdateService } func NewCiServiceImpl(Logger *zap.SugaredLogger, @@ -53,14 +55,16 @@ func NewCiServiceImpl(Logger *zap.SugaredLogger, eventFactory client.EventFactory, ciWorkflowRepository pipelineConfig.CiWorkflowRepository, transactionManager sql.TransactionWrapper, + workflowStatusUpdateService workflowStatusLatest.WorkflowStatusUpdateService, ) *CiServiceImpl { cis := &CiServiceImpl{ - Logger: Logger, - workflowStageStatusService: workflowStageStatusService, - eventClient: eventClient, - eventFactory: eventFactory, - ciWorkflowRepository: ciWorkflowRepository, - transactionManager: transactionManager, + Logger: Logger, + workflowStageStatusService: workflowStageStatusService, + eventClient: eventClient, + eventFactory: eventFactory, + ciWorkflowRepository: ciWorkflowRepository, + transactionManager: transactionManager, + workflowStatusUpdateService: workflowStatusUpdateService, } config, err := types.GetCiConfig() if err != nil { @@ -131,11 +135,24 @@ func (impl *CiServiceImpl) SaveCiWorkflowWithStage(wf *pipelineConfig.CiWorkflow return err } + // Update latest status table for CI workflow within the transaction + // Check if CiPipeline is loaded, if not pass 0 as appId to let the function fetch it + appId := 0 + if wf.CiPipeline != nil { + appId = wf.CiPipeline.AppId + } + err = impl.workflowStatusUpdateService.UpdateCiWorkflowStatusLatest(tx, wf.CiPipelineId, appId, wf.Id, wf.TriggeredBy) + if err != nil { + impl.Logger.Errorw("error in updating ci workflow status latest", "err", err, "pipelineId", wf.CiPipelineId, "workflowId", wf.Id) + return err + } + err = impl.transactionManager.CommitTx(tx) if err != nil { impl.Logger.Errorw("error in committing transaction", "workflowName", wf.Name, "error", err) return err } + return nil } @@ -167,11 +184,24 @@ func (impl *CiServiceImpl) UpdateCiWorkflowWithStage(wf *pipelineConfig.CiWorkfl return err } + // Update latest status table for CI workflow within the transaction + // Check if CiPipeline is loaded, if not pass 0 as appId to let the function fetch it + appId := 0 + if wf.CiPipeline != nil { + appId = wf.CiPipeline.AppId + } + err = impl.workflowStatusUpdateService.UpdateCiWorkflowStatusLatest(tx, wf.CiPipelineId, appId, wf.Id, wf.TriggeredBy) + if err != nil { + impl.Logger.Errorw("error in updating ci workflow status latest", "err", err, "pipelineId", wf.CiPipelineId, "workflowId", wf.Id) + return err + } + err = impl.transactionManager.CommitTx(tx) if err != nil { impl.Logger.Errorw("error in committing transaction", "workflowName", wf.Name, "error", err) return err } + return nil } diff --git a/pkg/workflow/cd/CdWorkflowRunnerService.go b/pkg/workflow/cd/CdWorkflowRunnerService.go index 49ea1f8928..b2276c0fd0 100644 --- a/pkg/workflow/cd/CdWorkflowRunnerService.go +++ b/pkg/workflow/cd/CdWorkflowRunnerService.go @@ -80,6 +80,21 @@ func (impl *CdWorkflowRunnerServiceImpl) UpdateWfr(dto *bean.CdWorkflowRunnerDto impl.logger.Errorw("error in updating runner status in db", "runnerId", runnerDbObj.Id, "err", err) return err } + + // Update latest status table for CD workflow + // Check if CdWorkflow and Pipeline are loaded, if not pass 0 as appId/environmentId to let the function fetch them + appId := 0 + environmentId := 0 + if runnerDbObj.CdWorkflow != nil && runnerDbObj.CdWorkflow.Pipeline != nil { + appId = runnerDbObj.CdWorkflow.Pipeline.AppId + environmentId = runnerDbObj.CdWorkflow.Pipeline.EnvironmentId + } + err = impl.workflowStatusUpdateService.UpdateCdWorkflowStatusLatest(nil, runnerDbObj.CdWorkflow.PipelineId, appId, environmentId, runnerDbObj.Id, runnerDbObj.WorkflowType.String(), int32(updatedBy)) + if err != nil { + impl.logger.Errorw("error in updating cd workflow status latest", "err", err, "pipelineId", runnerDbObj.CdWorkflow.PipelineId, "workflowRunnerId", runnerDbObj.Id) + // Don't return error here as the main workflow update was successful + } + return nil } @@ -169,6 +184,7 @@ func (impl *CdWorkflowRunnerServiceImpl) UpdateCdWorkflowRunnerWithStage(wfr *pi impl.logger.Errorw("error in committing transaction", "workflowName", wfr.Name, "error", err) return err } + return nil } diff --git a/pkg/workflow/dag/WorkflowDagExecutor.go b/pkg/workflow/dag/WorkflowDagExecutor.go index d705aafb0b..f6d41b8015 100644 --- a/pkg/workflow/dag/WorkflowDagExecutor.go +++ b/pkg/workflow/dag/WorkflowDagExecutor.go @@ -21,6 +21,12 @@ import ( "context" "encoding/json" "fmt" + "github.com/devtron-labs/devtron/pkg/workflow/workflowStatusLatest" + "net/http" + "strings" + "sync" + "time" + "github.com/argoproj/argo-workflows/v3/pkg/apis/workflow/v1alpha1" "github.com/devtron-labs/common-lib/async" "github.com/devtron-labs/common-lib/utils" @@ -71,10 +77,6 @@ import ( util2 "github.com/devtron-labs/devtron/util/event" errors2 "k8s.io/apimachinery/pkg/api/errors" "k8s.io/client-go/rest" - "net/http" - "strings" - "sync" - "time" "github.com/devtron-labs/devtron/api/bean" "github.com/devtron-labs/devtron/internal/sql/models" @@ -158,7 +160,8 @@ type WorkflowDagExecutorImpl struct { workflowService executor.WorkflowService ciHandlerService trigger.HandlerService workflowTriggerAuditService auditService.WorkflowTriggerAuditService - fluxApplicationService fluxApplication.FluxApplicationService + fluxApplicationService fluxApplication.FluxApplicationService + workflowStatusUpdateService workflowStatusLatest.WorkflowStatusUpdateService } func NewWorkflowDagExecutorImpl(Logger *zap.SugaredLogger, pipelineRepository pipelineConfig.PipelineRepository, @@ -195,6 +198,7 @@ func NewWorkflowDagExecutorImpl(Logger *zap.SugaredLogger, pipelineRepository pi ciHandlerService trigger.HandlerService, workflowTriggerAuditService auditService.WorkflowTriggerAuditService, fluxApplicationService fluxApplication.FluxApplicationService, + workflowStatusUpdateService workflowStatusLatest.WorkflowStatusUpdateService, ) *WorkflowDagExecutorImpl { wde := &WorkflowDagExecutorImpl{logger: Logger, pipelineRepository: pipelineRepository, @@ -231,6 +235,7 @@ func NewWorkflowDagExecutorImpl(Logger *zap.SugaredLogger, pipelineRepository pi ciHandlerService: ciHandlerService, workflowTriggerAuditService: workflowTriggerAuditService, fluxApplicationService: fluxApplicationService, + workflowStatusUpdateService: workflowStatusUpdateService, } config, err := types.GetCdConfig() if err != nil { @@ -939,7 +944,6 @@ func (impl *WorkflowDagExecutorImpl) UpdateCiWorkflowForCiSuccess(request *bean2 impl.logger.Errorw("update wf failed for id ", "err", err) return err } - return nil } diff --git a/pkg/workflow/workflowStatusLatest/WorkflowStatusLatestService.go b/pkg/workflow/workflowStatusLatest/WorkflowStatusLatestService.go index 1ae6e9dfd2..ffd685be8e 100644 --- a/pkg/workflow/workflowStatusLatest/WorkflowStatusLatestService.go +++ b/pkg/workflow/workflowStatusLatest/WorkflowStatusLatestService.go @@ -17,9 +17,11 @@ package workflowStatusLatest import ( - util2 "github.com/devtron-labs/devtron/internal/util" + "fmt" "time" + util2 "github.com/devtron-labs/devtron/internal/util" + "github.com/devtron-labs/devtron/api/bean" "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" "github.com/go-pg/pg" @@ -28,7 +30,7 @@ import ( type WorkflowStatusLatestService interface { // CI Workflow Status Latest methods - SaveOrUpdateCiWorkflowStatusLatest(pipelineId, appId, ciWorkflowId int, userId int32) error + SaveOrUpdateCiWorkflowStatusLatest(tx *pg.Tx, pipelineId, appId, ciWorkflowId int, userId int32) error GetCiWorkflowStatusLatestByPipelineId(pipelineId int) (*CiWorkflowStatusLatest, error) GetCiWorkflowStatusLatestByAppId(appId int) ([]*CiWorkflowStatusLatest, error) @@ -79,7 +81,46 @@ type CdWorkflowStatusLatest struct { } // CI Workflow Status Latest methods implementation -func (impl *WorkflowStatusLatestServiceImpl) SaveOrUpdateCiWorkflowStatusLatest(pipelineId, appId, ciWorkflowId int, userId int32) error { +func (impl *WorkflowStatusLatestServiceImpl) SaveOrUpdateCiWorkflowStatusLatest(tx *pg.Tx, pipelineId, appId, ciWorkflowId int, userId int32) error { + // Validate required parameters + if pipelineId <= 0 { + impl.logger.Errorw("invalid pipelineId provided", "pipelineId", pipelineId) + return fmt.Errorf("invalid pipelineId: %d", pipelineId) + } + + if ciWorkflowId <= 0 { + impl.logger.Errorw("invalid ciWorkflowId provided", "ciWorkflowId", ciWorkflowId) + return fmt.Errorf("invalid ciWorkflowId: %d", ciWorkflowId) + } + + // If appId is not provided (0), fetch it from the CiPipeline + if appId <= 0 { + ciPipeline, err := impl.ciWorkflowRepository.FindById(ciWorkflowId) + if err != nil { + impl.logger.Errorw("error in fetching ci workflow to get appId", "err", err, "ciWorkflowId", ciWorkflowId) + return err + } + + if ciPipeline == nil { + impl.logger.Errorw("ci workflow not found", "ciWorkflowId", ciWorkflowId) + return fmt.Errorf("ci workflow not found with id: %d", ciWorkflowId) + } + + // Check if CiPipeline is loaded + if ciPipeline.CiPipeline == nil { + impl.logger.Errorw("ci pipeline not loaded in ci workflow", "ciWorkflowId", ciWorkflowId, "ciPipelineId", ciPipeline.CiPipelineId) + return fmt.Errorf("ci pipeline not loaded for workflow id: %d", ciWorkflowId) + } + + appId = ciPipeline.CiPipeline.AppId + if appId <= 0 { + impl.logger.Errorw("invalid appId in ci pipeline", "ciWorkflowId", ciWorkflowId, "ciPipelineId", ciPipeline.CiPipelineId, "appId", appId) + return fmt.Errorf("invalid appId in ci pipeline: %d", appId) + } + + impl.logger.Debugw("fetched appId from ci workflow", "ciWorkflowId", ciWorkflowId, "appId", appId) + } + // Check if entry exists existingEntry, err := impl.workflowStatusLatestRepository.GetCiWorkflowStatusLatestByPipelineId(pipelineId) if err != nil && err != pg.ErrNoRows { @@ -100,14 +141,14 @@ func (impl *WorkflowStatusLatestServiceImpl) SaveOrUpdateCiWorkflowStatusLatest( model.UpdatedBy = userId model.UpdatedOn = now - return impl.workflowStatusLatestRepository.SaveCiWorkflowStatusLatest(model) + return impl.workflowStatusLatestRepository.SaveCiWorkflowStatusLatest(tx, model) } else { // Update existing entry existingEntry.CiWorkflowId = ciWorkflowId existingEntry.UpdatedBy = userId existingEntry.UpdatedOn = now - return impl.workflowStatusLatestRepository.UpdateCiWorkflowStatusLatest(existingEntry) + return impl.workflowStatusLatestRepository.UpdateCiWorkflowStatusLatest(tx, existingEntry) } } @@ -167,6 +208,61 @@ func (impl *WorkflowStatusLatestServiceImpl) GetCiWorkflowStatusLatestByAppId(ap // CD Workflow Status Latest methods implementation func (impl *WorkflowStatusLatestServiceImpl) SaveOrUpdateCdWorkflowStatusLatest(tx *pg.Tx, pipelineId, appId, environmentId, workflowRunnerId int, workflowType string, userId int32) error { + // Validate required parameters + if pipelineId <= 0 { + impl.logger.Errorw("invalid pipelineId provided", "pipelineId", pipelineId) + return fmt.Errorf("invalid pipelineId: %d", pipelineId) + } + + if workflowRunnerId <= 0 { + impl.logger.Errorw("invalid workflowRunnerId provided", "workflowRunnerId", workflowRunnerId) + return fmt.Errorf("invalid workflowRunnerId: %d", workflowRunnerId) + } + + // If appId or environmentId is not provided (0), fetch them from the CdWorkflow + if appId <= 0 || environmentId <= 0 { + cdWorkflowRunner, err := impl.cdWorkflowRepository.FindBasicWorkflowRunnerById(workflowRunnerId) + if err != nil { + impl.logger.Errorw("error in fetching cd workflow runner to get appId/environmentId", "err", err, "workflowRunnerId", workflowRunnerId) + return err + } + + if cdWorkflowRunner == nil { + impl.logger.Errorw("cd workflow runner not found", "workflowRunnerId", workflowRunnerId) + return fmt.Errorf("cd workflow runner not found with id: %d", workflowRunnerId) + } + + // Check if CdWorkflow is loaded + if cdWorkflowRunner.CdWorkflow == nil { + impl.logger.Errorw("cd workflow not loaded in cd workflow runner", "workflowRunnerId", workflowRunnerId, "cdWorkflowId", cdWorkflowRunner.CdWorkflowId) + return fmt.Errorf("cd workflow not loaded for workflow runner id: %d", workflowRunnerId) + } + + // Check if Pipeline is loaded + if cdWorkflowRunner.CdWorkflow.Pipeline == nil { + impl.logger.Errorw("pipeline not loaded in cd workflow", "workflowRunnerId", workflowRunnerId, "cdWorkflowId", cdWorkflowRunner.CdWorkflowId, "pipelineId", cdWorkflowRunner.CdWorkflow.PipelineId) + return fmt.Errorf("pipeline not loaded for workflow runner id: %d", workflowRunnerId) + } + + if appId <= 0 { + appId = cdWorkflowRunner.CdWorkflow.Pipeline.AppId + if appId <= 0 { + impl.logger.Errorw("invalid appId in pipeline", "workflowRunnerId", workflowRunnerId, "pipelineId", cdWorkflowRunner.CdWorkflow.PipelineId, "appId", appId) + return fmt.Errorf("invalid appId in pipeline: %d", appId) + } + impl.logger.Debugw("fetched appId from cd workflow runner", "workflowRunnerId", workflowRunnerId, "appId", appId) + } + + if environmentId <= 0 { + environmentId = cdWorkflowRunner.CdWorkflow.Pipeline.EnvironmentId + if environmentId <= 0 { + impl.logger.Errorw("invalid environmentId in pipeline", "workflowRunnerId", workflowRunnerId, "pipelineId", cdWorkflowRunner.CdWorkflow.PipelineId, "environmentId", environmentId) + return fmt.Errorf("invalid environmentId in pipeline: %d", environmentId) + } + impl.logger.Debugw("fetched environmentId from cd workflow runner", "workflowRunnerId", workflowRunnerId, "environmentId", environmentId) + } + } + // Check if entry exists existingEntry, err := impl.workflowStatusLatestRepository.GetCdWorkflowStatusLatestByPipelineIdAndWorkflowType(tx, pipelineId, workflowType) if err != nil && err != pg.ErrNoRows { diff --git a/pkg/workflow/workflowStatusLatest/WorkflowStatusUpdateService.go b/pkg/workflow/workflowStatusLatest/WorkflowStatusUpdateService.go index 1b1b56e9d8..785d5afc6a 100644 --- a/pkg/workflow/workflowStatusLatest/WorkflowStatusUpdateService.go +++ b/pkg/workflow/workflowStatusLatest/WorkflowStatusUpdateService.go @@ -24,7 +24,7 @@ import ( type WorkflowStatusUpdateService interface { // Methods to update latest status tables when workflow status changes - UpdateCiWorkflowStatusLatest(pipelineId, appId, ciWorkflowId int, userId int32) error + UpdateCiWorkflowStatusLatest(tx *pg.Tx, pipelineId, appId, ciWorkflowId int, userId int32) error UpdateCdWorkflowStatusLatest(tx *pg.Tx, pipelineId, appId, environmentId, workflowRunnerId int, workflowType string, userId int32) error // Methods to fetch optimized status for trigger view @@ -59,8 +59,8 @@ func NewWorkflowStatusUpdateServiceImpl( } } -func (impl *WorkflowStatusUpdateServiceImpl) UpdateCiWorkflowStatusLatest(pipelineId, appId, ciWorkflowId int, userId int32) error { - return impl.workflowStatusLatestService.SaveOrUpdateCiWorkflowStatusLatest(pipelineId, appId, ciWorkflowId, userId) +func (impl *WorkflowStatusUpdateServiceImpl) UpdateCiWorkflowStatusLatest(tx *pg.Tx, pipelineId, appId, ciWorkflowId int, userId int32) error { + return impl.workflowStatusLatestService.SaveOrUpdateCiWorkflowStatusLatest(tx, pipelineId, appId, ciWorkflowId, userId) } func (impl *WorkflowStatusUpdateServiceImpl) UpdateCdWorkflowStatusLatest(tx *pg.Tx, pipelineId, appId, environmentId, workflowRunnerId int, workflowType string, userId int32) error { diff --git a/wire_gen.go b/wire_gen.go index 1ba1dd8001..bd3bbdcc3a 100644 --- a/wire_gen.go +++ b/wire_gen.go @@ -692,7 +692,7 @@ func InitializeApp() (*App, error) { chartServiceImpl := chart.NewChartServiceImpl(chartRepositoryImpl, sugaredLogger, chartTemplateServiceImpl, chartRepoRepositoryImpl, appRepositoryImpl, mergeUtil, envConfigOverrideRepositoryImpl, pipelineConfigRepositoryImpl, environmentRepositoryImpl, deploymentTemplateHistoryServiceImpl, scopedVariableManagerImpl, deployedAppMetricsServiceImpl, chartRefServiceImpl, gitOpsConfigReadServiceImpl, deploymentConfigServiceImpl, envConfigOverrideReadServiceImpl, chartReadServiceImpl) ciCdPipelineOrchestratorImpl := pipeline.NewCiCdPipelineOrchestrator(appRepositoryImpl, sugaredLogger, materialRepositoryImpl, pipelineRepositoryImpl, ciPipelineRepositoryImpl, ciPipelineMaterialRepositoryImpl, cdWorkflowRepositoryImpl, clientImpl, ciCdConfig, appWorkflowRepositoryImpl, environmentRepositoryImpl, attributesServiceImpl, appCrudOperationServiceImpl, userAuthServiceImpl, prePostCdScriptHistoryServiceImpl, pipelineStageServiceImpl, gitMaterialHistoryServiceImpl, ciPipelineHistoryServiceImpl, ciTemplateReadServiceImpl, ciTemplateServiceImpl, dockerArtifactStoreRepositoryImpl, ciArtifactRepositoryImpl, configMapServiceImpl, customTagServiceImpl, genericNoteServiceImpl, chartServiceImpl, transactionUtilImpl, gitOpsConfigReadServiceImpl, deploymentConfigServiceImpl, deploymentConfigReadServiceImpl, chartReadServiceImpl) pluginInputVariableParserImpl := pipeline.NewPluginInputVariableParserImpl(sugaredLogger, dockerRegistryConfigImpl, customTagServiceImpl) - ciServiceImpl := pipeline.NewCiServiceImpl(sugaredLogger, workFlowStageStatusServiceImpl, eventRESTClientImpl, eventSimpleFactoryImpl, ciWorkflowRepositoryImpl, transactionUtilImpl) + ciServiceImpl := pipeline.NewCiServiceImpl(sugaredLogger, workFlowStageStatusServiceImpl, eventRESTClientImpl, eventSimpleFactoryImpl, ciWorkflowRepositoryImpl, transactionUtilImpl, workflowStatusUpdateServiceImpl) ciLogServiceImpl, err := pipeline.NewCiLogServiceImpl(sugaredLogger, k8sServiceImpl) if err != nil { return nil, err @@ -752,9 +752,8 @@ func InitializeApp() (*App, error) { deploymentTemplateValidationServiceEntImpl := validator.NewDeploymentTemplateValidationServiceEntImpl() deploymentTemplateValidationServiceImpl := validator.NewDeploymentTemplateValidationServiceImpl(sugaredLogger, chartRefServiceImpl, scopedVariableManagerImpl, deployedAppMetricsServiceImpl, deploymentTemplateValidationServiceEntImpl) devtronAppGitOpConfigServiceImpl := gitOpsConfig.NewDevtronAppGitOpConfigServiceImpl(sugaredLogger, chartRepositoryImpl, chartServiceImpl, gitOpsConfigReadServiceImpl, gitOpsValidationServiceImpl, argoClientWrapperServiceImpl, deploymentConfigServiceImpl, chartReadServiceImpl) - ciHandlerImpl := pipeline.NewCiHandlerImpl(sugaredLogger, ciServiceImpl, ciPipelineMaterialRepositoryImpl, clientImpl, ciWorkflowRepositoryImpl, ciArtifactRepositoryImpl, userServiceImpl, eventRESTClientImpl, eventSimpleFactoryImpl, ciPipelineRepositoryImpl, appListingRepositoryImpl, pipelineRepositoryImpl, enforcerUtilImpl, resourceGroupServiceImpl, environmentRepositoryImpl, imageTaggingServiceImpl, k8sCommonServiceImpl, appWorkflowRepositoryImpl, customTagServiceImpl, workFlowStageStatusServiceImpl, workflowStatusLatestRepositoryImpl) - cdWorkflowRunnerReadServiceImpl := read18.NewCdWorkflowRunnerReadServiceImpl(sugaredLogger, cdWorkflowRepositoryImpl, workflowStatusLatestServiceImpl, pipelineStageRepositoryImpl) - cdHandlerImpl := pipeline.NewCdHandlerImpl(sugaredLogger, userServiceImpl, cdWorkflowRepositoryImpl, ciArtifactRepositoryImpl, ciPipelineMaterialRepositoryImpl, pipelineRepositoryImpl, environmentRepositoryImpl, ciWorkflowRepositoryImpl, enforcerUtilImpl, resourceGroupServiceImpl, imageTaggingServiceImpl, k8sServiceImpl, customTagServiceImpl, deploymentConfigServiceImpl, workFlowStageStatusServiceImpl, cdWorkflowRunnerServiceImpl, workflowStatusLatestServiceImpl, pipelineStageRepositoryImpl, cdWorkflowRunnerReadServiceImpl) + ciHandlerImpl := pipeline.NewCiHandlerImpl(sugaredLogger, ciServiceImpl, ciPipelineMaterialRepositoryImpl, clientImpl, ciWorkflowRepositoryImpl, ciArtifactRepositoryImpl, userServiceImpl, eventRESTClientImpl, eventSimpleFactoryImpl, ciPipelineRepositoryImpl, appListingRepositoryImpl, pipelineRepositoryImpl, enforcerUtilImpl, resourceGroupServiceImpl, environmentRepositoryImpl, imageTaggingServiceImpl, k8sCommonServiceImpl, appWorkflowRepositoryImpl, customTagServiceImpl, workFlowStageStatusServiceImpl, workflowStatusLatestRepositoryImpl, workflowStatusUpdateServiceImpl) + cdHandlerImpl := pipeline.NewCdHandlerImpl(sugaredLogger, userServiceImpl, cdWorkflowRepositoryImpl, ciArtifactRepositoryImpl, ciPipelineMaterialRepositoryImpl, pipelineRepositoryImpl, environmentRepositoryImpl, ciWorkflowRepositoryImpl, enforcerUtilImpl, resourceGroupServiceImpl, imageTaggingServiceImpl, k8sServiceImpl, customTagServiceImpl, deploymentConfigServiceImpl, workFlowStageStatusServiceImpl, cdWorkflowRunnerServiceImpl, workflowStatusLatestServiceImpl, pipelineStageRepositoryImpl) appWorkflowServiceImpl := appWorkflow2.NewAppWorkflowServiceImpl(sugaredLogger, appWorkflowRepositoryImpl, ciCdPipelineOrchestratorImpl, ciPipelineRepositoryImpl, pipelineRepositoryImpl, enforcerUtilImpl, resourceGroupServiceImpl, appRepositoryImpl, userAuthServiceImpl, chartServiceImpl, deploymentConfigServiceImpl, pipelineBuilderImpl) appCloneServiceImpl := appClone.NewAppCloneServiceImpl(sugaredLogger, pipelineBuilderImpl, attributesServiceImpl, chartServiceImpl, configMapServiceImpl, appWorkflowServiceImpl, appListingServiceImpl, propertiesConfigServiceImpl, pipelineStageServiceImpl, ciTemplateReadServiceImpl, appRepositoryImpl, ciPipelineRepositoryImpl, pipelineRepositoryImpl, ciPipelineConfigServiceImpl, gitOpsConfigReadServiceImpl, chartReadServiceImpl) deploymentTemplateRepositoryImpl := repository2.NewDeploymentTemplateRepositoryImpl(db, sugaredLogger) @@ -792,7 +791,7 @@ func InitializeApp() (*App, error) { pipelineConfigRestHandlerImpl := configure.NewPipelineRestHandlerImpl(pipelineBuilderImpl, sugaredLogger, deploymentTemplateValidationServiceImpl, chartServiceImpl, devtronAppGitOpConfigServiceImpl, propertiesConfigServiceImpl, userServiceImpl, teamServiceImpl, enforcerImpl, ciHandlerImpl, validate, clientImpl, ciPipelineRepositoryImpl, pipelineRepositoryImpl, enforcerUtilImpl, dockerRegistryConfigImpl, cdHandlerImpl, appCloneServiceImpl, generateManifestDeploymentTemplateServiceImpl, appWorkflowServiceImpl, gitMaterialReadServiceImpl, policyServiceImpl, imageScanResultReadServiceImpl, ciPipelineMaterialRepositoryImpl, imageTaggingReadServiceImpl, imageTaggingServiceImpl, ciArtifactRepositoryImpl, deployedAppMetricsServiceImpl, chartRefServiceImpl, ciCdPipelineOrchestratorImpl, gitProviderReadServiceImpl, teamReadServiceImpl, environmentRepositoryImpl, chartReadServiceImpl, draftAwareConfigServiceImpl, handlerServiceImpl, devtronAppsHandlerServiceImpl) commonArtifactServiceImpl := artifacts.NewCommonArtifactServiceImpl(sugaredLogger, ciArtifactRepositoryImpl) fluxApplicationServiceImpl := fluxApplication.NewFluxApplicationServiceImpl(sugaredLogger, helmAppReadServiceImpl, clusterServiceImplExtended, helmAppClientImpl, pumpImpl, pipelineRepositoryImpl, installedAppRepositoryImpl) - workflowDagExecutorImpl := dag.NewWorkflowDagExecutorImpl(sugaredLogger, pipelineRepositoryImpl, pipelineOverrideRepositoryImpl, cdWorkflowRepositoryImpl, ciArtifactRepositoryImpl, enforcerUtilImpl, appWorkflowRepositoryImpl, pipelineStageServiceImpl, ciWorkflowRepositoryImpl, ciPipelineRepositoryImpl, pipelineStageRepositoryImpl, globalPluginRepositoryImpl, eventRESTClientImpl, eventSimpleFactoryImpl, customTagServiceImpl, pipelineStatusTimelineServiceImpl, cdWorkflowRunnerServiceImpl, ciServiceImpl, helmAppServiceImpl, cdWorkflowCommonServiceImpl, devtronAppsHandlerServiceImpl, userDeploymentRequestServiceImpl, manifestCreationServiceImpl, commonArtifactServiceImpl, deploymentConfigServiceImpl, runnable, imageScanHistoryRepositoryImpl, imageScanServiceImpl, k8sServiceImpl, environmentRepositoryImpl, k8sCommonServiceImpl, workflowServiceImpl, handlerServiceImpl, workflowTriggerAuditServiceImpl, fluxApplicationServiceImpl) + workflowDagExecutorImpl := dag.NewWorkflowDagExecutorImpl(sugaredLogger, pipelineRepositoryImpl, pipelineOverrideRepositoryImpl, cdWorkflowRepositoryImpl, ciArtifactRepositoryImpl, enforcerUtilImpl, appWorkflowRepositoryImpl, pipelineStageServiceImpl, ciWorkflowRepositoryImpl, ciPipelineRepositoryImpl, pipelineStageRepositoryImpl, globalPluginRepositoryImpl, eventRESTClientImpl, eventSimpleFactoryImpl, customTagServiceImpl, pipelineStatusTimelineServiceImpl, cdWorkflowRunnerServiceImpl, ciServiceImpl, helmAppServiceImpl, cdWorkflowCommonServiceImpl, devtronAppsHandlerServiceImpl, userDeploymentRequestServiceImpl, manifestCreationServiceImpl, commonArtifactServiceImpl, deploymentConfigServiceImpl, runnable, imageScanHistoryRepositoryImpl, imageScanServiceImpl, k8sServiceImpl, environmentRepositoryImpl, k8sCommonServiceImpl, workflowServiceImpl, handlerServiceImpl, workflowTriggerAuditServiceImpl, fluxApplicationServiceImpl, workflowStatusUpdateServiceImpl) externalCiRestHandlerImpl := restHandler.NewExternalCiRestHandlerImpl(sugaredLogger, validate, userServiceImpl, enforcerImpl, workflowDagExecutorImpl) pubSubClientRestHandlerImpl := restHandler.NewPubSubClientRestHandlerImpl(pubSubClientServiceImpl, sugaredLogger, ciCdConfig) webhookRouterImpl := router.NewWebhookRouterImpl(gitWebhookRestHandlerImpl, pipelineConfigRestHandlerImpl, externalCiRestHandlerImpl, pubSubClientRestHandlerImpl) From 13685590cc0c5c3acb82b5cd07c850712374091a Mon Sep 17 00:00:00 2001 From: prakhar katiyar Date: Wed, 23 Jul 2025 17:52:45 +0530 Subject: [PATCH 19/31] removed extra code --- pkg/workflow/dag/WorkflowDagExecutor.go | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/pkg/workflow/dag/WorkflowDagExecutor.go b/pkg/workflow/dag/WorkflowDagExecutor.go index f6d41b8015..a6e09f5295 100644 --- a/pkg/workflow/dag/WorkflowDagExecutor.go +++ b/pkg/workflow/dag/WorkflowDagExecutor.go @@ -21,7 +21,6 @@ import ( "context" "encoding/json" "fmt" - "github.com/devtron-labs/devtron/pkg/workflow/workflowStatusLatest" "net/http" "strings" "sync" @@ -161,7 +160,6 @@ type WorkflowDagExecutorImpl struct { ciHandlerService trigger.HandlerService workflowTriggerAuditService auditService.WorkflowTriggerAuditService fluxApplicationService fluxApplication.FluxApplicationService - workflowStatusUpdateService workflowStatusLatest.WorkflowStatusUpdateService } func NewWorkflowDagExecutorImpl(Logger *zap.SugaredLogger, pipelineRepository pipelineConfig.PipelineRepository, @@ -198,7 +196,6 @@ func NewWorkflowDagExecutorImpl(Logger *zap.SugaredLogger, pipelineRepository pi ciHandlerService trigger.HandlerService, workflowTriggerAuditService auditService.WorkflowTriggerAuditService, fluxApplicationService fluxApplication.FluxApplicationService, - workflowStatusUpdateService workflowStatusLatest.WorkflowStatusUpdateService, ) *WorkflowDagExecutorImpl { wde := &WorkflowDagExecutorImpl{logger: Logger, pipelineRepository: pipelineRepository, @@ -234,9 +231,7 @@ func NewWorkflowDagExecutorImpl(Logger *zap.SugaredLogger, pipelineRepository pi workflowService: workflowService, ciHandlerService: ciHandlerService, workflowTriggerAuditService: workflowTriggerAuditService, - fluxApplicationService: fluxApplicationService, - workflowStatusUpdateService: workflowStatusUpdateService, - } + fluxApplicationService: fluxApplicationService} config, err := types.GetCdConfig() if err != nil { return nil From 20dc98844150bbbbf476ebca07372a47f81af89b Mon Sep 17 00:00:00 2001 From: ayushmaheshwari Date: Wed, 23 Jul 2025 19:21:22 +0530 Subject: [PATCH 20/31] dev testing fixes --- .../pipelineConfig/CdWorfkflowRepository.go | 13 ++-- .../cd/read/CdWorkflowRunnerReadService.go | 59 ++++++++++++------- 2 files changed, 44 insertions(+), 28 deletions(-) diff --git a/internal/sql/repository/pipelineConfig/CdWorfkflowRepository.go b/internal/sql/repository/pipelineConfig/CdWorfkflowRepository.go index d07411be4e..26325aff56 100644 --- a/internal/sql/repository/pipelineConfig/CdWorfkflowRepository.go +++ b/internal/sql/repository/pipelineConfig/CdWorfkflowRepository.go @@ -71,7 +71,7 @@ type CdWorkflowRepository interface { IsLatestWf(pipelineId int, wfId int) (bool, error) FindLatestCdWorkflowByPipelineId(pipelineIds []int) (*CdWorkflow, error) FindLatestCdWorkflowByPipelineIdV2(pipelineIds []int) ([]*CdWorkflow, error) - FetchAllCdStagesLatestEntity(pipelineWorkflowPairs map[int]apiBean.WorkflowType) ([]*CdWorkflowStatus, error) + FetchAllCdStagesLatestEntity(pipelineWorkflowPairs map[int][]apiBean.WorkflowType) ([]*CdWorkflowStatus, error) FetchAllCdStagesLatestEntityStatus(wfrIds []int) ([]*CdWorkflowRunner, error) ExistsByStatus(status string) (bool, error) FetchEnvAllCdStagesLatestEntityStatus(wfrIds []int, envID int) ([]*CdWorkflowRunner, error) @@ -580,7 +580,7 @@ func (impl *CdWorkflowRepositoryImpl) IsLatestWf(pipelineId int, wfId int) (bool return !exists, err } -func (impl *CdWorkflowRepositoryImpl) FetchAllCdStagesLatestEntity(pipelineWorkflowPairs map[int]apiBean.WorkflowType) ([]*CdWorkflowStatus, error) { +func (impl *CdWorkflowRepositoryImpl) FetchAllCdStagesLatestEntity(pipelineWorkflowPairs map[int][]apiBean.WorkflowType) ([]*CdWorkflowStatus, error) { var cdWorkflowStatus []*CdWorkflowStatus if len(pipelineWorkflowPairs) == 0 { return cdWorkflowStatus, nil @@ -598,14 +598,15 @@ func (impl *CdWorkflowRepositoryImpl) FetchAllCdStagesLatestEntity(pipelineWorkf return cdWorkflowStatus, nil } -func buildPipelineTypeValuesList(pairs map[int]apiBean.WorkflowType) string { +func buildPipelineTypeValuesList(pairs map[int][]apiBean.WorkflowType) string { var values []string - for pipelineId, workflowType := range pairs { - values = append(values, fmt.Sprintf("(%d,'%s')", pipelineId, workflowType)) + for pipelineId, workflowTypes := range pairs { + for _, workflowType := range workflowTypes { + values = append(values, fmt.Sprintf("(%d,'%s')", pipelineId, workflowType)) + } } return strings.Join(values, ",") } - func (impl *CdWorkflowRepositoryImpl) FetchAllCdStagesLatestEntityStatus(wfrIds []int) ([]*CdWorkflowRunner, error) { var wfrList []*CdWorkflowRunner err := impl.dbConnection.Model(&wfrList). diff --git a/pkg/workflow/cd/read/CdWorkflowRunnerReadService.go b/pkg/workflow/cd/read/CdWorkflowRunnerReadService.go index 331077c090..c0ac8e0550 100644 --- a/pkg/workflow/cd/read/CdWorkflowRunnerReadService.go +++ b/pkg/workflow/cd/read/CdWorkflowRunnerReadService.go @@ -64,7 +64,7 @@ func (impl *CdWorkflowRunnerReadServiceImpl) GetWfrStatusForLatestRunners(pipeli return nil, err } - var pipelineIdToCiPipelineIdMap map[int]int + pipelineIdToCiPipelineIdMap := make(map[int]int) for _, item := range pipelines { pipelineIdToCiPipelineIdMap[item.Id] = item.CiPipelineId } @@ -78,12 +78,12 @@ func (impl *CdWorkflowRunnerReadServiceImpl) GetWfrStatusForLatestRunners(pipeli }) } - var cdWorfklowLatestMap map[int]map[bean2.WorkflowType]bool + cdWorfklowLatestMap := make(map[int][]bean2.WorkflowType) for _, item := range cdWorkflowLatest { if _, ok := cdWorfklowLatestMap[item.PipelineId]; !ok { - cdWorfklowLatestMap[item.PipelineId] = make(map[bean2.WorkflowType]bool) + cdWorfklowLatestMap[item.PipelineId] = make([]bean2.WorkflowType, 0) } - cdWorfklowLatestMap[item.PipelineId][bean2.WorkflowType(item.WorkflowType)] = true + cdWorfklowLatestMap[item.PipelineId] = append(cdWorfklowLatestMap[item.PipelineId], bean2.WorkflowType(item.WorkflowType)) } pipelineStage, err := impl.pipelineStageRepository.GetAllCdStagesByCdPipelineIds(pipelineIds) @@ -91,38 +91,53 @@ func (impl *CdWorkflowRunnerReadServiceImpl) GetWfrStatusForLatestRunners(pipeli impl.logger.Errorw("error in fetching pipeline stages", "pipelineId", pipelineIds, "err", err) return nil, err } - pipelineStageMap := make(map[int]map[bean2.WorkflowType]bool) + pipelineStageMap := make(map[int][]bean2.WorkflowType) for _, item := range pipelineStage { if _, ok := pipelineStageMap[item.CdPipelineId]; !ok { - pipelineStageMap[item.CdPipelineId] = make(map[bean2.WorkflowType]bool) + pipelineStageMap[item.CdPipelineId] = make([]bean2.WorkflowType, 0) } if item.Type == repository2.PIPELINE_STAGE_TYPE_PRE_CD { - pipelineStageMap[item.CdPipelineId][bean2.CD_WORKFLOW_TYPE_PRE] = true + pipelineStageMap[item.CdPipelineId] = append(pipelineStageMap[item.CdPipelineId], bean2.CD_WORKFLOW_TYPE_PRE) } else if item.Type == repository2.PIPELINE_STAGE_TYPE_POST_CD { - pipelineStageMap[item.CdPipelineId][bean2.CD_WORKFLOW_TYPE_POST] = true + pipelineStageMap[item.CdPipelineId] = append(pipelineStageMap[item.CdPipelineId], bean2.CD_WORKFLOW_TYPE_POST) } } // calculating all the pipelines not present in the index table cdWorkflowLatest - var pipelinesAbsentInCache map[int]bean2.WorkflowType + pipelinesAbsentInCache := make(map[int][]bean2.WorkflowType) for _, item := range pipelines { - if _, ok := cdWorfklowLatestMap[item.Id]; !ok { - pipelinesAbsentInCache[item.Id] = bean2.CD_WORKFLOW_TYPE_PRE - pipelinesAbsentInCache[item.Id] = bean2.CD_WORKFLOW_TYPE_DEPLOY - pipelinesAbsentInCache[item.Id] = bean2.CD_WORKFLOW_TYPE_POST + if stages, ok := cdWorfklowLatestMap[item.Id]; !ok || len(stages) == 0 { + pipelinesAbsentInCache[item.Id] = append(pipelinesAbsentInCache[item.Id], bean2.CD_WORKFLOW_TYPE_PRE, bean2.CD_WORKFLOW_TYPE_POST, bean2.CD_WORKFLOW_TYPE_DEPLOY) } else { - if _, ok := pipelineStageMap[item.Id][bean2.CD_WORKFLOW_TYPE_PRE]; ok { - if val, ok := cdWorfklowLatestMap[item.Id][bean2.CD_WORKFLOW_TYPE_PRE]; !ok || !val { - pipelinesAbsentInCache[item.Id] = bean2.CD_WORKFLOW_TYPE_PRE + isPreCDStageAbsent, isPostCdStageAbsent, isDeployStageAbsent := true, true, true + for _, stage := range stages { + switch stage { + case bean2.CD_WORKFLOW_TYPE_PRE: + isPreCDStageAbsent = false + case bean2.CD_WORKFLOW_TYPE_POST: + isPostCdStageAbsent = false + case bean2.CD_WORKFLOW_TYPE_DEPLOY: + isDeployStageAbsent = false } } - if _, ok := pipelineStageMap[item.Id][bean2.CD_WORKFLOW_TYPE_POST]; ok { - if val, ok := cdWorfklowLatestMap[item.Id][bean2.CD_WORKFLOW_TYPE_POST]; !ok || !val { - pipelinesAbsentInCache[item.Id] = bean2.CD_WORKFLOW_TYPE_POST - } + if isDeployStageAbsent { + pipelinesAbsentInCache[item.Id] = append(pipelinesAbsentInCache[item.Id], bean2.CD_WORKFLOW_TYPE_DEPLOY) } - if val, ok := cdWorfklowLatestMap[item.Id][bean2.CD_WORKFLOW_TYPE_DEPLOY]; !ok || !val { - pipelinesAbsentInCache[item.Id] = bean2.CD_WORKFLOW_TYPE_POST + if configuredStages, ok := pipelineStageMap[item.Id]; ok { + var isPreCDConfigured, isPostCDConfigured bool + for _, stage := range configuredStages { + if stage == bean2.CD_WORKFLOW_TYPE_PRE { + isPreCDConfigured = true + } else if stage == bean2.CD_WORKFLOW_TYPE_POST { + isPostCDConfigured = true + } + } + if isPreCDConfigured && isPreCDStageAbsent { + pipelinesAbsentInCache[item.Id] = append(pipelinesAbsentInCache[item.Id], bean2.CD_WORKFLOW_TYPE_PRE) + } + if isPostCDConfigured && isPostCdStageAbsent { + pipelinesAbsentInCache[item.Id] = append(pipelinesAbsentInCache[item.Id], bean2.CD_WORKFLOW_TYPE_POST) + } } } } From b6156f0e1c841fbda4870c951a977e70c0dd36ef Mon Sep 17 00:00:00 2001 From: ayushmaheshwari Date: Wed, 23 Jul 2025 19:28:16 +0530 Subject: [PATCH 21/31] dev testing changes --- .../cd/read/CdWorkflowRunnerReadService.go | 42 +++++++++++-------- 1 file changed, 25 insertions(+), 17 deletions(-) diff --git a/pkg/workflow/cd/read/CdWorkflowRunnerReadService.go b/pkg/workflow/cd/read/CdWorkflowRunnerReadService.go index c0ac8e0550..5db9313fb6 100644 --- a/pkg/workflow/cd/read/CdWorkflowRunnerReadService.go +++ b/pkg/workflow/cd/read/CdWorkflowRunnerReadService.go @@ -106,11 +106,29 @@ func (impl *CdWorkflowRunnerReadServiceImpl) GetWfrStatusForLatestRunners(pipeli // calculating all the pipelines not present in the index table cdWorkflowLatest pipelinesAbsentInCache := make(map[int][]bean2.WorkflowType) for _, item := range pipelines { - if stages, ok := cdWorfklowLatestMap[item.Id]; !ok || len(stages) == 0 { + + var isPreCDConfigured, isPostCDConfigured bool + if configuredStages, ok := pipelineStageMap[item.Id]; ok { + for _, stage := range configuredStages { + if stage == bean2.CD_WORKFLOW_TYPE_PRE { + isPreCDConfigured = true + } else if stage == bean2.CD_WORKFLOW_TYPE_POST { + isPostCDConfigured = true + } + } + } + + if _, ok := cdWorfklowLatestMap[item.Id]; !ok { pipelinesAbsentInCache[item.Id] = append(pipelinesAbsentInCache[item.Id], bean2.CD_WORKFLOW_TYPE_PRE, bean2.CD_WORKFLOW_TYPE_POST, bean2.CD_WORKFLOW_TYPE_DEPLOY) + if isPreCDConfigured { + pipelinesAbsentInCache[item.Id] = append(pipelinesAbsentInCache[item.Id], bean2.CD_WORKFLOW_TYPE_PRE) + } + if isPostCDConfigured { + pipelinesAbsentInCache[item.Id] = append(pipelinesAbsentInCache[item.Id], bean2.CD_WORKFLOW_TYPE_POST) + } } else { isPreCDStageAbsent, isPostCdStageAbsent, isDeployStageAbsent := true, true, true - for _, stage := range stages { + for _, stage := range cdWorfklowLatestMap[item.Id] { switch stage { case bean2.CD_WORKFLOW_TYPE_PRE: isPreCDStageAbsent = false @@ -123,21 +141,11 @@ func (impl *CdWorkflowRunnerReadServiceImpl) GetWfrStatusForLatestRunners(pipeli if isDeployStageAbsent { pipelinesAbsentInCache[item.Id] = append(pipelinesAbsentInCache[item.Id], bean2.CD_WORKFLOW_TYPE_DEPLOY) } - if configuredStages, ok := pipelineStageMap[item.Id]; ok { - var isPreCDConfigured, isPostCDConfigured bool - for _, stage := range configuredStages { - if stage == bean2.CD_WORKFLOW_TYPE_PRE { - isPreCDConfigured = true - } else if stage == bean2.CD_WORKFLOW_TYPE_POST { - isPostCDConfigured = true - } - } - if isPreCDConfigured && isPreCDStageAbsent { - pipelinesAbsentInCache[item.Id] = append(pipelinesAbsentInCache[item.Id], bean2.CD_WORKFLOW_TYPE_PRE) - } - if isPostCDConfigured && isPostCdStageAbsent { - pipelinesAbsentInCache[item.Id] = append(pipelinesAbsentInCache[item.Id], bean2.CD_WORKFLOW_TYPE_POST) - } + if isPreCDConfigured && isPreCDStageAbsent { + pipelinesAbsentInCache[item.Id] = append(pipelinesAbsentInCache[item.Id], bean2.CD_WORKFLOW_TYPE_PRE) + } + if isPostCDConfigured && isPostCdStageAbsent { + pipelinesAbsentInCache[item.Id] = append(pipelinesAbsentInCache[item.Id], bean2.CD_WORKFLOW_TYPE_POST) } } } From 4443e57daec6697f7faae6cd8028814a00c3b52b Mon Sep 17 00:00:00 2001 From: ayushmaheshwari Date: Wed, 23 Jul 2025 19:44:01 +0530 Subject: [PATCH 22/31] opt --- pkg/workflow/cd/read/CdWorkflowRunnerReadService.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/workflow/cd/read/CdWorkflowRunnerReadService.go b/pkg/workflow/cd/read/CdWorkflowRunnerReadService.go index 5db9313fb6..b12f3a13be 100644 --- a/pkg/workflow/cd/read/CdWorkflowRunnerReadService.go +++ b/pkg/workflow/cd/read/CdWorkflowRunnerReadService.go @@ -119,7 +119,7 @@ func (impl *CdWorkflowRunnerReadServiceImpl) GetWfrStatusForLatestRunners(pipeli } if _, ok := cdWorfklowLatestMap[item.Id]; !ok { - pipelinesAbsentInCache[item.Id] = append(pipelinesAbsentInCache[item.Id], bean2.CD_WORKFLOW_TYPE_PRE, bean2.CD_WORKFLOW_TYPE_POST, bean2.CD_WORKFLOW_TYPE_DEPLOY) + pipelinesAbsentInCache[item.Id] = append(pipelinesAbsentInCache[item.Id], bean2.CD_WORKFLOW_TYPE_DEPLOY) if isPreCDConfigured { pipelinesAbsentInCache[item.Id] = append(pipelinesAbsentInCache[item.Id], bean2.CD_WORKFLOW_TYPE_PRE) } From 714107a89401dffbd90756bb79934123df534dad Mon Sep 17 00:00:00 2001 From: Prakash Kumar Date: Thu, 24 Jul 2025 14:01:41 +0530 Subject: [PATCH 23/31] Remove redundant workflow status update logic for CI/CD workflows and refactor method signatures and repository integration in `WorkflowStatusLatestService`. --- pkg/pipeline/CiHandler.go | 12 -- pkg/pipeline/CiService.go | 20 +--- pkg/workflow/cd/CdWorkflowRunnerService.go | 15 --- .../WorkflowStatusLatestService.go | 48 +++----- .../WorkflowStatusUpdateService.go | 103 +----------------- wire_gen.go | 7 +- 6 files changed, 23 insertions(+), 182 deletions(-) diff --git a/pkg/pipeline/CiHandler.go b/pkg/pipeline/CiHandler.go index f222c2d894..6717214336 100644 --- a/pkg/pipeline/CiHandler.go +++ b/pkg/pipeline/CiHandler.go @@ -606,18 +606,6 @@ func (impl *CiHandlerImpl) UpdateWorkflow(workflowStatus eventProcessorBean.CiCd return savedWorkflow.Id, true, err } - // Update latest status table for CI workflow - // Check if CiPipeline is loaded, if not pass 0 as appId to let the function fetch it - appId := 0 - if savedWorkflow.CiPipeline != nil { - appId = savedWorkflow.CiPipeline.AppId - } - err = impl.workflowStatusUpdateService.UpdateCiWorkflowStatusLatest(nil, savedWorkflow.CiPipelineId, appId, savedWorkflow.Id, savedWorkflow.TriggeredBy) - if err != nil { - impl.Logger.Errorw("error in updating ci workflow status latest", "err", err, "pipelineId", savedWorkflow.CiPipelineId, "workflowId", savedWorkflow.Id) - // Don't return error here as the main workflow update was successful - } - impl.sendCIFailEvent(savedWorkflow, status, message) return savedWorkflow.Id, true, nil } diff --git a/pkg/pipeline/CiService.go b/pkg/pipeline/CiService.go index 08b82aa9fc..fb63f0a449 100644 --- a/pkg/pipeline/CiService.go +++ b/pkg/pipeline/CiService.go @@ -135,13 +135,7 @@ func (impl *CiServiceImpl) SaveCiWorkflowWithStage(wf *pipelineConfig.CiWorkflow return err } - // Update latest status table for CI workflow within the transaction - // Check if CiPipeline is loaded, if not pass 0 as appId to let the function fetch it - appId := 0 - if wf.CiPipeline != nil { - appId = wf.CiPipeline.AppId - } - err = impl.workflowStatusUpdateService.UpdateCiWorkflowStatusLatest(tx, wf.CiPipelineId, appId, wf.Id, wf.TriggeredBy) + err = impl.workflowStatusUpdateService.SaveCiWorkflowStatusLatest(tx, wf.CiPipelineId, wf.Id, wf.TriggeredBy) if err != nil { impl.Logger.Errorw("error in updating ci workflow status latest", "err", err, "pipelineId", wf.CiPipelineId, "workflowId", wf.Id) return err @@ -184,18 +178,6 @@ func (impl *CiServiceImpl) UpdateCiWorkflowWithStage(wf *pipelineConfig.CiWorkfl return err } - // Update latest status table for CI workflow within the transaction - // Check if CiPipeline is loaded, if not pass 0 as appId to let the function fetch it - appId := 0 - if wf.CiPipeline != nil { - appId = wf.CiPipeline.AppId - } - err = impl.workflowStatusUpdateService.UpdateCiWorkflowStatusLatest(tx, wf.CiPipelineId, appId, wf.Id, wf.TriggeredBy) - if err != nil { - impl.Logger.Errorw("error in updating ci workflow status latest", "err", err, "pipelineId", wf.CiPipelineId, "workflowId", wf.Id) - return err - } - err = impl.transactionManager.CommitTx(tx) if err != nil { impl.Logger.Errorw("error in committing transaction", "workflowName", wf.Name, "error", err) diff --git a/pkg/workflow/cd/CdWorkflowRunnerService.go b/pkg/workflow/cd/CdWorkflowRunnerService.go index b2276c0fd0..cdc6b890cf 100644 --- a/pkg/workflow/cd/CdWorkflowRunnerService.go +++ b/pkg/workflow/cd/CdWorkflowRunnerService.go @@ -80,21 +80,6 @@ func (impl *CdWorkflowRunnerServiceImpl) UpdateWfr(dto *bean.CdWorkflowRunnerDto impl.logger.Errorw("error in updating runner status in db", "runnerId", runnerDbObj.Id, "err", err) return err } - - // Update latest status table for CD workflow - // Check if CdWorkflow and Pipeline are loaded, if not pass 0 as appId/environmentId to let the function fetch them - appId := 0 - environmentId := 0 - if runnerDbObj.CdWorkflow != nil && runnerDbObj.CdWorkflow.Pipeline != nil { - appId = runnerDbObj.CdWorkflow.Pipeline.AppId - environmentId = runnerDbObj.CdWorkflow.Pipeline.EnvironmentId - } - err = impl.workflowStatusUpdateService.UpdateCdWorkflowStatusLatest(nil, runnerDbObj.CdWorkflow.PipelineId, appId, environmentId, runnerDbObj.Id, runnerDbObj.WorkflowType.String(), int32(updatedBy)) - if err != nil { - impl.logger.Errorw("error in updating cd workflow status latest", "err", err, "pipelineId", runnerDbObj.CdWorkflow.PipelineId, "workflowRunnerId", runnerDbObj.Id) - // Don't return error here as the main workflow update was successful - } - return nil } diff --git a/pkg/workflow/workflowStatusLatest/WorkflowStatusLatestService.go b/pkg/workflow/workflowStatusLatest/WorkflowStatusLatestService.go index ffd685be8e..303fae5273 100644 --- a/pkg/workflow/workflowStatusLatest/WorkflowStatusLatestService.go +++ b/pkg/workflow/workflowStatusLatest/WorkflowStatusLatestService.go @@ -30,12 +30,12 @@ import ( type WorkflowStatusLatestService interface { // CI Workflow Status Latest methods - SaveOrUpdateCiWorkflowStatusLatest(tx *pg.Tx, pipelineId, appId, ciWorkflowId int, userId int32) error + SaveCiWorkflowStatusLatest(tx *pg.Tx, pipelineId, ciWorkflowId int, userId int32) error GetCiWorkflowStatusLatestByPipelineId(pipelineId int) (*CiWorkflowStatusLatest, error) GetCiWorkflowStatusLatestByAppId(appId int) ([]*CiWorkflowStatusLatest, error) // CD Workflow Status Latest methods - SaveOrUpdateCdWorkflowStatusLatest(tx *pg.Tx, pipelineId, appId, environmentId, workflowRunnerId int, workflowType string, userId int32) error + SaveCdWorkflowStatusLatest(tx *pg.Tx, pipelineId, appId, environmentId, workflowRunnerId int, workflowType string, userId int32) error GetCdWorkflowStatusLatestByPipelineIdAndWorkflowType(pipelineId int, workflowType string) (*CdWorkflowStatusLatest, error) GetCdWorkflowStatusLatestByAppId(appId int) ([]*CdWorkflowStatusLatest, error) GetCdWorkflowStatusLatestByPipelineId(pipelineId int) ([]*CdWorkflowStatusLatest, error) @@ -47,6 +47,7 @@ type WorkflowStatusLatestServiceImpl struct { workflowStatusLatestRepository pipelineConfig.WorkflowStatusLatestRepository ciWorkflowRepository pipelineConfig.CiWorkflowRepository cdWorkflowRepository pipelineConfig.CdWorkflowRepository + ciPipelineRepository pipelineConfig.CiPipelineRepository } func NewWorkflowStatusLatestServiceImpl( @@ -54,12 +55,14 @@ func NewWorkflowStatusLatestServiceImpl( workflowStatusLatestRepository pipelineConfig.WorkflowStatusLatestRepository, ciWorkflowRepository pipelineConfig.CiWorkflowRepository, cdWorkflowRepository pipelineConfig.CdWorkflowRepository, + ciPipelineRepository pipelineConfig.CiPipelineRepository, ) *WorkflowStatusLatestServiceImpl { return &WorkflowStatusLatestServiceImpl{ logger: logger, workflowStatusLatestRepository: workflowStatusLatestRepository, ciWorkflowRepository: ciWorkflowRepository, cdWorkflowRepository: cdWorkflowRepository, + ciPipelineRepository: ciPipelineRepository, } } @@ -80,8 +83,7 @@ type CdWorkflowStatusLatest struct { Status string `json:"status"` // Derived from cd_workflow_runner table } -// CI Workflow Status Latest methods implementation -func (impl *WorkflowStatusLatestServiceImpl) SaveOrUpdateCiWorkflowStatusLatest(tx *pg.Tx, pipelineId, appId, ciWorkflowId int, userId int32) error { +func (impl *WorkflowStatusLatestServiceImpl) SaveCiWorkflowStatusLatest(tx *pg.Tx, pipelineId, ciWorkflowId int, userId int32) error { // Validate required parameters if pipelineId <= 0 { impl.logger.Errorw("invalid pipelineId provided", "pipelineId", pipelineId) @@ -92,44 +94,22 @@ func (impl *WorkflowStatusLatestServiceImpl) SaveOrUpdateCiWorkflowStatusLatest( impl.logger.Errorw("invalid ciWorkflowId provided", "ciWorkflowId", ciWorkflowId) return fmt.Errorf("invalid ciWorkflowId: %d", ciWorkflowId) } - - // If appId is not provided (0), fetch it from the CiPipeline - if appId <= 0 { - ciPipeline, err := impl.ciWorkflowRepository.FindById(ciWorkflowId) - if err != nil { - impl.logger.Errorw("error in fetching ci workflow to get appId", "err", err, "ciWorkflowId", ciWorkflowId) - return err - } - - if ciPipeline == nil { - impl.logger.Errorw("ci workflow not found", "ciWorkflowId", ciWorkflowId) - return fmt.Errorf("ci workflow not found with id: %d", ciWorkflowId) - } - - // Check if CiPipeline is loaded - if ciPipeline.CiPipeline == nil { - impl.logger.Errorw("ci pipeline not loaded in ci workflow", "ciWorkflowId", ciWorkflowId, "ciPipelineId", ciPipeline.CiPipelineId) - return fmt.Errorf("ci pipeline not loaded for workflow id: %d", ciWorkflowId) - } - - appId = ciPipeline.CiPipeline.AppId - if appId <= 0 { - impl.logger.Errorw("invalid appId in ci pipeline", "ciWorkflowId", ciWorkflowId, "ciPipelineId", ciPipeline.CiPipelineId, "appId", appId) - return fmt.Errorf("invalid appId in ci pipeline: %d", appId) - } - - impl.logger.Debugw("fetched appId from ci workflow", "ciWorkflowId", ciWorkflowId, "appId", appId) + ciPipeline, err := impl.ciPipelineRepository.FindOneWithAppData(pipelineId) + if err != nil { + impl.logger.Errorw("error in fetching ci pipeline for appId", "ciPipelineId", pipelineId, "err", err) + return err } + appId := ciPipeline.AppId // Check if entry exists existingEntry, err := impl.workflowStatusLatestRepository.GetCiWorkflowStatusLatestByPipelineId(pipelineId) - if err != nil && err != pg.ErrNoRows { + if err != nil && !util2.IsErrNoRows(err) { impl.logger.Errorw("error in getting ci workflow status latest", "err", err, "pipelineId", pipelineId) return err } now := time.Now() - if err == pg.ErrNoRows { + if util2.IsErrNoRows(err) { // Create new entry model := &pipelineConfig.CiWorkflowStatusLatest{ PipelineId: pipelineId, @@ -207,7 +187,7 @@ func (impl *WorkflowStatusLatestServiceImpl) GetCiWorkflowStatusLatestByAppId(ap } // CD Workflow Status Latest methods implementation -func (impl *WorkflowStatusLatestServiceImpl) SaveOrUpdateCdWorkflowStatusLatest(tx *pg.Tx, pipelineId, appId, environmentId, workflowRunnerId int, workflowType string, userId int32) error { +func (impl *WorkflowStatusLatestServiceImpl) SaveCdWorkflowStatusLatest(tx *pg.Tx, pipelineId, appId, environmentId, workflowRunnerId int, workflowType string, userId int32) error { // Validate required parameters if pipelineId <= 0 { impl.logger.Errorw("invalid pipelineId provided", "pipelineId", pipelineId) diff --git a/pkg/workflow/workflowStatusLatest/WorkflowStatusUpdateService.go b/pkg/workflow/workflowStatusLatest/WorkflowStatusUpdateService.go index 785d5afc6a..0183617fd3 100644 --- a/pkg/workflow/workflowStatusLatest/WorkflowStatusUpdateService.go +++ b/pkg/workflow/workflowStatusLatest/WorkflowStatusUpdateService.go @@ -24,12 +24,8 @@ import ( type WorkflowStatusUpdateService interface { // Methods to update latest status tables when workflow status changes - UpdateCiWorkflowStatusLatest(tx *pg.Tx, pipelineId, appId, ciWorkflowId int, userId int32) error + SaveCiWorkflowStatusLatest(tx *pg.Tx, pipelineId, ciWorkflowId int, userId int32) error UpdateCdWorkflowStatusLatest(tx *pg.Tx, pipelineId, appId, environmentId, workflowRunnerId int, workflowType string, userId int32) error - - // Methods to fetch optimized status for trigger view - FetchCiStatusForTriggerViewOptimized(appId int) ([]*pipelineConfig.CiWorkflowStatus, error) - FetchCdStatusForTriggerViewOptimized(appId int) ([]*pipelineConfig.CdWorkflowStatus, error) } type WorkflowStatusUpdateServiceImpl struct { @@ -59,101 +55,10 @@ func NewWorkflowStatusUpdateServiceImpl( } } -func (impl *WorkflowStatusUpdateServiceImpl) UpdateCiWorkflowStatusLatest(tx *pg.Tx, pipelineId, appId, ciWorkflowId int, userId int32) error { - return impl.workflowStatusLatestService.SaveOrUpdateCiWorkflowStatusLatest(tx, pipelineId, appId, ciWorkflowId, userId) +func (impl *WorkflowStatusUpdateServiceImpl) SaveCiWorkflowStatusLatest(tx *pg.Tx, pipelineId, ciWorkflowId int, userId int32) error { + return impl.workflowStatusLatestService.SaveCiWorkflowStatusLatest(tx, pipelineId, ciWorkflowId, userId) } func (impl *WorkflowStatusUpdateServiceImpl) UpdateCdWorkflowStatusLatest(tx *pg.Tx, pipelineId, appId, environmentId, workflowRunnerId int, workflowType string, userId int32) error { - return impl.workflowStatusLatestService.SaveOrUpdateCdWorkflowStatusLatest(tx, pipelineId, appId, environmentId, workflowRunnerId, workflowType, userId) -} - -func (impl *WorkflowStatusUpdateServiceImpl) FetchCiStatusForTriggerViewOptimized(appId int) ([]*pipelineConfig.CiWorkflowStatus, error) { - latestStatuses, err := impl.workflowStatusLatestService.GetCiWorkflowStatusLatestByAppId(appId) - if err != nil { - impl.logger.Errorw("error in getting ci workflow status latest by app id", "err", err, "appId", appId) - // Fallback to old method - return impl.ciWorkflowRepository.FIndCiWorkflowStatusesByAppId(appId) - } - - // Convert to the expected format - var ciWorkflowStatuses []*pipelineConfig.CiWorkflowStatus - for _, latestStatus := range latestStatuses { - ciPipeline, err := impl.ciPipelineRepository.FindById(latestStatus.PipelineId) - if err != nil { - impl.logger.Errorw("error in getting ci pipeline", "err", err, "pipelineId", latestStatus.PipelineId) - continue - } - - ciWorkflowStatus := &pipelineConfig.CiWorkflowStatus{ - CiPipelineId: latestStatus.PipelineId, - CiPipelineName: ciPipeline.Name, - CiStatus: latestStatus.Status, - CiWorkflowId: latestStatus.CiWorkflowId, - StorageConfigured: latestStatus.StorageConfigured, - } - ciWorkflowStatuses = append(ciWorkflowStatuses, ciWorkflowStatus) - } - - // If no entries found in latest status table, fallback to old method - if len(ciWorkflowStatuses) == 0 { - impl.logger.Infow("no entries found in ci workflow status latest table, falling back to old method", "appId", appId) - return impl.ciWorkflowRepository.FIndCiWorkflowStatusesByAppId(appId) - } - - return ciWorkflowStatuses, nil -} - -func (impl *WorkflowStatusUpdateServiceImpl) FetchCdStatusForTriggerViewOptimized(appId int) ([]*pipelineConfig.CdWorkflowStatus, error) { - // First try to get from the optimized latest status table - latestStatuses, err := impl.workflowStatusLatestService.GetCdWorkflowStatusLatestByAppId(appId) - if err != nil { - impl.logger.Errorw("error in getting cd workflow status latest by app id", "err", err, "appId", appId) - // Fallback to old method - would need to implement this based on existing CD status fetching logic - return nil, err - } - - // Convert to the expected format - var cdWorkflowStatuses []*pipelineConfig.CdWorkflowStatus - for _, latestStatus := range latestStatuses { - // Get pipeline info - pipeline, err := impl.pipelineRepository.FindById(latestStatus.PipelineId) - if err != nil { - impl.logger.Errorw("error in getting pipeline", "err", err, "pipelineId", latestStatus.PipelineId) - continue - } - - var status string - switch latestStatus.WorkflowType { - case "PRE": - status = latestStatus.Status - case "DEPLOY": - status = latestStatus.Status - case "POST": - status = latestStatus.Status - default: - status = latestStatus.Status - } - - cdWorkflowStatus := &pipelineConfig.CdWorkflowStatus{ - CiPipelineId: pipeline.CiPipelineId, - PipelineId: latestStatus.PipelineId, - PipelineName: pipeline.Name, - WorkflowType: latestStatus.WorkflowType, - WfrId: latestStatus.WorkflowRunnerId, - } - - // Set the appropriate status field based on workflow type - switch latestStatus.WorkflowType { - case "PRE": - cdWorkflowStatus.PreStatus = status - case "DEPLOY": - cdWorkflowStatus.DeployStatus = status - case "POST": - cdWorkflowStatus.PostStatus = status - } - - cdWorkflowStatuses = append(cdWorkflowStatuses, cdWorkflowStatus) - } - - return cdWorkflowStatuses, nil + return impl.workflowStatusLatestService.SaveCdWorkflowStatusLatest(tx, pipelineId, appId, environmentId, workflowRunnerId, workflowType, userId) } diff --git a/wire_gen.go b/wire_gen.go index bd3bbdcc3a..7273f957d1 100644 --- a/wire_gen.go +++ b/wire_gen.go @@ -625,7 +625,7 @@ func InitializeApp() (*App, error) { workflowStageRepositoryImpl := repository18.NewWorkflowStageRepositoryImpl(sugaredLogger, db) workFlowStageStatusServiceImpl := workflowStatus.NewWorkflowStageFlowStatusServiceImpl(sugaredLogger, workflowStageRepositoryImpl, ciWorkflowRepositoryImpl, cdWorkflowRepositoryImpl, transactionUtilImpl) workflowStatusLatestRepositoryImpl := pipelineConfig.NewWorkflowStatusLatestRepositoryImpl(db, sugaredLogger) - workflowStatusLatestServiceImpl := workflowStatusLatest.NewWorkflowStatusLatestServiceImpl(sugaredLogger, workflowStatusLatestRepositoryImpl, ciWorkflowRepositoryImpl, cdWorkflowRepositoryImpl) + workflowStatusLatestServiceImpl := workflowStatusLatest.NewWorkflowStatusLatestServiceImpl(sugaredLogger, workflowStatusLatestRepositoryImpl, ciWorkflowRepositoryImpl, cdWorkflowRepositoryImpl, ciPipelineRepositoryImpl) workflowStatusUpdateServiceImpl := workflowStatusLatest.NewWorkflowStatusUpdateServiceImpl(sugaredLogger, workflowStatusLatestServiceImpl, ciWorkflowRepositoryImpl, cdWorkflowRepositoryImpl, ciPipelineRepositoryImpl, pipelineRepositoryImpl) cdWorkflowRunnerServiceImpl := cd.NewCdWorkflowRunnerServiceImpl(sugaredLogger, cdWorkflowRepositoryImpl, workFlowStageStatusServiceImpl, transactionUtilImpl, workflowStatusUpdateServiceImpl) deploymentEventHandlerImpl := app2.NewDeploymentEventHandlerImpl(sugaredLogger, eventRESTClientImpl, eventSimpleFactoryImpl, runnable) @@ -753,7 +753,8 @@ func InitializeApp() (*App, error) { deploymentTemplateValidationServiceImpl := validator.NewDeploymentTemplateValidationServiceImpl(sugaredLogger, chartRefServiceImpl, scopedVariableManagerImpl, deployedAppMetricsServiceImpl, deploymentTemplateValidationServiceEntImpl) devtronAppGitOpConfigServiceImpl := gitOpsConfig.NewDevtronAppGitOpConfigServiceImpl(sugaredLogger, chartRepositoryImpl, chartServiceImpl, gitOpsConfigReadServiceImpl, gitOpsValidationServiceImpl, argoClientWrapperServiceImpl, deploymentConfigServiceImpl, chartReadServiceImpl) ciHandlerImpl := pipeline.NewCiHandlerImpl(sugaredLogger, ciServiceImpl, ciPipelineMaterialRepositoryImpl, clientImpl, ciWorkflowRepositoryImpl, ciArtifactRepositoryImpl, userServiceImpl, eventRESTClientImpl, eventSimpleFactoryImpl, ciPipelineRepositoryImpl, appListingRepositoryImpl, pipelineRepositoryImpl, enforcerUtilImpl, resourceGroupServiceImpl, environmentRepositoryImpl, imageTaggingServiceImpl, k8sCommonServiceImpl, appWorkflowRepositoryImpl, customTagServiceImpl, workFlowStageStatusServiceImpl, workflowStatusLatestRepositoryImpl, workflowStatusUpdateServiceImpl) - cdHandlerImpl := pipeline.NewCdHandlerImpl(sugaredLogger, userServiceImpl, cdWorkflowRepositoryImpl, ciArtifactRepositoryImpl, ciPipelineMaterialRepositoryImpl, pipelineRepositoryImpl, environmentRepositoryImpl, ciWorkflowRepositoryImpl, enforcerUtilImpl, resourceGroupServiceImpl, imageTaggingServiceImpl, k8sServiceImpl, customTagServiceImpl, deploymentConfigServiceImpl, workFlowStageStatusServiceImpl, cdWorkflowRunnerServiceImpl, workflowStatusLatestServiceImpl, pipelineStageRepositoryImpl) + cdWorkflowRunnerReadServiceImpl := read18.NewCdWorkflowRunnerReadServiceImpl(sugaredLogger, cdWorkflowRepositoryImpl, workflowStatusLatestServiceImpl, pipelineStageRepositoryImpl) + cdHandlerImpl := pipeline.NewCdHandlerImpl(sugaredLogger, userServiceImpl, cdWorkflowRepositoryImpl, ciArtifactRepositoryImpl, ciPipelineMaterialRepositoryImpl, pipelineRepositoryImpl, environmentRepositoryImpl, ciWorkflowRepositoryImpl, enforcerUtilImpl, resourceGroupServiceImpl, imageTaggingServiceImpl, k8sServiceImpl, customTagServiceImpl, deploymentConfigServiceImpl, workFlowStageStatusServiceImpl, cdWorkflowRunnerServiceImpl, workflowStatusLatestServiceImpl, pipelineStageRepositoryImpl, cdWorkflowRunnerReadServiceImpl) appWorkflowServiceImpl := appWorkflow2.NewAppWorkflowServiceImpl(sugaredLogger, appWorkflowRepositoryImpl, ciCdPipelineOrchestratorImpl, ciPipelineRepositoryImpl, pipelineRepositoryImpl, enforcerUtilImpl, resourceGroupServiceImpl, appRepositoryImpl, userAuthServiceImpl, chartServiceImpl, deploymentConfigServiceImpl, pipelineBuilderImpl) appCloneServiceImpl := appClone.NewAppCloneServiceImpl(sugaredLogger, pipelineBuilderImpl, attributesServiceImpl, chartServiceImpl, configMapServiceImpl, appWorkflowServiceImpl, appListingServiceImpl, propertiesConfigServiceImpl, pipelineStageServiceImpl, ciTemplateReadServiceImpl, appRepositoryImpl, ciPipelineRepositoryImpl, pipelineRepositoryImpl, ciPipelineConfigServiceImpl, gitOpsConfigReadServiceImpl, chartReadServiceImpl) deploymentTemplateRepositoryImpl := repository2.NewDeploymentTemplateRepositoryImpl(db, sugaredLogger) @@ -791,7 +792,7 @@ func InitializeApp() (*App, error) { pipelineConfigRestHandlerImpl := configure.NewPipelineRestHandlerImpl(pipelineBuilderImpl, sugaredLogger, deploymentTemplateValidationServiceImpl, chartServiceImpl, devtronAppGitOpConfigServiceImpl, propertiesConfigServiceImpl, userServiceImpl, teamServiceImpl, enforcerImpl, ciHandlerImpl, validate, clientImpl, ciPipelineRepositoryImpl, pipelineRepositoryImpl, enforcerUtilImpl, dockerRegistryConfigImpl, cdHandlerImpl, appCloneServiceImpl, generateManifestDeploymentTemplateServiceImpl, appWorkflowServiceImpl, gitMaterialReadServiceImpl, policyServiceImpl, imageScanResultReadServiceImpl, ciPipelineMaterialRepositoryImpl, imageTaggingReadServiceImpl, imageTaggingServiceImpl, ciArtifactRepositoryImpl, deployedAppMetricsServiceImpl, chartRefServiceImpl, ciCdPipelineOrchestratorImpl, gitProviderReadServiceImpl, teamReadServiceImpl, environmentRepositoryImpl, chartReadServiceImpl, draftAwareConfigServiceImpl, handlerServiceImpl, devtronAppsHandlerServiceImpl) commonArtifactServiceImpl := artifacts.NewCommonArtifactServiceImpl(sugaredLogger, ciArtifactRepositoryImpl) fluxApplicationServiceImpl := fluxApplication.NewFluxApplicationServiceImpl(sugaredLogger, helmAppReadServiceImpl, clusterServiceImplExtended, helmAppClientImpl, pumpImpl, pipelineRepositoryImpl, installedAppRepositoryImpl) - workflowDagExecutorImpl := dag.NewWorkflowDagExecutorImpl(sugaredLogger, pipelineRepositoryImpl, pipelineOverrideRepositoryImpl, cdWorkflowRepositoryImpl, ciArtifactRepositoryImpl, enforcerUtilImpl, appWorkflowRepositoryImpl, pipelineStageServiceImpl, ciWorkflowRepositoryImpl, ciPipelineRepositoryImpl, pipelineStageRepositoryImpl, globalPluginRepositoryImpl, eventRESTClientImpl, eventSimpleFactoryImpl, customTagServiceImpl, pipelineStatusTimelineServiceImpl, cdWorkflowRunnerServiceImpl, ciServiceImpl, helmAppServiceImpl, cdWorkflowCommonServiceImpl, devtronAppsHandlerServiceImpl, userDeploymentRequestServiceImpl, manifestCreationServiceImpl, commonArtifactServiceImpl, deploymentConfigServiceImpl, runnable, imageScanHistoryRepositoryImpl, imageScanServiceImpl, k8sServiceImpl, environmentRepositoryImpl, k8sCommonServiceImpl, workflowServiceImpl, handlerServiceImpl, workflowTriggerAuditServiceImpl, fluxApplicationServiceImpl, workflowStatusUpdateServiceImpl) + workflowDagExecutorImpl := dag.NewWorkflowDagExecutorImpl(sugaredLogger, pipelineRepositoryImpl, pipelineOverrideRepositoryImpl, cdWorkflowRepositoryImpl, ciArtifactRepositoryImpl, enforcerUtilImpl, appWorkflowRepositoryImpl, pipelineStageServiceImpl, ciWorkflowRepositoryImpl, ciPipelineRepositoryImpl, pipelineStageRepositoryImpl, globalPluginRepositoryImpl, eventRESTClientImpl, eventSimpleFactoryImpl, customTagServiceImpl, pipelineStatusTimelineServiceImpl, cdWorkflowRunnerServiceImpl, ciServiceImpl, helmAppServiceImpl, cdWorkflowCommonServiceImpl, devtronAppsHandlerServiceImpl, userDeploymentRequestServiceImpl, manifestCreationServiceImpl, commonArtifactServiceImpl, deploymentConfigServiceImpl, runnable, imageScanHistoryRepositoryImpl, imageScanServiceImpl, k8sServiceImpl, environmentRepositoryImpl, k8sCommonServiceImpl, workflowServiceImpl, handlerServiceImpl, workflowTriggerAuditServiceImpl, fluxApplicationServiceImpl) externalCiRestHandlerImpl := restHandler.NewExternalCiRestHandlerImpl(sugaredLogger, validate, userServiceImpl, enforcerImpl, workflowDagExecutorImpl) pubSubClientRestHandlerImpl := restHandler.NewPubSubClientRestHandlerImpl(pubSubClientServiceImpl, sugaredLogger, ciCdConfig) webhookRouterImpl := router.NewWebhookRouterImpl(gitWebhookRestHandlerImpl, pipelineConfigRestHandlerImpl, externalCiRestHandlerImpl, pubSubClientRestHandlerImpl) From ca578cfce53ddb01793ccbf5c8b1c8c214808224 Mon Sep 17 00:00:00 2001 From: ayushmaheshwari Date: Thu, 24 Jul 2025 14:11:53 +0530 Subject: [PATCH 24/31] changing slow query --- .../pipelineConfig/CdWorfkflowRepository.go | 19 +++---------- .../cd/read/CdWorkflowRunnerReadService.go | 27 +++++-------------- 2 files changed, 11 insertions(+), 35 deletions(-) diff --git a/internal/sql/repository/pipelineConfig/CdWorfkflowRepository.go b/internal/sql/repository/pipelineConfig/CdWorfkflowRepository.go index 26325aff56..6b6001a8dd 100644 --- a/internal/sql/repository/pipelineConfig/CdWorfkflowRepository.go +++ b/internal/sql/repository/pipelineConfig/CdWorfkflowRepository.go @@ -19,7 +19,6 @@ package pipelineConfig import ( "context" "errors" - "fmt" apiBean "github.com/devtron-labs/devtron/api/bean" "github.com/devtron-labs/devtron/internal/sql/repository" "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig/bean/workflow" @@ -29,7 +28,6 @@ import ( "github.com/go-pg/pg" "go.opentelemetry.io/otel" "go.uber.org/zap" - "strings" "time" ) @@ -71,7 +69,7 @@ type CdWorkflowRepository interface { IsLatestWf(pipelineId int, wfId int) (bool, error) FindLatestCdWorkflowByPipelineId(pipelineIds []int) (*CdWorkflow, error) FindLatestCdWorkflowByPipelineIdV2(pipelineIds []int) ([]*CdWorkflow, error) - FetchAllCdStagesLatestEntity(pipelineWorkflowPairs map[int][]apiBean.WorkflowType) ([]*CdWorkflowStatus, error) + FetchAllCdStagesLatestEntity(pipelineIds []int) ([]*CdWorkflowStatus, error) FetchAllCdStagesLatestEntityStatus(wfrIds []int) ([]*CdWorkflowRunner, error) ExistsByStatus(status string) (bool, error) FetchEnvAllCdStagesLatestEntityStatus(wfrIds []int, envID int) ([]*CdWorkflowRunner, error) @@ -580,15 +578,15 @@ func (impl *CdWorkflowRepositoryImpl) IsLatestWf(pipelineId int, wfId int) (bool return !exists, err } -func (impl *CdWorkflowRepositoryImpl) FetchAllCdStagesLatestEntity(pipelineWorkflowPairs map[int][]apiBean.WorkflowType) ([]*CdWorkflowStatus, error) { +func (impl *CdWorkflowRepositoryImpl) FetchAllCdStagesLatestEntity(pipelineIds []int) ([]*CdWorkflowStatus, error) { var cdWorkflowStatus []*CdWorkflowStatus - if len(pipelineWorkflowPairs) == 0 { + if len(pipelineIds) == 0 { return cdWorkflowStatus, nil } query := "select p.ci_pipeline_id, wf.pipeline_id, wfr.workflow_type, max(wfr.id) as wfr_id from cd_workflow_runner wfr" + " inner join cd_workflow wf on wf.id=wfr.cd_workflow_id" + " inner join pipeline p on p.id = wf.pipeline_id" + - " where (wf.pipeline_id, wfr.workflow_type) in (" + buildPipelineTypeValuesList(pipelineWorkflowPairs) + ")" + + " where wf.pipeline_id in (" + sqlIntSeq(pipelineIds) + ")" + " group by p.ci_pipeline_id, wf.pipeline_id, wfr.workflow_type order by wfr_id desc;" _, err := impl.dbConnection.Query(&cdWorkflowStatus, query) if err != nil { @@ -598,15 +596,6 @@ func (impl *CdWorkflowRepositoryImpl) FetchAllCdStagesLatestEntity(pipelineWorkf return cdWorkflowStatus, nil } -func buildPipelineTypeValuesList(pairs map[int][]apiBean.WorkflowType) string { - var values []string - for pipelineId, workflowTypes := range pairs { - for _, workflowType := range workflowTypes { - values = append(values, fmt.Sprintf("(%d,'%s')", pipelineId, workflowType)) - } - } - return strings.Join(values, ",") -} func (impl *CdWorkflowRepositoryImpl) FetchAllCdStagesLatestEntityStatus(wfrIds []int) ([]*CdWorkflowRunner, error) { var wfrList []*CdWorkflowRunner err := impl.dbConnection.Model(&wfrList). diff --git a/pkg/workflow/cd/read/CdWorkflowRunnerReadService.go b/pkg/workflow/cd/read/CdWorkflowRunnerReadService.go index b12f3a13be..ab44f45757 100644 --- a/pkg/workflow/cd/read/CdWorkflowRunnerReadService.go +++ b/pkg/workflow/cd/read/CdWorkflowRunnerReadService.go @@ -104,9 +104,8 @@ func (impl *CdWorkflowRunnerReadServiceImpl) GetWfrStatusForLatestRunners(pipeli } // calculating all the pipelines not present in the index table cdWorkflowLatest - pipelinesAbsentInCache := make(map[int][]bean2.WorkflowType) + absentPipelineIds := make([]int, 0) for _, item := range pipelines { - var isPreCDConfigured, isPostCDConfigured bool if configuredStages, ok := pipelineStageMap[item.Id]; ok { for _, stage := range configuredStages { @@ -119,13 +118,7 @@ func (impl *CdWorkflowRunnerReadServiceImpl) GetWfrStatusForLatestRunners(pipeli } if _, ok := cdWorfklowLatestMap[item.Id]; !ok { - pipelinesAbsentInCache[item.Id] = append(pipelinesAbsentInCache[item.Id], bean2.CD_WORKFLOW_TYPE_DEPLOY) - if isPreCDConfigured { - pipelinesAbsentInCache[item.Id] = append(pipelinesAbsentInCache[item.Id], bean2.CD_WORKFLOW_TYPE_PRE) - } - if isPostCDConfigured { - pipelinesAbsentInCache[item.Id] = append(pipelinesAbsentInCache[item.Id], bean2.CD_WORKFLOW_TYPE_POST) - } + absentPipelineIds = append(absentPipelineIds, item.Id) } else { isPreCDStageAbsent, isPostCdStageAbsent, isDeployStageAbsent := true, true, true for _, stage := range cdWorfklowLatestMap[item.Id] { @@ -138,21 +131,15 @@ func (impl *CdWorkflowRunnerReadServiceImpl) GetWfrStatusForLatestRunners(pipeli isDeployStageAbsent = false } } - if isDeployStageAbsent { - pipelinesAbsentInCache[item.Id] = append(pipelinesAbsentInCache[item.Id], bean2.CD_WORKFLOW_TYPE_DEPLOY) - } - if isPreCDConfigured && isPreCDStageAbsent { - pipelinesAbsentInCache[item.Id] = append(pipelinesAbsentInCache[item.Id], bean2.CD_WORKFLOW_TYPE_PRE) - } - if isPostCDConfigured && isPostCdStageAbsent { - pipelinesAbsentInCache[item.Id] = append(pipelinesAbsentInCache[item.Id], bean2.CD_WORKFLOW_TYPE_POST) + if isDeployStageAbsent || (isPreCDConfigured && isPreCDStageAbsent) || (isPostCDConfigured && isPostCdStageAbsent) { + absentPipelineIds = append(absentPipelineIds, item.Id) } } } - if len(pipelinesAbsentInCache) > 0 { - remainingRunners, err := impl.cdWorkflowRepository.FetchAllCdStagesLatestEntity(pipelinesAbsentInCache) + if len(absentPipelineIds) > 0 { + remainingRunners, err := impl.cdWorkflowRepository.FetchAllCdStagesLatestEntity(absentPipelineIds) if err != nil { - impl.logger.Errorw("error in fetching all cd stages latest entity", "pipelinesAbsentInCache", pipelinesAbsentInCache, "err", err) + impl.logger.Errorw("error in fetching all cd stages latest entity", "pipelineIds", absentPipelineIds, "err", err) return nil, err } result = append(result, remainingRunners...) From df5fb334d925f9fc4845ea168b1b189e717c3e30 Mon Sep 17 00:00:00 2001 From: Prakash Kumar Date: Thu, 24 Jul 2025 14:32:59 +0530 Subject: [PATCH 25/31] Remove `WorkflowStatusUpdateService` and update references to streamline workflow status handling in CI/CD services. --- .../trigger/devtronApps/HandlerService.go | 6 +- .../devtronApps/deployStageHandlerCode.go | 4 +- pkg/pipeline/CiHandler.go | 4 - pkg/pipeline/CiService.go | 21 ++- pkg/workflow/cd/CdWorkflowRunnerService.go | 8 +- .../WorkflowStatusLatestService.go | 143 +++--------------- .../WorkflowStatusUpdateService.go | 64 -------- .../wire_workflow_status_latest.go | 2 - wire_gen.go | 9 +- 9 files changed, 54 insertions(+), 207 deletions(-) delete mode 100644 pkg/workflow/workflowStatusLatest/WorkflowStatusUpdateService.go diff --git a/pkg/deployment/trigger/devtronApps/HandlerService.go b/pkg/deployment/trigger/devtronApps/HandlerService.go index dcc8fc3b0e..7a50014808 100644 --- a/pkg/deployment/trigger/devtronApps/HandlerService.go +++ b/pkg/deployment/trigger/devtronApps/HandlerService.go @@ -175,7 +175,7 @@ type HandlerServiceImpl struct { asyncRunnable *async.Runnable workflowTriggerAuditService service2.WorkflowTriggerAuditService fluxCdDeploymentService fluxcd.DeploymentService - workflowStatusUpdateService workflowStatusLatest.WorkflowStatusUpdateService + workflowStatusLatestService workflowStatusLatest.WorkflowStatusLatestService } func NewHandlerServiceImpl(logger *zap.SugaredLogger, @@ -241,7 +241,7 @@ func NewHandlerServiceImpl(logger *zap.SugaredLogger, asyncRunnable *async.Runnable, workflowTriggerAuditService service2.WorkflowTriggerAuditService, fluxCdDeploymentService fluxcd.DeploymentService, - workflowStatusUpdateService workflowStatusLatest.WorkflowStatusUpdateService) (*HandlerServiceImpl, error) { + workflowStatusLatestService workflowStatusLatest.WorkflowStatusLatestService) (*HandlerServiceImpl, error) { impl := &HandlerServiceImpl{ logger: logger, cdWorkflowCommonService: cdWorkflowCommonService, @@ -311,7 +311,7 @@ func NewHandlerServiceImpl(logger *zap.SugaredLogger, asyncRunnable: asyncRunnable, workflowTriggerAuditService: workflowTriggerAuditService, fluxCdDeploymentService: fluxCdDeploymentService, - workflowStatusUpdateService: workflowStatusUpdateService, + workflowStatusLatestService: workflowStatusLatestService, } config, err := types.GetCdConfig() if err != nil { diff --git a/pkg/deployment/trigger/devtronApps/deployStageHandlerCode.go b/pkg/deployment/trigger/devtronApps/deployStageHandlerCode.go index b99fa4265a..40f7e374b9 100644 --- a/pkg/deployment/trigger/devtronApps/deployStageHandlerCode.go +++ b/pkg/deployment/trigger/devtronApps/deployStageHandlerCode.go @@ -302,7 +302,7 @@ func (impl *HandlerServiceImpl) ManualCdTrigger(triggerContext bean.TriggerConte return 0, "", nil, err } - err = impl.workflowStatusUpdateService.UpdateCdWorkflowStatusLatest(tx, overrideRequest.PipelineId, overrideRequest.AppId, overrideRequest.EnvId, runner.Id, runner.WorkflowType.String(), overrideRequest.UserId) + err = impl.workflowStatusLatestService.SaveCdWorkflowStatusLatest(tx, overrideRequest.PipelineId, overrideRequest.AppId, overrideRequest.EnvId, runner.Id, runner.WorkflowType.String(), overrideRequest.UserId) if err != nil { impl.logger.Errorw("error in updating workflow status latest, ManualCdTrigger", "runnerId", overrideRequest.WfrId, "err", err) return 0, "", nil, err @@ -468,7 +468,7 @@ func (impl *HandlerServiceImpl) TriggerAutomaticDeployment(request bean.CdTrigge return err } - err = impl.workflowStatusUpdateService.UpdateCdWorkflowStatusLatest(tx, pipeline.Id, pipeline.AppId, pipeline.EnvironmentId, runner.Id, runner.WorkflowType.String(), request.TriggeredBy) + err = impl.workflowStatusLatestService.SaveCdWorkflowStatusLatest(tx, pipeline.Id, pipeline.AppId, pipeline.EnvironmentId, runner.Id, runner.WorkflowType.String(), request.TriggeredBy) if err != nil { impl.logger.Errorw("error in updating workflow status latest, ManualCdTrigger", "runnerId", runner.Id, "err", err) return err diff --git a/pkg/pipeline/CiHandler.go b/pkg/pipeline/CiHandler.go index 6717214336..d2d67802f7 100644 --- a/pkg/pipeline/CiHandler.go +++ b/pkg/pipeline/CiHandler.go @@ -30,7 +30,6 @@ import ( "github.com/devtron-labs/devtron/pkg/pipeline/adapter" "github.com/devtron-labs/devtron/pkg/pipeline/constants" "github.com/devtron-labs/devtron/pkg/pipeline/workflowStatus" - "github.com/devtron-labs/devtron/pkg/workflow/workflowStatusLatest" "regexp" "slices" "strconv" @@ -100,7 +99,6 @@ type CiHandlerImpl struct { k8sCommonService k8sPkg.K8sCommonService workFlowStageStatusService workflowStatus.WorkFlowStageStatusService workflowStatusLatestRepository pipelineConfig.WorkflowStatusLatestRepository - workflowStatusUpdateService workflowStatusLatest.WorkflowStatusUpdateService } func NewCiHandlerImpl(Logger *zap.SugaredLogger, ciService CiService, ciPipelineMaterialRepository pipelineConfig.CiPipelineMaterialRepository, gitSensorClient gitSensor.Client, ciWorkflowRepository pipelineConfig.CiWorkflowRepository, @@ -109,7 +107,6 @@ func NewCiHandlerImpl(Logger *zap.SugaredLogger, ciService CiService, ciPipeline imageTaggingService imageTagging.ImageTaggingService, k8sCommonService k8sPkg.K8sCommonService, appWorkflowRepository appWorkflow.AppWorkflowRepository, customTagService CustomTagService, workFlowStageStatusService workflowStatus.WorkFlowStageStatusService, workflowStatusLatestRepository pipelineConfig.WorkflowStatusLatestRepository, - workflowStatusUpdateService workflowStatusLatest.WorkflowStatusUpdateService, ) *CiHandlerImpl { cih := &CiHandlerImpl{ Logger: Logger, @@ -133,7 +130,6 @@ func NewCiHandlerImpl(Logger *zap.SugaredLogger, ciService CiService, ciPipeline k8sCommonService: k8sCommonService, workFlowStageStatusService: workFlowStageStatusService, workflowStatusLatestRepository: workflowStatusLatestRepository, - workflowStatusUpdateService: workflowStatusUpdateService, } config, err := types.GetCiConfig() if err != nil { diff --git a/pkg/pipeline/CiService.go b/pkg/pipeline/CiService.go index fb63f0a449..e610af74e0 100644 --- a/pkg/pipeline/CiService.go +++ b/pkg/pipeline/CiService.go @@ -46,16 +46,18 @@ type CiServiceImpl struct { eventFactory client.EventFactory config *types.CiConfig ciWorkflowRepository pipelineConfig.CiWorkflowRepository + ciPipelineRepository pipelineConfig.CiPipelineRepository transactionManager sql.TransactionWrapper - workflowStatusUpdateService workflowStatusLatest.WorkflowStatusUpdateService + workflowStatusLatestService workflowStatusLatest.WorkflowStatusLatestService } func NewCiServiceImpl(Logger *zap.SugaredLogger, workflowStageStatusService workflowStatus.WorkFlowStageStatusService, eventClient client.EventClient, eventFactory client.EventFactory, ciWorkflowRepository pipelineConfig.CiWorkflowRepository, + ciPipelineRepository pipelineConfig.CiPipelineRepository, transactionManager sql.TransactionWrapper, - workflowStatusUpdateService workflowStatusLatest.WorkflowStatusUpdateService, + workflowStatusLatestService workflowStatusLatest.WorkflowStatusLatestService, ) *CiServiceImpl { cis := &CiServiceImpl{ Logger: Logger, @@ -63,8 +65,9 @@ func NewCiServiceImpl(Logger *zap.SugaredLogger, eventClient: eventClient, eventFactory: eventFactory, ciWorkflowRepository: ciWorkflowRepository, + ciPipelineRepository: ciPipelineRepository, transactionManager: transactionManager, - workflowStatusUpdateService: workflowStatusUpdateService, + workflowStatusLatestService: workflowStatusLatestService, } config, err := types.GetCiConfig() if err != nil { @@ -135,9 +138,17 @@ func (impl *CiServiceImpl) SaveCiWorkflowWithStage(wf *pipelineConfig.CiWorkflow return err } - err = impl.workflowStatusUpdateService.SaveCiWorkflowStatusLatest(tx, wf.CiPipelineId, wf.Id, wf.TriggeredBy) + // Get appId from CI pipeline (not from workflow to avoid transaction issues) + ciPipeline, err := impl.ciPipelineRepository.FindById(wf.CiPipelineId) if err != nil { - impl.Logger.Errorw("error in updating ci workflow status latest", "err", err, "pipelineId", wf.CiPipelineId, "workflowId", wf.Id) + impl.Logger.Errorw("error in fetching ci pipeline for appId", "err", err, "ciPipelineId", wf.CiPipelineId) + return err + } + appId := ciPipeline.AppId + + err = impl.workflowStatusLatestService.SaveCiWorkflowStatusLatest(tx, wf.CiPipelineId, appId, wf.Id, wf.TriggeredBy) + if err != nil { + impl.Logger.Errorw("error in saving ci workflow status latest", "err", err, "pipelineId", wf.CiPipelineId, "workflowId", wf.Id) return err } diff --git a/pkg/workflow/cd/CdWorkflowRunnerService.go b/pkg/workflow/cd/CdWorkflowRunnerService.go index cdc6b890cf..4546dd6053 100644 --- a/pkg/workflow/cd/CdWorkflowRunnerService.go +++ b/pkg/workflow/cd/CdWorkflowRunnerService.go @@ -49,20 +49,20 @@ type CdWorkflowRunnerServiceImpl struct { workflowStageService workflowStatus.WorkFlowStageStatusService transactionManager sql.TransactionWrapper config *types.CiConfig - workflowStatusUpdateService workflowStatusLatest.WorkflowStatusUpdateService + workflowStatusLatestService workflowStatusLatest.WorkflowStatusLatestService } func NewCdWorkflowRunnerServiceImpl(logger *zap.SugaredLogger, cdWorkflowRepository pipelineConfig.CdWorkflowRepository, workflowStageService workflowStatus.WorkFlowStageStatusService, transactionManager sql.TransactionWrapper, - workflowStatusUpdateService workflowStatusLatest.WorkflowStatusUpdateService) *CdWorkflowRunnerServiceImpl { + workflowStatusLatestService workflowStatusLatest.WorkflowStatusLatestService) *CdWorkflowRunnerServiceImpl { impl := &CdWorkflowRunnerServiceImpl{ logger: logger, cdWorkflowRepository: cdWorkflowRepository, workflowStageService: workflowStageService, transactionManager: transactionManager, - workflowStatusUpdateService: workflowStatusUpdateService, + workflowStatusLatestService: workflowStatusLatestService, } ciConfig, err := types.GetCiConfig() if err != nil { @@ -121,7 +121,7 @@ func (impl *CdWorkflowRunnerServiceImpl) SaveCDWorkflowRunnerWithStage(wfr *pipe return wfr, err } - err = impl.workflowStatusUpdateService.UpdateCdWorkflowStatusLatest(tx, cdWf.PipelineId, pipeline.AppId, pipeline.EnvironmentId, wfr.Id, wfr.WorkflowType.String(), wfr.CreatedBy) + err = impl.workflowStatusLatestService.SaveCdWorkflowStatusLatest(tx, cdWf.PipelineId, pipeline.AppId, pipeline.EnvironmentId, wfr.Id, wfr.WorkflowType.String(), wfr.CreatedBy) if err != nil { impl.logger.Errorw("error in updating workflow status latest, ManualCdTrigger", "runnerId", wfr.CreatedBy, "err", err) return wfr, err diff --git a/pkg/workflow/workflowStatusLatest/WorkflowStatusLatestService.go b/pkg/workflow/workflowStatusLatest/WorkflowStatusLatestService.go index 303fae5273..0eb94eb48d 100644 --- a/pkg/workflow/workflowStatusLatest/WorkflowStatusLatestService.go +++ b/pkg/workflow/workflowStatusLatest/WorkflowStatusLatestService.go @@ -30,7 +30,7 @@ import ( type WorkflowStatusLatestService interface { // CI Workflow Status Latest methods - SaveCiWorkflowStatusLatest(tx *pg.Tx, pipelineId, ciWorkflowId int, userId int32) error + SaveCiWorkflowStatusLatest(tx *pg.Tx, pipelineId, appId, ciWorkflowId int, userId int32) error GetCiWorkflowStatusLatestByPipelineId(pipelineId int) (*CiWorkflowStatusLatest, error) GetCiWorkflowStatusLatestByAppId(appId int) ([]*CiWorkflowStatusLatest, error) @@ -83,53 +83,25 @@ type CdWorkflowStatusLatest struct { Status string `json:"status"` // Derived from cd_workflow_runner table } -func (impl *WorkflowStatusLatestServiceImpl) SaveCiWorkflowStatusLatest(tx *pg.Tx, pipelineId, ciWorkflowId int, userId int32) error { +func (impl *WorkflowStatusLatestServiceImpl) SaveCiWorkflowStatusLatest(tx *pg.Tx, pipelineId, appId, ciWorkflowId int, userId int32) error { // Validate required parameters - if pipelineId <= 0 { - impl.logger.Errorw("invalid pipelineId provided", "pipelineId", pipelineId) - return fmt.Errorf("invalid pipelineId: %d", pipelineId) - } - if ciWorkflowId <= 0 { impl.logger.Errorw("invalid ciWorkflowId provided", "ciWorkflowId", ciWorkflowId) return fmt.Errorf("invalid ciWorkflowId: %d", ciWorkflowId) } - ciPipeline, err := impl.ciPipelineRepository.FindOneWithAppData(pipelineId) - if err != nil { - impl.logger.Errorw("error in fetching ci pipeline for appId", "ciPipelineId", pipelineId, "err", err) - return err - } - appId := ciPipeline.AppId - - // Check if entry exists - existingEntry, err := impl.workflowStatusLatestRepository.GetCiWorkflowStatusLatestByPipelineId(pipelineId) - if err != nil && !util2.IsErrNoRows(err) { - impl.logger.Errorw("error in getting ci workflow status latest", "err", err, "pipelineId", pipelineId) - return err - } now := time.Now() - if util2.IsErrNoRows(err) { - // Create new entry - model := &pipelineConfig.CiWorkflowStatusLatest{ - PipelineId: pipelineId, - AppId: appId, - CiWorkflowId: ciWorkflowId, - } - model.CreatedBy = userId - model.CreatedOn = now - model.UpdatedBy = userId - model.UpdatedOn = now - - return impl.workflowStatusLatestRepository.SaveCiWorkflowStatusLatest(tx, model) - } else { - // Update existing entry - existingEntry.CiWorkflowId = ciWorkflowId - existingEntry.UpdatedBy = userId - existingEntry.UpdatedOn = now - - return impl.workflowStatusLatestRepository.UpdateCiWorkflowStatusLatest(tx, existingEntry) + model := &pipelineConfig.CiWorkflowStatusLatest{ + PipelineId: pipelineId, + AppId: appId, + CiWorkflowId: ciWorkflowId, } + model.CreatedBy = userId + model.CreatedOn = now + model.UpdatedBy = userId + model.UpdatedOn = now + + return impl.workflowStatusLatestRepository.SaveCiWorkflowStatusLatest(tx, model) } func (impl *WorkflowStatusLatestServiceImpl) GetCiWorkflowStatusLatestByPipelineId(pipelineId int) (*CiWorkflowStatusLatest, error) { @@ -189,91 +161,26 @@ func (impl *WorkflowStatusLatestServiceImpl) GetCiWorkflowStatusLatestByAppId(ap // CD Workflow Status Latest methods implementation func (impl *WorkflowStatusLatestServiceImpl) SaveCdWorkflowStatusLatest(tx *pg.Tx, pipelineId, appId, environmentId, workflowRunnerId int, workflowType string, userId int32) error { // Validate required parameters - if pipelineId <= 0 { - impl.logger.Errorw("invalid pipelineId provided", "pipelineId", pipelineId) - return fmt.Errorf("invalid pipelineId: %d", pipelineId) - } - if workflowRunnerId <= 0 { impl.logger.Errorw("invalid workflowRunnerId provided", "workflowRunnerId", workflowRunnerId) return fmt.Errorf("invalid workflowRunnerId: %d", workflowRunnerId) } - // If appId or environmentId is not provided (0), fetch them from the CdWorkflow - if appId <= 0 || environmentId <= 0 { - cdWorkflowRunner, err := impl.cdWorkflowRepository.FindBasicWorkflowRunnerById(workflowRunnerId) - if err != nil { - impl.logger.Errorw("error in fetching cd workflow runner to get appId/environmentId", "err", err, "workflowRunnerId", workflowRunnerId) - return err - } - - if cdWorkflowRunner == nil { - impl.logger.Errorw("cd workflow runner not found", "workflowRunnerId", workflowRunnerId) - return fmt.Errorf("cd workflow runner not found with id: %d", workflowRunnerId) - } - - // Check if CdWorkflow is loaded - if cdWorkflowRunner.CdWorkflow == nil { - impl.logger.Errorw("cd workflow not loaded in cd workflow runner", "workflowRunnerId", workflowRunnerId, "cdWorkflowId", cdWorkflowRunner.CdWorkflowId) - return fmt.Errorf("cd workflow not loaded for workflow runner id: %d", workflowRunnerId) - } - - // Check if Pipeline is loaded - if cdWorkflowRunner.CdWorkflow.Pipeline == nil { - impl.logger.Errorw("pipeline not loaded in cd workflow", "workflowRunnerId", workflowRunnerId, "cdWorkflowId", cdWorkflowRunner.CdWorkflowId, "pipelineId", cdWorkflowRunner.CdWorkflow.PipelineId) - return fmt.Errorf("pipeline not loaded for workflow runner id: %d", workflowRunnerId) - } - - if appId <= 0 { - appId = cdWorkflowRunner.CdWorkflow.Pipeline.AppId - if appId <= 0 { - impl.logger.Errorw("invalid appId in pipeline", "workflowRunnerId", workflowRunnerId, "pipelineId", cdWorkflowRunner.CdWorkflow.PipelineId, "appId", appId) - return fmt.Errorf("invalid appId in pipeline: %d", appId) - } - impl.logger.Debugw("fetched appId from cd workflow runner", "workflowRunnerId", workflowRunnerId, "appId", appId) - } - - if environmentId <= 0 { - environmentId = cdWorkflowRunner.CdWorkflow.Pipeline.EnvironmentId - if environmentId <= 0 { - impl.logger.Errorw("invalid environmentId in pipeline", "workflowRunnerId", workflowRunnerId, "pipelineId", cdWorkflowRunner.CdWorkflow.PipelineId, "environmentId", environmentId) - return fmt.Errorf("invalid environmentId in pipeline: %d", environmentId) - } - impl.logger.Debugw("fetched environmentId from cd workflow runner", "workflowRunnerId", workflowRunnerId, "environmentId", environmentId) - } - } - - // Check if entry exists - existingEntry, err := impl.workflowStatusLatestRepository.GetCdWorkflowStatusLatestByPipelineIdAndWorkflowType(tx, pipelineId, workflowType) - if err != nil && err != pg.ErrNoRows { - impl.logger.Errorw("error in getting cd workflow status latest", "err", err, "pipelineId", pipelineId, "workflowType", workflowType) - return err - } - + // Create new entry (always save, don't update) now := time.Now() - if err == pg.ErrNoRows { - // Create new entry - model := &pipelineConfig.CdWorkflowStatusLatest{ - PipelineId: pipelineId, - AppId: appId, - EnvironmentId: environmentId, - WorkflowType: workflowType, - WorkflowRunnerId: workflowRunnerId, - } - model.CreatedBy = userId - model.CreatedOn = now - model.UpdatedBy = userId - model.UpdatedOn = now - - return impl.workflowStatusLatestRepository.SaveCdWorkflowStatusLatest(tx, model) - } else { - // Update existing entry - existingEntry.WorkflowRunnerId = workflowRunnerId - existingEntry.UpdatedBy = userId - existingEntry.UpdatedOn = now - - return impl.workflowStatusLatestRepository.UpdateCdWorkflowStatusLatest(tx, existingEntry) + model := &pipelineConfig.CdWorkflowStatusLatest{ + PipelineId: pipelineId, + AppId: appId, + EnvironmentId: environmentId, + WorkflowType: workflowType, + WorkflowRunnerId: workflowRunnerId, } + model.CreatedBy = userId + model.CreatedOn = now + model.UpdatedBy = userId + model.UpdatedOn = now + + return impl.workflowStatusLatestRepository.SaveCdWorkflowStatusLatest(tx, model) } func (impl *WorkflowStatusLatestServiceImpl) GetCdWorkflowStatusLatestByPipelineIdAndWorkflowType(pipelineId int, workflowType string) (*CdWorkflowStatusLatest, error) { diff --git a/pkg/workflow/workflowStatusLatest/WorkflowStatusUpdateService.go b/pkg/workflow/workflowStatusLatest/WorkflowStatusUpdateService.go deleted file mode 100644 index 0183617fd3..0000000000 --- a/pkg/workflow/workflowStatusLatest/WorkflowStatusUpdateService.go +++ /dev/null @@ -1,64 +0,0 @@ -/* - * Copyright (c) 2024. Devtron Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package workflowStatusLatest - -import ( - "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" - "github.com/go-pg/pg" - "go.uber.org/zap" -) - -type WorkflowStatusUpdateService interface { - // Methods to update latest status tables when workflow status changes - SaveCiWorkflowStatusLatest(tx *pg.Tx, pipelineId, ciWorkflowId int, userId int32) error - UpdateCdWorkflowStatusLatest(tx *pg.Tx, pipelineId, appId, environmentId, workflowRunnerId int, workflowType string, userId int32) error -} - -type WorkflowStatusUpdateServiceImpl struct { - logger *zap.SugaredLogger - workflowStatusLatestService WorkflowStatusLatestService - ciWorkflowRepository pipelineConfig.CiWorkflowRepository - cdWorkflowRepository pipelineConfig.CdWorkflowRepository - ciPipelineRepository pipelineConfig.CiPipelineRepository - pipelineRepository pipelineConfig.PipelineRepository -} - -func NewWorkflowStatusUpdateServiceImpl( - logger *zap.SugaredLogger, - workflowStatusLatestService WorkflowStatusLatestService, - ciWorkflowRepository pipelineConfig.CiWorkflowRepository, - cdWorkflowRepository pipelineConfig.CdWorkflowRepository, - ciPipelineRepository pipelineConfig.CiPipelineRepository, - pipelineRepository pipelineConfig.PipelineRepository, -) *WorkflowStatusUpdateServiceImpl { - return &WorkflowStatusUpdateServiceImpl{ - logger: logger, - workflowStatusLatestService: workflowStatusLatestService, - ciWorkflowRepository: ciWorkflowRepository, - cdWorkflowRepository: cdWorkflowRepository, - ciPipelineRepository: ciPipelineRepository, - pipelineRepository: pipelineRepository, - } -} - -func (impl *WorkflowStatusUpdateServiceImpl) SaveCiWorkflowStatusLatest(tx *pg.Tx, pipelineId, ciWorkflowId int, userId int32) error { - return impl.workflowStatusLatestService.SaveCiWorkflowStatusLatest(tx, pipelineId, ciWorkflowId, userId) -} - -func (impl *WorkflowStatusUpdateServiceImpl) UpdateCdWorkflowStatusLatest(tx *pg.Tx, pipelineId, appId, environmentId, workflowRunnerId int, workflowType string, userId int32) error { - return impl.workflowStatusLatestService.SaveCdWorkflowStatusLatest(tx, pipelineId, appId, environmentId, workflowRunnerId, workflowType, userId) -} diff --git a/pkg/workflow/workflowStatusLatest/wire_workflow_status_latest.go b/pkg/workflow/workflowStatusLatest/wire_workflow_status_latest.go index a0b18557f7..aebe5f18ce 100644 --- a/pkg/workflow/workflowStatusLatest/wire_workflow_status_latest.go +++ b/pkg/workflow/workflowStatusLatest/wire_workflow_status_latest.go @@ -26,6 +26,4 @@ var WorkflowStatusLatestWireSet = wire.NewSet( wire.Bind(new(pipelineConfig.WorkflowStatusLatestRepository), new(*pipelineConfig.WorkflowStatusLatestRepositoryImpl)), NewWorkflowStatusLatestServiceImpl, wire.Bind(new(WorkflowStatusLatestService), new(*WorkflowStatusLatestServiceImpl)), - NewWorkflowStatusUpdateServiceImpl, - wire.Bind(new(WorkflowStatusUpdateService), new(*WorkflowStatusUpdateServiceImpl)), ) diff --git a/wire_gen.go b/wire_gen.go index 7273f957d1..53d97340a1 100644 --- a/wire_gen.go +++ b/wire_gen.go @@ -626,8 +626,7 @@ func InitializeApp() (*App, error) { workFlowStageStatusServiceImpl := workflowStatus.NewWorkflowStageFlowStatusServiceImpl(sugaredLogger, workflowStageRepositoryImpl, ciWorkflowRepositoryImpl, cdWorkflowRepositoryImpl, transactionUtilImpl) workflowStatusLatestRepositoryImpl := pipelineConfig.NewWorkflowStatusLatestRepositoryImpl(db, sugaredLogger) workflowStatusLatestServiceImpl := workflowStatusLatest.NewWorkflowStatusLatestServiceImpl(sugaredLogger, workflowStatusLatestRepositoryImpl, ciWorkflowRepositoryImpl, cdWorkflowRepositoryImpl, ciPipelineRepositoryImpl) - workflowStatusUpdateServiceImpl := workflowStatusLatest.NewWorkflowStatusUpdateServiceImpl(sugaredLogger, workflowStatusLatestServiceImpl, ciWorkflowRepositoryImpl, cdWorkflowRepositoryImpl, ciPipelineRepositoryImpl, pipelineRepositoryImpl) - cdWorkflowRunnerServiceImpl := cd.NewCdWorkflowRunnerServiceImpl(sugaredLogger, cdWorkflowRepositoryImpl, workFlowStageStatusServiceImpl, transactionUtilImpl, workflowStatusUpdateServiceImpl) + cdWorkflowRunnerServiceImpl := cd.NewCdWorkflowRunnerServiceImpl(sugaredLogger, cdWorkflowRepositoryImpl, workFlowStageStatusServiceImpl, transactionUtilImpl, workflowStatusLatestServiceImpl) deploymentEventHandlerImpl := app2.NewDeploymentEventHandlerImpl(sugaredLogger, eventRESTClientImpl, eventSimpleFactoryImpl, runnable) appServiceImpl := app2.NewAppService(pipelineOverrideRepositoryImpl, utilMergeUtil, sugaredLogger, pipelineRepositoryImpl, eventRESTClientImpl, eventSimpleFactoryImpl, appRepositoryImpl, configMapRepositoryImpl, chartRepositoryImpl, cdWorkflowRepositoryImpl, commonServiceImpl, chartTemplateServiceImpl, pipelineStatusTimelineRepositoryImpl, pipelineStatusTimelineResourcesServiceImpl, pipelineStatusSyncDetailServiceImpl, pipelineStatusTimelineServiceImpl, appServiceConfig, appStatusServiceImpl, installedAppReadServiceImpl, installedAppVersionHistoryRepositoryImpl, scopedVariableCMCSManagerImpl, acdConfig, gitOpsConfigReadServiceImpl, gitOperationServiceImpl, deploymentTemplateServiceImpl, appListingServiceImpl, deploymentConfigServiceImpl, envConfigOverrideReadServiceImpl, cdWorkflowRunnerServiceImpl, deploymentEventHandlerImpl) scopedVariableManagerImpl, err := variables.NewScopedVariableManagerImpl(sugaredLogger, scopedVariableServiceImpl, variableEntityMappingServiceImpl, variableSnapshotHistoryServiceImpl, variableTemplateParserImpl) @@ -692,7 +691,7 @@ func InitializeApp() (*App, error) { chartServiceImpl := chart.NewChartServiceImpl(chartRepositoryImpl, sugaredLogger, chartTemplateServiceImpl, chartRepoRepositoryImpl, appRepositoryImpl, mergeUtil, envConfigOverrideRepositoryImpl, pipelineConfigRepositoryImpl, environmentRepositoryImpl, deploymentTemplateHistoryServiceImpl, scopedVariableManagerImpl, deployedAppMetricsServiceImpl, chartRefServiceImpl, gitOpsConfigReadServiceImpl, deploymentConfigServiceImpl, envConfigOverrideReadServiceImpl, chartReadServiceImpl) ciCdPipelineOrchestratorImpl := pipeline.NewCiCdPipelineOrchestrator(appRepositoryImpl, sugaredLogger, materialRepositoryImpl, pipelineRepositoryImpl, ciPipelineRepositoryImpl, ciPipelineMaterialRepositoryImpl, cdWorkflowRepositoryImpl, clientImpl, ciCdConfig, appWorkflowRepositoryImpl, environmentRepositoryImpl, attributesServiceImpl, appCrudOperationServiceImpl, userAuthServiceImpl, prePostCdScriptHistoryServiceImpl, pipelineStageServiceImpl, gitMaterialHistoryServiceImpl, ciPipelineHistoryServiceImpl, ciTemplateReadServiceImpl, ciTemplateServiceImpl, dockerArtifactStoreRepositoryImpl, ciArtifactRepositoryImpl, configMapServiceImpl, customTagServiceImpl, genericNoteServiceImpl, chartServiceImpl, transactionUtilImpl, gitOpsConfigReadServiceImpl, deploymentConfigServiceImpl, deploymentConfigReadServiceImpl, chartReadServiceImpl) pluginInputVariableParserImpl := pipeline.NewPluginInputVariableParserImpl(sugaredLogger, dockerRegistryConfigImpl, customTagServiceImpl) - ciServiceImpl := pipeline.NewCiServiceImpl(sugaredLogger, workFlowStageStatusServiceImpl, eventRESTClientImpl, eventSimpleFactoryImpl, ciWorkflowRepositoryImpl, transactionUtilImpl, workflowStatusUpdateServiceImpl) + ciServiceImpl := pipeline.NewCiServiceImpl(sugaredLogger, workFlowStageStatusServiceImpl, eventRESTClientImpl, eventSimpleFactoryImpl, ciWorkflowRepositoryImpl, ciPipelineRepositoryImpl, transactionUtilImpl, workflowStatusLatestServiceImpl) ciLogServiceImpl, err := pipeline.NewCiLogServiceImpl(sugaredLogger, k8sServiceImpl) if err != nil { return nil, err @@ -752,7 +751,7 @@ func InitializeApp() (*App, error) { deploymentTemplateValidationServiceEntImpl := validator.NewDeploymentTemplateValidationServiceEntImpl() deploymentTemplateValidationServiceImpl := validator.NewDeploymentTemplateValidationServiceImpl(sugaredLogger, chartRefServiceImpl, scopedVariableManagerImpl, deployedAppMetricsServiceImpl, deploymentTemplateValidationServiceEntImpl) devtronAppGitOpConfigServiceImpl := gitOpsConfig.NewDevtronAppGitOpConfigServiceImpl(sugaredLogger, chartRepositoryImpl, chartServiceImpl, gitOpsConfigReadServiceImpl, gitOpsValidationServiceImpl, argoClientWrapperServiceImpl, deploymentConfigServiceImpl, chartReadServiceImpl) - ciHandlerImpl := pipeline.NewCiHandlerImpl(sugaredLogger, ciServiceImpl, ciPipelineMaterialRepositoryImpl, clientImpl, ciWorkflowRepositoryImpl, ciArtifactRepositoryImpl, userServiceImpl, eventRESTClientImpl, eventSimpleFactoryImpl, ciPipelineRepositoryImpl, appListingRepositoryImpl, pipelineRepositoryImpl, enforcerUtilImpl, resourceGroupServiceImpl, environmentRepositoryImpl, imageTaggingServiceImpl, k8sCommonServiceImpl, appWorkflowRepositoryImpl, customTagServiceImpl, workFlowStageStatusServiceImpl, workflowStatusLatestRepositoryImpl, workflowStatusUpdateServiceImpl) + ciHandlerImpl := pipeline.NewCiHandlerImpl(sugaredLogger, ciServiceImpl, ciPipelineMaterialRepositoryImpl, clientImpl, ciWorkflowRepositoryImpl, ciArtifactRepositoryImpl, userServiceImpl, eventRESTClientImpl, eventSimpleFactoryImpl, ciPipelineRepositoryImpl, appListingRepositoryImpl, pipelineRepositoryImpl, enforcerUtilImpl, resourceGroupServiceImpl, environmentRepositoryImpl, imageTaggingServiceImpl, k8sCommonServiceImpl, appWorkflowRepositoryImpl, customTagServiceImpl, workFlowStageStatusServiceImpl, workflowStatusLatestRepositoryImpl) cdWorkflowRunnerReadServiceImpl := read18.NewCdWorkflowRunnerReadServiceImpl(sugaredLogger, cdWorkflowRepositoryImpl, workflowStatusLatestServiceImpl, pipelineStageRepositoryImpl) cdHandlerImpl := pipeline.NewCdHandlerImpl(sugaredLogger, userServiceImpl, cdWorkflowRepositoryImpl, ciArtifactRepositoryImpl, ciPipelineMaterialRepositoryImpl, pipelineRepositoryImpl, environmentRepositoryImpl, ciWorkflowRepositoryImpl, enforcerUtilImpl, resourceGroupServiceImpl, imageTaggingServiceImpl, k8sServiceImpl, customTagServiceImpl, deploymentConfigServiceImpl, workFlowStageStatusServiceImpl, cdWorkflowRunnerServiceImpl, workflowStatusLatestServiceImpl, pipelineStageRepositoryImpl, cdWorkflowRunnerReadServiceImpl) appWorkflowServiceImpl := appWorkflow2.NewAppWorkflowServiceImpl(sugaredLogger, appWorkflowRepositoryImpl, ciCdPipelineOrchestratorImpl, ciPipelineRepositoryImpl, pipelineRepositoryImpl, enforcerUtilImpl, resourceGroupServiceImpl, appRepositoryImpl, userAuthServiceImpl, chartServiceImpl, deploymentConfigServiceImpl, pipelineBuilderImpl) @@ -785,7 +784,7 @@ func InitializeApp() (*App, error) { scanToolExecutionHistoryMappingRepositoryImpl := repository25.NewScanToolExecutionHistoryMappingRepositoryImpl(db, sugaredLogger) cdWorkflowReadServiceImpl := read18.NewCdWorkflowReadServiceImpl(sugaredLogger, cdWorkflowRepositoryImpl) imageScanServiceImpl := imageScanning.NewImageScanServiceImpl(sugaredLogger, imageScanHistoryRepositoryImpl, imageScanResultRepositoryImpl, imageScanObjectMetaRepositoryImpl, cveStoreRepositoryImpl, imageScanDeployInfoRepositoryImpl, userServiceImpl, appRepositoryImpl, environmentServiceImpl, ciArtifactRepositoryImpl, policyServiceImpl, pipelineRepositoryImpl, ciPipelineRepositoryImpl, scanToolMetadataRepositoryImpl, scanToolExecutionHistoryMappingRepositoryImpl, cvePolicyRepositoryImpl, cdWorkflowReadServiceImpl) - devtronAppsHandlerServiceImpl, err := devtronApps.NewHandlerServiceImpl(sugaredLogger, cdWorkflowCommonServiceImpl, gitOpsManifestPushServiceImpl, gitOpsConfigReadServiceImpl, argoK8sClientImpl, acdConfig, argoClientWrapperServiceImpl, pipelineStatusTimelineServiceImpl, chartTemplateServiceImpl, workflowEventPublishServiceImpl, manifestCreationServiceImpl, deployedConfigurationHistoryServiceImpl, pipelineStageServiceImpl, globalPluginServiceImpl, customTagServiceImpl, pluginInputVariableParserImpl, prePostCdScriptHistoryServiceImpl, scopedVariableCMCSManagerImpl, imageDigestPolicyServiceImpl, userServiceImpl, helmAppServiceImpl, enforcerUtilImpl, userDeploymentRequestServiceImpl, helmAppClientImpl, eventSimpleFactoryImpl, eventRESTClientImpl, environmentVariables, appRepositoryImpl, ciPipelineMaterialRepositoryImpl, imageScanHistoryReadServiceImpl, imageScanDeployInfoReadServiceImpl, imageScanDeployInfoServiceImpl, pipelineRepositoryImpl, pipelineOverrideRepositoryImpl, manifestPushConfigRepositoryImpl, chartRepositoryImpl, environmentRepositoryImpl, cdWorkflowRepositoryImpl, ciWorkflowRepositoryImpl, ciArtifactRepositoryImpl, ciTemplateReadServiceImpl, gitMaterialReadServiceImpl, appLabelRepositoryImpl, ciPipelineRepositoryImpl, appWorkflowRepositoryImpl, dockerArtifactStoreRepositoryImpl, imageScanServiceImpl, k8sServiceImpl, transactionUtilImpl, deploymentConfigServiceImpl, ciCdPipelineOrchestratorImpl, gitOperationServiceImpl, attributesServiceImpl, clusterRepositoryImpl, cdWorkflowRunnerServiceImpl, clusterServiceImplExtended, ciLogServiceImpl, workflowServiceImpl, blobStorageConfigServiceImpl, deploymentEventHandlerImpl, runnable, workflowTriggerAuditServiceImpl, deploymentServiceImpl, workflowStatusUpdateServiceImpl) + devtronAppsHandlerServiceImpl, err := devtronApps.NewHandlerServiceImpl(sugaredLogger, cdWorkflowCommonServiceImpl, gitOpsManifestPushServiceImpl, gitOpsConfigReadServiceImpl, argoK8sClientImpl, acdConfig, argoClientWrapperServiceImpl, pipelineStatusTimelineServiceImpl, chartTemplateServiceImpl, workflowEventPublishServiceImpl, manifestCreationServiceImpl, deployedConfigurationHistoryServiceImpl, pipelineStageServiceImpl, globalPluginServiceImpl, customTagServiceImpl, pluginInputVariableParserImpl, prePostCdScriptHistoryServiceImpl, scopedVariableCMCSManagerImpl, imageDigestPolicyServiceImpl, userServiceImpl, helmAppServiceImpl, enforcerUtilImpl, userDeploymentRequestServiceImpl, helmAppClientImpl, eventSimpleFactoryImpl, eventRESTClientImpl, environmentVariables, appRepositoryImpl, ciPipelineMaterialRepositoryImpl, imageScanHistoryReadServiceImpl, imageScanDeployInfoReadServiceImpl, imageScanDeployInfoServiceImpl, pipelineRepositoryImpl, pipelineOverrideRepositoryImpl, manifestPushConfigRepositoryImpl, chartRepositoryImpl, environmentRepositoryImpl, cdWorkflowRepositoryImpl, ciWorkflowRepositoryImpl, ciArtifactRepositoryImpl, ciTemplateReadServiceImpl, gitMaterialReadServiceImpl, appLabelRepositoryImpl, ciPipelineRepositoryImpl, appWorkflowRepositoryImpl, dockerArtifactStoreRepositoryImpl, imageScanServiceImpl, k8sServiceImpl, transactionUtilImpl, deploymentConfigServiceImpl, ciCdPipelineOrchestratorImpl, gitOperationServiceImpl, attributesServiceImpl, clusterRepositoryImpl, cdWorkflowRunnerServiceImpl, clusterServiceImplExtended, ciLogServiceImpl, workflowServiceImpl, blobStorageConfigServiceImpl, deploymentEventHandlerImpl, runnable, workflowTriggerAuditServiceImpl, deploymentServiceImpl, workflowStatusLatestServiceImpl) if err != nil { return nil, err } From 6ecb09bf7bdbc2af6f022f7827e942b1bcc9192e Mon Sep 17 00:00:00 2001 From: Prakash Kumar Date: Thu, 24 Jul 2025 14:49:10 +0530 Subject: [PATCH 26/31] Remove legacy CI/CD workflow status fetching methods and streamline integration with `WorkflowStatusLatestService`. Update `CiHandlerImpl` references and clean up unused code. --- pkg/pipeline/CiHandler.go | 93 ++++---- .../WorkflowStatusLatestService.go | 215 +----------------- wire_gen.go | 2 +- 3 files changed, 51 insertions(+), 259 deletions(-) diff --git a/pkg/pipeline/CiHandler.go b/pkg/pipeline/CiHandler.go index d2d67802f7..3bc8ab1c6f 100644 --- a/pkg/pipeline/CiHandler.go +++ b/pkg/pipeline/CiHandler.go @@ -30,6 +30,7 @@ import ( "github.com/devtron-labs/devtron/pkg/pipeline/adapter" "github.com/devtron-labs/devtron/pkg/pipeline/constants" "github.com/devtron-labs/devtron/pkg/pipeline/workflowStatus" + "github.com/devtron-labs/devtron/pkg/workflow/workflowStatusLatest" "regexp" "slices" "strconv" @@ -77,28 +78,28 @@ type CiHandler interface { } type CiHandlerImpl struct { - Logger *zap.SugaredLogger - ciPipelineMaterialRepository pipelineConfig.CiPipelineMaterialRepository - ciService CiService - gitSensorClient gitSensor.Client - ciWorkflowRepository pipelineConfig.CiWorkflowRepository - ciArtifactRepository repository.CiArtifactRepository - userService user.UserService - eventClient client.EventClient - eventFactory client.EventFactory - ciPipelineRepository pipelineConfig.CiPipelineRepository - appListingRepository repository.AppListingRepository - cdPipelineRepository pipelineConfig.PipelineRepository - enforcerUtil rbac.EnforcerUtil - resourceGroupService resourceGroup.ResourceGroupService - envRepository repository2.EnvironmentRepository - imageTaggingService imageTagging.ImageTaggingService - customTagService CustomTagService - appWorkflowRepository appWorkflow.AppWorkflowRepository - config *types.CiConfig - k8sCommonService k8sPkg.K8sCommonService - workFlowStageStatusService workflowStatus.WorkFlowStageStatusService - workflowStatusLatestRepository pipelineConfig.WorkflowStatusLatestRepository + Logger *zap.SugaredLogger + ciPipelineMaterialRepository pipelineConfig.CiPipelineMaterialRepository + ciService CiService + gitSensorClient gitSensor.Client + ciWorkflowRepository pipelineConfig.CiWorkflowRepository + ciArtifactRepository repository.CiArtifactRepository + userService user.UserService + eventClient client.EventClient + eventFactory client.EventFactory + ciPipelineRepository pipelineConfig.CiPipelineRepository + appListingRepository repository.AppListingRepository + cdPipelineRepository pipelineConfig.PipelineRepository + enforcerUtil rbac.EnforcerUtil + resourceGroupService resourceGroup.ResourceGroupService + envRepository repository2.EnvironmentRepository + imageTaggingService imageTagging.ImageTaggingService + customTagService CustomTagService + appWorkflowRepository appWorkflow.AppWorkflowRepository + config *types.CiConfig + k8sCommonService k8sPkg.K8sCommonService + workFlowStageStatusService workflowStatus.WorkFlowStageStatusService + workflowStatusLatestService workflowStatusLatest.WorkflowStatusLatestService } func NewCiHandlerImpl(Logger *zap.SugaredLogger, ciService CiService, ciPipelineMaterialRepository pipelineConfig.CiPipelineMaterialRepository, gitSensorClient gitSensor.Client, ciWorkflowRepository pipelineConfig.CiWorkflowRepository, @@ -106,30 +107,30 @@ func NewCiHandlerImpl(Logger *zap.SugaredLogger, ciService CiService, ciPipeline appListingRepository repository.AppListingRepository, cdPipelineRepository pipelineConfig.PipelineRepository, enforcerUtil rbac.EnforcerUtil, resourceGroupService resourceGroup.ResourceGroupService, envRepository repository2.EnvironmentRepository, imageTaggingService imageTagging.ImageTaggingService, k8sCommonService k8sPkg.K8sCommonService, appWorkflowRepository appWorkflow.AppWorkflowRepository, customTagService CustomTagService, workFlowStageStatusService workflowStatus.WorkFlowStageStatusService, - workflowStatusLatestRepository pipelineConfig.WorkflowStatusLatestRepository, + workflowStatusLatestService workflowStatusLatest.WorkflowStatusLatestService, ) *CiHandlerImpl { cih := &CiHandlerImpl{ - Logger: Logger, - ciService: ciService, - ciPipelineMaterialRepository: ciPipelineMaterialRepository, - gitSensorClient: gitSensorClient, - ciWorkflowRepository: ciWorkflowRepository, - ciArtifactRepository: ciArtifactRepository, - userService: userService, - eventClient: eventClient, - eventFactory: eventFactory, - ciPipelineRepository: ciPipelineRepository, - appListingRepository: appListingRepository, - cdPipelineRepository: cdPipelineRepository, - enforcerUtil: enforcerUtil, - resourceGroupService: resourceGroupService, - envRepository: envRepository, - imageTaggingService: imageTaggingService, - customTagService: customTagService, - appWorkflowRepository: appWorkflowRepository, - k8sCommonService: k8sCommonService, - workFlowStageStatusService: workFlowStageStatusService, - workflowStatusLatestRepository: workflowStatusLatestRepository, + Logger: Logger, + ciService: ciService, + ciPipelineMaterialRepository: ciPipelineMaterialRepository, + gitSensorClient: gitSensorClient, + ciWorkflowRepository: ciWorkflowRepository, + ciArtifactRepository: ciArtifactRepository, + userService: userService, + eventClient: eventClient, + eventFactory: eventFactory, + ciPipelineRepository: ciPipelineRepository, + appListingRepository: appListingRepository, + cdPipelineRepository: cdPipelineRepository, + enforcerUtil: enforcerUtil, + resourceGroupService: resourceGroupService, + envRepository: envRepository, + imageTaggingService: imageTaggingService, + customTagService: customTagService, + appWorkflowRepository: appWorkflowRepository, + k8sCommonService: k8sCommonService, + workFlowStageStatusService: workFlowStageStatusService, + workflowStatusLatestService: workflowStatusLatestService, } config, err := types.GetCiConfig() if err != nil { @@ -659,7 +660,7 @@ func (impl *CiHandlerImpl) FetchCiStatusForTriggerViewV1(appId int) ([]*pipeline return []*pipelineConfig.CiWorkflowStatus{}, nil } - latestStatusEntries, err := impl.workflowStatusLatestRepository.GetCiWorkflowStatusLatestByPipelineIds(allPipelineIds) + latestStatusEntries, err := impl.workflowStatusLatestService.GetCiWorkflowStatusLatestByPipelineIds(allPipelineIds) if err != nil { impl.Logger.Errorw("error in checking latest status table, falling back to old method", "appId", appId, "err", err) return impl.ciWorkflowRepository.FIndCiWorkflowStatusesByAppId(appId) @@ -747,7 +748,7 @@ func (impl *CiHandlerImpl) fetchLastTriggeredWorkflowsHybrid(pipelineIds []int) return []*pipelineConfig.CiWorkflow{}, nil } - latestStatusEntries, err := impl.workflowStatusLatestRepository.GetCiWorkflowStatusLatestByPipelineIds(pipelineIds) + latestStatusEntries, err := impl.workflowStatusLatestService.GetCiWorkflowStatusLatestByPipelineIds(pipelineIds) if err != nil { impl.Logger.Errorw("error in checking latest status table, falling back to complex query", "pipelineIds", pipelineIds, "err", err) return impl.ciWorkflowRepository.FindLastTriggeredWorkflowByCiIds(pipelineIds) diff --git a/pkg/workflow/workflowStatusLatest/WorkflowStatusLatestService.go b/pkg/workflow/workflowStatusLatest/WorkflowStatusLatestService.go index 0eb94eb48d..7c9a561fc1 100644 --- a/pkg/workflow/workflowStatusLatest/WorkflowStatusLatestService.go +++ b/pkg/workflow/workflowStatusLatest/WorkflowStatusLatestService.go @@ -20,9 +20,6 @@ import ( "fmt" "time" - util2 "github.com/devtron-labs/devtron/internal/util" - - "github.com/devtron-labs/devtron/api/bean" "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" "github.com/go-pg/pg" "go.uber.org/zap" @@ -31,14 +28,10 @@ import ( type WorkflowStatusLatestService interface { // CI Workflow Status Latest methods SaveCiWorkflowStatusLatest(tx *pg.Tx, pipelineId, appId, ciWorkflowId int, userId int32) error - GetCiWorkflowStatusLatestByPipelineId(pipelineId int) (*CiWorkflowStatusLatest, error) - GetCiWorkflowStatusLatestByAppId(appId int) ([]*CiWorkflowStatusLatest, error) + GetCiWorkflowStatusLatestByPipelineIds(pipelineIds []int) ([]*pipelineConfig.CiWorkflowStatusLatest, error) // CD Workflow Status Latest methods SaveCdWorkflowStatusLatest(tx *pg.Tx, pipelineId, appId, environmentId, workflowRunnerId int, workflowType string, userId int32) error - GetCdWorkflowStatusLatestByPipelineIdAndWorkflowType(pipelineId int, workflowType string) (*CdWorkflowStatusLatest, error) - GetCdWorkflowStatusLatestByAppId(appId int) ([]*CdWorkflowStatusLatest, error) - GetCdWorkflowStatusLatestByPipelineId(pipelineId int) ([]*CdWorkflowStatusLatest, error) GetCdWorkflowLatestByPipelineIds(pipelineIds []int) ([]*CdWorkflowStatusLatest, error) } @@ -104,58 +97,8 @@ func (impl *WorkflowStatusLatestServiceImpl) SaveCiWorkflowStatusLatest(tx *pg.T return impl.workflowStatusLatestRepository.SaveCiWorkflowStatusLatest(tx, model) } -func (impl *WorkflowStatusLatestServiceImpl) GetCiWorkflowStatusLatestByPipelineId(pipelineId int) (*CiWorkflowStatusLatest, error) { - model, err := impl.workflowStatusLatestRepository.GetCiWorkflowStatusLatestByPipelineId(pipelineId) - if err != nil { - if err == pg.ErrNoRows { - // Fallback to old method - return impl.getCiWorkflowStatusFromOldMethod(pipelineId) - } - impl.logger.Errorw("error in getting ci workflow status latest", "err", err, "pipelineId", pipelineId) - return nil, err - } - - // Get status from ci_workflow table - ciWorkflow, err := impl.ciWorkflowRepository.FindById(model.CiWorkflowId) - if err != nil { - impl.logger.Errorw("error in getting ci workflow", "err", err, "ciWorkflowId", model.CiWorkflowId) - return nil, err - } - - return &CiWorkflowStatusLatest{ - PipelineId: model.PipelineId, - AppId: model.AppId, - CiWorkflowId: model.CiWorkflowId, - Status: ciWorkflow.Status, - }, nil -} - -func (impl *WorkflowStatusLatestServiceImpl) GetCiWorkflowStatusLatestByAppId(appId int) ([]*CiWorkflowStatusLatest, error) { - models, err := impl.workflowStatusLatestRepository.GetCiWorkflowStatusLatestByAppId(appId) - if err != nil { - impl.logger.Errorw("error in getting ci workflow status latest by app id", "err", err, "appId", appId) - return nil, err - } - - var result []*CiWorkflowStatusLatest - for _, model := range models { - // Get status from ci_workflow table - ciWorkflow, err := impl.ciWorkflowRepository.FindById(model.CiWorkflowId) - if err != nil { - impl.logger.Errorw("error in getting ci workflow", "err", err, "ciWorkflowId", model.CiWorkflowId) - continue // Skip this entry if we can't get the workflow - } - - result = append(result, &CiWorkflowStatusLatest{ - PipelineId: model.PipelineId, - AppId: model.AppId, - CiWorkflowId: model.CiWorkflowId, - Status: ciWorkflow.Status, - StorageConfigured: ciWorkflow.BlobStorageEnabled, - }) - } - - return result, nil +func (impl *WorkflowStatusLatestServiceImpl) GetCiWorkflowStatusLatestByPipelineIds(pipelineIds []int) ([]*pipelineConfig.CiWorkflowStatusLatest, error) { + return impl.workflowStatusLatestRepository.GetCiWorkflowStatusLatestByPipelineIds(pipelineIds) } // CD Workflow Status Latest methods implementation @@ -183,158 +126,6 @@ func (impl *WorkflowStatusLatestServiceImpl) SaveCdWorkflowStatusLatest(tx *pg.T return impl.workflowStatusLatestRepository.SaveCdWorkflowStatusLatest(tx, model) } -func (impl *WorkflowStatusLatestServiceImpl) GetCdWorkflowStatusLatestByPipelineIdAndWorkflowType(pipelineId int, workflowType string) (*CdWorkflowStatusLatest, error) { - model, err := impl.workflowStatusLatestRepository.GetCdWorkflowStatusLatestByPipelineIdAndWorkflowType(nil, pipelineId, workflowType) - if err != nil { - if err == pg.ErrNoRows { - // Fallback to old method - return impl.getCdWorkflowStatusFromOldMethod(pipelineId, workflowType) - } - impl.logger.Errorw("error in getting cd workflow status latest", "err", err, "pipelineId", pipelineId, "workflowType", workflowType) - return nil, err - } - - // Get status from cd_workflow_runner table - cdWorkflowRunner, err := impl.cdWorkflowRepository.FindBasicWorkflowRunnerById(model.WorkflowRunnerId) - if err != nil { - impl.logger.Errorw("error in getting cd workflow runner", "err", err, "workflowRunnerId", model.WorkflowRunnerId) - return nil, err - } - - return &CdWorkflowStatusLatest{ - PipelineId: model.PipelineId, - AppId: model.AppId, - EnvironmentId: model.EnvironmentId, - WorkflowType: model.WorkflowType, - WorkflowRunnerId: model.WorkflowRunnerId, - Status: cdWorkflowRunner.Status, - }, nil -} - -func (impl *WorkflowStatusLatestServiceImpl) GetCdWorkflowStatusLatestByAppId(appId int) ([]*CdWorkflowStatusLatest, error) { - models, err := impl.workflowStatusLatestRepository.GetCdWorkflowStatusLatestByAppId(appId) - if err != nil { - impl.logger.Errorw("error in getting cd workflow status latest by app id", "err", err, "appId", appId) - return nil, err - } - - var result []*CdWorkflowStatusLatest - for _, model := range models { - // Get status from cd_workflow_runner table - cdWorkflowRunner, err := impl.cdWorkflowRepository.FindBasicWorkflowRunnerById(model.WorkflowRunnerId) - if err != nil { - impl.logger.Errorw("error in getting cd workflow runner", "err", err, "workflowRunnerId", model.WorkflowRunnerId) - continue // Skip this entry if we can't get the workflow runner - } - - result = append(result, &CdWorkflowStatusLatest{ - PipelineId: model.PipelineId, - AppId: model.AppId, - EnvironmentId: model.EnvironmentId, - WorkflowType: model.WorkflowType, - WorkflowRunnerId: model.WorkflowRunnerId, - Status: cdWorkflowRunner.Status, - }) - } - - return result, nil -} - -func (impl *WorkflowStatusLatestServiceImpl) GetCdWorkflowStatusLatestByPipelineId(pipelineId int) ([]*CdWorkflowStatusLatest, error) { - models, err := impl.workflowStatusLatestRepository.GetCdWorkflowStatusLatestByPipelineId(pipelineId) - if err != nil { - impl.logger.Errorw("error in getting cd workflow status latest by pipeline id", "err", err, "pipelineId", pipelineId) - return nil, err - } - - var result []*CdWorkflowStatusLatest - for _, model := range models { - // Get status from cd_workflow_runner table - cdWorkflowRunner, err := impl.cdWorkflowRepository.FindBasicWorkflowRunnerById(model.WorkflowRunnerId) - if err != nil { - impl.logger.Errorw("error in getting cd workflow runner", "err", err, "workflowRunnerId", model.WorkflowRunnerId) - continue // Skip this entry if we can't get the workflow runner - } - - result = append(result, &CdWorkflowStatusLatest{ - PipelineId: model.PipelineId, - AppId: model.AppId, - EnvironmentId: model.EnvironmentId, - WorkflowType: model.WorkflowType, - WorkflowRunnerId: model.WorkflowRunnerId, - Status: cdWorkflowRunner.Status, - }) - } - - return result, nil -} - -// Fallback methods to old implementation when no entry found in latest status tables -func (impl *WorkflowStatusLatestServiceImpl) getCiWorkflowStatusFromOldMethod(pipelineId int) (*CiWorkflowStatusLatest, error) { - // Get the latest CI workflow for this pipeline using the old method - workflow, err := impl.ciWorkflowRepository.FindLastTriggeredWorkflow(pipelineId) - if err != nil { - if util2.IsErrNoRows(err) { - return &CiWorkflowStatusLatest{ - PipelineId: pipelineId, - AppId: 0, // Will need to be populated from pipeline info - CiWorkflowId: 0, - Status: "Not Triggered", - }, nil - } - impl.logger.Errorw("error in getting last triggered workflow", "err", err, "pipelineId", pipelineId) - return nil, err - } - - return &CiWorkflowStatusLatest{ - PipelineId: pipelineId, - AppId: workflow.CiPipeline.AppId, - CiWorkflowId: workflow.Id, - Status: workflow.Status, - }, nil -} - -func (impl *WorkflowStatusLatestServiceImpl) getCdWorkflowStatusFromOldMethod(pipelineId int, workflowType string) (*CdWorkflowStatusLatest, error) { - // Convert workflowType to the appropriate enum - var runnerType bean.WorkflowType - switch workflowType { - case "PRE": - runnerType = bean.CD_WORKFLOW_TYPE_PRE - case "DEPLOY": - runnerType = bean.CD_WORKFLOW_TYPE_DEPLOY - case "POST": - runnerType = bean.CD_WORKFLOW_TYPE_POST - default: - runnerType = bean.WorkflowType(workflowType) - } - - // Get the latest CD workflow runner for this pipeline and type using the old method - wfr, err := impl.cdWorkflowRepository.FindLatestByPipelineIdAndRunnerType(pipelineId, runnerType) - if err != nil { - if err == pg.ErrNoRows { - return &CdWorkflowStatusLatest{ - PipelineId: pipelineId, - AppId: 0, // Will need to be populated from pipeline info - EnvironmentId: 0, - WorkflowType: workflowType, - WorkflowRunnerId: 0, - Status: "Not Triggered", - }, nil - } - impl.logger.Errorw("error in getting latest cd workflow runner", "err", err, "pipelineId", pipelineId, "runnerType", runnerType) - return nil, err - } - - return &CdWorkflowStatusLatest{ - PipelineId: pipelineId, - AppId: wfr.CdWorkflow.Pipeline.AppId, - EnvironmentId: wfr.CdWorkflow.Pipeline.EnvironmentId, - WorkflowType: workflowType, - WorkflowRunnerId: wfr.Id, - Status: wfr.Status, - }, nil -} - func (impl *WorkflowStatusLatestServiceImpl) GetCdWorkflowLatestByPipelineIds(pipelineIds []int) ([]*CdWorkflowStatusLatest, error) { cdWorkflowStatusLatest, err := impl.workflowStatusLatestRepository.GetCdWorkflowStatusLatestByPipelineIds(pipelineIds) if err != nil { diff --git a/wire_gen.go b/wire_gen.go index 53d97340a1..39fb103244 100644 --- a/wire_gen.go +++ b/wire_gen.go @@ -751,7 +751,7 @@ func InitializeApp() (*App, error) { deploymentTemplateValidationServiceEntImpl := validator.NewDeploymentTemplateValidationServiceEntImpl() deploymentTemplateValidationServiceImpl := validator.NewDeploymentTemplateValidationServiceImpl(sugaredLogger, chartRefServiceImpl, scopedVariableManagerImpl, deployedAppMetricsServiceImpl, deploymentTemplateValidationServiceEntImpl) devtronAppGitOpConfigServiceImpl := gitOpsConfig.NewDevtronAppGitOpConfigServiceImpl(sugaredLogger, chartRepositoryImpl, chartServiceImpl, gitOpsConfigReadServiceImpl, gitOpsValidationServiceImpl, argoClientWrapperServiceImpl, deploymentConfigServiceImpl, chartReadServiceImpl) - ciHandlerImpl := pipeline.NewCiHandlerImpl(sugaredLogger, ciServiceImpl, ciPipelineMaterialRepositoryImpl, clientImpl, ciWorkflowRepositoryImpl, ciArtifactRepositoryImpl, userServiceImpl, eventRESTClientImpl, eventSimpleFactoryImpl, ciPipelineRepositoryImpl, appListingRepositoryImpl, pipelineRepositoryImpl, enforcerUtilImpl, resourceGroupServiceImpl, environmentRepositoryImpl, imageTaggingServiceImpl, k8sCommonServiceImpl, appWorkflowRepositoryImpl, customTagServiceImpl, workFlowStageStatusServiceImpl, workflowStatusLatestRepositoryImpl) + ciHandlerImpl := pipeline.NewCiHandlerImpl(sugaredLogger, ciServiceImpl, ciPipelineMaterialRepositoryImpl, clientImpl, ciWorkflowRepositoryImpl, ciArtifactRepositoryImpl, userServiceImpl, eventRESTClientImpl, eventSimpleFactoryImpl, ciPipelineRepositoryImpl, appListingRepositoryImpl, pipelineRepositoryImpl, enforcerUtilImpl, resourceGroupServiceImpl, environmentRepositoryImpl, imageTaggingServiceImpl, k8sCommonServiceImpl, appWorkflowRepositoryImpl, customTagServiceImpl, workFlowStageStatusServiceImpl, workflowStatusLatestServiceImpl) cdWorkflowRunnerReadServiceImpl := read18.NewCdWorkflowRunnerReadServiceImpl(sugaredLogger, cdWorkflowRepositoryImpl, workflowStatusLatestServiceImpl, pipelineStageRepositoryImpl) cdHandlerImpl := pipeline.NewCdHandlerImpl(sugaredLogger, userServiceImpl, cdWorkflowRepositoryImpl, ciArtifactRepositoryImpl, ciPipelineMaterialRepositoryImpl, pipelineRepositoryImpl, environmentRepositoryImpl, ciWorkflowRepositoryImpl, enforcerUtilImpl, resourceGroupServiceImpl, imageTaggingServiceImpl, k8sServiceImpl, customTagServiceImpl, deploymentConfigServiceImpl, workFlowStageStatusServiceImpl, cdWorkflowRunnerServiceImpl, workflowStatusLatestServiceImpl, pipelineStageRepositoryImpl, cdWorkflowRunnerReadServiceImpl) appWorkflowServiceImpl := appWorkflow2.NewAppWorkflowServiceImpl(sugaredLogger, appWorkflowRepositoryImpl, ciCdPipelineOrchestratorImpl, ciPipelineRepositoryImpl, pipelineRepositoryImpl, enforcerUtilImpl, resourceGroupServiceImpl, appRepositoryImpl, userAuthServiceImpl, chartServiceImpl, deploymentConfigServiceImpl, pipelineBuilderImpl) From aae32f6e799b744b5673394a0c357f611653d8ee Mon Sep 17 00:00:00 2001 From: Prakash Kumar Date: Thu, 24 Jul 2025 15:29:53 +0530 Subject: [PATCH 27/31] Remove unused methods from `WorkflowStatusLatestRepository` to streamline CI/CD workflow status handling. --- .../WorkflowStatusLatestRepository.go | 128 ------------------ 1 file changed, 128 deletions(-) diff --git a/internal/sql/repository/pipelineConfig/WorkflowStatusLatestRepository.go b/internal/sql/repository/pipelineConfig/WorkflowStatusLatestRepository.go index d0ffdac7c3..58ee30f8ac 100644 --- a/internal/sql/repository/pipelineConfig/WorkflowStatusLatestRepository.go +++ b/internal/sql/repository/pipelineConfig/WorkflowStatusLatestRepository.go @@ -26,19 +26,10 @@ import ( type WorkflowStatusLatestRepository interface { // CI Workflow Status Latest methods SaveCiWorkflowStatusLatest(tx *pg.Tx, model *CiWorkflowStatusLatest) error - UpdateCiWorkflowStatusLatest(tx *pg.Tx, model *CiWorkflowStatusLatest) error - GetCiWorkflowStatusLatestByPipelineId(pipelineId int) (*CiWorkflowStatusLatest, error) - GetCiWorkflowStatusLatestByAppId(appId int) ([]*CiWorkflowStatusLatest, error) GetCiWorkflowStatusLatestByPipelineIds(pipelineIds []int) ([]*CiWorkflowStatusLatest, error) - DeleteCiWorkflowStatusLatestByPipelineId(pipelineId int) error // CD Workflow Status Latest methods SaveCdWorkflowStatusLatest(tx *pg.Tx, model *CdWorkflowStatusLatest) error - UpdateCdWorkflowStatusLatest(tx *pg.Tx, model *CdWorkflowStatusLatest) error - GetCdWorkflowStatusLatestByPipelineIdAndWorkflowType(tx *pg.Tx, pipelineId int, workflowType string) (*CdWorkflowStatusLatest, error) - GetCdWorkflowStatusLatestByAppId(appId int) ([]*CdWorkflowStatusLatest, error) - GetCdWorkflowStatusLatestByPipelineId(pipelineId int) ([]*CdWorkflowStatusLatest, error) - DeleteCdWorkflowStatusLatestByPipelineId(pipelineId int) error GetCdWorkflowStatusLatestByPipelineIds(pipelineIds []int) ([]*CdWorkflowStatusLatest, error) } @@ -92,56 +83,6 @@ func (impl *WorkflowStatusLatestRepositoryImpl) SaveCiWorkflowStatusLatest(tx *p return nil } -func (impl *WorkflowStatusLatestRepositoryImpl) UpdateCiWorkflowStatusLatest(tx *pg.Tx, model *CiWorkflowStatusLatest) error { - var connection orm.DB - if tx != nil { - connection = tx - } else { - connection = impl.dbConnection - } - _, err := connection.Model(model).WherePK().UpdateNotNull() - if err != nil { - impl.logger.Errorw("error in updating ci workflow status latest", "err", err, "model", model) - return err - } - return nil -} - -func (impl *WorkflowStatusLatestRepositoryImpl) GetCiWorkflowStatusLatestByPipelineId(pipelineId int) (*CiWorkflowStatusLatest, error) { - model := &CiWorkflowStatusLatest{} - err := impl.dbConnection.Model(model). - Where("pipeline_id = ?", pipelineId). - Select() - if err != nil { - impl.logger.Errorw("error in getting ci workflow status latest by pipeline id", "err", err, "pipelineId", pipelineId) - return nil, err - } - return model, nil -} - -func (impl *WorkflowStatusLatestRepositoryImpl) GetCiWorkflowStatusLatestByAppId(appId int) ([]*CiWorkflowStatusLatest, error) { - var models []*CiWorkflowStatusLatest - err := impl.dbConnection.Model(&models). - Where("app_id = ?", appId). - Select() - if err != nil { - impl.logger.Errorw("error in getting ci workflow status latest by app id", "err", err, "appId", appId) - return nil, err - } - return models, nil -} - -func (impl *WorkflowStatusLatestRepositoryImpl) DeleteCiWorkflowStatusLatestByPipelineId(pipelineId int) error { - _, err := impl.dbConnection.Model(&CiWorkflowStatusLatest{}). - Where("pipeline_id = ?", pipelineId). - Delete() - if err != nil { - impl.logger.Errorw("error in deleting ci workflow status latest by pipeline id", "err", err, "pipelineId", pipelineId) - return err - } - return nil -} - func (impl *WorkflowStatusLatestRepositoryImpl) GetCiWorkflowStatusLatestByPipelineIds(pipelineIds []int) ([]*CiWorkflowStatusLatest, error) { if len(pipelineIds) == 0 { return []*CiWorkflowStatusLatest{}, nil @@ -174,75 +115,6 @@ func (impl *WorkflowStatusLatestRepositoryImpl) SaveCdWorkflowStatusLatest(tx *p return nil } -func (impl *WorkflowStatusLatestRepositoryImpl) UpdateCdWorkflowStatusLatest(tx *pg.Tx, model *CdWorkflowStatusLatest) error { - var connection orm.DB - if tx != nil { - connection = tx - } else { - connection = impl.dbConnection - } - _, err := connection.Model(model).WherePK().UpdateNotNull() - if err != nil { - impl.logger.Errorw("error in updating cd workflow status latest", "err", err, "model", model) - return err - } - return nil -} - -func (impl *WorkflowStatusLatestRepositoryImpl) GetCdWorkflowStatusLatestByPipelineIdAndWorkflowType(tx *pg.Tx, pipelineId int, workflowType string) (*CdWorkflowStatusLatest, error) { - model := &CdWorkflowStatusLatest{} - var connection orm.DB - if tx != nil { - connection = tx - } else { - connection = impl.dbConnection - } - err := connection.Model(model). - Where("pipeline_id = ?", pipelineId). - Where("workflow_type = ?", workflowType). - Select() - if err != nil { - impl.logger.Errorw("error in getting cd workflow status latest by pipeline id and workflow type", "err", err, "pipelineId", pipelineId, "workflowType", workflowType) - return nil, err - } - return model, nil -} - -func (impl *WorkflowStatusLatestRepositoryImpl) GetCdWorkflowStatusLatestByAppId(appId int) ([]*CdWorkflowStatusLatest, error) { - var models []*CdWorkflowStatusLatest - err := impl.dbConnection.Model(&models). - Where("app_id = ?", appId). - Select() - if err != nil { - impl.logger.Errorw("error in getting cd workflow status latest by app id", "err", err, "appId", appId) - return nil, err - } - return models, nil -} - -func (impl *WorkflowStatusLatestRepositoryImpl) GetCdWorkflowStatusLatestByPipelineId(pipelineId int) ([]*CdWorkflowStatusLatest, error) { - var models []*CdWorkflowStatusLatest - err := impl.dbConnection.Model(&models). - Where("pipeline_id = ?", pipelineId). - Select() - if err != nil { - impl.logger.Errorw("error in getting cd workflow status latest by pipeline id", "err", err, "pipelineId", pipelineId) - return nil, err - } - return models, nil -} - -func (impl *WorkflowStatusLatestRepositoryImpl) DeleteCdWorkflowStatusLatestByPipelineId(pipelineId int) error { - _, err := impl.dbConnection.Model(&CdWorkflowStatusLatest{}). - Where("pipeline_id = ?", pipelineId). - Delete() - if err != nil { - impl.logger.Errorw("error in deleting cd workflow status latest by pipeline id", "err", err, "pipelineId", pipelineId) - return err - } - return nil -} - func (impl *WorkflowStatusLatestRepositoryImpl) GetCdWorkflowStatusLatestByPipelineIds(pipelineIds []int) ([]*CdWorkflowStatusLatest, error) { var models []*CdWorkflowStatusLatest err := impl.dbConnection.Model(&models). From ee5672ec6a3aee6e69320d977951e955d2e839d1 Mon Sep 17 00:00:00 2001 From: Prakash Kumar Date: Thu, 24 Jul 2025 15:56:37 +0530 Subject: [PATCH 28/31] Update unique constraints in workflow status tables to use `ci_workflow_id` and `workflow_runner_id` --- scripts/sql/34203900_workflow_status_latest_tables.up.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/sql/34203900_workflow_status_latest_tables.up.sql b/scripts/sql/34203900_workflow_status_latest_tables.up.sql index 6c87a7ec1d..d39726cfd5 100644 --- a/scripts/sql/34203900_workflow_status_latest_tables.up.sql +++ b/scripts/sql/34203900_workflow_status_latest_tables.up.sql @@ -14,7 +14,7 @@ CREATE TABLE IF NOT EXISTS "public"."ci_workflow_status_latest" ( "updated_on" timestamptz NOT NULL, "updated_by" int4 NOT NULL, PRIMARY KEY ("id"), - UNIQUE ("pipeline_id") + UNIQUE ("ci_workflow_id") ); -- Create Sequence for cd_workflow_status_latest @@ -33,7 +33,7 @@ CREATE TABLE IF NOT EXISTS "public"."cd_workflow_status_latest" ( "updated_on" timestamptz NOT NULL, "updated_by" int4 NOT NULL, PRIMARY KEY ("id"), - UNIQUE ("pipeline_id", "workflow_type") + UNIQUE ("workflow_runner_id", "workflow_type") ); COMMIT; From b86f98fe2b0815ef4ab64b6c17c08a0adb8c38b6 Mon Sep 17 00:00:00 2001 From: Prakash Kumar Date: Thu, 24 Jul 2025 16:14:54 +0530 Subject: [PATCH 29/31] Update `WorkflowStatusLatestService` to handle both create and update for CI/CD workflow statuses, refactor repository methods, and modify unique constraints in workflow status tables. --- .../WorkflowStatusLatestRepository.go | 65 +++++++++++++++ .../WorkflowStatusLatestService.go | 80 +++++++++++++------ ...03900_workflow_status_latest_tables.up.sql | 4 +- 3 files changed, 124 insertions(+), 25 deletions(-) diff --git a/internal/sql/repository/pipelineConfig/WorkflowStatusLatestRepository.go b/internal/sql/repository/pipelineConfig/WorkflowStatusLatestRepository.go index 58ee30f8ac..e9c78e0866 100644 --- a/internal/sql/repository/pipelineConfig/WorkflowStatusLatestRepository.go +++ b/internal/sql/repository/pipelineConfig/WorkflowStatusLatestRepository.go @@ -26,10 +26,14 @@ import ( type WorkflowStatusLatestRepository interface { // CI Workflow Status Latest methods SaveCiWorkflowStatusLatest(tx *pg.Tx, model *CiWorkflowStatusLatest) error + UpdateCiWorkflowStatusLatest(tx *pg.Tx, model *CiWorkflowStatusLatest) error + GetCiWorkflowStatusLatestByPipelineId(pipelineId int) (*CiWorkflowStatusLatest, error) GetCiWorkflowStatusLatestByPipelineIds(pipelineIds []int) ([]*CiWorkflowStatusLatest, error) // CD Workflow Status Latest methods SaveCdWorkflowStatusLatest(tx *pg.Tx, model *CdWorkflowStatusLatest) error + UpdateCdWorkflowStatusLatest(tx *pg.Tx, model *CdWorkflowStatusLatest) error + GetCdWorkflowStatusLatestByPipelineIdAndWorkflowType(tx *pg.Tx, pipelineId int, workflowType string) (*CdWorkflowStatusLatest, error) GetCdWorkflowStatusLatestByPipelineIds(pipelineIds []int) ([]*CdWorkflowStatusLatest, error) } @@ -99,6 +103,33 @@ func (impl *WorkflowStatusLatestRepositoryImpl) GetCiWorkflowStatusLatestByPipel return models, nil } +func (impl *WorkflowStatusLatestRepositoryImpl) UpdateCiWorkflowStatusLatest(tx *pg.Tx, model *CiWorkflowStatusLatest) error { + var connection orm.DB + if tx != nil { + connection = tx + } else { + connection = impl.dbConnection + } + _, err := connection.Model(model).WherePK().Update() + if err != nil { + impl.logger.Errorw("error in updating ci workflow status latest", "err", err, "model", model) + return err + } + return nil +} + +func (impl *WorkflowStatusLatestRepositoryImpl) GetCiWorkflowStatusLatestByPipelineId(pipelineId int) (*CiWorkflowStatusLatest, error) { + var model CiWorkflowStatusLatest + err := impl.dbConnection.Model(&model). + Where("pipeline_id = ?", pipelineId). + Select() + if err != nil { + impl.logger.Errorw("error in getting ci workflow status latest by pipeline id", "err", err, "pipelineId", pipelineId) + return nil, err + } + return &model, nil +} + // CD Workflow Status Latest methods implementation func (impl *WorkflowStatusLatestRepositoryImpl) SaveCdWorkflowStatusLatest(tx *pg.Tx, model *CdWorkflowStatusLatest) error { var connection orm.DB @@ -126,3 +157,37 @@ func (impl *WorkflowStatusLatestRepositoryImpl) GetCdWorkflowStatusLatestByPipel } return models, nil } + +func (impl *WorkflowStatusLatestRepositoryImpl) UpdateCdWorkflowStatusLatest(tx *pg.Tx, model *CdWorkflowStatusLatest) error { + var connection orm.DB + if tx != nil { + connection = tx + } else { + connection = impl.dbConnection + } + _, err := connection.Model(model).WherePK().Update() + if err != nil { + impl.logger.Errorw("error in updating cd workflow status latest", "err", err, "model", model) + return err + } + return nil +} + +func (impl *WorkflowStatusLatestRepositoryImpl) GetCdWorkflowStatusLatestByPipelineIdAndWorkflowType(tx *pg.Tx, pipelineId int, workflowType string) (*CdWorkflowStatusLatest, error) { + var connection orm.DB + if tx != nil { + connection = tx + } else { + connection = impl.dbConnection + } + + var model CdWorkflowStatusLatest + err := connection.Model(&model). + Where("pipeline_id = ? AND workflow_type = ?", pipelineId, workflowType). + Select() + if err != nil { + impl.logger.Errorw("error in getting cd workflow status latest by pipeline id and workflow type", "err", err, "pipelineId", pipelineId, "workflowType", workflowType) + return nil, err + } + return &model, nil +} diff --git a/pkg/workflow/workflowStatusLatest/WorkflowStatusLatestService.go b/pkg/workflow/workflowStatusLatest/WorkflowStatusLatestService.go index 7c9a561fc1..e2c318e5b4 100644 --- a/pkg/workflow/workflowStatusLatest/WorkflowStatusLatestService.go +++ b/pkg/workflow/workflowStatusLatest/WorkflowStatusLatestService.go @@ -18,6 +18,7 @@ package workflowStatusLatest import ( "fmt" + util2 "github.com/devtron-labs/devtron/internal/util" "time" "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" @@ -83,18 +84,35 @@ func (impl *WorkflowStatusLatestServiceImpl) SaveCiWorkflowStatusLatest(tx *pg.T return fmt.Errorf("invalid ciWorkflowId: %d", ciWorkflowId) } - now := time.Now() - model := &pipelineConfig.CiWorkflowStatusLatest{ - PipelineId: pipelineId, - AppId: appId, - CiWorkflowId: ciWorkflowId, + // Check if entry exists + existingEntry, err := impl.workflowStatusLatestRepository.GetCiWorkflowStatusLatestByPipelineId(pipelineId) + if err != nil && !util2.IsErrNoRows(err) { + impl.logger.Errorw("error in getting ci workflow status latest", "err", err, "pipelineId", pipelineId) + return err } - model.CreatedBy = userId - model.CreatedOn = now - model.UpdatedBy = userId - model.UpdatedOn = now - return impl.workflowStatusLatestRepository.SaveCiWorkflowStatusLatest(tx, model) + now := time.Now() + if util2.IsErrNoRows(err) { + // Create new entry + model := &pipelineConfig.CiWorkflowStatusLatest{ + PipelineId: pipelineId, + AppId: appId, + CiWorkflowId: ciWorkflowId, + } + model.CreatedBy = userId + model.CreatedOn = now + model.UpdatedBy = userId + model.UpdatedOn = now + + return impl.workflowStatusLatestRepository.SaveCiWorkflowStatusLatest(tx, model) + } else { + // Update existing entry with latest workflow ID + existingEntry.CiWorkflowId = ciWorkflowId + existingEntry.UpdatedBy = userId + existingEntry.UpdatedOn = now + + return impl.workflowStatusLatestRepository.UpdateCiWorkflowStatusLatest(tx, existingEntry) + } } func (impl *WorkflowStatusLatestServiceImpl) GetCiWorkflowStatusLatestByPipelineIds(pipelineIds []int) ([]*pipelineConfig.CiWorkflowStatusLatest, error) { @@ -109,21 +127,37 @@ func (impl *WorkflowStatusLatestServiceImpl) SaveCdWorkflowStatusLatest(tx *pg.T return fmt.Errorf("invalid workflowRunnerId: %d", workflowRunnerId) } - // Create new entry (always save, don't update) - now := time.Now() - model := &pipelineConfig.CdWorkflowStatusLatest{ - PipelineId: pipelineId, - AppId: appId, - EnvironmentId: environmentId, - WorkflowType: workflowType, - WorkflowRunnerId: workflowRunnerId, + // Check if entry exists + existingEntry, err := impl.workflowStatusLatestRepository.GetCdWorkflowStatusLatestByPipelineIdAndWorkflowType(tx, pipelineId, workflowType) + if err != nil && err != pg.ErrNoRows { + impl.logger.Errorw("error in getting cd workflow status latest", "err", err, "pipelineId", pipelineId, "workflowType", workflowType) + return err } - model.CreatedBy = userId - model.CreatedOn = now - model.UpdatedBy = userId - model.UpdatedOn = now - return impl.workflowStatusLatestRepository.SaveCdWorkflowStatusLatest(tx, model) + now := time.Now() + if err == pg.ErrNoRows { + // Create new entry + model := &pipelineConfig.CdWorkflowStatusLatest{ + PipelineId: pipelineId, + AppId: appId, + EnvironmentId: environmentId, + WorkflowType: workflowType, + WorkflowRunnerId: workflowRunnerId, + } + model.CreatedBy = userId + model.CreatedOn = now + model.UpdatedBy = userId + model.UpdatedOn = now + + return impl.workflowStatusLatestRepository.SaveCdWorkflowStatusLatest(tx, model) + } else { + // Update existing entry with latest workflow runner ID + existingEntry.WorkflowRunnerId = workflowRunnerId + existingEntry.UpdatedBy = userId + existingEntry.UpdatedOn = now + + return impl.workflowStatusLatestRepository.UpdateCdWorkflowStatusLatest(tx, existingEntry) + } } func (impl *WorkflowStatusLatestServiceImpl) GetCdWorkflowLatestByPipelineIds(pipelineIds []int) ([]*CdWorkflowStatusLatest, error) { diff --git a/scripts/sql/34203900_workflow_status_latest_tables.up.sql b/scripts/sql/34203900_workflow_status_latest_tables.up.sql index d39726cfd5..6c87a7ec1d 100644 --- a/scripts/sql/34203900_workflow_status_latest_tables.up.sql +++ b/scripts/sql/34203900_workflow_status_latest_tables.up.sql @@ -14,7 +14,7 @@ CREATE TABLE IF NOT EXISTS "public"."ci_workflow_status_latest" ( "updated_on" timestamptz NOT NULL, "updated_by" int4 NOT NULL, PRIMARY KEY ("id"), - UNIQUE ("ci_workflow_id") + UNIQUE ("pipeline_id") ); -- Create Sequence for cd_workflow_status_latest @@ -33,7 +33,7 @@ CREATE TABLE IF NOT EXISTS "public"."cd_workflow_status_latest" ( "updated_on" timestamptz NOT NULL, "updated_by" int4 NOT NULL, PRIMARY KEY ("id"), - UNIQUE ("workflow_runner_id", "workflow_type") + UNIQUE ("pipeline_id", "workflow_type") ); COMMIT; From 197a9dc8ddd0b9c815f707b8925885efc926d0bb Mon Sep 17 00:00:00 2001 From: satya_prakash <155617493+SATYAsasini@users.noreply.github.com> Date: Mon, 28 Jul 2025 15:45:36 +0530 Subject: [PATCH 30/31] fix: refactored unit conversion for cluster details (#6768) --- pkg/k8s/capacity/k8sCapacityService.go | 60 ++++++++++++++++++-------- 1 file changed, 41 insertions(+), 19 deletions(-) diff --git a/pkg/k8s/capacity/k8sCapacityService.go b/pkg/k8s/capacity/k8sCapacityService.go index dd70446852..08c2246533 100644 --- a/pkg/k8s/capacity/k8sCapacityService.go +++ b/pkg/k8s/capacity/k8sCapacityService.go @@ -42,6 +42,7 @@ import ( "k8s.io/client-go/kubernetes" resourcehelper "k8s.io/kubectl/pkg/util/resource" metrics "k8s.io/metrics/pkg/client/clientset/versioned" + "math" "net/http" "strings" "time" @@ -931,30 +932,51 @@ func getResourceString(quantity resource.Quantity, resourceName corev1.ResourceN //not a standard resource, we do not know if conversion would be valid or not //for example - pods: "250", this is not in bytes but an integer so conversion is invalid return quantity.String() - } else { + // exempting CPU resource as CPU's resource unit is in cores + } else if resourceName != corev1.ResourceCPU { var quantityStr string value := quantity.Value() - valueGi := value / bean.Gibibyte - //allowing remainder 0 only, because for Gi rounding off will be highly erroneous - if valueGi > 1 && value%bean.Gibibyte == 0 { - quantityStr = fmt.Sprintf("%dGi", valueGi) - } else { - valueMi := value / bean.Mebibyte - if valueMi > 10 { - if value%bean.Mebibyte != 0 { - valueMi++ - } - quantityStr = fmt.Sprintf("%dMi", valueMi) - } else if value > 1000 { - valueKi := value / bean.Kibibyte - if value%bean.Kibibyte != 0 { - valueKi++ - } - quantityStr = fmt.Sprintf("%dKi", valueKi) + //allowing remainder 0 only, because for Gi rounding off will be highly erroneous - + //despite rounding allowing decimal value upto 2 decimal places + + // first check for Gi + valueGi := float64(value) / (bean.Gibibyte * 1.0) + if valueGi >= 1 { + if valueGi == math.Floor(valueGi) { // if the converted value is a whole number + quantityStr = fmt.Sprintf("%dGi", int64(valueGi)) } else { - quantityStr = fmt.Sprintf("%dm", quantity.MilliValue()) + quantityStr = fmt.Sprintf("%.2fGi", valueGi) + } + } else if value >= bean.Mebibyte { // fall back to check for Mi + valueMi := value / bean.Mebibyte + if value%bean.Mebibyte != 0 { + valueMi++ + } + quantityStr = fmt.Sprintf("%dMi", valueMi) + } else if value >= bean.Kibibyte { // fall back to check for Ki + valueKi := value / bean.Kibibyte + if value%bean.Kibibyte != 0 { + valueKi++ } + quantityStr = fmt.Sprintf("%dKi", valueKi) + } else { // else better to show in Bytes + quantityStr = fmt.Sprintf("%dB", value) + } + return quantityStr + } else { + var quantityStr string + cpuValueMilli := quantity.MilliValue() // it is safe to use MilliValue here as in real world the value would not exceed int64 range + cpuValueCore := float64(cpuValueMilli) / 1000.0 + quantityStr = fmt.Sprintf("%.2f", cpuValueCore) + // if cpuValueCore is less than 1 then show in milli core only + if cpuValueCore < 1 { + return fmt.Sprintf("%dm", cpuValueMilli) + } + // if the core value is a whole number then returning int else float + if cpuValueCore == math.Floor(cpuValueCore) { + return fmt.Sprintf("%d", int64(cpuValueCore)) } + // showing values in cores upto 2 decimal value return quantityStr } } From 35f4e38acff7c85cefc7943978ac47770344fa4e Mon Sep 17 00:00:00 2001 From: ayushmaheshwari Date: Tue, 29 Jul 2025 16:14:43 +0530 Subject: [PATCH 31/31] develop merge --- api/cluster/ClusterRestHandler.go | 70 +++++++++++++++++-- api/cluster/ClusterRouter.go | 5 ++ pkg/cluster/ClusterService.go | 37 ++++++++-- pkg/cluster/ClusterServiceExtended.go | 15 +++- .../WorkflowStageStatusService.go | 39 +++++++++-- specs/cluster_api_spec.yaml | 15 +++- specs/swagger/openapi.yaml | 15 +++- .../workflow-stage-status.internal.yaml | 4 +- 8 files changed, 178 insertions(+), 22 deletions(-) diff --git a/api/cluster/ClusterRestHandler.go b/api/cluster/ClusterRestHandler.go index c6baecb85d..2d2f946cb9 100644 --- a/api/cluster/ClusterRestHandler.go +++ b/api/cluster/ClusterRestHandler.go @@ -20,14 +20,15 @@ import ( "context" "encoding/json" "errors" - bean2 "github.com/devtron-labs/devtron/pkg/cluster/bean" - "github.com/devtron-labs/devtron/pkg/cluster/environment" - "github.com/devtron-labs/devtron/pkg/cluster/rbac" "net/http" "strconv" "strings" "time" + bean2 "github.com/devtron-labs/devtron/pkg/cluster/bean" + "github.com/devtron-labs/devtron/pkg/cluster/environment" + "github.com/devtron-labs/devtron/pkg/cluster/rbac" + "github.com/devtron-labs/devtron/pkg/auth/authorisation/casbin" "github.com/devtron-labs/devtron/pkg/auth/user" "github.com/devtron-labs/devtron/pkg/genericNotes" @@ -60,6 +61,7 @@ type ClusterRestHandler interface { GetClusterNamespaces(w http.ResponseWriter, r *http.Request) GetAllClusterNamespaces(w http.ResponseWriter, r *http.Request) FindAllForClusterPermission(w http.ResponseWriter, r *http.Request) + FindByIds(w http.ResponseWriter, r *http.Request) } type ClusterRestHandlerImpl struct { @@ -296,6 +298,59 @@ func (impl ClusterRestHandlerImpl) FindAll(w http.ResponseWriter, r *http.Reques common.WriteJsonResp(w, err, result, http.StatusOK) } +func (impl ClusterRestHandlerImpl) FindByIds(w http.ResponseWriter, r *http.Request) { + token := r.Header.Get("token") + + // Parse clusterId query parameter + clusterIdsStr := r.URL.Query().Get("clusterId") + if clusterIdsStr == "" { + // If no clusterId parameter, return all clusters (same as FindAll) + impl.FindAll(w, r) + return + } + + // Parse comma-separated cluster IDs + var clusterIds []int + clusterIdStrs := strings.Split(clusterIdsStr, ",") + for _, idStr := range clusterIdStrs { + idStr = strings.TrimSpace(idStr) + if idStr == "" { + continue + } + id, err := strconv.Atoi(idStr) + if err != nil { + impl.logger.Errorw("request err, FindByIds", "error", err, "clusterId", idStr) + common.WriteJsonResp(w, err, nil, http.StatusBadRequest) + return + } + clusterIds = append(clusterIds, id) + } + + if len(clusterIds) == 0 { + // If no valid cluster IDs, return empty result + common.WriteJsonResp(w, nil, []*bean2.ClusterBean{}, http.StatusOK) + return + } + + clusterList, err := impl.clusterService.FindByIdsWithoutConfig(clusterIds) + if err != nil { + impl.logger.Errorw("service err, FindByIds", "err", err) + common.WriteJsonResp(w, err, nil, http.StatusInternalServerError) + return + } + + // RBAC enforcer applying + var result []*bean2.ClusterBean + for _, item := range clusterList { + if ok := impl.enforcer.Enforce(token, casbin.ResourceCluster, casbin.ActionGet, item.ClusterName); ok { + result = append(result, item) + } + } + //RBAC enforcer Ends + + common.WriteJsonResp(w, nil, result, http.StatusOK) +} + func (impl ClusterRestHandlerImpl) FindById(w http.ResponseWriter, r *http.Request) { vars := mux.Vars(r) id := vars["id"] @@ -671,7 +726,14 @@ func (impl ClusterRestHandlerImpl) GetClusterNamespaces(w http.ResponseWriter, r allClusterNamespaces, err := impl.clusterService.FindAllNamespacesByUserIdAndClusterId(userId, clusterId, isActionUserSuperAdmin) if err != nil { - common.WriteJsonResp(w, err, nil, http.StatusInternalServerError) + // Check if it's a cluster connectivity error and return appropriate status code + if err.Error() == cluster.ErrClusterNotReachable { + impl.logger.Errorw("cluster connectivity error in GetClusterNamespaces", "error", err, "clusterId", clusterId) + common.WriteJsonResp(w, err, nil, http.StatusBadRequest) + } else { + impl.logger.Errorw("error in GetClusterNamespaces", "error", err, "clusterId", clusterId) + common.WriteJsonResp(w, err, nil, http.StatusInternalServerError) + } return } common.WriteJsonResp(w, nil, allClusterNamespaces, http.StatusOK) diff --git a/api/cluster/ClusterRouter.go b/api/cluster/ClusterRouter.go index abd233e953..07fb4f80ba 100644 --- a/api/cluster/ClusterRouter.go +++ b/api/cluster/ClusterRouter.go @@ -57,6 +57,11 @@ func (impl ClusterRouterImpl) InitClusterRouter(clusterRouter *mux.Router) { Queries("id", "{id}"). HandlerFunc(impl.clusterRestHandler.FindNoteByClusterId) + clusterRouter.Path(""). + Methods("GET"). + Queries("clusterId", "{clusterId}"). + HandlerFunc(impl.clusterRestHandler.FindByIds) + clusterRouter.Path(""). Methods("GET"). HandlerFunc(impl.clusterRestHandler.FindAll) diff --git a/pkg/cluster/ClusterService.go b/pkg/cluster/ClusterService.go index 6a1741e9c2..669487d541 100644 --- a/pkg/cluster/ClusterService.go +++ b/pkg/cluster/ClusterService.go @@ -20,6 +20,11 @@ import ( "context" "encoding/json" "fmt" + "log" + "net/url" + "sync" + "time" + "github.com/devtron-labs/common-lib/async" informerBean "github.com/devtron-labs/common-lib/informer" "github.com/devtron-labs/common-lib/utils/k8s/commonBean" @@ -32,10 +37,6 @@ import ( "github.com/devtron-labs/devtron/pkg/cluster/read" cronUtil "github.com/devtron-labs/devtron/util/cron" "github.com/robfig/cron/v3" - "log" - "net/url" - "sync" - "time" "github.com/argoproj/argo-cd/v2/pkg/apis/application/v1alpha1" "github.com/devtron-labs/common-lib/utils/k8s" @@ -75,6 +76,7 @@ type ClusterService interface { FindById(id int) (*bean.ClusterBean, error) FindByIdWithoutConfig(id int) (*bean.ClusterBean, error) FindByIds(id []int) ([]bean.ClusterBean, error) + FindByIdsWithoutConfig(ids []int) ([]*bean.ClusterBean, error) Update(ctx context.Context, bean *bean.ClusterBean, userId int32) (*bean.ClusterBean, error) Delete(bean *bean.ClusterBean, userId int32) error @@ -355,6 +357,21 @@ func (impl *ClusterServiceImpl) FindByIds(ids []int) ([]bean.ClusterBean, error) return beans, nil } +func (impl *ClusterServiceImpl) FindByIdsWithoutConfig(ids []int) ([]*bean.ClusterBean, error) { + models, err := impl.clusterRepository.FindByIds(ids) + if err != nil { + return nil, err + } + var beans []*bean.ClusterBean + for _, model := range models { + bean := adapter.GetClusterBean(model) + //empty bearer token as it will be hidden for user + bean.Config = map[string]string{commonBean.BearerToken: ""} + beans = append(beans, &bean) + } + return beans, nil +} + func (impl *ClusterServiceImpl) Update(ctx context.Context, bean *bean.ClusterBean, userId int32) (*bean.ClusterBean, error) { model, err := impl.clusterRepository.FindById(bean.Id) if err != nil { @@ -640,6 +657,11 @@ func (impl *ClusterServiceImpl) GetAllClusterNamespaces() map[string][]string { return result } +const ( + // Cluster connectivity error constants + ErrClusterNotReachable = "cluster is not reachable" +) + func (impl *ClusterServiceImpl) FindAllNamespacesByUserIdAndClusterId(userId int32, clusterId int, isActionUserSuperAdmin bool) ([]string, error) { result := make([]string, 0) clusterBean, err := impl.clusterReadService.FindById(clusterId) @@ -647,6 +669,13 @@ func (impl *ClusterServiceImpl) FindAllNamespacesByUserIdAndClusterId(userId int impl.logger.Errorw("failed to find cluster for id", "error", err, "clusterId", clusterId) return nil, err } + + // Check if cluster has connection errors + if len(clusterBean.ErrorInConnecting) > 0 { + impl.logger.Errorw("cluster is not reachable", "clusterId", clusterId, "clusterName", clusterBean.ClusterName, "error", clusterBean.ErrorInConnecting) + return nil, fmt.Errorf("%s: %s", ErrClusterNotReachable, clusterBean.ErrorInConnecting) + } + namespaceListGroupByCLuster := impl.K8sInformerFactory.GetLatestNamespaceListGroupByCLuster() namespaces := namespaceListGroupByCLuster[clusterBean.ClusterName] if len(namespaces) == 0 { diff --git a/pkg/cluster/ClusterServiceExtended.go b/pkg/cluster/ClusterServiceExtended.go index e4465c07cb..8fe04bf571 100644 --- a/pkg/cluster/ClusterServiceExtended.go +++ b/pkg/cluster/ClusterServiceExtended.go @@ -19,14 +19,15 @@ package cluster import ( "context" "fmt" + "net/http" + "strings" + "time" + "github.com/devtron-labs/common-lib/utils/k8s/commonBean" "github.com/devtron-labs/devtron/client/argocdServer" "github.com/devtron-labs/devtron/pkg/cluster/bean" "github.com/devtron-labs/devtron/pkg/cluster/environment/repository" "github.com/devtron-labs/devtron/pkg/deployment/gitOps/config" - "net/http" - "strings" - "time" cluster3 "github.com/argoproj/argo-cd/v2/pkg/apiclient/cluster" "github.com/devtron-labs/devtron/client/grafana" @@ -75,6 +76,14 @@ func (impl *ClusterServiceImplExtended) FindAllWithoutConfig() ([]*bean.ClusterB return beans, nil } +func (impl *ClusterServiceImplExtended) FindByIdsWithoutConfig(ids []int) ([]*bean.ClusterBean, error) { + beans, err := impl.ClusterServiceImpl.FindByIdsWithoutConfig(ids) + if err != nil { + return nil, err + } + return impl.GetClusterFullModeDTO(beans) +} + func (impl *ClusterServiceImplExtended) GetClusterFullModeDTO(beans []*bean.ClusterBean) ([]*bean.ClusterBean, error) { //devtron full mode logic var clusterIds []int diff --git a/pkg/pipeline/workflowStatus/WorkflowStageStatusService.go b/pkg/pipeline/workflowStatus/WorkflowStageStatusService.go index d583ef9261..df576a806f 100644 --- a/pkg/pipeline/workflowStatus/WorkflowStageStatusService.go +++ b/pkg/pipeline/workflowStatus/WorkflowStageStatusService.go @@ -2,11 +2,17 @@ package workflowStatus import ( "encoding/json" + "slices" + "strconv" + "strings" + "time" + "github.com/argoproj/argo-workflows/v3/pkg/apis/workflow/v1alpha1" "github.com/devtron-labs/devtron/api/bean" "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig/bean/workflow/cdWorkflow" bean3 "github.com/devtron-labs/devtron/pkg/bean" + envRepository "github.com/devtron-labs/devtron/pkg/cluster/environment/repository" "github.com/devtron-labs/devtron/pkg/pipeline/constants" "github.com/devtron-labs/devtron/pkg/pipeline/types" "github.com/devtron-labs/devtron/pkg/pipeline/workflowStatus/adapter" @@ -16,9 +22,6 @@ import ( "github.com/devtron-labs/devtron/pkg/sql" "github.com/go-pg/pg" "go.uber.org/zap" - "slices" - "strings" - "time" ) type WorkFlowStageStatusService interface { @@ -35,6 +38,7 @@ type WorkFlowStageStatusServiceImpl struct { workflowStatusRepository repository.WorkflowStageRepository ciWorkflowRepository pipelineConfig.CiWorkflowRepository cdWorkflowRepository pipelineConfig.CdWorkflowRepository + envRepository envRepository.EnvironmentRepository transactionManager sql.TransactionWrapper config *types.CiConfig } @@ -43,6 +47,7 @@ func NewWorkflowStageFlowStatusServiceImpl(logger *zap.SugaredLogger, workflowStatusRepository repository.WorkflowStageRepository, ciWorkflowRepository pipelineConfig.CiWorkflowRepository, cdWorkflowRepository pipelineConfig.CdWorkflowRepository, + envRepository envRepository.EnvironmentRepository, transactionManager sql.TransactionWrapper, ) *WorkFlowStageStatusServiceImpl { wfStageServiceImpl := &WorkFlowStageStatusServiceImpl{ @@ -50,6 +55,7 @@ func NewWorkflowStageFlowStatusServiceImpl(logger *zap.SugaredLogger, workflowStatusRepository: workflowStatusRepository, ciWorkflowRepository: ciWorkflowRepository, cdWorkflowRepository: cdWorkflowRepository, + envRepository: envRepository, transactionManager: transactionManager, } ciConfig, err := types.GetCiConfig() @@ -109,9 +115,32 @@ func (impl *WorkFlowStageStatusServiceImpl) updatePodStages(currentWorkflowStage //update pod stage status by using convertPodStatusToDevtronStatus for _, stage := range currentWorkflowStages { if stage.StatusFor == bean2.WORKFLOW_STAGE_STATUS_TYPE_POD { - // add pod name in stage metadata if not empty + // add pod name and clusterId in stage metadata if not empty if len(podName) > 0 { - marshalledMetadata, _ := json.Marshal(map[string]string{"podName": podName}) + metadata := map[string]string{"podName": podName} + + // Try to get clusterId from the workflow + if stage.WorkflowType == bean.CI_WORKFLOW_TYPE.String() { + // For CI workflows, get clusterId from environment + ciWorkflow, err := impl.ciWorkflowRepository.FindById(stage.WorkflowId) + if err == nil && ciWorkflow.EnvironmentId != 0 { + env, err := impl.envRepository.FindById(ciWorkflow.EnvironmentId) + if err == nil && env != nil && env.Cluster != nil { + metadata["clusterId"] = strconv.Itoa(env.Cluster.Id) + } + } + } else if stage.WorkflowType == bean.CD_WORKFLOW_TYPE_PRE.String() || stage.WorkflowType == bean.CD_WORKFLOW_TYPE_POST.String() || stage.WorkflowType == bean.CD_WORKFLOW_TYPE_DEPLOY.String() { + // For CD workflows, get clusterId from environment + cdWorkflowRunner, err := impl.cdWorkflowRepository.FindWorkflowRunnerById(stage.WorkflowId) + if err == nil && cdWorkflowRunner != nil && cdWorkflowRunner.CdWorkflow != nil && cdWorkflowRunner.CdWorkflow.Pipeline != nil { + env, err := impl.envRepository.FindById(cdWorkflowRunner.CdWorkflow.Pipeline.EnvironmentId) + if err == nil && env != nil && env.Cluster != nil { + metadata["clusterId"] = strconv.Itoa(env.Cluster.Id) + } + } + } + + marshalledMetadata, _ := json.Marshal(metadata) stage.Metadata = string(marshalledMetadata) } switch podStatus { diff --git a/specs/cluster_api_spec.yaml b/specs/cluster_api_spec.yaml index d1fa91e613..78cd252547 100644 --- a/specs/cluster_api_spec.yaml +++ b/specs/cluster_api_spec.yaml @@ -49,13 +49,24 @@ paths: required: true schema: type: integer + - name: clusterId + in: query + description: comma-separated list of cluster IDs to filter clusters. If not provided, returns all clusters. + required: false + schema: + type: string + example: "1,2,3" responses: '200': - description: Successfully get cluster + description: Successfully get cluster(s) content: application/json: schema: - $ref: '#/components/schemas/ClusterBean' + oneOf: + - $ref: '#/components/schemas/ClusterBean' + - type: array + items: + $ref: '#/components/schemas/ClusterBean' '400': description: Bad Request. Input Validation(decode) error/wrong request body. content: diff --git a/specs/swagger/openapi.yaml b/specs/swagger/openapi.yaml index da87266b2e..96bf7a84e7 100644 --- a/specs/swagger/openapi.yaml +++ b/specs/swagger/openapi.yaml @@ -1832,13 +1832,24 @@ paths: required: true schema: type: integer + - name: clusterId + in: query + description: comma-separated list of cluster IDs to filter clusters. If not provided, returns all clusters. + required: false + schema: + type: string + example: "1,2,3" responses: '200': - description: Successfully get cluster + description: Successfully get cluster(s) content: application/json: schema: - $ref: '#/components/schemas/ClusterBean' + oneOf: + - $ref: '#/components/schemas/ClusterBean' + - type: array + items: + $ref: '#/components/schemas/ClusterBean' '400': description: Bad Request. Input Validation(decode) error/wrong request body. content: diff --git a/specs/workflow/workflow-stage-status.internal.yaml b/specs/workflow/workflow-stage-status.internal.yaml index 549951a0f9..999e0af05a 100644 --- a/specs/workflow/workflow-stage-status.internal.yaml +++ b/specs/workflow/workflow-stage-status.internal.yaml @@ -13,7 +13,7 @@ paths: application/json: schema: $ref: '#/components/schemas/GetWorkflowStatusResponse' - example: '{"status":"In progress","startTime":"1","endTime":"","message":"e-message","podStatus":"Running","podName":"pod-name","workflowExecutionStages":{"workflow":[{"stageName":"Preparation","status":"SUCCESS","startTime":"1","endTime":"2","message":"p-message","metadata":{}},{"stageName":"Execution","status":"STARTED","startTime":"2","endTime":"","message":"e-message","metadata":{}}],"pod":[{"stageName":"Execution","status":"STARTED","startTime":"2","endTime":"","message":"e-message","metadata":{"ClusterID":"?? (possible?)","podName":"pod-name"}}]}}' + example: '{"status":"In progress","startTime":"1","endTime":"","message":"e-message","podStatus":"Running","podName":"pod-name","workflowExecutionStages":{"workflow":[{"stageName":"Preparation","status":"SUCCESS","startTime":"1","endTime":"2","message":"p-message","metadata":{}},{"stageName":"Execution","status":"STARTED","startTime":"2","endTime":"","message":"e-message","metadata":{}}],"pod":[{"stageName":"Execution","status":"STARTED","startTime":"2","endTime":"","message":"e-message","metadata":{"clusterId":"123","podName":"pod-name"}}]}}' components: schemas: @@ -89,7 +89,7 @@ components: metadata: type: object properties: - ClusterID: + clusterId: type: string description: Cluster ID podName: