feat: Adds support for -server mode and concurrently downloading dashboards (#21)

* adds error logger to logger package

* sets up flag to run detection on a periodic interval

* Working now. Added flag for -server mode. Added a web server with an /output route which returns a JSON payload of dashboard objects that are still using angular. Updates the in memory representation of that list every (default) 10m interval. This can be hooked up directly to grafana via the infinity data source plugin

* moves flag handling to flags.go package. cleans up http handler

* comment

* setup flag for server port

* add comment

* add readiness probe since full detections can take 5+ minutes. refactors for loop on channel to be after http server is setup

* update readiness check to use sync.Once. Creates struct with mutex for output data

* add support for concurrently download dashboards to speed up detection runs

* update test

* refactor

* Fix CLI mode not working

* refactor

* more refactoring

* moved run channel inside loop goroutine

* removed context with cancel

* incorporate PR feedback

* do not set feault server args

* add log when running dector

* add a log statement when interval ticker is setup for periodic runs

* sets up runServer method with graceful termintion

* Update flags/flags.go

Co-authored-by: Giuseppe Guerra <giuseppe@guerra.in>

* Update flags/flags.go

Co-authored-by: Giuseppe Guerra <giuseppe@guerra.in>

* Update main.go

Co-authored-by: Giuseppe Guerra <giuseppe@guerra.in>

* rename handleOutputRequest to handleDetectionsRequest

* Update flags/flags.go

Co-authored-by: Giuseppe Guerra <giuseppe@guerra.in>

---------

Co-authored-by: Giuseppe Guerra <giuseppe@guerra.in>

Author: vanugrah

.gitignore

@@ -1,3 +1,4 @@
 .idea
 detect-angular-dashboards
 dist
+.vscode

detector/detector.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"net/url"
 	"strings"
+	"sync"
 
 	"github.com/grafana/detect-angular-dashboards/api/gcom"
 	"github.com/grafana/detect-angular-dashboards/api/grafana"
@@ -38,14 +39,17 @@ type Detector struct {
 
 	angularDetected     map[string]bool
 	datasourcePluginIDs map[string]string
+	maxConcurrency      int
 }
 
 // NewDetector returns a new Detector.
-func NewDetector(log *logger.LeveledLogger, grafanaClient GrafanaDetectorAPIClient, gcomClient gcom.APIClient) *Detector {
+func NewDetector(log *logger.LeveledLogger, grafanaClient GrafanaDetectorAPIClient, gcomClient gcom.APIClient, maxConcurrency int) *Detector {
 	return &Detector{
-		log:           log,
-		grafanaClient: grafanaClient,
-		gcomClient:    gcomClient,
+		log:             log,
+		grafanaClient:   grafanaClient,
+		gcomClient:      gcomClient,
+		angularDetected: map[string]bool{},
+		maxConcurrency:  maxConcurrency,
 	}
 }
 
@@ -60,7 +64,6 @@ func (d *Detector) Run(ctx context.Context) ([]output.Dashboard, error) {
 	// Determine if plugins are angular.
 	// This can be done from frontendsettings (faster and works with private plugins, but only works with >= 10.1.0)
 	// or from GCOM (slower, but always available, but public plugins only)
-	d.angularDetected = map[string]bool{}
 	frontendSettings, err := d.grafanaClient.GetFrontendSettings(ctx)
 	if err != nil {
 		return []output.Dashboard{}, fmt.Errorf("get frontend settings: %w", err)
@@ -154,33 +157,59 @@ func (d *Detector) Run(ctx context.Context) ([]output.Dashboard, error) {
 		return []output.Dashboard{}, fmt.Errorf("get dashboards: %w", err)
 	}
 
+	// Create a semaphore to limit concurrency
+	semaphore := make(chan struct{}, d.maxConcurrency)
+	var wg sync.WaitGroup
+	var mu sync.Mutex
+	var downloadErrors []error
+
 	for _, dash := range dashboards {
-		// Determine absolute dashboard URL for output
-		dashboardAbsURL, err := url.JoinPath(strings.TrimSuffix(d.grafanaClient.BaseURL(), "/api"), dash.URL)
-		if err != nil {
-			// Silently ignore errors
-			dashboardAbsURL = ""
-		}
-		dashboardDefinition, err := d.grafanaClient.GetDashboard(ctx, dash.UID)
-		if err != nil {
-			return []output.Dashboard{}, fmt.Errorf("get dashboard %q: %w", dash.UID, err)
-		}
-		dashboardOutput := output.Dashboard{
-			Detections: []output.Detection{},
-			URL:        dashboardAbsURL,
-			Title:      dash.Title,
-			Folder:     dashboardDefinition.Meta.FolderTitle,
-			CreatedBy:  dashboardDefinition.Meta.CreatedBy,
-			UpdatedBy:  dashboardDefinition.Meta.UpdatedBy,
-			Created:    dashboardDefinition.Meta.Created,
-			Updated:    dashboardDefinition.Meta.Updated,
-		}
-		dashboardOutput.Detections, err = d.checkPanels(dashboardDefinition, dashboardDefinition.Dashboard.Panels)
-		if err != nil {
-			return []output.Dashboard{}, fmt.Errorf("check panels: %w", err)
-		}
-		finalOutput = append(finalOutput, dashboardOutput)
+		wg.Add(1)
+		go func(dash grafana.ListedDashboard) {
+			defer wg.Done()
+			semaphore <- struct{}{}        // Acquire semaphore
+			defer func() { <-semaphore }() // Release semaphore
+
+			dashboardAbsURL, err := url.JoinPath(strings.TrimSuffix(d.grafanaClient.BaseURL(), "/api"), dash.URL)
+			if err != nil {
+				dashboardAbsURL = ""
+			}
+			dashboardDefinition, err := d.grafanaClient.GetDashboard(ctx, dash.UID)
+			if err != nil {
+				mu.Lock()
+				downloadErrors = append(downloadErrors, fmt.Errorf("get dashboard %q: %w", dash.UID, err))
+				mu.Unlock()
+				return
+			}
+			dashboardOutput := output.Dashboard{
+				Detections: []output.Detection{},
+				URL:        dashboardAbsURL,
+				Title:      dash.Title,
+				Folder:     dashboardDefinition.Meta.FolderTitle,
+				CreatedBy:  dashboardDefinition.Meta.CreatedBy,
+				UpdatedBy:  dashboardDefinition.Meta.UpdatedBy,
+				Created:    dashboardDefinition.Meta.Created,
+				Updated:    dashboardDefinition.Meta.Updated,
+			}
+			dashboardOutput.Detections, err = d.checkPanels(dashboardDefinition, dashboardDefinition.Dashboard.Panels)
+			if err != nil {
+				mu.Lock()
+				downloadErrors = append(downloadErrors, fmt.Errorf("check panels: %w", err))
+				mu.Unlock()
+				return
+			}
+			mu.Lock()
+			finalOutput = append(finalOutput, dashboardOutput)
+			mu.Unlock()
+		}(dash)
 	}
+
+	wg.Wait()
+
+	if len(downloadErrors) > 0 {
+		return finalOutput, fmt.Errorf("errors occurred during dashboard download: %v", downloadErrors)
+	}
+
 	return finalOutput, nil
 }
 

detector/detector_test.go

@@ -19,7 +19,7 @@ import (
 func TestDetector(t *testing.T) {
 	t.Run("meta", func(t *testing.T) {
 		cl := NewTestAPIClient(filepath.Join("testdata", "dashboards", "graph-old.json"))
-		d := NewDetector(logger.NewLeveledLogger(false), cl, gcom.NewAPIClient())
+		d := NewDetector(logger.NewLeveledLogger(false), cl, gcom.NewAPIClient(), 5)
 		out, err := d.Run(context.Background())
 		require.NoError(t, err)
 		require.Len(t, out, 1)
@@ -31,7 +31,7 @@ func TestDetector(t *testing.T) {
 		require.Equal(t, "2023-11-07T11:13:24+01:00", out[0].Created)
 		require.Equal(t, "2024-02-21T13:09:27+01:00", out[0].Updated)
 	})
-	
+
 	type expDetection struct {
 		pluginID      string
 		detectionType output.DetectionType
@@ -112,7 +112,7 @@ func TestDetector(t *testing.T) {
 	} {
 		t.Run(tc.name, func(t *testing.T) {
 			cl := NewTestAPIClient(filepath.Join("testdata", "dashboards", tc.file))
-			d := NewDetector(logger.NewLeveledLogger(false), cl, gcom.NewAPIClient())
+			d := NewDetector(logger.NewLeveledLogger(false), cl, gcom.NewAPIClient(), 5)
 			out, err := d.Run(context.Background())
 			require.NoError(t, err)
 			require.Len(t, out, 1, "should have result for one dashboard")

flags/flags.go

@@ -0,0 +1,32 @@
+package flags
+
+import (
+	"flag"
+	"time"
+)
+
+// Flags holds the command-line flags.
+type Flags struct {
+	Version        bool
+	Verbose        bool
+	JSONOutput     bool
+	SkipTLS        bool
+	Server         string
+	Interval       time.Duration
+	MaxConcurrency int
+}
+
+// Parse parses the command-line flags.
+func Parse() Flags {
+	var flags Flags
+	flag.BoolVar(&flags.Version, "version", false, "print version number")
+	flag.BoolVar(&flags.Verbose, "v", false, "verbose output")
+	flag.BoolVar(&flags.JSONOutput, "j", false, "json output")
+	flag.BoolVar(&flags.SkipTLS, "insecure", false, "skip TLS verification")
+	flag.DurationVar(&flags.Interval, "interval", 5*time.Minute, "detection refresh interval when running in HTTP server mode")
+	flag.StringVar(&flags.Server, "server", "", "Run as HTTP server instead of CLI. Value must be a listen address (e.g.: 0.0.0.0:5000. Output is exposed as JSON at /detections.")
+	flag.IntVar(&flags.MaxConcurrency, "max-concurrency", 10, "maximum number of concurrent dashboard downloads")
+	flag.Parse()
+
+	return flags
+}

logger/logger.go

@@ -8,12 +8,16 @@ import (
 type Logger interface {
 	Log(format string, v ...any)
 	Warn(format string, v ...any)
+	Error(format string, v ...any)
+	Errorf(format string, v ...any)
 }
 
 type nopLogger struct{}
 
-func (nopLogger) Log(string, ...any)  {}
-func (nopLogger) Warn(string, ...any) {}
+func (nopLogger) Log(string, ...any)    {}
+func (nopLogger) Warn(string, ...any)   {}
+func (nopLogger) Error(string, ...any)  {}
+func (nopLogger) Errorf(string, ...any) {}
 
 // NewNopLogger returns a new logger whose methods are no-ops and don't log anything.
 func NewNopLogger() Logger {
@@ -23,26 +27,36 @@ func NewNopLogger() Logger {
 type LeveledLogger struct {
 	isVerbose bool
 
-	Logger     *log.Logger
-	WarnLogger *log.Logger
+	Logger      *log.Logger
+	WarnLogger  *log.Logger
+	ErrorLogger *log.Logger
 }
 
 func NewLeveledLogger(verbose bool) *LeveledLogger {
 	return &LeveledLogger{
-		isVerbose:  verbose,
-		Logger:     log.New(os.Stdout, "", log.LstdFlags),
-		WarnLogger: log.New(os.Stderr, "", log.LstdFlags),
+		isVerbose:   verbose,
+		Logger:      log.New(os.Stdout, "INFO: ", log.LstdFlags),
+		WarnLogger:  log.New(os.Stderr, "WARN: ", log.LstdFlags),
+		ErrorLogger: log.New(os.Stderr, "ERROR: ", log.LstdFlags),
 	}
 }
 
 func (l *LeveledLogger) Log(format string, v ...any) {
-	log.Printf(format, v...)
+	l.Logger.Printf(format, v...)
 }
 
 func (l *LeveledLogger) Warn(format string, v ...any) {
 	l.WarnLogger.Printf(format, v...)
 }
 
+func (l *LeveledLogger) Error(format string, v ...any) {
+	l.ErrorLogger.Printf(format, v...)
+}
+
+func (l *LeveledLogger) Errorf(format string, v ...any) {
+	l.ErrorLogger.Printf(format, v...)
+}
+
 func (l *LeveledLogger) Verbose() Logger {
 	if !l.isVerbose {
 		return NewNopLogger()