Feat: Generate a manifest file for ihealth ingestion

* Job.Collect now returns filenames in addition to error and skipped status
* Some files from the nim-dqlite-job were written directly in the job instead of being returned as part of the JobResult, this was corrected in a previous commit
* Added GenerateManifest and associated structures to generate a manifest.json file for iHealth ingestion

Author: mrajagopal

cmd/nginx-supportpkg.go

@@ -23,7 +23,6 @@ import (
 	"os"
 	"path/filepath"
 	"slices"
-	"strings"
 	"time"
 
 	"github.com/nginxinc/nginx-k8s-supportpkg/pkg/data_collector"
@@ -70,49 +69,39 @@ func Execute() {
 			if collector.AllNamespacesExist() {
 				failedJobs := 0
 				totalJobs := len(jobList)
-				var jobTimings []data_collector.JobTiming
+				var jobTimings []data_collector.JobInfo
 
 				for _, job := range jobList {
 					fmt.Printf("Running job %s...", job.Name)
 
-					// Record job start time
+					// Record job start and end time to calculate duration
 					jobStartTime := time.Now()
-					jobResult := job.Collect(&collector)
-
-					// Record job end time and calculate duration
+					err, skipped, files := job.Collect(&collector)
 					jobEndTime := time.Now()
 					duration := jobEndTime.Sub(jobStartTime)
 
-					// Create job timing record
-					files := make([]string, 0, len(jobResult.Files))
-					for filename := range jobResult.Files {
-						if len(filename) > 0 {
-							packagePath := strings.TrimPrefix(filename, collector.BaseDir)
-							files = append(files, packagePath)
-						}
-					}
-
-					jobTiming := data_collector.JobTiming{
+					// Create job info record
+					jobInfo := data_collector.JobInfo{
 						Name:      job.Name,
-						StartTime: jobStartTime.UTC().Format(time.RFC3339),
-						EndTime:   jobEndTime.UTC().Format(time.RFC3339),
+						StartTime: jobStartTime.UTC().Format(time.RFC3339Nano),
+						EndTime:   jobEndTime.UTC().Format(time.RFC3339Nano),
 						Duration:  duration.String(),
 						Files:     files,
 					}
 
-					if jobResult.Skipped {
+					if skipped {
 						fmt.Print(" SKIPPED\n")
-					} else if jobResult.Error != nil {
-						fmt.Printf(" FAILED: %s\n", jobResult.Error)
+					} else if err != nil {
+						fmt.Printf(" FAILED: %s\n", err)
 						failedJobs++
 					} else {
 						fmt.Print(" COMPLETED\n")
 					}
 
-					jobTimings = append(jobTimings, jobTiming)
+					jobTimings = append(jobTimings, jobInfo)
 				}
 
-				// Generate manifest with job timings - UPDATE THIS LINE
+				// Generate manifest with job timings
 				manifestData, err := collector.GenerateManifest(product, startTime, totalJobs, failedJobs, jobTimings)
 				if err != nil {
 					fmt.Printf("Warning: Failed to generate manifest: %v\n", err)

pkg/data_collector/data_collector.go

@@ -110,7 +110,7 @@ type SubPackage struct {
 	ID             string        `json:"id,omitempty"`
 }
 
-type JobTiming struct {
+type JobInfo struct {
 	Name      string   `json:"name"`
 	StartTime string   `json:"start_time"`
 	EndTime   string   `json:"end_time"`
@@ -168,7 +168,7 @@ func (c *DataCollector) WrapUp(product string) (string, error) {
 	unixTime := time.Now().Unix()
 	unixTimeString := strconv.FormatInt(unixTime, 10)
 	tarballName := fmt.Sprintf("%s-supportpkg-%s.tar.gz", product, unixTimeString)
-	tarballRootDirName := fmt.Sprintf("%s-supportpkg-%s", product, unixTimeString)
+	tarballRootDirName := "."
 
 	err := c.LogFile.Close()
 	if err != nil {
@@ -327,12 +327,12 @@ func (c *DataCollector) AllNamespacesExist() bool {
 	return allExist
 }
 
-func (c *DataCollector) GenerateManifest(product string, startTime time.Time, jobsRun, jobsFailed int, jobTimings []JobTiming) ([]byte, error) {
+func (c *DataCollector) GenerateManifest(product string, startTime time.Time, jobsRun, jobsFailed int, jobTimings []JobInfo) ([]byte, error) {
 	manifest := Manifest{
 		Version: "1.2", // Match the schema version
 		Timestamp: TimestampInfo{
-			Start: startTime.UTC().Format(time.RFC3339),
-			Stop:  time.Now().UTC().Format(time.RFC3339),
+			Start: startTime.UTC().Format(time.RFC3339Nano),
+			Stop:  time.Now().UTC().Format(time.RFC3339Nano),
 		},
 		PackageType: "root", // As defined in schema enum
 		RootDir:     ".",

pkg/jobs/job.go

@@ -23,6 +23,7 @@ import (
 	"fmt"
 	"os"
 	"path/filepath"
+	"strings"
 	"time"
 
 	"github.com/nginxinc/nginx-k8s-supportpkg/pkg/data_collector"
@@ -40,7 +41,7 @@ type JobResult struct {
 	Skipped bool
 }
 
-func (j Job) Collect(dc *data_collector.DataCollector) JobResult {
+func (j Job) Collect(dc *data_collector.DataCollector) (error, bool, []string) {
 	ch := make(chan JobResult, 1)
 
 	ctx, cancel := context.WithTimeout(context.Background(), j.Timeout)
@@ -52,32 +53,44 @@ func (j Job) Collect(dc *data_collector.DataCollector) JobResult {
 	select {
 	case <-ctx.Done():
 		dc.Logger.Printf("\tJob %s has timed out: %s\n---\n", j.Name, ctx.Err())
-		return JobResult{Error: fmt.Errorf("Context cancelled: %v", ctx.Err()), Skipped: false}
+		return fmt.Errorf("Context cancelled: %v", ctx.Err()), false, nil
 
 	case jobResults := <-ch:
+		files := j.GetFilesFromJobResult(dc, jobResults)
 		if jobResults.Skipped {
 			dc.Logger.Printf("\tJob %s has been skipped\n---\n", j.Name)
-			return JobResult{Error: nil, Skipped: true}
+			return nil, true, files
 		}
 		if jobResults.Error != nil {
 			dc.Logger.Printf("\tJob %s has failed: %s\n", j.Name, jobResults.Error)
-			return JobResult{Error: jobResults.Error, Skipped: false}
+			return jobResults.Error, false, files
 		}
 
 		for fileName, fileValue := range jobResults.Files {
 			err := os.MkdirAll(filepath.Dir(fileName), os.ModePerm)
 			if err != nil {
-				return JobResult{Error: fmt.Errorf("MkdirAll failed: %v", err), Skipped: jobResults.Skipped}
+				return fmt.Errorf("MkdirAll failed: %v", err), jobResults.Skipped, files
 			}
 			file, _ := os.Create(fileName)
 			_, err = file.Write(fileValue)
 			if err != nil {
-				return JobResult{Error: fmt.Errorf("Write failed: %v", err), Skipped: jobResults.Skipped}
+				return fmt.Errorf("Write failed: %v", err), jobResults.Skipped, files
 			}
 			_ = file.Close()
 			dc.Logger.Printf("\tJob %s wrote %d bytes to %s\n", j.Name, len(fileValue), fileName)
 		}
 		dc.Logger.Printf("\tJob %s completed successfully\n---\n", j.Name)
-		return JobResult{Files: jobResults.Files, Error: nil, Skipped: false}
+		return nil, false, files
 	}
 }
+
+func (j Job) GetFilesFromJobResult(dc *data_collector.DataCollector, jobResult JobResult) []string {
+	files := make([]string, 0, len(jobResult.Files))
+	for filename := range jobResult.Files {
+		if len(filename) > 0 {
+			packagePath := strings.TrimPrefix(filename, dc.BaseDir)
+			files = append(files, packagePath)
+		}
+	}
+	return files
+}