Embedded OTel Collector Config

.codecov.yml

@@ -22,10 +22,9 @@ coverage:
 
     # Patch-level coverage settings
     patch:
-
       default:
-
-        target: 80%
+        informational: true
+        target: auto
         threshold: 0%
         only_pulls: false
 

.github/workflows/ci.yml

@@ -20,7 +20,7 @@ permissions:
 
 env:
   NFPM_VERSION: 'v2.35.3'
-  GOPROXY: "direct"
+  GOPROXY: "https://${{ secrets.ARTIFACTORY_USER }}:${{ secrets.ARTIFACTORY_TOKEN }}@azr.artifactory.f5net.com/artifactory/api/go/f5-nginx-go-dev"
 
 jobs:
   proxy-sanity-check:

internal/collector/nginxossreceiver/internal/config/config.go

@@ -24,6 +24,7 @@ const (
 type Config struct {
 	confighttp.ClientConfig        `mapstructure:",squash"`
 	APIDetails                     APIDetails                    `mapstructure:"api_details"`
+	InstanceID                     string                        `mapstructure:"instance_id"`
 	AccessLogs                     []AccessLog                   `mapstructure:"access_logs"`
 	MetricsBuilderConfig           metadata.MetricsBuilderConfig `mapstructure:",squash"`
 	scraperhelper.ControllerConfig `mapstructure:",squash"`

internal/collector/nginxossreceiver/internal/scraper/accesslog/nginx_log_scraper.go

@@ -171,7 +171,7 @@ func (nls *NginxLogScraper) Scrape(_ context.Context) (pmetric.Metrics, error) {
 	nls.entries = make([]*entry.Entry, 0)
 	timeNow := pcommon.NewTimestampFromTime(time.Now())
 
-	nls.rb.SetInstanceID(nls.settings.ID.Name())
+	nls.rb.SetInstanceID(nls.cfg.InstanceID)
 	nls.rb.SetInstanceType("nginx")
 	nls.logger.Debug("NGINX OSS access log resource info", zap.Any("resource", nls.rb))
 

internal/collector/nginxossreceiver/internal/scraper/stubstatus/stub_status_scraper.go

@@ -138,7 +138,7 @@ func (s *NginxStubStatusScraper) Scrape(context.Context) (pmetric.Metrics, error
 		return pmetric.Metrics{}, err
 	}
 
-	s.rb.SetInstanceID(s.settings.ID.Name())
+	s.rb.SetInstanceID(s.cfg.InstanceID)
 	s.rb.SetInstanceType("nginx")
 	s.settings.Logger.Debug("NGINX OSS stub status resource info", zap.Any("resource", s.rb))
 

internal/collector/nginxplusreceiver/config.go

@@ -21,6 +21,7 @@ const defaultCollectInterval = 10 * time.Second
 type Config struct {
 	confighttp.ClientConfig        `mapstructure:",squash"`
 	APIDetails                     APIDetails                    `mapstructure:"api_details"`
+	InstanceID                     string                        `mapstructure:"instance_id"`
 	MetricsBuilderConfig           metadata.MetricsBuilderConfig `mapstructure:",squash"`
 	scraperhelper.ControllerConfig `mapstructure:",squash"`
 }

internal/collector/nginxplusreceiver/scraper.go

@@ -127,7 +127,7 @@ func (nps *NginxPlusScraper) Scrape(ctx context.Context) (pmetric.Metrics, error
 		return pmetric.Metrics{}, fmt.Errorf("failed to get stats from plus API: %w", err)
 	}
 
-	nps.rb.SetInstanceID(nps.settings.ID.Name())
+	nps.rb.SetInstanceID(nps.cfg.InstanceID)
 	nps.rb.SetInstanceType("nginxplus")
 	nps.logger.Debug("NGINX Plus resource info", zap.Any("resource", nps.rb))
 

internal/collector/otel_collector_plugin.go

@@ -2,6 +2,7 @@
 //
 // This source code is licensed under the Apache License, Version 2.0 license found in the
 // LICENSE file in the root directory of this source tree.
+
 package collector
 
 import (
@@ -16,7 +17,9 @@ import (
 	"sync"
 	"time"
 
+	"github.com/goccy/go-yaml"
 	pkgConfig "github.com/nginx/agent/v3/pkg/config"
+	"go.opentelemetry.io/collector/confmap"
 
 	"github.com/nginx/agent/v3/api/grpc/mpi/v1"
 	"github.com/nginx/agent/v3/internal/backoff"
@@ -43,6 +46,7 @@ const (
 		`? (let utcTime = ` +
 		`date(timestamp).UTC(); utcTime.Format("Jan  2 15:04:05")) : date(timestamp).Format("Jan 02 15:04:05"); ` +
 		`split(body, ">")[0] + ">" + newTimestamp + " " + split(body, " ", 2)[1])'`
+	debugOTelConfigPath = "/var/lib/nginx-agent/opentelemetry-collector-agent-debug.yaml"
 )
 
 type (
@@ -53,6 +57,7 @@ type (
 		mu                      *sync.Mutex
 		cancel                  context.CancelFunc
 		previousNAPSysLogServer string
+		debugOTelConfigPath     string
 		stopped                 bool
 	}
 )
@@ -94,6 +99,7 @@ func NewCollector(conf *config.Config) (*Collector, error) {
 		stopped:                 true,
 		mu:                      &sync.Mutex{},
 		previousNAPSysLogServer: "",
+		debugOTelConfigPath:     debugOTelConfigPath,
 	}, nil
 }
 
@@ -384,6 +390,31 @@ func (oc *Collector) updateHeadersSetterExtension(
 	return headersSetterExtensionUpdated
 }
 
+func (oc *Collector) writeRunningConfig(ctx context.Context, settings otelcol.CollectorSettings) error {
+	slog.DebugContext(ctx, "Writing running OTel collector config", "path",
+		oc.debugOTelConfigPath)
+	resolver, err := confmap.NewResolver(settings.ConfigProviderSettings.ResolverSettings)
+	if err != nil {
+		return fmt.Errorf("unable to create resolver: %w", err)
+	}
+
+	con, err := resolver.Resolve(ctx)
+	if err != nil {
+		return fmt.Errorf("error while resolving config: %w", err)
+	}
+	b, err := yaml.Marshal(con.ToStringMap())
+	if err != nil {
+		return fmt.Errorf("error while marshaling to YAML: %w", err)
+	}
+
+	writeErr := os.WriteFile(oc.debugOTelConfigPath, b, filePermission)
+	if writeErr != nil {
+		return fmt.Errorf("error while writing debug config: %w", writeErr)
+	}
+
+	return nil
+}
+
 func (oc *Collector) restartCollector(ctx context.Context) {
 	err := oc.Close(ctx)
 	if err != nil {
@@ -392,6 +423,14 @@ func (oc *Collector) restartCollector(ctx context.Context) {
 	}
 
 	settings := OTelCollectorSettings(oc.config)
+
+	if strings.ToLower(oc.config.Log.Level) == "debug" {
+		writeErr := oc.writeRunningConfig(ctx, settings)
+		if writeErr != nil {
+			slog.ErrorContext(ctx, "Failed to write debug OTel Collector config", "error", writeErr)
+		}
+	}
+
 	oTelCollector, err := otelcol.NewCollector(settings)
 	if err != nil {
 		slog.ErrorContext(ctx, "Failed to create OTel Collector", "error", err)

internal/collector/otel_collector_plugin_test.go

@@ -18,6 +18,13 @@ import (
 	"github.com/nginx/agent/v3/test/stub"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
+	"go.opentelemetry.io/collector/confmap"
+	"go.opentelemetry.io/collector/confmap/provider/envprovider"
+	"go.opentelemetry.io/collector/confmap/provider/fileprovider"
+	"go.opentelemetry.io/collector/confmap/provider/httpprovider"
+	"go.opentelemetry.io/collector/confmap/provider/httpsprovider"
+	"go.opentelemetry.io/collector/confmap/provider/yamlprovider"
+
 	"go.opentelemetry.io/collector/otelcol"
 
 	"github.com/nginx/agent/v3/internal/bus"
@@ -994,6 +1001,73 @@ func TestCollector_findAvailableSyslogServers(t *testing.T) {
 	}
 }
 
+func TestCollector_writeRunningConfig(t *testing.T) {
+	tempDir := t.TempDir()
+
+	tests := []struct {
+		name           string
+		writeConfigErr error
+		settings       otelcol.CollectorSettings
+	}{
+		{
+			name: "Test 1: Write Config Success",
+			settings: otelcol.CollectorSettings{
+				ConfigProviderSettings: otelcol.ConfigProviderSettings{
+					ResolverSettings: confmap.ResolverSettings{
+						URIs: []string{"./testdata/otel_config.yaml", "./testdata/custom_otel_config.yaml"},
+						ProviderFactories: []confmap.ProviderFactory{
+							envprovider.NewFactory(),
+							fileprovider.NewFactory(),
+							httpprovider.NewFactory(),
+							httpsprovider.NewFactory(),
+							yamlprovider.NewFactory(),
+						},
+						DefaultScheme:      "",
+						ProviderSettings:   confmap.ProviderSettings{},
+						ConverterFactories: nil,
+						ConverterSettings:  confmap.ConverterSettings{},
+					},
+				},
+			},
+			writeConfigErr: nil,
+		},
+		{
+			name: "Test 2: Write Config Failed",
+			settings: otelcol.CollectorSettings{
+				ConfigProviderSettings: otelcol.ConfigProviderSettings{
+					ResolverSettings: confmap.ResolverSettings{},
+				},
+			},
+			writeConfigErr: errors.New("unable to create resolver: invalid " +
+				"'confmap.ResolverSettings' configuration: no URIs"),
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			conf := types.OTelConfig(t)
+			conf.Collector.Log.Path = filepath.Join(tempDir, "otel-collector-test.log")
+			newCollector, err := NewCollector(conf)
+			newCollector.debugOTelConfigPath = filepath.Join(tempDir, "otel-collector-debug-config.yaml")
+			require.NoError(t, err)
+
+			writeErr := newCollector.writeRunningConfig(context.Background(), tt.settings)
+
+			if tt.writeConfigErr == nil {
+				actual, readErr := os.ReadFile(newCollector.debugOTelConfigPath)
+				require.NoError(t, readErr)
+
+				expected, expectedFileErr := os.ReadFile("./testdata/merge_config.yaml")
+				require.NoError(t, expectedFileErr)
+
+				assert.Equal(t, string(expected), string(actual))
+			} else {
+				assert.Equal(t, tt.writeConfigErr.Error(), writeErr.Error())
+			}
+		})
+	}
+}
+
 func createFakeCollector() *typesfakes.FakeCollectorInterface {
 	fakeCollector := &typesfakes.FakeCollectorInterface{}
 	fakeCollector.RunStub = func(ctx context.Context) error { return nil }

internal/collector/otelcol.tmpl

@@ -79,7 +79,12 @@ receivers:
 {{- end }}
 
 {{- range .Receivers.NginxReceivers }}
+{{- if gt (len $.Receivers.NginxReceivers) 1 }}
   nginx/{{- .InstanceID -}}:
+{{- else }}
+  nginx:
+{{- end}}
+    instance_id: "{{- .InstanceID -}}"
     api_details:
       url: "{{- .StubStatus.URL -}}"
       listen: "{{- .StubStatus.Listen -}}"
@@ -98,12 +103,17 @@ receivers:
 {{- end }}
 
 {{- range .Receivers.NginxPlusReceivers }}
+{{- if gt (len $.Receivers.NginxPlusReceivers) 1 }}
   nginxplus/{{- .InstanceID -}}:
+{{- else }}
+  nginxplus:
+{{- end}}
+    instance_id: "{{- .InstanceID -}}"
     api_details:
-        url: "{{- .PlusAPI.URL -}}"
-        listen: "{{- .PlusAPI.Listen -}}"
-        location: "{{- .PlusAPI.Location -}}"
-        ca: "{{- .PlusAPI.Ca -}}"
+      url: "{{- .PlusAPI.URL -}}"
+      listen: "{{- .PlusAPI.Listen -}}"
+      location: "{{- .PlusAPI.Location -}}"
+      ca: "{{- .PlusAPI.Ca -}}"
     {{- if .CollectionInterval }}
     collection_interval: {{ .CollectionInterval }}
     {{- end }}
@@ -274,10 +284,18 @@ service:
             {{- end }}
           {{- else if eq $receiver "nginx_metrics" }}
             {{- range $.Receivers.NginxReceivers }}
+            {{- if gt (len $.Receivers.NginxReceivers) 1 }}
         - nginx/{{- .InstanceID -}}
+            {{- else }}
+        - nginx
+            {{- end }}
             {{- end }}
             {{- range $.Receivers.NginxPlusReceivers }}
+            {{- if gt (len $.Receivers.NginxReceivers) 1 }}
         - nginxplus/{{- .InstanceID -}}
+            {{- else }}
+        - nginxplus
+            {{- end }}
             {{- end }}
           {{- else }}
         - {{ $receiver }}

internal/collector/settings.go

@@ -71,7 +71,11 @@ func createConverterFactories() []confmap.ConverterFactory {
 }
 
 func createURIs(cfg *config.Config) []string {
-	return []string{cfg.Collector.ConfigPath}
+	configFiles := []string{cfg.Collector.ConfigPath}
+	configFiles = slices.Concat(configFiles, cfg.Collector.AdditionalConfigPaths)
+	slog.Info("Merging additional OTel config files", "config_files", configFiles)
+
+	return configFiles
 }
 
 func createFile(confPath string) error {

internal/collector/settings_test.go

@@ -7,6 +7,7 @@ package collector
 import (
 	"os"
 	"path/filepath"
+	"slices"
 	"testing"
 	"time"
 
@@ -107,6 +108,28 @@ func TestTemplateWrite(t *testing.T) {
 			},
 		},
 	})
+
+	cfg.Collector.Receivers.NginxPlusReceivers = slices.Concat(cfg.Collector.Receivers.NginxPlusReceivers,
+		[]config.NginxPlusReceiver{
+			{
+				InstanceID: "456",
+				PlusAPI: config.APIDetails{
+					URL:      "http://localhost:80/api",
+					Listen:   "",
+					Location: "",
+				},
+				CollectionInterval: 20 * time.Second,
+			},
+			{
+				InstanceID: "789",
+				PlusAPI: config.APIDetails{
+					URL:      "http://localhost:90/api",
+					Listen:   "",
+					Location: "",
+				},
+				CollectionInterval: 20 * time.Second,
+			},
+		})
 	// Clear default config and test collector with TLS enabled
 	cfg.Collector.Receivers.OtlpReceivers["default"] = &config.OtlpReceiver{
 		Server: &config.ServerConfig{
@@ -165,7 +188,10 @@ func TestTemplateWrite(t *testing.T) {
 
 	cfg.Collector.Pipelines.Metrics = make(map[string]*config.Pipeline)
 	cfg.Collector.Pipelines.Metrics["default"] = &config.Pipeline{
-		Receivers:  []string{"hostmetrics", "containermetrics", "otlp/default", "nginx/123"},
+		Receivers: []string{
+			"hostmetrics", "containermetrics",
+			"otlp/default", "nginx", "nginxplus/456", "nginxplus/789",
+		},
 		Processors: []string{"resource/default", "batch/default"},
 		Exporters:  []string{"otlp/default", "prometheus", "debug"},
 	}

internal/collector/testdata/custom_otel_config.yaml

@@ -0,0 +1,15 @@
+exporters:
+  prometheus:
+    endpoint: "127.0.0.1:5643"
+    resource_to_telemetry_conversion:
+      enabled: true
+
+service:
+  pipelines:
+    metrics/prometheus-example-pipeline:
+      receivers:
+        - nginxplus
+      processors:
+        - resource/default
+      exporters:
+        - prometheus