NGINX App Protect OTel syslog processor (#1262)

Author: devbgv

go.mod

@@ -16,6 +16,7 @@ require (
 	github.com/google/uuid v1.6.0
 	github.com/grpc-ecosystem/go-grpc-middleware v1.4.0
 	github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.3.2
+	github.com/leodido/go-syslog/v4 v4.2.0
 	github.com/mitchellh/mapstructure v1.5.1-0.20231216201459-8508981c8b6c
 	github.com/nginx/nginx-plus-go-client/v3 v3.0.1
 	github.com/nginxinc/nginx-prometheus-exporter v1.3.0
@@ -180,7 +181,6 @@ require (
 	github.com/knadh/koanf/providers/confmap v1.0.0 // indirect
 	github.com/knadh/koanf/v2 v2.3.0 // indirect
 	github.com/kylelemons/godebug v1.1.0 // indirect
-	github.com/leodido/go-syslog/v4 v4.2.0 // indirect
 	github.com/leodido/go-urn v1.4.0 // indirect
 	github.com/leodido/ragel-machinery v0.0.0-20190525184631-5f46317e436b // indirect
 	github.com/lightstep/go-expohisto v1.0.0 // indirect

internal/collector/factories.go

@@ -10,6 +10,7 @@ import (
 	"github.com/nginx/agent/v3/internal/collector/logsgzipprocessor"
 	nginxreceiver "github.com/nginx/agent/v3/internal/collector/nginxossreceiver"
 	"github.com/nginx/agent/v3/internal/collector/nginxplusreceiver"
+	"github.com/nginx/agent/v3/internal/collector/securityviolationsprocessor"
 
 	"github.com/open-telemetry/opentelemetry-collector-contrib/exporter/prometheusexporter"
 	"github.com/open-telemetry/opentelemetry-collector-contrib/extension/headerssetterextension"
@@ -104,6 +105,7 @@ func createProcessorFactories() map[component.Type]processor.Factory {
 		memorylimiterprocessor.NewFactory(),
 		redactionprocessor.NewFactory(),
 		resourceprocessor.NewFactory(),
+		securityviolationsprocessor.NewFactory(),
 		transformprocessor.NewFactory(),
 		logsgzipprocessor.NewFactory(),
 	}

internal/collector/factories_test.go

@@ -19,7 +19,7 @@ func TestOTelComponentFactoriesDefault(t *testing.T) {
 	assert.NotNil(t, factories, "factories should not be nil")
 
 	assert.Len(t, factories.Receivers, 6)
-	assert.Len(t, factories.Processors, 9)
+	assert.Len(t, factories.Processors, 10)
 	assert.Len(t, factories.Exporters, 4)
 	assert.Len(t, factories.Extensions, 3)
 	assert.Empty(t, factories.Connectors)

internal/collector/otel_collector_plugin_test.go

@@ -423,6 +423,7 @@ func TestCollector_ProcessResourceUpdateTopicFails(t *testing.T) {
 	conf.Collector.Processors.Attribute = nil
 	conf.Collector.Processors.Resource = nil
 	conf.Collector.Processors.LogsGzip = nil
+	conf.Collector.Processors.SecurityViolations = nil
 	conf.Collector.Exporters.OtlpExporters = nil
 	conf.Collector.Exporters.PrometheusExporter = &config.PrometheusExporter{
 		Server: &config.ServerConfig{
@@ -747,6 +748,7 @@ func TestCollector_updateNginxAppProtectTcplogReceivers(t *testing.T) {
 	conf.Collector.Processors.Attribute = nil
 	conf.Collector.Processors.Resource = nil
 	conf.Collector.Processors.LogsGzip = nil
+	conf.Collector.Processors.SecurityViolations = nil
 	collector, err := NewCollector(conf)
 	require.NoError(t, err)
 
@@ -929,6 +931,9 @@ func TestCollector_findAvailableSyslogServers(t *testing.T) {
 	conf.Collector.Processors.Attribute = nil
 	conf.Collector.Processors.Resource = nil
 	conf.Collector.Processors.LogsGzip = nil
+	conf.Collector.Processors.SecurityViolations = nil
+	collector, err := NewCollector(conf)
+	require.NoError(t, err)
 
 	tests := []struct {
 		name                    string
@@ -983,7 +988,6 @@ func TestCollector_findAvailableSyslogServers(t *testing.T) {
 
 	for _, test := range tests {
 		t.Run(test.name, func(tt *testing.T) {
-			collector, err := NewCollector(conf)
 			require.NoError(t, err)
 
 			collector.previousNAPSysLogServer = test.previousNAPSysLogServer

internal/collector/otelcol.tmpl

@@ -166,6 +166,11 @@ processors:
     send_batch_max_size: {{ $batch.SendBatchMaxSize }}
 {{- end }}
 {{- end }}
+{{- if ne .Processors.SecurityViolations nil }}
+{{- range $key, $value := .Processors.SecurityViolations }}
+  securityviolations/{{$key}}: {}
+{{- end }}
+{{- end }}
 {{- if ne .Processors.LogsGzip nil }}
 {{ range $key, $value := .Processors.LogsGzip }}
   logsgzip/{{$key}}: {}

internal/collector/securityviolationsprocessor/README.md

@@ -0,0 +1,5 @@
+# SecurityViolations Processor
+
+Internal component of the NGINX Agent that processes security violation syslog messages. Parses RFC3164 formatted syslog entries from log records and extracts structured attributes. Successfully parsed messages have their body replaced with the clean message content.
+
+Part of the NGINX Agent's log collection pipeline.

internal/collector/securityviolationsprocessor/factory.go

@@ -0,0 +1,41 @@
+// Copyright (c) F5, Inc.
+//
+// This source code is licensed under the Apache License, Version 2.0 license found in the
+// LICENSE file in the root directory of this source tree.
+
+package securityviolationsprocessor
+
+import (
+	"context"
+
+	"go.opentelemetry.io/collector/component"
+	"go.opentelemetry.io/collector/consumer"
+	"go.opentelemetry.io/collector/processor"
+)
+
+const typeStr = "securityviolations"
+
+// NewFactory creates a factory for the securityviolations processor.
+//
+//nolint:ireturn // factory methods return interfaces by design
+func NewFactory() processor.Factory {
+	return processor.NewFactory(
+		component.MustNewType(typeStr),
+		func() component.Config { return &struct{}{} },
+		processor.WithLogs(createSecurityViolationsProcessor, component.StabilityLevelAlpha),
+	)
+}
+
+// createSecurityViolationsProcessor instantiates the logs processor.
+//
+//nolint:ireturn // required to comply with component factory interface
+func createSecurityViolationsProcessor(
+	_ context.Context,
+	settings processor.Settings,
+	_ component.Config,
+	next consumer.Logs,
+) (processor.Logs, error) {
+	settings.Logger.Info("Creating security violations processor")
+
+	return newSecurityViolationsProcessor(next, settings), nil
+}

internal/collector/securityviolationsprocessor/model.go

@@ -0,0 +1,70 @@
+// Copyright (c) F5, Inc.
+//
+// This source code is licensed under the Apache License, Version 2.0 license found in the
+// LICENSE file in the root directory of this source tree.
+
+package securityviolationsprocessor
+
+// SecurityViolationEvent represents the structured NGINX App Protect security violation data
+type SecurityViolationEvent struct {
+	PolicyName               string          `json:"policy_name"`
+	SupportID                string          `json:"support_id"`
+	Outcome                  string          `json:"outcome"`
+	OutcomeReason            string          `json:"outcome_reason"`
+	BlockingExceptionReason  string          `json:"blocking_exception_reason"`
+	Method                   string          `json:"method"`
+	Protocol                 string          `json:"protocol"`
+	XForwardedForHeaderValue string          `json:"xff_header_value"`
+	URI                      string          `json:"uri"`
+	Request                  string          `json:"request"`
+	IsTruncated              string          `json:"is_truncated"`
+	RequestStatus            string          `json:"request_status"`
+	ResponseCode             string          `json:"response_code"`
+	ServerAddr               string          `json:"server_addr"`
+	VSName                   string          `json:"vs_name"`
+	RemoteAddr               string          `json:"remote_addr"`
+	RemotePort               string          `json:"destination_port"`
+	ServerPort               string          `json:"server_port"`
+	Violations               string          `json:"violations"`
+	SubViolations            string          `json:"sub_violations"`
+	ViolationRating          string          `json:"violation_rating"`
+	SigSetNames              string          `json:"sig_set_names"`
+	SigCVEs                  string          `json:"sig_cves"`
+	ClientClass              string          `json:"client_class"`
+	ClientApplication        string          `json:"client_application"`
+	ClientApplicationVersion string          `json:"client_application_version"`
+	Severity                 string          `json:"severity"`
+	ThreatCampaignNames      string          `json:"threat_campaign_names"`
+	BotAnomalies             string          `json:"bot_anomalies"`
+	BotCategory              string          `json:"bot_category"`
+	EnforcedBotAnomalies     string          `json:"enforced_bot_anomalies"`
+	BotSignatureName         string          `json:"bot_signature_name"`
+	SystemID                 string          `json:"system_id"`
+	InstanceTags             string          `json:"instance_tags"`
+	InstanceGroup            string          `json:"instance_group"`
+	ParentHostname           string          `json:"parent_hostname"`
+	DisplayName              string          `json:"display_name"`
+	ViolationsData           []ViolationData `json:"violations_data"`
+}
+
+type ViolationData struct {
+	Name        string          `json:"violation_data_name"`
+	Context     string          `json:"violation_data_context"`
+	ContextData ContextData     `json:"violation_data_context_data"`
+	Signatures  []SignatureData `json:"violation_data_signatures"`
+}
+
+// SignatureData represents signature data contained within each violation
+type SignatureData struct {
+	ID           string `json:"sig_data_id"`
+	BlockingMask string `json:"sig_data_blocking_mask"`
+	Buffer       string `json:"sig_data_buffer"`
+	Offset       string `json:"sig_data_offset"`
+	Length       string `json:"sig_data_length"`
+}
+
+// ContextData represents the context data of the violation
+type ContextData struct {
+	Name  string `json:"context_data_name"`
+	Value string `json:"context_data_value"`
+}