Add variant for Kibana >= 8.7.0 with experimental toggles enabled

.ci/Jenkinsfile

@@ -99,6 +99,7 @@ pipeline {
                     'stack-command-default': generateTestCommandStage(command: 'test-stack-command-default', artifacts: ['build/elastic-stack-dump/stack/*/logs/*.log', 'build/elastic-stack-dump/stack/*/logs/fleet-server-internal/*', 'build/elastic-stack-status/*/*']),
                     'stack-command-oldest': generateTestCommandStage(command: 'test-stack-command-oldest', artifacts: ['build/elastic-stack-dump/stack/*/logs/*.log', 'build/elastic-stack-dump/stack/*/logs/fleet-server-internal/*', 'build/elastic-stack-status/*/*']),
                     'stack-command-7x': generateTestCommandStage(command: 'test-stack-command-7x', artifacts: ['build/elastic-stack-dump/stack/*/logs/*.log', 'build/elastic-stack-dump/stack/*/logs/fleet-server-internal/*', 'build/elastic-stack-status/*/*']),
+                    'stack-command-86': generateTestCommandStage(command: 'test-stack-command-86', artifacts: ['build/elastic-stack-dump/stack/*/logs/*.log', 'build/elastic-stack-dump/stack/*/logs/fleet-server-internal/*', 'build/elastic-stack-status/*/*']),
                     'stack-command-8x': generateTestCommandStage(command: 'test-stack-command-8x', artifacts: ['build/elastic-stack-dump/stack/*/logs/*.log', 'build/elastic-stack-dump/stack/*/logs/fleet-server-internal/*', 'build/elastic-stack-status/*/*']),
                     'check-packages-with-kind': generateTestCommandStage(command: 'test-check-packages-with-kind', artifacts: ['build/test-results/*.xml', 'build/kubectl-dump.txt', 'build/elastic-stack-dump/check-*/logs/*.log', 'build/elastic-stack-dump/check-*/logs/fleet-server-internal/*'], junitArtifacts: true, publishCoverage: true),
                     'check-packages-other': generateTestCommandStage(command: 'test-check-packages-other', artifacts: ['build/test-results/*.xml', 'build/elastic-stack-dump/check-*/logs/*.log', 'build/elastic-stack-dump/check-*/logs/fleet-server-internal/*'], junitArtifacts: true, publishCoverage: true),

Makefile

@@ -58,10 +58,14 @@ test-stack-command-oldest:
 	./scripts/test-stack-command.sh 7.14.2
 
 test-stack-command-7x:
-	./scripts/test-stack-command.sh 7.17.3-SNAPSHOT
+	./scripts/test-stack-command.sh 7.17.8
+
+# Keeping a test for 8.6 because it has an specific configuration file.
+test-stack-command-86:
+	./scripts/test-stack-command.sh 8.6.1
 
 test-stack-command-8x:
-	./scripts/test-stack-command.sh 8.6.0-SNAPSHOT
+	./scripts/test-stack-command.sh 8.7.0-SNAPSHOT
 
 test-stack-command: test-stack-command-default test-stack-command-7x test-stack-command-800 test-stack-command-8x
 

internal/profile/_static/kibana_config_86.yml

@@ -0,0 +1,61 @@
+server.name: kibana
+server.host: "0.0.0.0"
+server.ssl.enabled: true
+server.ssl.certificate: "/usr/share/kibana/config/certs/cert.pem"
+server.ssl.key: "/usr/share/kibana/config/certs/key.pem"
+server.ssl.certificateAuthorities: ["/usr/share/kibana/config/certs/ca-cert.pem"]
+
+elasticsearch.hosts: [ "https://elasticsearch:9200" ]
+elasticsearch.ssl.certificateAuthorities: "/usr/share/kibana/config/certs/ca-cert.pem"
+elasticsearch.serviceAccountToken: "AAEAAWVsYXN0aWMva2liYW5hL2VsYXN0aWMtcGFja2FnZS1raWJhbmEtdG9rZW46b2x4b051SWNRa0tYMHdXazdLWmFBdw"
+
+monitoring.ui.container.elasticsearch.enabled: true
+
+xpack.fleet.registryUrl: "https://package-registry:8080"
+xpack.fleet.agents.enabled: true
+xpack.fleet.agents.elasticsearch.hosts: ["https://elasticsearch:9200"]
+xpack.fleet.agents.fleet_server.hosts: ["https://fleet-server:8220"]
+
+xpack.encryptedSavedObjects.encryptionKey: "12345678901234567890123456789012"
+
+xpack.cloudSecurityPosture.enabled: true
+
+xpack.fleet.packages:
+  - name: system
+    version: latest
+  - name: elastic_agent
+    version: latest
+  - name: fleet_server
+    version: latest
+xpack.fleet.agentPolicies:
+  - name: Elastic-Agent (elastic-package)
+    id: elastic-agent-managed-ep
+    is_default: true
+    is_managed: false
+    namespace: default
+    monitoring_enabled:
+      - logs
+      - metrics
+    package_policies:
+      - name: system-1
+        id: default-system
+        package:
+          name: system
+  - name: Fleet Server (elastic-package)
+    id: fleet-server-policy
+    is_default_fleet_server: true
+    is_managed: false
+    namespace: default
+    package_policies:
+      - name: fleet_server-1
+        id: default-fleet-server
+        package:
+          name: fleet_server
+xpack.fleet.outputs:
+  - id: fleet-default-output
+    name: default
+    type: elasticsearch
+    hosts: [ https://elasticsearch:9200 ]
+    ca_trusted_fingerprint: "${ELASTIC_PACKAGE_CA_TRUSTED_FINGERPRINT}"
+    is_default: true
+    is_default_monitoring: true

internal/profile/_static/kibana_config_8x.yml

@@ -15,6 +15,7 @@ xpack.fleet.registryUrl: "https://package-registry:8080"
 xpack.fleet.agents.enabled: true
 xpack.fleet.agents.elasticsearch.hosts: ["https://elasticsearch:9200"]
 xpack.fleet.agents.fleet_server.hosts: ["https://fleet-server:8220"]
+xpack.fleet.enableExperimental: ["experimentalDataStreamSettings"] # Enable experimental toggles in Fleet UI
 
 xpack.encryptedSavedObjects.encryptionKey: "12345678901234567890123456789012"
 

internal/profile/profile.go

@@ -40,12 +40,15 @@ type configFile string
 var managedProfileFiles = map[configFile]NewConfig{
 	ElasticAgentDefaultEnvFile:     newElasticAgentDefaultEnv,
 	ElasticAgent8xEnvFile:          newElasticAgent8xEnv,
+	ElasticAgent86EnvFile:          newElasticAgent86Env,
 	ElasticAgent80EnvFile:          newElasticAgent80Env,
 	ElasticsearchConfigDefaultFile: newElasticsearchConfigDefault,
 	ElasticsearchConfig8xFile:      newElasticsearchConfig8x,
+	ElasticsearchConfig86File:      newElasticsearchConfig86,
 	ElasticsearchConfig80File:      newElasticsearchConfig80,
 	KibanaConfigDefaultFile:        newKibanaConfigDefault,
 	KibanaConfig8xFile:             newKibanaConfig8x,
+	KibanaConfig86File:             newKibanaConfig86,
 	KibanaConfig80File:             newKibanaConfig80,
 	PackageRegistryDockerfileFile:  newPackageRegistryDockerfile,
 	PackageRegistryConfigFile:      newPackageRegistryConfig,

internal/profile/static.go

@@ -53,6 +53,20 @@ func newKibanaConfig8x(_ string, profilePath string) (*simpleFile, error) {
 	}, nil
 }
 
+// KibanaConfig86File is the Kibana config file for the 8.x stack family (8.2 to 8.6)
+const KibanaConfig86File configFile = "kibana.config.86.yml"
+
+//go:embed _static/kibana_config_86.yml
+var kibanaConfig86Yml string
+
+func newKibanaConfig86(_ string, profilePath string) (*simpleFile, error) {
+	return &simpleFile{
+		name: string(KibanaConfig86File),
+		path: filepath.Join(profilePath, profileStackPath, string(KibanaConfig86File)),
+		body: kibanaConfig86Yml,
+	}, nil
+}
+
 // KibanaConfig80File is the Kibana config file for 8.0 stack family (8.0 to 8.1)
 const KibanaConfig80File configFile = "kibana.config.80.yml"
 
@@ -95,6 +109,17 @@ func newElasticsearchConfig8x(_ string, profilePath string) (*simpleFile, error)
 	}, nil
 }
 
+// ElasticsearchConfig8xFile is the Elasticsearch config file for 8.x stack family (8.2 to 8.6)
+const ElasticsearchConfig86File configFile = "elasticsearch.config.86.yml"
+
+func newElasticsearchConfig86(_ string, profilePath string) (*simpleFile, error) {
+	return &simpleFile{
+		name: string(ElasticsearchConfig86File),
+		path: filepath.Join(profilePath, profileStackPath, string(ElasticsearchConfig86File)),
+		body: elasticsearchConfig8xYml,
+	}, nil
+}
+
 // ElasticsearchConfig80File is the Elasticsearch virtual config file name for 8.0 stack family (8.0 to 8.1)
 // This file does not exist in the source code, since it's identical to the 8x config file.
 const ElasticsearchConfig80File configFile = "elasticsearch.config.80.yml"
@@ -154,6 +179,18 @@ func newElasticAgent80Env(_ string, profilePath string) (*simpleFile, error) {
 	}, nil
 }
 
+// ElasticAgent86EnvFile is the .env for the 8.6 stack.
+// This file does not exist in the source code, since it's identical to the 8x env file.
+const ElasticAgent86EnvFile configFile = "elastic-agent.86.env"
+
+func newElasticAgent86Env(_ string, profilePath string) (*simpleFile, error) {
+	return &simpleFile{
+		name: string(ElasticAgent86EnvFile),
+		path: filepath.Join(profilePath, profileStackPath, string(ElasticAgent86EnvFile)),
+		body: elasticAgent8xEnv,
+	}, nil
+}
+
 // ElasticAgent8xEnvFile is the .env for the 8x stack.
 const ElasticAgent8xEnvFile configFile = "elastic-agent.8x.env"
 

internal/stack/variants.go

@@ -14,13 +14,15 @@ import (
 // This map is used to deploy different versions of the Elastic stack with matching configurations.
 var configurationVariantMap = map[string]string{
 	"8.0-0 - 8.1.x-x": "80",
-	"^8.2-0":          "8x",
+	"8.2-0 - 8.6.x-x": "86",
+	"^8.7-0":          "8x",
 }
 
 // stackVariantAsEnv function returns a stack variant based on the given stack version.
 // We identified three variants:
 // * default, covers all of 7.x branches
 // * 80, covers stack versions 8.0.0 to 8.1.x
+// * 86, covers stack versions 8.2.0 to 8.6.x, they don't support experimental toggles in Fleet
 // * 8x, supports different configuration options in Kibana, covers stack versions 8.2.0+
 func stackVariantAsEnv(version string) string {
 	return fmt.Sprintf("STACK_VERSION_VARIANT=%s", selectStackVersion(version))

internal/stack/variants_test.go

@@ -32,14 +32,18 @@ var tests = []struct {
 	{"8.1.58", "80"},
 	{"8.1.99-beta", "80"},
 	{"8.1.999-SNAPSHOT", "80"},
-	{"8.2-0", "8x"},
-	{"8.2", "8x"},
-	{"8.2.0-alpha", "8x"},
-	{"8.2.0", "8x"},
-	{"8.2.58", "8x"},
-	{"8.2.99-gamma", "8x"},
-	{"8.2.777-SNAPSHOT+arm64", "8x"},
-	{"8.5", "8x"},
+	{"8.2-0", "86"},
+	{"8.2", "86"},
+	{"8.2.0-alpha", "86"},
+	{"8.2.0", "86"},
+	{"8.2.58", "86"},
+	{"8.2.99-gamma", "86"},
+	{"8.2.777-SNAPSHOT+arm64", "86"},
+	{"8.5", "86"},
+	{"8.6.1", "86"},
+	{"8.7.0", "8x"},
+	{"8.7.0-SNAPSHOT", "8x"},
+	{"8.7.1-SNAPSHOT", "8x"},
 	{"9", "default"},
 }