[8.19] (backport #18423) Rewrite Env2yaml in java instead of Go (#18456)

* Rewrite Env2yaml in java instead of Go (#18423)

* WIP: Rewrite Env2yaml in java instead of Go

Managing a Go toolchain for persisting ENV vars in logstash container artifacts
has become cumbersome. We already manage a java runtime so this commit presents
a path forward to use that instead of Go. The Go binary is faster than java (in
my testing Go would complete in around less than 200ms while java takes over
300ms). Given the container startup time is on the order of magnitute of seconds
this change should be inperceptable to consumers. The benefit from consolidating
in Java is worth the slightly lower performance.

* Use TreeMap in java to try to replicate lexicographical order

* Explicit imports and TreeMap everywher

* Go removals and ironbank workflow update

* More non-code removals

* Update based on codereview

Use snakeyaml-engine and some java flags for faster execution

* Build env2yaml in stage

Build env2yaml in a separate build stage for container artifacts. Include
its dependencies and manage separately from logstash. Continue to use the
java runtime in the final container to run the program, but manage the classpath
separately. Note this did not use gradle for dependency management because installing
that as a depdendcy was not worth it compared with downloading a jar directly.

* Use gradle to manage snakeyaml-engine dependency

Use gradle (and a dedicated gradle base image) for building env2yaml

* Refactor to build env2yaml with gradle rather than in docker build

Dont rely on compiling at docker build time, rather do it when logstash
compilation is done.

* Dont try to use snakeyaml from jruby

The complexity around trying to copy over the jar shipped with jruby is not worth
how easy it is to just manage it with gradle. This helps with keeping env2yaml contained.

* Add license for snakeyaml-engine

Licence from https://bitbucket.org/snakeyaml/snakeyaml-engine/src/master/LICENSE.txt

* Cleanup and bugfix

* Stop skipping empty env vars

I mistakenly thought I had observed this behavior in the go version.

* Remove quotes from interpolated values

Even though we set `.setDefaultScalarStyle(ScalarStyle.PLAIN)` snakeyaml-engine
ends up quoting `${}` values. This commit removes them as this was not the behavior
with the go version.

(cherry picked from commit b15c6c5)

# Conflicts:
#	docker/Makefile
#	docker/data/logstash/env2yaml/env2yaml.go
#	docker/templates/Dockerfile.erb
#	docker/templates/IronbankDockerfile.erb
#	rakelib/artifacts.rake
#	settings.gradle

* fix merge conflicts

* Actually delete unused go source for env2yaml

---------

Co-authored-by: Cas Donoghue <cas.donoghue@gmail.com>

Author: mergify[bot]

.gitignore

@@ -16,7 +16,6 @@ out
 local
 test/setup/elasticsearch/elasticsearch-*
 vendor
-!docker/ironbank/go/src/env2yaml/vendor
 .sass-cache
 /data
 .buildpath

build.gradle

@@ -299,6 +299,13 @@ tasks.register("bootstrap") {
 }
 
 
+
+task dockerBootstrap {
+    description = "Docker bootstrap ensures env2yaml java is compiled and staged for inclusion in tarballs"
+    dependsOn bootstrap
+    dependsOn ':docker:data:logstash:env2yaml:compileJava'
+}
+
 tasks.register("installDefaultGems") {
     dependsOn bootstrap
     doLast {

docker/Makefile

@@ -22,11 +22,11 @@ DEFAULT_IMAGE_FLAVOR ?= full
 IMAGE_TAG := $(ELASTIC_REGISTRY)/logstash/logstash
 HTTPD ?= logstash-docker-artifact-server
 
-all: build-from-local-artifacts build-from-local-oss-artifacts build-from-local-observability-sre-artifacts public-dockerfiles
+all: build-from-local-full-artifacts build-from-local-oss-artifacts build-from-local-observability-sre-artifacts public-dockerfiles
 
 # Build from artifacts on the local filesystem, using an http server (running
 # in a container) to provide the artifacts to the Dockerfile.
-build-from-local-full-artifacts: dockerfile env2yaml
+build-from-local-full-artifacts: dockerfile
 	docker run --rm -d --name=$(HTTPD) \
 	           -p 8000:8000 --expose=8000 -v $(ARTIFACTS_DIR):/mnt \
 	           python:3 bash -c 'cd /mnt && python3 -m http.server'
@@ -36,7 +36,7 @@ build-from-local-full-artifacts: dockerfile env2yaml
 	docker tag $(IMAGE_TAG)-full:$(VERSION_TAG) $(IMAGE_TAG):$(VERSION_TAG);
 	docker kill $(HTTPD)
 
-build-from-local-oss-artifacts: dockerfile env2yaml
+build-from-local-oss-artifacts: dockerfile
 	docker run --rm -d --name=$(HTTPD) \
 	           -p 8000:8000 --expose=8000 -v $(ARTIFACTS_DIR):/mnt \
 	           python:3 bash -c 'cd /mnt && python3 -m http.server'
@@ -45,7 +45,7 @@ build-from-local-oss-artifacts: dockerfile env2yaml
 	  (docker kill $(HTTPD); false);
 	-docker kill $(HTTPD)
 
-build-from-local-ubi8-artifacts: dockerfile env2yaml
+build-from-local-ubi8-artifacts: dockerfile
 	docker run --rm -d --name=$(HTTPD) \
 	           -p 8000:8000 --expose=8000 -v $(ARTIFACTS_DIR):/mnt \
 	           python:3 bash -c 'cd /mnt && python3 -m http.server'
@@ -54,7 +54,7 @@ build-from-local-ubi8-artifacts: dockerfile env2yaml
 	  (docker kill $(HTTPD); false);
 	-docker kill $(HTTPD)
 
-build-from-local-observability-sre-artifacts: dockerfile env2yaml
+build-from-local-observability-sre-artifacts: dockerfile
 	docker run --rm -d --name=$(HTTPD) \
 	           -p 8000:8000 --expose=8000 -v $(ARTIFACTS_DIR):/mnt \
 	           python:3 bash -c 'cd /mnt && python3 -m http.server'
@@ -74,9 +74,8 @@ build-from-local-wolfi-artifacts: dockerfile
 
 COPY_FILES := $(ARTIFACTS_DIR)/docker/config/pipelines.yml $(ARTIFACTS_DIR)/docker/config/logstash-oss.yml $(ARTIFACTS_DIR)/docker/config/logstash-full.yml
 COPY_FILES += $(ARTIFACTS_DIR)/docker/config/log4j2.file.properties $(ARTIFACTS_DIR)/docker/config/log4j2.properties
+COPY_FILES += $(ARTIFACTS_DIR)/docker/env2yaml/env2yaml
 COPY_FILES += $(ARTIFACTS_DIR)/docker/pipeline/default.conf $(ARTIFACTS_DIR)/docker/bin/docker-entrypoint
-COPY_FILES += $(ARTIFACTS_DIR)/docker/env2yaml/env2yaml-arm64 
-COPY_FILES += $(ARTIFACTS_DIR)/docker/env2yaml/env2yaml-amd64
 
 $(ARTIFACTS_DIR)/docker/config/pipelines.yml: data/logstash/config/pipelines.yml
 $(ARTIFACTS_DIR)/docker/config/logstash-oss.yml: data/logstash/config/logstash-oss.yml
@@ -85,8 +84,7 @@ $(ARTIFACTS_DIR)/docker/config/log4j2.file.properties: data/logstash/config/log4
 $(ARTIFACTS_DIR)/docker/config/log4j2.properties: data/logstash/config/log4j2.properties
 $(ARTIFACTS_DIR)/docker/pipeline/default.conf: data/logstash/pipeline/default.conf
 $(ARTIFACTS_DIR)/docker/bin/docker-entrypoint: data/logstash/bin/docker-entrypoint
-$(ARTIFACTS_DIR)/docker/env2yaml/env2yaml-arm64: data/logstash/env2yaml/env2yaml-arm64
-$(ARTIFACTS_DIR)/docker/env2yaml/env2yaml-amd64: data/logstash/env2yaml/env2yaml-amd64
+$(ARTIFACTS_DIR)/docker/env2yaml/env2yaml: data/logstash/env2yaml/env2yaml
 
 $(ARTIFACTS_DIR)/docker/%:
 	cp -f $< $@
@@ -97,22 +95,22 @@ docker_paths:
 	mkdir -p $(ARTIFACTS_DIR)/docker/config
 	mkdir -p $(ARTIFACTS_DIR)/docker/env2yaml
 	mkdir -p $(ARTIFACTS_DIR)/docker/pipeline
+	cp -r data/logstash/env2yaml/classes $(ARTIFACTS_DIR)/docker/env2yaml/
+	cp -r data/logstash/env2yaml/lib $(ARTIFACTS_DIR)/docker/env2yaml/
 
 COPY_IRONBANK_FILES := $(ARTIFACTS_DIR)/ironbank/scripts/config/pipelines.yml $(ARTIFACTS_DIR)/ironbank/scripts/config/logstash.yml
 COPY_IRONBANK_FILES += $(ARTIFACTS_DIR)/ironbank/scripts/config/log4j2.file.properties $(ARTIFACTS_DIR)/ironbank/scripts/config/log4j2.properties
-COPY_IRONBANK_FILES += $(ARTIFACTS_DIR)/ironbank/scripts/pipeline/default.conf $(ARTIFACTS_DIR)/ironbank/scripts/bin/docker-entrypoint $(ARTIFACTS_DIR)/ironbank/scripts/go/src/env2yaml/env2yaml.go
-COPY_IRONBANK_FILES += $(ARTIFACTS_DIR)/ironbank/scripts/go/src/env2yaml/go.mod $(ARTIFACTS_DIR)/ironbank/scripts/go/src/env2yaml/go.sum $(ARTIFACTS_DIR)/ironbank/scripts/go/src/env2yaml/vendor/modules.txt $(ARTIFACTS_DIR)/ironbank/LICENSE $(ARTIFACTS_DIR)/ironbank/README.md
+COPY_IRONBANK_FILES += $(ARTIFACTS_DIR)/ironbank/scripts/pipeline/default.conf $(ARTIFACTS_DIR)/ironbank/scripts/bin/docker-entrypoint
+COPY_IRONBANK_FILES += $(ARTIFACTS_DIR)/ironbank/scripts/env2yaml/env2yaml
+COPY_IRONBANK_FILES += $(ARTIFACTS_DIR)/ironbank/LICENSE $(ARTIFACTS_DIR)/ironbank/README.md
 
 $(ARTIFACTS_DIR)/ironbank/scripts/config/pipelines.yml: data/logstash/config/pipelines.yml
 $(ARTIFACTS_DIR)/ironbank/scripts/config/logstash.yml: data/logstash/config/logstash-full.yml
 $(ARTIFACTS_DIR)/ironbank/scripts/config/log4j2.file.properties: data/logstash/config/log4j2.file.properties
 $(ARTIFACTS_DIR)/ironbank/scripts/config/log4j2.properties: data/logstash/config/log4j2.properties
 $(ARTIFACTS_DIR)/ironbank/scripts/pipeline/default.conf: data/logstash/pipeline/default.conf
 $(ARTIFACTS_DIR)/ironbank/scripts/bin/docker-entrypoint: data/logstash/bin/docker-entrypoint
-$(ARTIFACTS_DIR)/ironbank/scripts/go/src/env2yaml/env2yaml.go: data/logstash/env2yaml/env2yaml.go
-$(ARTIFACTS_DIR)/ironbank/scripts/go/src/env2yaml/go.mod: ironbank/go/src/env2yaml/go.mod
-$(ARTIFACTS_DIR)/ironbank/scripts/go/src/env2yaml/go.sum: ironbank/go/src/env2yaml/go.sum
-$(ARTIFACTS_DIR)/ironbank/scripts/go/src/env2yaml/vendor/modules.txt: ironbank/go/src/env2yaml/vendor/modules.txt
+$(ARTIFACTS_DIR)/ironbank/scripts/env2yaml/env2yaml: data/logstash/env2yaml/env2yaml
 $(ARTIFACTS_DIR)/ironbank/LICENSE: ironbank/LICENSE
 $(ARTIFACTS_DIR)/ironbank/README.md: ironbank/README.md
 
@@ -124,8 +122,10 @@ ironbank_docker_paths:
 	mkdir -p $(ARTIFACTS_DIR)/ironbank/scripts
 	mkdir -p $(ARTIFACTS_DIR)/ironbank/scripts/bin
 	mkdir -p $(ARTIFACTS_DIR)/ironbank/scripts/config
-	mkdir -p $(ARTIFACTS_DIR)/ironbank/scripts/go/src/env2yaml/vendor
+	mkdir -p $(ARTIFACTS_DIR)/ironbank/scripts/env2yaml
 	mkdir -p $(ARTIFACTS_DIR)/ironbank/scripts/pipeline
+	cp -r data/logstash/env2yaml/classes $(ARTIFACTS_DIR)/ironbank/scripts/env2yaml/
+	cp -r data/logstash/env2yaml/lib $(ARTIFACTS_DIR)/ironbank/scripts/env2yaml/
 
 public-dockerfiles: public-dockerfiles_oss public-dockerfiles_full public-dockerfiles_ubi8 public-dockerfiles_wolfi public-dockerfiles_observability-sre public-dockerfiles_ironbank
 
@@ -228,18 +228,6 @@ push:
 	docker push push.$(IMAGE_TAG):$(VERSION_TAG);
 	docker rmi push.$(IMAGE_TAG):$(VERSION_TAG);
 
-# Compile "env2yaml", the helper for configuring logstash.yml via environment
-# variables.
-env2yaml:
-	docker run --rm \
-	  -v "$(PWD)/data/logstash/env2yaml:/usr/src/env2yaml" \
-		-e GOARCH=arm64 -e GOOS=linux \
-		-w /usr/src/env2yaml golang:1 go build -o /usr/src/env2yaml/env2yaml-arm64
-	docker run --rm \
-	  -v "$(PWD)/data/logstash/env2yaml:/usr/src/env2yaml" \
-		-e GOARCH=amd64 -e GOOS=linux \
-		-w /usr/src/env2yaml golang:1 go build -o /usr/src/env2yaml/env2yaml-amd64
-
 # Generate the Dockerfiles from ERB templates.
 dockerfile: templates/Dockerfile.erb
 	$(foreach FLAVOR, $(IMAGE_FLAVORS), \
@@ -254,7 +242,7 @@ dockerfile: templates/Dockerfile.erb
 	)
 
 clean:
-	rm -f ${ARTIFACTS_DIR}/env2yaml/env2yaml-* ${ARTIFACTS_DIR}/Dockerfile
+	rm -f ${ARTIFACTS_DIR}/Dockerfile
 
 
 .PHONY: clean push

docker/data/logstash/env2yaml/build.gradle

@@ -0,0 +1,23 @@
+plugins {
+    id 'java'
+}
+
+compileJava {
+    destinationDirectory = file("${projectDir}/classes")
+}
+
+repositories {
+    mavenCentral()
+}
+
+dependencies {
+    implementation 'org.snakeyaml:snakeyaml-engine:2.9'
+}
+
+tasks.register('copyRuntimeLibs', Copy) {
+    from configurations.runtimeClasspath
+    into "${projectDir}/lib"
+ }
+
+compileJava.finalizedBy copyRuntimeLibs
+jar.enabled = false

docker/data/logstash/env2yaml/env2yaml

@@ -0,0 +1,12 @@
+#!/bin/bash
+
+# Execute the env2yaml java program. Ensure the snakeyaml-engine jar is in the classpath.
+
+exec /usr/share/logstash/jdk/bin/java \
+    -XX:+UseSerialGC \
+    -Xms32m \
+    -Xmx32m \
+    -cp "/usr/share/logstash/env2yaml/classes:/usr/share/logstash/env2yaml/lib/*" \
+    org.logstash.env2yaml.Env2Yaml "$@"
+
+