Rewrite Env2yaml in java instead of Go (#18423)

* WIP: Rewrite Env2yaml in java instead of Go

Managing a Go toolchain for persisting ENV vars in logstash container artifacts
has become cumbersome. We already manage a java runtime so this commit presents
a path forward to use that instead of Go. The Go binary is faster than java (in
my testing Go would complete in around less than 200ms while java takes over
300ms). Given the container startup time is on the order of magnitute of seconds
this change should be inperceptable to consumers. The benefit from consolidating
in Java is worth the slightly lower performance.

* Use TreeMap in java to try to replicate lexicographical order

* Explicit imports and TreeMap everywher

* Go removals and ironbank workflow update

* More non-code removals

* Update based on codereview

Use snakeyaml-engine and some java flags for faster execution

* Build env2yaml in stage

Build env2yaml in a separate build stage for container artifacts. Include
its dependencies and manage separately from logstash. Continue to use the
java runtime in the final container to run the program, but manage the classpath
separately. Note this did not use gradle for dependency management because installing
that as a depdendcy was not worth it compared with downloading a jar directly.

* Use gradle to manage snakeyaml-engine dependency

Use gradle (and a dedicated gradle base image) for building env2yaml

* Refactor to build env2yaml with gradle rather than in docker build

Dont rely on compiling at docker build time, rather do it when logstash
compilation is done.

* Dont try to use snakeyaml from jruby

The complexity around trying to copy over the jar shipped with jruby is not worth
how easy it is to just manage it with gradle. This helps with keeping env2yaml contained.

* Add license for snakeyaml-engine

Licence from https://bitbucket.org/snakeyaml/snakeyaml-engine/src/master/LICENSE.txt

* Cleanup and bugfix

* Stop skipping empty env vars

I mistakenly thought I had observed this behavior in the go version.

* Remove quotes from interpolated values

Even though we set `.setDefaultScalarStyle(ScalarStyle.PLAIN)` snakeyaml-engine
ends up quoting `${}` values. This commit removes them as this was not the behavior
with the go version.

(cherry picked from commit b15c6c5)

# Conflicts:
#	docker/data/logstash/env2yaml/env2yaml.go
#	docker/templates/Dockerfile.erb
#	rakelib/artifacts.rake

Author: donoghuc

.gitignore

@@ -16,7 +16,6 @@ out
 local
 test/setup/elasticsearch/elasticsearch-*
 vendor
-!docker/ironbank/go/src/env2yaml/vendor
 .sass-cache
 /data
 .buildpath

build.gradle

@@ -297,6 +297,13 @@ tasks.register("bootstrap") {
 }
 
 
+
+task dockerBootstrap {
+    description = "Docker bootstrap ensures env2yaml java is compiled and staged for inclusion in tarballs"
+    dependsOn bootstrap
+    dependsOn ':docker:data:logstash:env2yaml:compileJava'
+}
+
 tasks.register("installDefaultGems") {
     dependsOn bootstrap
     doLast {

docker/Makefile

@@ -60,7 +60,7 @@ build-from-local-wolfi-artifacts: dockerfile
 
 COPY_FILES := $(ARTIFACTS_DIR)/docker/config/pipelines.yml $(ARTIFACTS_DIR)/docker/config/logstash-oss.yml $(ARTIFACTS_DIR)/docker/config/logstash-full.yml
 COPY_FILES += $(ARTIFACTS_DIR)/docker/config/log4j2.file.properties $(ARTIFACTS_DIR)/docker/config/log4j2.properties
-COPY_FILES += $(ARTIFACTS_DIR)/docker/env2yaml/env2yaml.go $(ARTIFACTS_DIR)/docker/env2yaml/go.mod $(ARTIFACTS_DIR)/docker/env2yaml/go.sum
+COPY_FILES += $(ARTIFACTS_DIR)/docker/env2yaml/env2yaml
 COPY_FILES += $(ARTIFACTS_DIR)/docker/pipeline/default.conf $(ARTIFACTS_DIR)/docker/bin/docker-entrypoint
 
 $(ARTIFACTS_DIR)/docker/config/pipelines.yml: data/logstash/config/pipelines.yml
@@ -70,9 +70,7 @@ $(ARTIFACTS_DIR)/docker/config/log4j2.file.properties: data/logstash/config/log4
 $(ARTIFACTS_DIR)/docker/config/log4j2.properties: data/logstash/config/log4j2.properties
 $(ARTIFACTS_DIR)/docker/pipeline/default.conf: data/logstash/pipeline/default.conf
 $(ARTIFACTS_DIR)/docker/bin/docker-entrypoint: data/logstash/bin/docker-entrypoint
-$(ARTIFACTS_DIR)/docker/env2yaml/env2yaml.go: data/logstash/env2yaml/env2yaml.go
-$(ARTIFACTS_DIR)/docker/env2yaml/go.mod: data/logstash/env2yaml/go.mod
-$(ARTIFACTS_DIR)/docker/env2yaml/go.sum: data/logstash/env2yaml/go.sum
+$(ARTIFACTS_DIR)/docker/env2yaml/env2yaml: data/logstash/env2yaml/env2yaml
 
 $(ARTIFACTS_DIR)/docker/%:
 	cp -f $< $@
@@ -83,22 +81,22 @@ docker_paths:
 	mkdir -p $(ARTIFACTS_DIR)/docker/config
 	mkdir -p $(ARTIFACTS_DIR)/docker/env2yaml
 	mkdir -p $(ARTIFACTS_DIR)/docker/pipeline
+	cp -r data/logstash/env2yaml/classes $(ARTIFACTS_DIR)/docker/env2yaml/
+	cp -r data/logstash/env2yaml/lib $(ARTIFACTS_DIR)/docker/env2yaml/
 
 COPY_IRONBANK_FILES := $(ARTIFACTS_DIR)/ironbank/scripts/config/pipelines.yml $(ARTIFACTS_DIR)/ironbank/scripts/config/logstash.yml
 COPY_IRONBANK_FILES += $(ARTIFACTS_DIR)/ironbank/scripts/config/log4j2.file.properties $(ARTIFACTS_DIR)/ironbank/scripts/config/log4j2.properties
-COPY_IRONBANK_FILES += $(ARTIFACTS_DIR)/ironbank/scripts/pipeline/default.conf $(ARTIFACTS_DIR)/ironbank/scripts/bin/docker-entrypoint $(ARTIFACTS_DIR)/ironbank/scripts/go/src/env2yaml/env2yaml.go
-COPY_IRONBANK_FILES += $(ARTIFACTS_DIR)/ironbank/scripts/go/src/env2yaml/go.mod $(ARTIFACTS_DIR)/ironbank/scripts/go/src/env2yaml/go.sum $(ARTIFACTS_DIR)/ironbank/scripts/go/src/env2yaml/vendor/modules.txt $(ARTIFACTS_DIR)/ironbank/LICENSE $(ARTIFACTS_DIR)/ironbank/README.md
+COPY_IRONBANK_FILES += $(ARTIFACTS_DIR)/ironbank/scripts/pipeline/default.conf $(ARTIFACTS_DIR)/ironbank/scripts/bin/docker-entrypoint
+COPY_IRONBANK_FILES += $(ARTIFACTS_DIR)/ironbank/scripts/env2yaml/env2yaml
+COPY_IRONBANK_FILES += $(ARTIFACTS_DIR)/ironbank/LICENSE $(ARTIFACTS_DIR)/ironbank/README.md
 
 $(ARTIFACTS_DIR)/ironbank/scripts/config/pipelines.yml: data/logstash/config/pipelines.yml
 $(ARTIFACTS_DIR)/ironbank/scripts/config/logstash.yml: data/logstash/config/logstash-full.yml
 $(ARTIFACTS_DIR)/ironbank/scripts/config/log4j2.file.properties: data/logstash/config/log4j2.file.properties
 $(ARTIFACTS_DIR)/ironbank/scripts/config/log4j2.properties: data/logstash/config/log4j2.properties
 $(ARTIFACTS_DIR)/ironbank/scripts/pipeline/default.conf: data/logstash/pipeline/default.conf
 $(ARTIFACTS_DIR)/ironbank/scripts/bin/docker-entrypoint: data/logstash/bin/docker-entrypoint
-$(ARTIFACTS_DIR)/ironbank/scripts/go/src/env2yaml/env2yaml.go: data/logstash/env2yaml/env2yaml.go
-$(ARTIFACTS_DIR)/ironbank/scripts/go/src/env2yaml/go.mod: ironbank/go/src/env2yaml/go.mod
-$(ARTIFACTS_DIR)/ironbank/scripts/go/src/env2yaml/go.sum: ironbank/go/src/env2yaml/go.sum
-$(ARTIFACTS_DIR)/ironbank/scripts/go/src/env2yaml/vendor/modules.txt: ironbank/go/src/env2yaml/vendor/modules.txt
+$(ARTIFACTS_DIR)/ironbank/scripts/env2yaml/env2yaml: data/logstash/env2yaml/env2yaml
 $(ARTIFACTS_DIR)/ironbank/LICENSE: ironbank/LICENSE
 $(ARTIFACTS_DIR)/ironbank/README.md: ironbank/README.md
 
@@ -110,8 +108,10 @@ ironbank_docker_paths:
 	mkdir -p $(ARTIFACTS_DIR)/ironbank/scripts
 	mkdir -p $(ARTIFACTS_DIR)/ironbank/scripts/bin
 	mkdir -p $(ARTIFACTS_DIR)/ironbank/scripts/config
-	mkdir -p $(ARTIFACTS_DIR)/ironbank/scripts/go/src/env2yaml/vendor
+	mkdir -p $(ARTIFACTS_DIR)/ironbank/scripts/env2yaml
 	mkdir -p $(ARTIFACTS_DIR)/ironbank/scripts/pipeline
+	cp -r data/logstash/env2yaml/classes $(ARTIFACTS_DIR)/ironbank/scripts/env2yaml/
+	cp -r data/logstash/env2yaml/lib $(ARTIFACTS_DIR)/ironbank/scripts/env2yaml/
 
 public-dockerfiles: public-dockerfiles_oss public-dockerfiles_full public-dockerfiles_wolfi public-dockerfiles_ironbank
 

docker/data/logstash/env2yaml/build.gradle

@@ -0,0 +1,23 @@
+plugins {
+    id 'java'
+}
+
+compileJava {
+    destinationDirectory = file("${projectDir}/classes")
+}
+
+repositories {
+    mavenCentral()
+}
+
+dependencies {
+    implementation 'org.snakeyaml:snakeyaml-engine:2.9'
+}
+
+tasks.register('copyRuntimeLibs', Copy) {
+    from configurations.runtimeClasspath
+    into "${projectDir}/lib"
+ }
+
+compileJava.finalizedBy copyRuntimeLibs
+jar.enabled = false

docker/data/logstash/env2yaml/env2yaml

@@ -0,0 +1,12 @@
+#!/bin/bash
+
+# Execute the env2yaml java program. Ensure the snakeyaml-engine jar is in the classpath.
+
+exec /usr/share/logstash/jdk/bin/java \
+    -XX:+UseSerialGC \
+    -Xms32m \
+    -Xmx32m \
+    -cp "/usr/share/logstash/env2yaml/classes:/usr/share/logstash/env2yaml/lib/*" \
+    org.logstash.env2yaml.Env2Yaml "$@"
+
+

docker/data/logstash/env2yaml/go.mod

@@ -0,0 +1,206 @@
+package org.logstash.env2yaml;
+
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.nio.file.StandardOpenOption;
+import java.nio.file.attribute.PosixFilePermission;
+import java.time.LocalDateTime;
+import java.time.format.DateTimeFormatter;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeMap;
+import org.snakeyaml.engine.v2.api.Dump;
+import org.snakeyaml.engine.v2.api.DumpSettings;
+import org.snakeyaml.engine.v2.api.Load;
+import org.snakeyaml.engine.v2.api.LoadSettings;
+import org.snakeyaml.engine.v2.common.FlowStyle;
+import org.snakeyaml.engine.v2.common.ScalarStyle;
+
+/**
+ * Environment variable to YAML configuration merger
+ *
+ * Takes environment variables and merges them into logstash.yml
+ * Example: docker run -e pipeline.workers=6
+ * or: docker run -e PIPELINE_WORKERS=6
+ * Result: pipeline.workers: 6 in logstash.yml
+ */
+public class Env2Yaml {
+    private static final DateTimeFormatter TIMESTAMP_FORMAT = DateTimeFormatter.ofPattern("yyyy/MM/dd HH:mm:ss");
+    private static class SettingValidator {
+        private final Map<String, String> normalizedToCanonical;
+
+        public SettingValidator() {
+            this.normalizedToCanonical = buildSettingMap();
+        }
+
+        private Map<String, String> buildSettingMap() {
+            Map<String, String> map = new TreeMap<>();
+            String[] allowedConfigs = {
+                "api.enabled", "api.http.host", "api.http.port", "api.environment",
+                "node.name", "path.data", "pipeline.id", "pipeline.workers",
+                "pipeline.output.workers", "pipeline.batch.size", "pipeline.batch.delay",
+                "pipeline.unsafe_shutdown", "pipeline.ecs_compatibility", "pipeline.ordered",
+                "pipeline.plugin_classloaders", "pipeline.separate_logs", "path.config",
+                "config.string", "config.test_and_exit", "config.reload.automatic",
+                "config.reload.interval", "config.debug", "config.support_escapes",
+                "config.field_reference.escape_style", "queue.type", "path.queue",
+                "queue.page_capacity", "queue.max_events", "queue.max_bytes",
+                "queue.checkpoint.acks", "queue.checkpoint.writes", "queue.checkpoint.interval",
+                "queue.compression", "queue.drain", "dead_letter_queue.enable",
+                "dead_letter_queue.max_bytes", "dead_letter_queue.flush_interval",
+                "dead_letter_queue.storage_policy", "dead_letter_queue.retain.age",
+                "path.dead_letter_queue", "log.level", "log.format",
+                "log.format.json.fix_duplicate_message_fields", "metric.collect",
+                "path.logs", "path.plugins", "api.auth.type", "api.auth.basic.username",
+                "api.auth.basic.password", "api.auth.basic.password_policy.mode",
+                "api.auth.basic.password_policy.length.minimum", "api.auth.basic.password_policy.include.upper",
+                "api.auth.basic.password_policy.include.lower", "api.auth.basic.password_policy.include.digit",
+                "api.auth.basic.password_policy.include.symbol", "allow_superuser",
+                "monitoring.cluster_uuid", "xpack.monitoring.allow_legacy_collection",
+                "xpack.monitoring.enabled", "xpack.monitoring.collection.interval",
+                "xpack.monitoring.elasticsearch.hosts", "xpack.monitoring.elasticsearch.username",
+                "xpack.monitoring.elasticsearch.password", "xpack.monitoring.elasticsearch.proxy",
+                "xpack.monitoring.elasticsearch.api_key", "xpack.monitoring.elasticsearch.cloud_auth",
+                "xpack.monitoring.elasticsearch.cloud_id", "xpack.monitoring.elasticsearch.sniffing",
+                "xpack.monitoring.elasticsearch.ssl.certificate_authority", "xpack.monitoring.elasticsearch.ssl.ca_trusted_fingerprint",
+                "xpack.monitoring.elasticsearch.ssl.verification_mode", "xpack.monitoring.elasticsearch.ssl.truststore.path",
+                "xpack.monitoring.elasticsearch.ssl.truststore.password", "xpack.monitoring.elasticsearch.ssl.keystore.path",
+                "xpack.monitoring.elasticsearch.ssl.keystore.password", "xpack.monitoring.elasticsearch.ssl.certificate",
+                "xpack.monitoring.elasticsearch.ssl.key", "xpack.monitoring.elasticsearch.ssl.cipher_suites",
+                "xpack.management.enabled", "xpack.management.logstash.poll_interval",
+                "xpack.management.pipeline.id", "xpack.management.elasticsearch.hosts",
+                "xpack.management.elasticsearch.username", "xpack.management.elasticsearch.password",
+                "xpack.management.elasticsearch.proxy", "xpack.management.elasticsearch.api_key",
+                "xpack.management.elasticsearch.cloud_auth", "xpack.management.elasticsearch.cloud_id",
+                "xpack.management.elasticsearch.sniffing", "xpack.management.elasticsearch.ssl.certificate_authority",
+                "xpack.management.elasticsearch.ssl.ca_trusted_fingerprint", "xpack.management.elasticsearch.ssl.verification_mode",
+                "xpack.management.elasticsearch.ssl.truststore.path", "xpack.management.elasticsearch.ssl.truststore.password",
+                "xpack.management.elasticsearch.ssl.keystore.path", "xpack.management.elasticsearch.ssl.keystore.password",
+                "xpack.management.elasticsearch.ssl.certificate", "xpack.management.elasticsearch.ssl.key",
+                "xpack.management.elasticsearch.ssl.cipher_suites", "xpack.geoip.download.endpoint",
+                "xpack.geoip.downloader.enabled"
+            };
+
+            for (String configName : allowedConfigs) {
+                String normalizedKey = normalizeKey(configName);
+                map.put(normalizedKey, configName);
+            }
+            return map;
+        }
+
+        public String findCanonicalSetting(String envVarName) {
+            String normalized = normalizeKey(envVarName);
+            return normalizedToCanonical.get(normalized);
+        }
+    }
+
+    private static String normalizeKey(String key) {
+        return key.toLowerCase()
+                 .replace(".", "")
+                 .replace("_", "");
+    }
+
+    public static void main(String[] args) {
+        if (args.length != 1) {
+            System.err.println("usage: env2yaml FILENAME");
+            System.exit(1);
+        }
+
+        try {
+            String configPath = args[0];
+            new Env2Yaml().processConfigFile(configPath);
+        } catch (Exception e) {
+            System.err.println("error: " + e.getMessage());
+            System.exit(1);
+        }
+    }
+
+    private void processConfigFile(String configPath) throws Exception {
+        Path fileLocation = Paths.get(configPath);
+        Map<String, Object> configData = loadExistingConfig(fileLocation);
+
+        SettingValidator validator = new SettingValidator();
+        boolean addedNewConfigs = incorporateEnvironmentVars(configData, validator);
+
+        if (addedNewConfigs) {
+            saveUpdatedConfig(fileLocation, configData);
+        }
+    }
+
+    @SuppressWarnings("unchecked")
+    private Map<String, Object> loadExistingConfig(Path fileLocation) throws Exception {
+        if (!Files.exists(fileLocation)) {
+            return new TreeMap<>();
+        }
+        LoadSettings loadSettings = LoadSettings.builder().build();
+        Load loader = new Load(loadSettings);
+        try (InputStream fileInput = Files.newInputStream(fileLocation)) {
+            Object parsedData = loader.loadFromInputStream(fileInput);
+            if (parsedData instanceof Map) {
+                // Convert to TreeMap to ensure alphabetical ordering like Go version
+                return new TreeMap<>((Map<String, Object>) parsedData);
+            }
+            return new TreeMap<>();
+        }
+    }
+
+    private boolean incorporateEnvironmentVars(Map<String, Object> configData, SettingValidator validator) {
+        boolean addedNewConfigs = false;
+
+        for (Map.Entry<String, String> envEntry : System.getenv().entrySet()) {
+            String envVarName = envEntry.getKey();
+            String envValue = envEntry.getValue();
+            String canonicalSetting = validator.findCanonicalSetting(envVarName);
+
+            if (canonicalSetting != null) {
+                addedNewConfigs = true;
+                System.err.println(LocalDateTime.now().format(TIMESTAMP_FORMAT) + " Setting '" + canonicalSetting + "' from environment.");
+                configData.put(canonicalSetting, "${" + envVarName + "}");
+            }
+        }
+
+        return addedNewConfigs;
+    }
+
+    private void saveUpdatedConfig(Path fileLocation, Map<String, Object> configData) throws Exception {
+        // Configure YAML output to match Go version formatting (block style)
+        DumpSettings dumpSettings = DumpSettings.builder()
+            .setDefaultFlowStyle(FlowStyle.BLOCK)
+            .setDefaultScalarStyle(ScalarStyle.PLAIN)
+            .setIndent(2)
+            .build();
+        Dump dumper = new Dump(dumpSettings);
+        String yamlOutput = dumper.dumpToString(configData);
+        // Remove quotes (single or double) around ${VAR} to match Go behavior
+        // https://github.com/snakeyaml/snakeyaml-engine/blob/2070eb4e3d23bb1d81097875526a071003067877/src/main/java/org/snakeyaml/engine/v2/emitter/Emitter.java#L969-L996
+        yamlOutput = yamlOutput.replaceAll("(['\"])(\\$\\{[^}]+\\})\\1", "$2");
+        Set<PosixFilePermission> existingPermissions = getFilePermissions(fileLocation);
+
+        Files.write(fileLocation, yamlOutput.getBytes(StandardCharsets.UTF_8), StandardOpenOption.WRITE, StandardOpenOption.TRUNCATE_EXISTING);
+
+        applyFilePermissions(fileLocation, existingPermissions);
+    }
+
+    private Set<PosixFilePermission> getFilePermissions(Path fileLocation) {
+        try {
+            return Files.getPosixFilePermissions(fileLocation);
+        } catch (UnsupportedOperationException e) {
+            return null;
+        } catch (Exception e) {
+            return null;
+        }
+    }
+
+    private void applyFilePermissions(Path fileLocation, Set<PosixFilePermission> existingPermissions) {
+        if (existingPermissions != null) {
+            try {
+                Files.setPosixFilePermissions(fileLocation, existingPermissions);
+            } catch (Exception e) {
+                // Ignore failures
+            }
+        }
+    }
+}