diff --git a/.gitignore b/.gitignore index cc0ab0db4c0..883206a8ab3 100644 --- a/.gitignore +++ b/.gitignore @@ -16,7 +16,6 @@ out local test/setup/elasticsearch/elasticsearch-* vendor -!docker/ironbank/go/src/env2yaml/vendor .sass-cache /data .buildpath diff --git a/build.gradle b/build.gradle index a238e899729..bf7e8ae6577 100644 --- a/build.gradle +++ b/build.gradle @@ -302,6 +302,13 @@ tasks.register("bootstrap") { } + +task dockerBootstrap { + description = "Docker bootstrap ensures env2yaml java is compiled and staged for inclusion in tarballs" + dependsOn bootstrap + dependsOn ':docker:data:logstash:env2yaml:compileJava' +} + tasks.register("installDefaultGems") { dependsOn bootstrap doLast { diff --git a/docker/Makefile b/docker/Makefile index c220f57f898..a074ec2067a 100644 --- a/docker/Makefile +++ b/docker/Makefile @@ -69,7 +69,7 @@ build-from-local-observability-sre-artifacts: dockerfile COPY_FILES := $(ARTIFACTS_DIR)/docker/config/pipelines.yml $(ARTIFACTS_DIR)/docker/config/logstash-oss.yml $(ARTIFACTS_DIR)/docker/config/logstash-full.yml COPY_FILES += $(ARTIFACTS_DIR)/docker/config/log4j2.file.properties $(ARTIFACTS_DIR)/docker/config/log4j2.properties -COPY_FILES += $(ARTIFACTS_DIR)/docker/env2yaml/env2yaml.go $(ARTIFACTS_DIR)/docker/env2yaml/go.mod $(ARTIFACTS_DIR)/docker/env2yaml/go.sum +COPY_FILES += $(ARTIFACTS_DIR)/docker/env2yaml/env2yaml COPY_FILES += $(ARTIFACTS_DIR)/docker/pipeline/default.conf $(ARTIFACTS_DIR)/docker/bin/docker-entrypoint $(ARTIFACTS_DIR)/docker/config/pipelines.yml: data/logstash/config/pipelines.yml @@ -79,9 +79,7 @@ $(ARTIFACTS_DIR)/docker/config/log4j2.file.properties: data/logstash/config/log4 $(ARTIFACTS_DIR)/docker/config/log4j2.properties: data/logstash/config/log4j2.properties $(ARTIFACTS_DIR)/docker/pipeline/default.conf: data/logstash/pipeline/default.conf $(ARTIFACTS_DIR)/docker/bin/docker-entrypoint: data/logstash/bin/docker-entrypoint -$(ARTIFACTS_DIR)/docker/env2yaml/env2yaml.go: data/logstash/env2yaml/env2yaml.go -$(ARTIFACTS_DIR)/docker/env2yaml/go.mod: data/logstash/env2yaml/go.mod -$(ARTIFACTS_DIR)/docker/env2yaml/go.sum: data/logstash/env2yaml/go.sum +$(ARTIFACTS_DIR)/docker/env2yaml/env2yaml: data/logstash/env2yaml/env2yaml $(ARTIFACTS_DIR)/docker/%: cp -f $< $@ @@ -92,11 +90,14 @@ docker_paths: mkdir -p $(ARTIFACTS_DIR)/docker/config mkdir -p $(ARTIFACTS_DIR)/docker/env2yaml mkdir -p $(ARTIFACTS_DIR)/docker/pipeline + cp -r data/logstash/env2yaml/classes $(ARTIFACTS_DIR)/docker/env2yaml/ + cp -r data/logstash/env2yaml/lib $(ARTIFACTS_DIR)/docker/env2yaml/ COPY_IRONBANK_FILES := $(ARTIFACTS_DIR)/ironbank/scripts/config/pipelines.yml $(ARTIFACTS_DIR)/ironbank/scripts/config/logstash.yml COPY_IRONBANK_FILES += $(ARTIFACTS_DIR)/ironbank/scripts/config/log4j2.file.properties $(ARTIFACTS_DIR)/ironbank/scripts/config/log4j2.properties -COPY_IRONBANK_FILES += $(ARTIFACTS_DIR)/ironbank/scripts/pipeline/default.conf $(ARTIFACTS_DIR)/ironbank/scripts/bin/docker-entrypoint $(ARTIFACTS_DIR)/ironbank/scripts/go/src/env2yaml/env2yaml.go -COPY_IRONBANK_FILES += $(ARTIFACTS_DIR)/ironbank/scripts/go/src/env2yaml/go.mod $(ARTIFACTS_DIR)/ironbank/scripts/go/src/env2yaml/go.sum $(ARTIFACTS_DIR)/ironbank/scripts/go/src/env2yaml/vendor/modules.txt $(ARTIFACTS_DIR)/ironbank/LICENSE $(ARTIFACTS_DIR)/ironbank/README.md +COPY_IRONBANK_FILES += $(ARTIFACTS_DIR)/ironbank/scripts/pipeline/default.conf $(ARTIFACTS_DIR)/ironbank/scripts/bin/docker-entrypoint +COPY_IRONBANK_FILES += $(ARTIFACTS_DIR)/ironbank/scripts/env2yaml/env2yaml +COPY_IRONBANK_FILES += $(ARTIFACTS_DIR)/ironbank/LICENSE $(ARTIFACTS_DIR)/ironbank/README.md $(ARTIFACTS_DIR)/ironbank/scripts/config/pipelines.yml: data/logstash/config/pipelines.yml $(ARTIFACTS_DIR)/ironbank/scripts/config/logstash.yml: data/logstash/config/logstash-full.yml @@ -104,10 +105,7 @@ $(ARTIFACTS_DIR)/ironbank/scripts/config/log4j2.file.properties: data/logstash/c $(ARTIFACTS_DIR)/ironbank/scripts/config/log4j2.properties: data/logstash/config/log4j2.properties $(ARTIFACTS_DIR)/ironbank/scripts/pipeline/default.conf: data/logstash/pipeline/default.conf $(ARTIFACTS_DIR)/ironbank/scripts/bin/docker-entrypoint: data/logstash/bin/docker-entrypoint -$(ARTIFACTS_DIR)/ironbank/scripts/go/src/env2yaml/env2yaml.go: data/logstash/env2yaml/env2yaml.go -$(ARTIFACTS_DIR)/ironbank/scripts/go/src/env2yaml/go.mod: ironbank/go/src/env2yaml/go.mod -$(ARTIFACTS_DIR)/ironbank/scripts/go/src/env2yaml/go.sum: ironbank/go/src/env2yaml/go.sum -$(ARTIFACTS_DIR)/ironbank/scripts/go/src/env2yaml/vendor/modules.txt: ironbank/go/src/env2yaml/vendor/modules.txt +$(ARTIFACTS_DIR)/ironbank/scripts/env2yaml/env2yaml: data/logstash/env2yaml/env2yaml $(ARTIFACTS_DIR)/ironbank/LICENSE: ironbank/LICENSE $(ARTIFACTS_DIR)/ironbank/README.md: ironbank/README.md @@ -119,8 +117,10 @@ ironbank_docker_paths: mkdir -p $(ARTIFACTS_DIR)/ironbank/scripts mkdir -p $(ARTIFACTS_DIR)/ironbank/scripts/bin mkdir -p $(ARTIFACTS_DIR)/ironbank/scripts/config - mkdir -p $(ARTIFACTS_DIR)/ironbank/scripts/go/src/env2yaml/vendor + mkdir -p $(ARTIFACTS_DIR)/ironbank/scripts/env2yaml mkdir -p $(ARTIFACTS_DIR)/ironbank/scripts/pipeline + cp -r data/logstash/env2yaml/classes $(ARTIFACTS_DIR)/ironbank/scripts/env2yaml/ + cp -r data/logstash/env2yaml/lib $(ARTIFACTS_DIR)/ironbank/scripts/env2yaml/ public-dockerfiles: public-dockerfiles_oss public-dockerfiles_full public-dockerfiles_wolfi public-dockerfiles_observability-sre public-dockerfiles_ironbank diff --git a/docker/data/logstash/env2yaml/build.gradle b/docker/data/logstash/env2yaml/build.gradle new file mode 100644 index 00000000000..406b55c73bf --- /dev/null +++ b/docker/data/logstash/env2yaml/build.gradle @@ -0,0 +1,23 @@ +plugins { + id 'java' +} + +compileJava { + destinationDirectory = file("${projectDir}/classes") +} + +repositories { + mavenCentral() +} + +dependencies { + implementation 'org.snakeyaml:snakeyaml-engine:2.9' +} + +tasks.register('copyRuntimeLibs', Copy) { + from configurations.runtimeClasspath + into "${projectDir}/lib" + } + +compileJava.finalizedBy copyRuntimeLibs +jar.enabled = false \ No newline at end of file diff --git a/docker/data/logstash/env2yaml/env2yaml b/docker/data/logstash/env2yaml/env2yaml new file mode 100644 index 00000000000..c709c30f86c --- /dev/null +++ b/docker/data/logstash/env2yaml/env2yaml @@ -0,0 +1,12 @@ +#!/bin/bash + +# Execute the env2yaml java program. Ensure the snakeyaml-engine jar is in the classpath. + +exec /usr/share/logstash/jdk/bin/java \ + -XX:+UseSerialGC \ + -Xms32m \ + -Xmx32m \ + -cp "/usr/share/logstash/env2yaml/classes:/usr/share/logstash/env2yaml/lib/*" \ + org.logstash.env2yaml.Env2Yaml "$@" + + diff --git a/docker/data/logstash/env2yaml/env2yaml.go b/docker/data/logstash/env2yaml/env2yaml.go deleted file mode 100644 index d1e976bbab1..00000000000 --- a/docker/data/logstash/env2yaml/env2yaml.go +++ /dev/null @@ -1,199 +0,0 @@ -// env2yaml -// -// Merge environment variables into logstash.yml. -// For example, running Docker with: -// -// docker run -e pipeline.workers=6 -// -// or -// -// docker run -e PIPELINE_WORKERS=6 -// -// will cause logstash.yml to contain the line: -// -// pipeline.workers: 6 -package main - -import ( - "errors" - "fmt" - "io/ioutil" - "log" - "os" - "strings" - - "gopkg.in/yaml.v2" -) - -var validSettings = []string{ - "api.enabled", - "api.http.host", - "api.http.port", - "api.environment", - "node.name", - "path.data", - "pipeline.id", - "pipeline.workers", - "pipeline.output.workers", - "pipeline.batch.size", - "pipeline.batch.delay", - "pipeline.unsafe_shutdown", - "pipeline.ecs_compatibility", - "pipeline.ordered", - "pipeline.plugin_classloaders", - "pipeline.separate_logs", - "path.config", - "config.string", - "config.test_and_exit", - "config.reload.automatic", - "config.reload.interval", - "config.debug", - "config.support_escapes", - "config.field_reference.escape_style", - "queue.type", - "path.queue", - "queue.page_capacity", - "queue.max_events", - "queue.max_bytes", - "queue.checkpoint.acks", - "queue.checkpoint.writes", - "queue.checkpoint.interval", // remove it for #17155 - "queue.compression", - "queue.drain", - "dead_letter_queue.enable", - "dead_letter_queue.max_bytes", - "dead_letter_queue.flush_interval", - "dead_letter_queue.storage_policy", - "dead_letter_queue.retain.age", - "path.dead_letter_queue", - "log.level", - "log.format", - "log.format.json.fix_duplicate_message_fields", - "metric.collect", - "path.logs", - "path.plugins", - "api.auth.type", - "api.auth.basic.username", - "api.auth.basic.password", - "api.auth.basic.password_policy.mode", - "api.auth.basic.password_policy.length.minimum", - "api.auth.basic.password_policy.include.upper", - "api.auth.basic.password_policy.include.lower", - "api.auth.basic.password_policy.include.digit", - "api.auth.basic.password_policy.include.symbol", - "allow_superuser", - "monitoring.cluster_uuid", - "xpack.monitoring.allow_legacy_collection", - "xpack.monitoring.enabled", - "xpack.monitoring.collection.interval", - "xpack.monitoring.elasticsearch.hosts", - "xpack.monitoring.elasticsearch.username", - "xpack.monitoring.elasticsearch.password", - "xpack.monitoring.elasticsearch.proxy", - "xpack.monitoring.elasticsearch.api_key", - "xpack.monitoring.elasticsearch.cloud_auth", - "xpack.monitoring.elasticsearch.cloud_id", - "xpack.monitoring.elasticsearch.sniffing", - "xpack.monitoring.elasticsearch.ssl.certificate_authority", - "xpack.monitoring.elasticsearch.ssl.ca_trusted_fingerprint", - "xpack.monitoring.elasticsearch.ssl.verification_mode", - "xpack.monitoring.elasticsearch.ssl.truststore.path", - "xpack.monitoring.elasticsearch.ssl.truststore.password", - "xpack.monitoring.elasticsearch.ssl.keystore.path", - "xpack.monitoring.elasticsearch.ssl.keystore.password", - "xpack.monitoring.elasticsearch.ssl.certificate", - "xpack.monitoring.elasticsearch.ssl.key", - "xpack.monitoring.elasticsearch.ssl.cipher_suites", - "xpack.management.enabled", - "xpack.management.logstash.poll_interval", - "xpack.management.pipeline.id", - "xpack.management.elasticsearch.hosts", - "xpack.management.elasticsearch.username", - "xpack.management.elasticsearch.password", - "xpack.management.elasticsearch.proxy", - "xpack.management.elasticsearch.api_key", - "xpack.management.elasticsearch.cloud_auth", - "xpack.management.elasticsearch.cloud_id", - "xpack.management.elasticsearch.sniffing", - "xpack.management.elasticsearch.ssl.certificate_authority", - "xpack.management.elasticsearch.ssl.ca_trusted_fingerprint", - "xpack.management.elasticsearch.ssl.verification_mode", - "xpack.management.elasticsearch.ssl.truststore.path", - "xpack.management.elasticsearch.ssl.truststore.password", - "xpack.management.elasticsearch.ssl.keystore.path", - "xpack.management.elasticsearch.ssl.keystore.password", - "xpack.management.elasticsearch.ssl.certificate", - "xpack.management.elasticsearch.ssl.key", - "xpack.management.elasticsearch.ssl.cipher_suites", - "xpack.geoip.download.endpoint", - "xpack.geoip.downloader.enabled", -} - -// Given a setting name, return a downcased version with delimiters removed. -func squashSetting(setting string) string { - downcased := strings.ToLower(setting) - de_dotted := strings.Replace(downcased, ".", "", -1) - de_underscored := strings.Replace(de_dotted, "_", "", -1) - return de_underscored -} - -// Given a setting name like "pipeline.workers" or "PIPELINE_UNSAFE_SHUTDOWN", -// return the canonical setting name. eg. 'pipeline.unsafe_shutdown' -func normalizeSetting(setting string) (string, error) { - for _, validSetting := range validSettings { - if squashSetting(setting) == squashSetting(validSetting) { - return validSetting, nil - } - } - return "", errors.New("Invalid setting: " + setting) -} - -func main() { - if len(os.Args) != 2 { - log.Fatalf("usage: env2yaml FILENAME") - } - settingsFilePath := os.Args[1] - - settingsFile, err := ioutil.ReadFile(settingsFilePath) - if err != nil { - log.Fatalf("error: %v", err) - } - - // Read the original settings file into a map. - settings := make(map[string]interface{}) - err = yaml.Unmarshal(settingsFile, &settings) - if err != nil { - log.Fatalf("error: %v", err) - } - - // Merge any valid settings found in the environment. - foundNewSettings := false - for _, line := range os.Environ() { - kv := strings.SplitN(line, "=", 2) - key := kv[0] - setting, err := normalizeSetting(key) - if err == nil { - foundNewSettings = true - log.Printf("Setting '%s' from environment.", setting) - // we need to keep ${KEY} in the logstash.yml to let Logstash decide using ${KEY}'s value from either keystore or environment - settings[setting] = fmt.Sprintf("${%s}", key) - } - } - - if foundNewSettings { - output, err := yaml.Marshal(&settings) - if err != nil { - log.Fatalf("error: %v", err) - } - - stat, err := os.Stat(settingsFilePath) - if err != nil { - log.Fatalf("error: %v", err) - } - - err = ioutil.WriteFile(settingsFilePath, output, stat.Mode()) - if err != nil { - log.Fatalf("error: %v", err) - } - } -} diff --git a/docker/data/logstash/env2yaml/go.mod b/docker/data/logstash/env2yaml/go.mod deleted file mode 100644 index 7600007420d..00000000000 --- a/docker/data/logstash/env2yaml/go.mod +++ /dev/null @@ -1,5 +0,0 @@ -module logstash/env2yaml - -go 1.21 - -require gopkg.in/yaml.v2 v2.4.0 diff --git a/docker/data/logstash/env2yaml/go.sum b/docker/data/logstash/env2yaml/go.sum deleted file mode 100644 index 75346616b19..00000000000 --- a/docker/data/logstash/env2yaml/go.sum +++ /dev/null @@ -1,3 +0,0 @@ -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= -gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= diff --git a/docker/data/logstash/env2yaml/src/main/java/org/logstash/env2yaml/Env2Yaml.java b/docker/data/logstash/env2yaml/src/main/java/org/logstash/env2yaml/Env2Yaml.java new file mode 100644 index 00000000000..a697f12f9fb --- /dev/null +++ b/docker/data/logstash/env2yaml/src/main/java/org/logstash/env2yaml/Env2Yaml.java @@ -0,0 +1,206 @@ +package org.logstash.env2yaml; + +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.StandardOpenOption; +import java.nio.file.attribute.PosixFilePermission; +import java.time.LocalDateTime; +import java.time.format.DateTimeFormatter; +import java.util.Map; +import java.util.Set; +import java.util.TreeMap; +import org.snakeyaml.engine.v2.api.Dump; +import org.snakeyaml.engine.v2.api.DumpSettings; +import org.snakeyaml.engine.v2.api.Load; +import org.snakeyaml.engine.v2.api.LoadSettings; +import org.snakeyaml.engine.v2.common.FlowStyle; +import org.snakeyaml.engine.v2.common.ScalarStyle; + +/** + * Environment variable to YAML configuration merger + * + * Takes environment variables and merges them into logstash.yml + * Example: docker run -e pipeline.workers=6 + * or: docker run -e PIPELINE_WORKERS=6 + * Result: pipeline.workers: 6 in logstash.yml + */ +public class Env2Yaml { + private static final DateTimeFormatter TIMESTAMP_FORMAT = DateTimeFormatter.ofPattern("yyyy/MM/dd HH:mm:ss"); + private static class SettingValidator { + private final Map normalizedToCanonical; + + public SettingValidator() { + this.normalizedToCanonical = buildSettingMap(); + } + + private Map buildSettingMap() { + Map map = new TreeMap<>(); + String[] allowedConfigs = { + "api.enabled", "api.http.host", "api.http.port", "api.environment", + "node.name", "path.data", "pipeline.id", "pipeline.workers", + "pipeline.output.workers", "pipeline.batch.size", "pipeline.batch.delay", + "pipeline.unsafe_shutdown", "pipeline.ecs_compatibility", "pipeline.ordered", + "pipeline.plugin_classloaders", "pipeline.separate_logs", "path.config", + "config.string", "config.test_and_exit", "config.reload.automatic", + "config.reload.interval", "config.debug", "config.support_escapes", + "config.field_reference.escape_style", "queue.type", "path.queue", + "queue.page_capacity", "queue.max_events", "queue.max_bytes", + "queue.checkpoint.acks", "queue.checkpoint.writes", "queue.checkpoint.interval", + "queue.compression", "queue.drain", "dead_letter_queue.enable", + "dead_letter_queue.max_bytes", "dead_letter_queue.flush_interval", + "dead_letter_queue.storage_policy", "dead_letter_queue.retain.age", + "path.dead_letter_queue", "log.level", "log.format", + "log.format.json.fix_duplicate_message_fields", "metric.collect", + "path.logs", "path.plugins", "api.auth.type", "api.auth.basic.username", + "api.auth.basic.password", "api.auth.basic.password_policy.mode", + "api.auth.basic.password_policy.length.minimum", "api.auth.basic.password_policy.include.upper", + "api.auth.basic.password_policy.include.lower", "api.auth.basic.password_policy.include.digit", + "api.auth.basic.password_policy.include.symbol", "allow_superuser", + "monitoring.cluster_uuid", "xpack.monitoring.allow_legacy_collection", + "xpack.monitoring.enabled", "xpack.monitoring.collection.interval", + "xpack.monitoring.elasticsearch.hosts", "xpack.monitoring.elasticsearch.username", + "xpack.monitoring.elasticsearch.password", "xpack.monitoring.elasticsearch.proxy", + "xpack.monitoring.elasticsearch.api_key", "xpack.monitoring.elasticsearch.cloud_auth", + "xpack.monitoring.elasticsearch.cloud_id", "xpack.monitoring.elasticsearch.sniffing", + "xpack.monitoring.elasticsearch.ssl.certificate_authority", "xpack.monitoring.elasticsearch.ssl.ca_trusted_fingerprint", + "xpack.monitoring.elasticsearch.ssl.verification_mode", "xpack.monitoring.elasticsearch.ssl.truststore.path", + "xpack.monitoring.elasticsearch.ssl.truststore.password", "xpack.monitoring.elasticsearch.ssl.keystore.path", + "xpack.monitoring.elasticsearch.ssl.keystore.password", "xpack.monitoring.elasticsearch.ssl.certificate", + "xpack.monitoring.elasticsearch.ssl.key", "xpack.monitoring.elasticsearch.ssl.cipher_suites", + "xpack.management.enabled", "xpack.management.logstash.poll_interval", + "xpack.management.pipeline.id", "xpack.management.elasticsearch.hosts", + "xpack.management.elasticsearch.username", "xpack.management.elasticsearch.password", + "xpack.management.elasticsearch.proxy", "xpack.management.elasticsearch.api_key", + "xpack.management.elasticsearch.cloud_auth", "xpack.management.elasticsearch.cloud_id", + "xpack.management.elasticsearch.sniffing", "xpack.management.elasticsearch.ssl.certificate_authority", + "xpack.management.elasticsearch.ssl.ca_trusted_fingerprint", "xpack.management.elasticsearch.ssl.verification_mode", + "xpack.management.elasticsearch.ssl.truststore.path", "xpack.management.elasticsearch.ssl.truststore.password", + "xpack.management.elasticsearch.ssl.keystore.path", "xpack.management.elasticsearch.ssl.keystore.password", + "xpack.management.elasticsearch.ssl.certificate", "xpack.management.elasticsearch.ssl.key", + "xpack.management.elasticsearch.ssl.cipher_suites", "xpack.geoip.download.endpoint", + "xpack.geoip.downloader.enabled" + }; + + for (String configName : allowedConfigs) { + String normalizedKey = normalizeKey(configName); + map.put(normalizedKey, configName); + } + return map; + } + + public String findCanonicalSetting(String envVarName) { + String normalized = normalizeKey(envVarName); + return normalizedToCanonical.get(normalized); + } + } + + private static String normalizeKey(String key) { + return key.toLowerCase() + .replace(".", "") + .replace("_", ""); + } + + public static void main(String[] args) { + if (args.length != 1) { + System.err.println("usage: env2yaml FILENAME"); + System.exit(1); + } + + try { + String configPath = args[0]; + new Env2Yaml().processConfigFile(configPath); + } catch (Exception e) { + System.err.println("error: " + e.getMessage()); + System.exit(1); + } + } + + private void processConfigFile(String configPath) throws Exception { + Path fileLocation = Paths.get(configPath); + Map configData = loadExistingConfig(fileLocation); + + SettingValidator validator = new SettingValidator(); + boolean addedNewConfigs = incorporateEnvironmentVars(configData, validator); + + if (addedNewConfigs) { + saveUpdatedConfig(fileLocation, configData); + } + } + + @SuppressWarnings("unchecked") + private Map loadExistingConfig(Path fileLocation) throws Exception { + if (!Files.exists(fileLocation)) { + return new TreeMap<>(); + } + LoadSettings loadSettings = LoadSettings.builder().build(); + Load loader = new Load(loadSettings); + try (InputStream fileInput = Files.newInputStream(fileLocation)) { + Object parsedData = loader.loadFromInputStream(fileInput); + if (parsedData instanceof Map) { + // Convert to TreeMap to ensure alphabetical ordering like Go version + return new TreeMap<>((Map) parsedData); + } + return new TreeMap<>(); + } + } + + private boolean incorporateEnvironmentVars(Map configData, SettingValidator validator) { + boolean addedNewConfigs = false; + + for (Map.Entry envEntry : System.getenv().entrySet()) { + String envVarName = envEntry.getKey(); + String envValue = envEntry.getValue(); + String canonicalSetting = validator.findCanonicalSetting(envVarName); + + if (canonicalSetting != null) { + addedNewConfigs = true; + System.err.println(LocalDateTime.now().format(TIMESTAMP_FORMAT) + " Setting '" + canonicalSetting + "' from environment."); + configData.put(canonicalSetting, "${" + envVarName + "}"); + } + } + + return addedNewConfigs; + } + + private void saveUpdatedConfig(Path fileLocation, Map configData) throws Exception { + // Configure YAML output to match Go version formatting (block style) + DumpSettings dumpSettings = DumpSettings.builder() + .setDefaultFlowStyle(FlowStyle.BLOCK) + .setDefaultScalarStyle(ScalarStyle.PLAIN) + .setIndent(2) + .build(); + Dump dumper = new Dump(dumpSettings); + String yamlOutput = dumper.dumpToString(configData); + // Remove quotes (single or double) around ${VAR} to match Go behavior + // https://github.com/snakeyaml/snakeyaml-engine/blob/2070eb4e3d23bb1d81097875526a071003067877/src/main/java/org/snakeyaml/engine/v2/emitter/Emitter.java#L969-L996 + yamlOutput = yamlOutput.replaceAll("(['\"])(\\$\\{[^}]+\\})\\1", "$2"); + Set existingPermissions = getFilePermissions(fileLocation); + + Files.write(fileLocation, yamlOutput.getBytes(StandardCharsets.UTF_8), StandardOpenOption.WRITE, StandardOpenOption.TRUNCATE_EXISTING); + + applyFilePermissions(fileLocation, existingPermissions); + } + + private Set getFilePermissions(Path fileLocation) { + try { + return Files.getPosixFilePermissions(fileLocation); + } catch (UnsupportedOperationException e) { + return null; + } catch (Exception e) { + return null; + } + } + + private void applyFilePermissions(Path fileLocation, Set existingPermissions) { + if (existingPermissions != null) { + try { + Files.setPosixFilePermissions(fileLocation, existingPermissions); + } catch (Exception e) { + // Ignore failures + } + } + } +} diff --git a/docker/ironbank/go/src/env2yaml/go.mod b/docker/ironbank/go/src/env2yaml/go.mod deleted file mode 100644 index a21d1f1af29..00000000000 --- a/docker/ironbank/go/src/env2yaml/go.mod +++ /dev/null @@ -1,5 +0,0 @@ -module env2yaml - -go 1.13 - -require gopkg.in/yaml.v2 v2.3.0 diff --git a/docker/ironbank/go/src/env2yaml/go.sum b/docker/ironbank/go/src/env2yaml/go.sum deleted file mode 100644 index 8fabe8daafe..00000000000 --- a/docker/ironbank/go/src/env2yaml/go.sum +++ /dev/null @@ -1,3 +0,0 @@ -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU= -gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/docker/ironbank/go/src/env2yaml/vendor/modules.txt b/docker/ironbank/go/src/env2yaml/vendor/modules.txt deleted file mode 100644 index bdd2db2d793..00000000000 --- a/docker/ironbank/go/src/env2yaml/vendor/modules.txt +++ /dev/null @@ -1,2 +0,0 @@ -# gopkg.in/yaml.v2 v2.3.0 -gopkg.in/yaml.v2 diff --git a/docker/templates/Dockerfile.erb b/docker/templates/Dockerfile.erb index 355a76cff25..1747a3c2386 100644 --- a/docker/templates/Dockerfile.erb +++ b/docker/templates/Dockerfile.erb @@ -20,30 +20,16 @@ <% end -%> <% if image_flavor == 'full' || image_flavor == 'oss' -%> <% base_image = 'redhat/ubi9-minimal:latest' -%> - <% go_image = 'golang:1.25' -%> <% package_manager = 'microdnf' -%> <% elsif image_flavor == 'observability-sre' -%> <% base_image = 'docker.elastic.co/wolfi/chainguard-base-fips' -%> - <% go_image = 'docker.elastic.co/wolfi/go:1.25' -%> <% package_manager = 'apk' -%> <% else -%> <% base_image = 'docker.elastic.co/wolfi/chainguard-base' -%> - <% go_image = 'docker.elastic.co/wolfi/go:1.25' -%> <% package_manager = 'apk' -%> <% end -%> <% locale = 'C.UTF-8' -%> -# Build env2yaml -FROM <%= go_image %> AS builder-env2yaml - -COPY env2yaml/env2yaml.go env2yaml/go.mod env2yaml/go.sum /tmp/go/src/env2yaml/ - -WORKDIR /tmp/go/src/env2yaml - -RUN go build -trimpath - -# Build main image -# Minimal distributions do not ship with en language packs. FROM <%= base_image %> ENV ELASTIC_CONTAINER=true @@ -96,7 +82,10 @@ RUN addgroup -g 1000 logstash && \ find /usr/share/logstash -type d -exec chmod g+s {} \; && \ ln -s /usr/share/logstash /opt/logstash -COPY --from=builder-env2yaml /tmp/go/src/env2yaml/env2yaml /usr/local/bin/env2yaml +<%# Copy env2yaml from tarball (pre-compiled by Gradle) %> +COPY --chown=logstash:root env2yaml/classes /usr/share/logstash/env2yaml/classes/ +COPY --chown=logstash:root env2yaml/lib /usr/share/logstash/env2yaml/lib/ +COPY --chmod=0755 env2yaml/env2yaml /usr/local/bin/env2yaml COPY --chown=logstash:root config/pipelines.yml config/log4j2.properties config/log4j2.file.properties /usr/share/logstash/config/ <% if image_flavor == 'oss' -%> COPY --chown=logstash:root config/logstash-oss.yml /usr/share/logstash/config/logstash.yml diff --git a/docker/templates/IronbankDockerfile.erb b/docker/templates/IronbankDockerfile.erb index 1f46007382a..8c4eb27b6da 100644 --- a/docker/templates/IronbankDockerfile.erb +++ b/docker/templates/IronbankDockerfile.erb @@ -4,22 +4,7 @@ ARG BASE_REGISTRY=registry1.dso.mil ARG BASE_IMAGE=ironbank/redhat/ubi/ubi9 ARG BASE_TAG=9.6 ARG LOGSTASH_VERSION=<%= elastic_version %> -ARG GOLANG_VERSION=1.25.0 -# stage 1: build env2yaml -FROM ${BASE_REGISTRY}/google/golang/ubi9/golang-1.25:${GOLANG_VERSION} AS env2yaml - -ENV GOPATH=/go - -COPY scripts/go /go - -USER root - -RUN dnf-3 -y upgrade && dnf-3 install -y git && \ - cd /go/src/env2yaml && \ - go build - -# Final stage FROM ${BASE_REGISTRY}/${BASE_IMAGE}:${BASE_TAG} ARG LOGSTASH_VERSION @@ -29,7 +14,6 @@ ENV PATH=/usr/share/logstash/bin:$PATH WORKDIR /usr/share -COPY --from=env2yaml /go/src/env2yaml/env2yaml /usr/local/bin/env2yaml COPY scripts/config/* config/ COPY scripts/pipeline/default.conf pipeline/logstash.conf COPY scripts/bin/docker-entrypoint /usr/local/bin/ @@ -56,6 +40,11 @@ RUN dnf -y upgrade && \ rmdir config && \ rm /tmp/logstash.tar.gz +<%# Copy env2yaml from tarball (pre-compiled by Gradle) %> +COPY --chown=logstash:root scripts/env2yaml/classes /usr/share/logstash/env2yaml/classes/ +COPY --chown=logstash:root scripts/env2yaml/lib /usr/share/logstash/env2yaml/lib/ +COPY --chmod=0755 scripts/env2yaml/env2yaml /usr/local/bin/env2yaml + WORKDIR /usr/share/logstash USER 1000 diff --git a/docker/templates/hardening_manifest.yaml.erb b/docker/templates/hardening_manifest.yaml.erb index d289077b7d6..aaf88afe3a5 100644 --- a/docker/templates/hardening_manifest.yaml.erb +++ b/docker/templates/hardening_manifest.yaml.erb @@ -16,7 +16,6 @@ args: BASE_IMAGE: "redhat/ubi/ubi9" BASE_TAG: "9.6" LOGSTASH_VERSION: "<%= elastic_version %>" - GOLANG_VERSION: "1.21.8" # Docker image labels labels: diff --git a/rakelib/artifacts.rake b/rakelib/artifacts.rake index 2739e1fc6a7..248553b2708 100644 --- a/rakelib/artifacts.rake +++ b/rakelib/artifacts.rake @@ -113,6 +113,10 @@ namespace "artifact" do @exclude_paths << 'vendor/jruby/lib/ruby/gems/shared/specifications/net-imap-0.2.3.gemspec' @exclude_paths << 'vendor/jruby/lib/ruby/gems/shared/gems/net-imap-0.2.3/**/*' + # Exclude env2yaml source files - only compiled classes should be in tarball + @exclude_paths << 'docker/data/logstash/env2yaml/**/*.java' + @exclude_paths << 'docker/data/logstash/env2yaml/build.gradle' + @exclude_paths << 'docker/data/logstash/env2yaml/settings.gradle' @exclude_paths.freeze end @@ -192,8 +196,9 @@ namespace "artifact" do task "archives_docker" => ["prepare", "generate_build_metadata"] do license_details = ['ELASTIC-LICENSE'] @bundles_jdk = true + @building_docker = true create_archive_pack(license_details, ARCH, "linux") - safe_system("./gradlew bootstrap") # force the build of Logstash jars + safe_system("./gradlew dockerBootstrap") # force the build of Logstash jars + env2yaml end def create_archive_pack(license_details, arch, *oses, &tar_interceptor) @@ -259,15 +264,17 @@ namespace "artifact" do task "archives_docker_oss" => ["prepare-oss", "generate_build_metadata"] do #with bundled JDKs @bundles_jdk = true + @building_docker = true license_details = ['APACHE-LICENSE-2.0', "-oss", oss_exclude_paths] create_archive_pack(license_details, ARCH, "linux") - safe_system("./gradlew bootstrap") # force the build of Logstash jars + safe_system("./gradlew dockerBootstrap") # force the build of Logstash jars + env2yaml end desc "Build jdk bundled tar.gz of observabilitySRE logstash plugins with all dependencies for docker" task "archives_docker_observabilitySRE" => ["prepare-observabilitySRE", "generate_build_metadata"] do #with bundled JDKs @bundles_jdk = true + @building_docker = true exclude_paths = default_exclude_paths + %w( bin/logstash-plugin bin/logstash-plugin.bat @@ -280,7 +287,7 @@ namespace "artifact" do # copy additional files into the tarball puts "HELLO(#{dedicated_directory_tar})" end - safe_system("./gradlew bootstrap") # force the build of Logstash jars + safe_system("./gradlew dockerBootstrap") # force the build of Logstash jars + env2yaml end desc "Build an RPM of logstash with all dependencies" @@ -563,7 +570,22 @@ namespace "artifact" do # add build.rb to tar metadata_file_path_in_tar = File.join("logstash-core", "lib", "logstash", "build.rb") dedicated_directory_tarball.write(BUILD_METADATA_FILE.path, metadata_file_path_in_tar) - + # add env2yaml for docker builds + if @building_docker + env2yaml_classes = "docker/data/logstash/env2yaml/classes" + if File.directory?(env2yaml_classes) + # Add compiled class files + Dir.glob("#{env2yaml_classes}/**/*.class").each do |class_file| + relative_path = class_file.sub("#{env2yaml_classes}/", "") + dedicated_directory_tarball.write(class_file, "env2yaml/classes/#{relative_path}") + end + # Add dependency JARs + Dir.glob("docker/data/logstash/env2yaml/lib/*.jar").each do |jar_file| + dedicated_directory_tarball.write(jar_file, "env2yaml/lib/#{File.basename(jar_file)}") + end + end + dedicated_directory_tarball.write("docker/data/logstash/env2yaml/env2yaml", "env2yaml/env2yaml") if File.exist?("docker/data/logstash/env2yaml/env2yaml") + end # yield to the tar interceptor if we have one yield(dedicated_directory_tarball) if block_given? end diff --git a/settings.gradle b/settings.gradle index d538972b85c..4ccc34a3c44 100644 --- a/settings.gradle +++ b/settings.gradle @@ -1,6 +1,6 @@ rootProject.name = "logstash" -include ':logstash-core', 'logstash-core-benchmarks', 'benchmark-cli', 'jvm-options-parser', 'logstash-integration-tests', 'dependencies-report' +include ':logstash-core', 'logstash-core-benchmarks', 'benchmark-cli', 'jvm-options-parser', 'logstash-integration-tests', 'dependencies-report', 'docker:data:logstash:env2yaml' project(':logstash-core').projectDir = new File('./logstash-core') project(':logstash-core-benchmarks').projectDir = new File('./logstash-core/benchmarks') project(':logstash-integration-tests').projectDir = new File('./qa/integration') diff --git a/tools/dependencies-report/src/main/resources/licenseMapping.csv b/tools/dependencies-report/src/main/resources/licenseMapping.csv index 14fb7c8c28a..6133921b5fa 100644 --- a/tools/dependencies-report/src/main/resources/licenseMapping.csv +++ b/tools/dependencies-report/src/main/resources/licenseMapping.csv @@ -174,6 +174,7 @@ dependency,dependencyUrl,licenseOverride,copyright,sourceURL "org.logstash:jvm-options-parser:",http://github.com/elastic/logstash,Apache-2.0 "org.reflections:reflections:",https://github.com/ronmamo/reflections,BSD-2-Clause "org.slf4j:slf4j-api:",http://www.slf4j.org/,MIT +"org.snakeyaml:snakeyaml-engine:2.9",https://bitbucket.org/snakeyaml/snakeyaml-engine,Apache-2.0 "org.yaml:snakeyaml:",https://bitbucket.org/snakeyaml/snakeyaml/src/master/,Apache-2.0 "ostruct:",https://github.com/ruby/ostruct,BSD-2-Clause "paquet:",https://github.com/elastic/logstash,Apache-2.0 diff --git a/tools/dependencies-report/src/main/resources/notices/org.snakeyaml!snakeyaml-engine-NOTICE.txt b/tools/dependencies-report/src/main/resources/notices/org.snakeyaml!snakeyaml-engine-NOTICE.txt new file mode 100644 index 00000000000..2bb9ad240fa --- /dev/null +++ b/tools/dependencies-report/src/main/resources/notices/org.snakeyaml!snakeyaml-engine-NOTICE.txt @@ -0,0 +1,176 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS \ No newline at end of file