Add setting to configure a directory with geoip databases (#1211)

Author: jsoriano

README.md

@@ -266,7 +266,7 @@ Use this command to add, remove, and manage multiple config profiles.
 
 Individual user profiles appear in ~/.elastic-package/stack, and contain all the config files needed by the "stack" subcommand. 
 Once a new profile is created, it can be specified with the -p flag, or the ELASTIC_PACKAGE_PROFILE environment variable.
-User profiles are not overwritten on upgrade of elastic-stack, and can be freely modified to allow for different stack configs.
+User profiles can be configured with a "config.yml" file in the profile directory.
 
 ### `elastic-package profiles create`
 
@@ -474,6 +474,28 @@ Use this command to print the version of elastic-package that you have installed
 
 
 
+## Elastic Package profiles
+
+The `profiles` subcommand allows to work with different configurations. By default,
+`elastic-package` uses the "default" profile. Other profiles can be created with the
+`elastic-package profiles create` command. Once a profile is created, it will have its
+own directory inside the elastic-package data directory. Once you have more profiles,
+you can change the default with `elastic-package profiles use`.
+
+You can find the profiles in your system with `elastic-package profiles list`.
+
+You can delete profiles with `elastic-package profiles delete`.
+
+Each profile can have a `config.yml` file that allows to persist configuration settings
+that apply only to commands using this profile. You can find a `config.yml.example` that
+you can copy to start.
+
+The following settings are available per profile:
+
+* `stack.geoip_dir` defines a directory with GeoIP databases that can be used by
+  Elasticsearch in stacks managed by elastic-package. It is recommended to use
+  an absolute path, out of the `.elastic-package` directory.
+
 ## Development
 
 Even though the project is "go-gettable", there is the `Makefile` present, which can be used to build, format or vendor

cmd/profiles.go

@@ -31,7 +31,7 @@ func setupProfilesCommand() *cobraext.Command {
 	
 Individual user profiles appear in ~/.elastic-package/stack, and contain all the config files needed by the "stack" subcommand. 
 Once a new profile is created, it can be specified with the -p flag, or the ELASTIC_PACKAGE_PROFILE environment variable.
-User profiles are not overwritten on upgrade of elastic-stack, and can be freely modified to allow for different stack configs.`
+User profiles can be configured with a "config.yml" file in the profile directory.`
 
 	profileCommand := &cobra.Command{
 		Use:   "profiles",

internal/profile/_static/config.yml.example

@@ -0,0 +1,2 @@
+# Directory containing GeoIP databases for stacks managed by elastic-agent.
+# stack.geoip_dir: "/path/to/geoip_dir/"

internal/profile/_testdata/config.yml

@@ -0,0 +1,18 @@
+# An expected setting.
+stack.geoip_dir: "/home/foo/Documents/ingest-geoip"
+
+# An empty string, should exist, but return empty.
+other.empty: ""
+
+# A nested value, should work as "other.nested".
+other:
+  nested: "foo"
+
+# A number. Will be parsed as string.
+other.number: 42
+
+# A float. Will be parsed as string.
+other.float: 0.12345
+
+# A bool. Will be parsed as string.
+other.bool: false

internal/profile/config.go

@@ -0,0 +1,50 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License;
+// you may not use this file except in compliance with the Elastic License.
+
+package profile
+
+import (
+	"errors"
+	"fmt"
+	"os"
+
+	"github.com/elastic/go-ucfg/yaml"
+
+	"github.com/elastic/elastic-package/internal/common"
+)
+
+type config struct {
+	settings common.MapStr
+}
+
+func loadProfileConfig(path string) (config, error) {
+	cfg, err := yaml.NewConfigWithFile(path)
+	if errors.Is(err, os.ErrNotExist) {
+		return config{}, nil
+	}
+	if err != nil {
+		return config{}, fmt.Errorf("can't load profile configuration (%s): %w", path, err)
+	}
+
+	settings := make(common.MapStr)
+	err = cfg.Unpack(settings)
+	if err != nil {
+		return config{}, fmt.Errorf("can't unpack configuration: %w", err)
+	}
+
+	return config{settings: settings}, nil
+}
+
+func (c *config) get(name string) (string, bool) {
+	raw, err := c.settings.GetValue(name)
+	if err != nil {
+		return "", false
+	}
+	switch v := raw.(type) {
+	case string:
+		return v, true
+	default:
+		return fmt.Sprintf("%v", v), true
+	}
+}

internal/profile/config_test.go

@@ -0,0 +1,67 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License;
+// you may not use this file except in compliance with the Elastic License.
+
+package profile
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestLoadProfileConfig(t *testing.T) {
+	cases := []struct {
+		name     string
+		expected string
+		found    bool
+	}{
+		{
+			name:     "stack.geoip_dir",
+			expected: "/home/foo/Documents/ingest-geoip",
+			found:    true,
+		},
+		{
+			name:     "other.empty",
+			expected: "",
+			found:    true,
+		},
+		{
+			name:     "other.nested",
+			expected: "foo",
+			found:    true,
+		},
+		{
+			name:     "other.number",
+			expected: "42",
+			found:    true,
+		},
+		{
+			name:     "other.float",
+			expected: "0.12345",
+			found:    true,
+		},
+		{
+			name:     "other.bool",
+			expected: "false",
+			found:    true,
+		},
+		{
+			name:  "not.present",
+			found: false,
+		},
+	}
+
+	config, err := loadProfileConfig("_testdata/config.yml")
+	require.NoError(t, err)
+
+	for _, c := range cases {
+		t.Run(c.name, func(t *testing.T) {
+			value, found := config.get(c.name)
+			if assert.Equal(t, c.found, found) {
+				assert.Equal(t, c.expected, value)
+			}
+		})
+	}
+}

internal/profile/profile.go

@@ -5,6 +5,7 @@
 package profile
 
 import (
+	"embed"
 	"errors"
 	"fmt"
 	"os"
@@ -21,16 +22,27 @@ const (
 	// PackageProfileMetaFile is the filename of the profile metadata file
 	PackageProfileMetaFile = "profile.json"
 
+	// PackageProfileConfigFile is the filename of the profile configuration file
+	PackageProfileConfigFile = "config.yml"
+
 	// DefaultProfile is the name of the default profile.
 	DefaultProfile = "default"
 )
 
+//go:embed _static
+var static embed.FS
+
 var (
+	staticSource     = resource.NewSourceFS(static)
 	profileResources = []resource.Resource{
 		&resource.File{
 			Path:    PackageProfileMetaFile,
 			Content: profileMetadataContent,
 		},
+		&resource.File{
+			Path:    PackageProfileConfigFile + ".example",
+			Content: staticSource.File("_static/config.yml.example"),
+		},
 	}
 )
 
@@ -123,6 +135,8 @@ type Profile struct {
 	// ProfilePath is the absolute path to the profile
 	ProfilePath string
 	ProfileName string
+
+	config config
 }
 
 // Path returns an absolute path to the given file
@@ -131,6 +145,15 @@ func (profile Profile) Path(names ...string) string {
 	return filepath.Join(elems...)
 }
 
+// Config returns a configuration setting, or its default if setting not found
+func (profile Profile) Config(name string, def string) string {
+	v, found := profile.config.get(name)
+	if !found {
+		return def
+	}
+	return v
+}
+
 // ErrNotAProfile is returned in cases where we don't have a valid profile directory
 var ErrNotAProfile = errors.New("not a profile")
 
@@ -211,9 +234,16 @@ func loadProfile(elasticPackagePath string, profileName string) (*Profile, error
 		return nil, ErrNotAProfile
 	}
 
+	configPath := filepath.Join(profilePath, PackageProfileConfigFile)
+	config, err := loadProfileConfig(configPath)
+	if err != nil {
+		return nil, fmt.Errorf("error loading configuration for profile %q: %w", profileName, err)
+	}
+
 	profile := Profile{
 		ProfileName: profileName,
 		ProfilePath: profilePath,
+		config:      config,
 	}
 
 	return &profile, nil

internal/stack/_static/docker-compose-stack.yml.tmpl

@@ -14,7 +14,7 @@ services:
     volumes:
       - "./elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml"
       - "../certs/elasticsearch:/usr/share/elasticsearch/config/certs"
-      - "./ingest-geoip:/usr/share/elasticsearch/config/ingest-geoip"
+      - "{{ fact "geoip_dir" }}:/usr/share/elasticsearch/config/ingest-geoip"
       - "./service_tokens:/usr/share/elasticsearch/config/service_tokens"
     ports:
       - "127.0.0.1:9200:9200"

internal/stack/resources.go

@@ -117,6 +117,8 @@ func applyResources(profile *profile.Profile, stackVersion string) error {
 
 		"username": elasticsearchUsername,
 		"password": elasticsearchPassword,
+
+		"geoip_dir": profile.Config("stack.geoip_dir", "./ingest-geoip"),
 	})
 
 	os.MkdirAll(stackDir, 0755)

internal/stack/resources_test.go

@@ -0,0 +1,71 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License;
+// you may not use this file except in compliance with the Elastic License.
+
+package stack
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"gopkg.in/yaml.v3"
+
+	"github.com/elastic/elastic-package/internal/profile"
+)
+
+func TestApplyResourcesWithCustomGeoipDir(t *testing.T) {
+	const expectedGeoipPath = "/some/path/ingest-geoip"
+	const profileName = "custom_geoip"
+
+	elasticPackagePath := t.TempDir()
+	profilesPath := filepath.Join(elasticPackagePath, "profiles")
+
+	os.Setenv("ELASTIC_PACKAGE_DATA_HOME", elasticPackagePath)
+
+	// Create profile.
+	err := profile.CreateProfile(profile.Options{
+		// PackagePath is actually the profiles path, what is a bit counterintuitive.
+		PackagePath: profilesPath,
+		Name:        profileName,
+	})
+	require.NoError(t, err)
+
+	// Write configuration to the profile.
+	configPath := filepath.Join(profilesPath, profileName, profile.PackageProfileConfigFile)
+	config := fmt.Sprintf("stack.geoip_dir: %q", expectedGeoipPath)
+	err = os.WriteFile(configPath, []byte(config), 0644)
+	require.NoError(t, err)
+
+	p, err := profile.LoadProfile(profileName)
+	require.NoError(t, err)
+	t.Logf("Profile name: %s, path: %s", p.ProfileName, p.ProfilePath)
+
+	// Smoke test to check that we are actually loading the profile we want and it has the setting.
+	v := p.Config("stack.geoip_dir", "")
+	require.Equal(t, expectedGeoipPath, v)
+
+	// Now, apply resources and check that the variable has been used.
+	err = applyResources(p, "8.6.1")
+	require.NoError(t, err)
+
+	d, err := os.ReadFile(p.Path(profileStackPath, SnapshotFile))
+	require.NoError(t, err)
+
+	var composeFile struct {
+		Services struct {
+			Elasticsearch struct {
+				Volumes []string `yaml:"volumes"`
+			} `yaml:"elasticsearch"`
+		} `yaml:"services"`
+	}
+	err = yaml.Unmarshal(d, &composeFile)
+	require.NoError(t, err)
+
+	volumes := composeFile.Services.Elasticsearch.Volumes
+	expectedVolume := fmt.Sprintf("%s:/usr/share/elasticsearch/config/ingest-geoip", expectedGeoipPath)
+	assert.Contains(t, volumes, expectedVolume)
+}