Add find and foreach sub commands for bulk operations

.gitignore

@@ -23,6 +23,9 @@ elastic-package
 # IDEA
 .idea
 
+# VSCode
+.vscode
+
 # Build directory
 /build
 

README.md

@@ -366,6 +366,24 @@ Use this command to export ingest pipelines with referenced pipelines from the E
 
 Use this command to download selected ingest pipelines and its referenced processor pipelines from Elasticsearch. Select data stream or the package root directories to download the pipelines. Pipelines are downloaded as is and will need adjustment to meet your package needs.
 
+### `elastic-package filter [flags]`
+
+_Context: package_
+
+This command gives you a list of all packages based on the given query
+
+### `elastic-package foreach [flags] -- <SUBCOMMAND>`
+
+_Context: package_
+
+Execute a command for each package matching the given filter criteria.
+
+This command combines filtering capabilities with command execution, allowing you to run
+any elastic-package subcommand across multiple packages in a single operation.
+
+The command uses the same filter flags as the 'filter' command to select packages, 
+then executes the specified subcommand for each matched package.
+
 ### `elastic-package format`
 
 _Context: package_

cmd/filter.go

@@ -0,0 +1,98 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License;
+// you may not use this file except in compliance with the Elastic License.
+
+package cmd
+
+import (
+	"encoding/json"
+	"fmt"
+	"io"
+	"os"
+	"slices"
+
+	"github.com/spf13/cobra"
+
+	"github.com/elastic/elastic-package/internal/cobraext"
+	"github.com/elastic/elastic-package/internal/filter"
+	"github.com/elastic/elastic-package/internal/packages"
+)
+
+const filterLongDescription = `This command gives you a list of all packages based on the given query`
+
+func setupFilterCommand() *cobraext.Command {
+	cmd := &cobra.Command{
+		Use:   "filter [flags]",
+		Short: "filter integrations based on given flags",
+		Long:  filterLongDescription,
+		Args:  cobra.NoArgs,
+		RunE:  filterCommandAction,
+	}
+
+	// add filter flags to the command (input, code owner, kibana version, categories)
+	filter.SetFilterFlags(cmd)
+
+	// add the output package name flag to the command
+	cmd.Flags().BoolP(cobraext.FilterOutputPackageNameFlagName, cobraext.FilterOutputPackageNameFlagShorthand, false, cobraext.FilterOutputPackageNameFlagDescription)
+
+	return cobraext.NewCommand(cmd, cobraext.ContextPackage)
+}
+
+func filterCommandAction(cmd *cobra.Command, args []string) error {
+	filtered, err := filterPackage(cmd)
+	if err != nil {
+		return fmt.Errorf("filtering packages failed: %w", err)
+	}
+
+	printPackageName, err := cmd.Flags().GetBool(cobraext.FilterOutputPackageNameFlagName)
+	if err != nil {
+		return fmt.Errorf("getting output package name flag failed: %w", err)
+	}
+
+	if err = printPkgList(filtered, printPackageName, os.Stdout); err != nil {
+		return fmt.Errorf("printing JSON failed: %w", err)
+	}
+
+	return nil
+}
+
+func filterPackage(cmd *cobra.Command) ([]packages.PackageDirNameAndManifest, error) {
+	filters := filter.NewFilterRegistry()
+
+	if err := filters.Parse(cmd); err != nil {
+		return nil, fmt.Errorf("parsing filter options failed: %w", err)
+	}
+
+	if err := filters.Validate(); err != nil {
+		return nil, fmt.Errorf("validating filter options failed: %w", err)
+	}
+
+	filtered, errors := filters.Execute()
+	if errors != nil {
+		return nil, fmt.Errorf("filtering packages failed: %s", errors.Error())
+	}
+
+	return filtered, nil
+}
+
+func printPkgList(pkgs []packages.PackageDirNameAndManifest, printPackageName bool, w io.Writer) error {
+	enc := json.NewEncoder(w)
+	enc.SetEscapeHTML(false)
+	if len(pkgs) == 0 {
+		return nil
+	}
+
+	names := make([]string, 0, len(pkgs))
+	if printPackageName {
+		for _, pkg := range pkgs {
+			names = append(names, pkg.Manifest.Name)
+		}
+	} else {
+		for _, pkg := range pkgs {
+			names = append(names, pkg.DirName)
+		}
+	}
+
+	slices.Sort(names)
+	return enc.Encode(names)
+}

cmd/foreach.go

@@ -0,0 +1,162 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License;
+// you may not use this file except in compliance with the Elastic License.
+
+package cmd
+
+import (
+	"fmt"
+	"io"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"slices"
+	"strings"
+	"sync"
+
+	"github.com/spf13/cobra"
+
+	"github.com/elastic/elastic-package/internal/cobraext"
+	"github.com/elastic/elastic-package/internal/filter"
+	"github.com/elastic/elastic-package/internal/logger"
+	"github.com/elastic/elastic-package/internal/multierror"
+	"github.com/elastic/elastic-package/internal/packages"
+)
+
+const foreachLongDescription = `Execute a command for each package matching the given filter criteria.
+
+This command combines filtering capabilities with command execution, allowing you to run
+any elastic-package subcommand across multiple packages in a single operation.
+
+The command uses the same filter flags as the 'filter' command to select packages, 
+then executes the specified subcommand for each matched package.`
+
+// getAllowedSubCommands returns the list of allowed subcommands for the foreach command.
+func getAllowedSubCommands() []string {
+	return []string{
+		"build",
+		"check",
+		"clean",
+		"format",
+		"install",
+		"lint",
+		"test",
+		"uninstall",
+	}
+}
+
+func setupForeachCommand() *cobraext.Command {
+	cmd := &cobra.Command{
+		Use:   "foreach [flags] -- <SUBCOMMAND>",
+		Short: "Execute a command for filtered packages",
+		Long:  foreachLongDescription,
+		Example: `  # Run system tests for packages with specific inputs
+  elastic-package foreach --input tcp,udp --parallel 10 -- test system -g`,
+		RunE: foreachCommandAction,
+		Args: cobra.MinimumNArgs(1),
+	}
+
+	// Add filter flags
+	filter.SetFilterFlags(cmd)
+
+	// Add pool size flag
+	cmd.Flags().IntP(cobraext.ForeachPoolSizeFlagName, "", 1, cobraext.ForeachPoolSizeFlagDescription)
+
+	return cobraext.NewCommand(cmd, cobraext.ContextPackage)
+}
+
+func foreachCommandAction(cmd *cobra.Command, args []string) error {
+	poolSize, err := cmd.Flags().GetInt(cobraext.ForeachPoolSizeFlagName)
+	if err != nil {
+		return fmt.Errorf("getting pool size failed: %w", err)
+	}
+
+	if err := validateSubCommand(args[0]); err != nil {
+		return fmt.Errorf("validating sub command failed: %w", err)
+	}
+
+	// Find integration root
+	root, err := packages.MustFindIntegrationRoot()
+	if err != nil {
+		return fmt.Errorf("can't find integration root: %w", err)
+	}
+
+	// reuse filterPackage from cmd/filter.go
+	filtered, err := filterPackage(cmd)
+	if err != nil {
+		return fmt.Errorf("filtering packages failed: %w", err)
+	}
+
+	wg := sync.WaitGroup{}
+	mu := sync.Mutex{}
+	errs := multierror.Error{}
+	successes := 0
+
+	packageChan := make(chan string, poolSize)
+
+	for range poolSize {
+		wg.Add(1)
+		go func(packageChan <-chan string) {
+			defer wg.Done()
+			for packageName := range packageChan {
+				path := filepath.Join(root, "packages", packageName)
+				err := executeCommand(args, path)
+
+				mu.Lock()
+				if err != nil {
+					errs = append(errs, fmt.Errorf("executing command for package %s failed: %w", packageName, err))
+				} else {
+					successes++
+				}
+				mu.Unlock()
+			}
+		}(packageChan)
+	}
+
+	for _, pkg := range filtered {
+		packageChan <- pkg.DirName
+	}
+	close(packageChan)
+
+	wg.Wait()
+
+	logger.Infof("Successfully executed command for %d packages\n", successes)
+	logger.Infof("Errors occurred while executing command for %d packages\n", len(errs))
+
+	if errs.Error() != "" {
+		return fmt.Errorf("errors occurred while executing command for packages: \n%s", errs.Error())
+	}
+
+	return nil
+}
+
+func executeCommand(args []string, path string) error {
+	// Look up the elastic-package binary in PATH
+	execPath, err := exec.LookPath("elastic-package")
+	if err != nil {
+		return fmt.Errorf("elastic-package binary not found in PATH: %w", err)
+	}
+
+	cmd := &exec.Cmd{
+		Path:   execPath,
+		Args:   append([]string{execPath}, args...),
+		Dir:    path,
+		Stdout: io.Discard,
+		Stderr: os.Stderr,
+		Env:    os.Environ(),
+	}
+
+	if err := cmd.Run(); err != nil {
+		return fmt.Errorf("executing command for package %s failed: %w", path, err)
+	}
+
+	return nil
+}
+
+func validateSubCommand(subCommand string) error {
+	if !slices.Contains(getAllowedSubCommands(), subCommand) {
+		return fmt.Errorf("invalid subcommand: %s. Allowed subcommands are: %s", subCommand, strings.Join(getAllowedSubCommands(), ", "))
+	}
+
+	return nil
+}

cmd/root.go

@@ -26,7 +26,9 @@ var commands = []*cobraext.Command{
 	setupDumpCommand(),
 	setupEditCommand(),
 	setupExportCommand(),
+	setupFilterCommand(),
 	setupFormatCommand(),
+	setupForeachCommand(),
 	setupInstallCommand(),
 	setupLinksCommand(),
 	setupLintCommand(),

go.mod

@@ -21,6 +21,7 @@ require (
 	github.com/elastic/package-spec/v3 v3.5.0
 	github.com/fatih/color v1.18.0
 	github.com/go-viper/mapstructure/v2 v2.4.0
+	github.com/gobwas/glob v0.2.3
 	github.com/google/go-cmp v0.7.0
 	github.com/google/go-github/v32 v32.1.0
 	github.com/google/go-querystring v1.1.0

go.sum

@@ -179,6 +179,8 @@ github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1v
 github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8=
 github.com/go-viper/mapstructure/v2 v2.4.0 h1:EBsztssimR/CONLSZZ04E8qAkxNYq4Qp9LvH92wZUgs=
 github.com/go-viper/mapstructure/v2 v2.4.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM=
+github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y=
+github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8=
 github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
 github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=

internal/cobraext/flags.go

@@ -133,8 +133,37 @@ const (
 	FailOnMissingFlagName        = "fail-on-missing"
 	FailOnMissingFlagDescription = "fail if tests are missing"
 
-	FailFastFlagName                  = "fail-fast"
-	FailFastFlagDescription           = "fail immediately if any file requires updates (do not overwrite)"
+	FailFastFlagName        = "fail-fast"
+	FailFastFlagDescription = "fail immediately if any file requires updates (do not overwrite)"
+
+	FilterCategoriesFlagName        = "category"
+	FilterCategoriesFlagDescription = "integration categories to filter by (comma-separated values)"
+
+	FilterCodeOwnerFlagName        = "code-owner"
+	FilterCodeOwnerFlagDescription = "code owners to filter by (comma-separated values)"
+
+	FilterIntegrationTypeFlagName        = "integration-type"
+	FilterIntegrationTypeFlagDescription = "integration types to filter by (comma-separated values)"
+
+	FilterInputFlagName        = "input"
+	FilterInputFlagDescription = "name of the inputs to filter by (comma-separated values)"
+
+	FilterKibanaVersionFlagName        = "kibana-version"
+	FilterKibanaVersionFlagDescription = "kibana version to filter by (semver)"
+
+	FilterOutputPackageNameFlagName        = "output-package-name"
+	FilterOutputPackageNameFlagShorthand   = "p"
+	FilterOutputPackageNameFlagDescription = "print the package name instead of the directory name in the output"
+
+	FilterPackagesFlagName        = "package-name"
+	FilterPackagesFlagDescription = "package names to filter by (comma-separated values)"
+
+	FilterSpecVersionFlagName        = "spec-version"
+	FilterSpecVersionFlagDescription = "Package spec version to filter by (semver)"
+
+	ForeachPoolSizeFlagName        = "parallel"
+	ForeachPoolSizeFlagDescription = "number of packages to execute in parallel (defaults to serial execution)"
+
 	GenerateTestResultFlagName        = "generate"
 	GenerateTestResultFlagDescription = "generate test result file"