- add way to filter packages in arbitrary directory.

- update --packages to use only package name; added another filter to filter by package dirs.
- Added auto discovery of package for configurable depth.
- Added flag to exclude dirs from filter process.

Author: vinit-chauhan

cmd/filter.go

@@ -32,8 +32,9 @@ func setupFilterCommand() *cobraext.Command {
 	// add filter flags to the command (input, code owner, kibana version, categories)
 	filter.SetFilterFlags(cmd)
 
-	// add the output package name flag to the command
-	cmd.Flags().BoolP(cobraext.FilterOutputPackageNameFlagName, cobraext.FilterOutputPackageNameFlagShorthand, false, cobraext.FilterOutputPackageNameFlagDescription)
+	// add the output package name and absolute path flags to the command
+	cmd.Flags().BoolP(cobraext.FilterOutputPackageNameFlagName, "", false, cobraext.FilterOutputPackageNameFlagDescription)
+	cmd.Flags().BoolP(cobraext.FilterOutputAbsolutePathFlagName, "", false, cobraext.FilterOutputAbsolutePathFlagDescription)
 
 	return cobraext.NewCommand(cmd, cobraext.ContextPackage)
 }
@@ -49,15 +50,30 @@ func filterCommandAction(cmd *cobra.Command, args []string) error {
 		return fmt.Errorf("getting output package name flag failed: %w", err)
 	}
 
-	if err = printPkgList(filtered, printPackageName, os.Stdout); err != nil {
+	outputAbsolutePath, err := cmd.Flags().GetBool("output-absolute-path")
+	if err != nil {
+		return fmt.Errorf("getting output absolute path flag failed: %w", err)
+	}
+
+	if err = printPkgList(filtered, printPackageName, outputAbsolutePath, os.Stdout); err != nil {
 		return fmt.Errorf("printing JSON failed: %w", err)
 	}
 
 	return nil
 }
 
 func filterPackage(cmd *cobra.Command) ([]packages.PackageDirNameAndManifest, error) {
-	filters := filter.NewFilterRegistry()
+	depth, err := cmd.Flags().GetInt(cobraext.FilterDepthFlagName)
+	if err != nil {
+		return nil, fmt.Errorf("getting depth flag failed: %w", err)
+	}
+
+	excludeDirs, err := cmd.Flags().GetString(cobraext.FilterExcludeDirFlagName)
+	if err != nil {
+		return nil, fmt.Errorf("getting exclude-dir flag failed: %w", err)
+	}
+
+	filters := filter.NewFilterRegistry(depth, excludeDirs)
 
 	if err := filters.Parse(cmd); err != nil {
 		return nil, fmt.Errorf("parsing filter options failed: %w", err)
@@ -75,7 +91,7 @@ func filterPackage(cmd *cobra.Command) ([]packages.PackageDirNameAndManifest, er
 	return filtered, nil
 }
 
-func printPkgList(pkgs []packages.PackageDirNameAndManifest, printPackageName bool, w io.Writer) error {
+func printPkgList(pkgs []packages.PackageDirNameAndManifest, printPackageName bool, outputAbsolutePath bool, w io.Writer) error {
 	enc := json.NewEncoder(w)
 	enc.SetEscapeHTML(false)
 	if len(pkgs) == 0 {
@@ -87,6 +103,10 @@ func printPkgList(pkgs []packages.PackageDirNameAndManifest, printPackageName bo
 		for _, pkg := range pkgs {
 			names = append(names, pkg.Manifest.Name)
 		}
+	} else if outputAbsolutePath {
+		for _, pkg := range pkgs {
+			names = append(names, pkg.Path)
+		}
 	} else {
 		for _, pkg := range pkgs {
 			names = append(names, pkg.DirName)

cmd/foreach.go

@@ -9,7 +9,6 @@ import (
 	"io"
 	"os"
 	"os/exec"
-	"path/filepath"
 	"slices"
 	"strings"
 	"sync"
@@ -20,7 +19,6 @@ import (
 	"github.com/elastic/elastic-package/internal/filter"
 	"github.com/elastic/elastic-package/internal/logger"
 	"github.com/elastic/elastic-package/internal/multierror"
-	"github.com/elastic/elastic-package/internal/packages"
 )
 
 const foreachLongDescription = `Execute a command for each package matching the given filter criteria.
@@ -60,7 +58,7 @@ func setupForeachCommand() *cobraext.Command {
 	filter.SetFilterFlags(cmd)
 
 	// Add pool size flag
-	cmd.Flags().IntP(cobraext.ForeachPoolSizeFlagName, "", 1, cobraext.ForeachPoolSizeFlagDescription)
+	cmd.Flags().IntP(cobraext.ForeachPoolSizeFlagName, cobraext.ForeachPoolSizeFlagShorthand, 1, cobraext.ForeachPoolSizeFlagDescription)
 
 	return cobraext.NewCommand(cmd, cobraext.ContextPackage)
 }
@@ -75,12 +73,6 @@ func foreachCommandAction(cmd *cobra.Command, args []string) error {
 		return fmt.Errorf("validating sub command failed: %w", err)
 	}
 
-	// Find integration root
-	root, err := packages.MustFindIntegrationRoot()
-	if err != nil {
-		return fmt.Errorf("can't find integration root: %w", err)
-	}
-
 	// reuse filterPackage from cmd/filter.go
 	filtered, err := filterPackage(cmd)
 	if err != nil {
@@ -92,31 +84,30 @@ func foreachCommandAction(cmd *cobra.Command, args []string) error {
 	errs := multierror.Error{}
 	successes := 0
 
-	packageChan := make(chan string, poolSize)
+	packagePathChan := make(chan string, poolSize)
 
 	for range poolSize {
 		wg.Add(1)
-		go func(packageChan <-chan string) {
+		go func(packagePathChan <-chan string) {
 			defer wg.Done()
-			for packageName := range packageChan {
-				path := filepath.Join(root, "packages", packageName)
-				err := executeCommand(args, path)
+			for packagePath := range packagePathChan {
+				err := executeCommand(args, packagePath)
 
 				mu.Lock()
 				if err != nil {
-					errs = append(errs, fmt.Errorf("executing command for package %s failed: %w", packageName, err))
+					errs = append(errs, fmt.Errorf("executing command for package %s failed: %w", packagePath, err))
 				} else {
 					successes++
 				}
 				mu.Unlock()
 			}
-		}(packageChan)
+		}(packagePathChan)
 	}
 
 	for _, pkg := range filtered {
-		packageChan <- pkg.DirName
+		packagePathChan <- pkg.Path
 	}
-	close(packageChan)
+	close(packagePathChan)
 
 	wg.Wait()
 

internal/cobraext/flags.go

@@ -136,32 +136,46 @@ const (
 	FailFastFlagName        = "fail-fast"
 	FailFastFlagDescription = "fail immediately if any file requires updates (do not overwrite)"
 
-	FilterCategoriesFlagName        = "category"
+	FilterCategoriesFlagName        = "categories"
 	FilterCategoriesFlagDescription = "integration categories to filter by (comma-separated values)"
 
-	FilterCodeOwnerFlagName        = "code-owner"
+	FilterCodeOwnerFlagName        = "code-owners"
 	FilterCodeOwnerFlagDescription = "code owners to filter by (comma-separated values)"
 
-	FilterPackageTypeFlagName        = "package-type"
+	FilterPackageTypeFlagName        = "package-types"
 	FilterPackageTypeFlagDescription = "package types to filter by (comma-separated values)"
 
-	FilterInputFlagName        = "input"
+	FilterInputFlagName        = "inputs"
 	FilterInputFlagDescription = "name of the inputs to filter by (comma-separated values)"
 
 	FilterKibanaVersionFlagName        = "kibana-version"
 	FilterKibanaVersionFlagDescription = "kibana version to filter by (semver)"
 
+	FilterOutputAbsolutePathFlagName        = "output-absolute-path"
+	FilterOutputAbsolutePathFlagDescription = "output the absolute path of the package"
+
 	FilterOutputPackageNameFlagName        = "output-package-name"
-	FilterOutputPackageNameFlagShorthand   = "p"
 	FilterOutputPackageNameFlagDescription = "print the package name instead of the directory name in the output"
 
+	FilterPackageDirNameFlagName        = "package-dirs"
+	FilterPackageDirNameFlagDescription = "package directories to filter by (comma-separated values)"
+
 	FilterPackagesFlagName        = "packages"
 	FilterPackagesFlagDescription = "package names to filter by (comma-separated values)"
 
 	FilterSpecVersionFlagName        = "spec-version"
 	FilterSpecVersionFlagDescription = "Package spec version to filter by (semver)"
 
+	FilterDepthFlagName        = "depth"
+	FilterDepthFlagDescription = "maximum depth to search for packages (default: 2)"
+	FilterDepthFlagDefault     = 2
+	FilterDepthFlagShorthand   = "d"
+
+	FilterExcludeDirFlagName        = "exclude-dirs"
+	FilterExcludeDirFlagDescription = "comma-separated list of directories to exclude from search"
+
 	ForeachPoolSizeFlagName        = "parallel"
+	ForeachPoolSizeFlagShorthand   = "p"
 	ForeachPoolSizeFlagDescription = "Number of subcommands to execute in parallel (defaults to serial execution)"
 
 	GenerateTestResultFlagName        = "generate"

internal/filter/packagedirname.go

@@ -0,0 +1,77 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License;
+// you may not use this file except in compliance with the Elastic License.
+
+package filter
+
+import (
+	"fmt"
+
+	"github.com/gobwas/glob"
+	"github.com/spf13/cobra"
+
+	"github.com/elastic/elastic-package/internal/cobraext"
+	"github.com/elastic/elastic-package/internal/packages"
+)
+
+type PackageDirNameFlag struct {
+	FilterFlagBase
+
+	patterns []glob.Glob
+}
+
+func (f *PackageDirNameFlag) Parse(cmd *cobra.Command) error {
+	packageDirNamePatterns, err := cmd.Flags().GetString(cobraext.FilterPackageDirNameFlagName)
+	if err != nil {
+		return cobraext.FlagParsingError(err, cobraext.FilterPackageDirNameFlagName)
+	}
+
+	patterns := splitAndTrim(packageDirNamePatterns, ",")
+	for _, patternString := range patterns {
+		pattern, err := glob.Compile(patternString)
+		if err != nil {
+			return fmt.Errorf("invalid package dir name pattern: %s: %w", patternString, err)
+		}
+		f.patterns = append(f.patterns, pattern)
+	}
+
+	if len(f.patterns) > 0 {
+		f.isApplied = true
+	}
+
+	return nil
+}
+
+func (f *PackageDirNameFlag) Validate() error {
+	return nil
+}
+
+func (f *PackageDirNameFlag) Matches(dirName string, manifest *packages.PackageManifest) bool {
+	for _, pattern := range f.patterns {
+		if pattern.Match(dirName) {
+			return true
+		}
+	}
+	return false
+}
+
+func (f *PackageDirNameFlag) ApplyTo(pkgs []packages.PackageDirNameAndManifest) ([]packages.PackageDirNameAndManifest, error) {
+	filtered := make([]packages.PackageDirNameAndManifest, 0, len(pkgs))
+	for _, pkg := range pkgs {
+		if f.Matches(pkg.DirName, pkg.Manifest) {
+			filtered = append(filtered, pkg)
+		}
+	}
+	return filtered, nil
+}
+
+func initPackageDirNameFlag() *PackageDirNameFlag {
+	return &PackageDirNameFlag{
+		FilterFlagBase: FilterFlagBase{
+			name:         cobraext.FilterPackageDirNameFlagName,
+			description:  cobraext.FilterPackageDirNameFlagDescription,
+			shorthand:    "",
+			defaultValue: "",
+		},
+	}
+}

internal/filter/packagename.go

@@ -48,7 +48,7 @@ func (f *PackageNameFlag) Validate() error {
 
 func (f *PackageNameFlag) Matches(dirName string, manifest *packages.PackageManifest) bool {
 	for _, pattern := range f.patterns {
-		if pattern.Match(dirName) {
+		if pattern.Match(manifest.Name) {
 			return true
 		}
 	}

internal/filter/registry.go

@@ -6,9 +6,11 @@ package filter
 
 import (
 	"fmt"
+	"os"
 
 	"github.com/spf13/cobra"
 
+	"github.com/elastic/elastic-package/internal/cobraext"
 	"github.com/elastic/elastic-package/internal/logger"
 	"github.com/elastic/elastic-package/internal/multierror"
 	"github.com/elastic/elastic-package/internal/packages"
@@ -18,27 +20,35 @@ var registry = []Filter{
 	initCategoryFlag(),
 	initCodeOwnerFlag(),
 	initInputFlag(),
+	initPackageDirNameFlag(),
 	initPackageNameFlag(),
 	initPackageTypeFlag(),
 	initSpecVersionFlag(),
 }
 
 // SetFilterFlags registers all filter flags with the given command.
 func SetFilterFlags(cmd *cobra.Command) {
+	cmd.Flags().IntP(cobraext.FilterDepthFlagName, cobraext.FilterDepthFlagShorthand, cobraext.FilterDepthFlagDefault, cobraext.FilterDepthFlagDescription)
+	cmd.Flags().StringP(cobraext.FilterExcludeDirFlagName, "", "", cobraext.FilterExcludeDirFlagDescription)
+
 	for _, filterFlag := range registry {
 		filterFlag.Register(cmd)
 	}
 }
 
 // FilterRegistry manages a collection of filters for package filtering.
 type FilterRegistry struct {
-	filters []Filter
+	filters     []Filter
+	depth       int
+	excludeDirs string
 }
 
 // NewFilterRegistry creates a new FilterRegistry instance.
-func NewFilterRegistry() *FilterRegistry {
+func NewFilterRegistry(depth int, excludeDirs string) *FilterRegistry {
 	return &FilterRegistry{
-		filters: []Filter{},
+		filters:     []Filter{},
+		depth:       depth,
+		excludeDirs: excludeDirs,
 	}
 }
 
@@ -71,12 +81,12 @@ func (r *FilterRegistry) Validate() error {
 }
 
 func (r *FilterRegistry) Execute() (filtered []packages.PackageDirNameAndManifest, errors multierror.Error) {
-	root, err := packages.MustFindIntegrationRoot()
+	currentDir, err := os.Getwd()
 	if err != nil {
-		return nil, multierror.Error{err}
+		return nil, multierror.Error{fmt.Errorf("getting current directory failed: %w", err)}
 	}
 
-	pkgs, err := packages.ReadAllPackageManifests(root)
+	pkgs, err := packages.ReadAllPackageManifestsFromRepo(currentDir, r.depth, r.excludeDirs)
 	if err != nil {
 		return nil, multierror.Error{err}
 	}