adds autoload of registries list and pull registry (#19)

with this commit the cli when run
  will auto create a default registry list
  if one does not exist

  it will also pull down all configured registry
  files in the list if none yet are pulled.

  [#16]

Author: xchapter7x

internal/commands/pull.go

@@ -6,6 +6,7 @@ import (
 	"io/ioutil"
 
 	"github.com/open-policy-agent/conftest/downloader"
+	"github.com/policy-hub/policy-hub-cli/pkg/helpers"
 	"github.com/policy-hub/policy-hub-cli/pkg/metaconfig"
 	"github.com/spf13/cobra"
 )
@@ -20,7 +21,7 @@ func newPullCommand() *cobra.Command {
 		},
 	}
 
-	cmd.Flags().StringVarP(&c.Repository, "repository", "r", "metadata/registries.yml", "Location of the repository to search.")
+	cmd.Flags().StringVarP(&c.Repository, "repository", "r", helpers.DefaultMetaDataFile(), "Location of the repository to search.")
 
 	return cmd
 }

internal/commands/search.go

@@ -5,6 +5,7 @@ import (
 	"io/ioutil"
 	"os"
 
+	"github.com/policy-hub/policy-hub-cli/pkg/helpers"
 	"github.com/policy-hub/policy-hub-cli/pkg/metaconfig"
 	"github.com/policy-hub/policy-hub-cli/pkg/search"
 	"github.com/spf13/cobra"
@@ -20,7 +21,7 @@ func newSearchCommand() *cobra.Command {
 		},
 	}
 
-	cmd.Flags().StringVarP(&c.Repository, "repository", "r", "metadata/registries.yml", "Location of the repository to search.")
+	cmd.Flags().StringVarP(&c.Repository, "repository", "r", helpers.DefaultMetaDataFile(), "Location of the repository to search.")
 
 	return cmd
 }
@@ -37,7 +38,7 @@ func (c *searchConfig) run(query string) error {
 
 	e, err := search.Load()
 	defer e.Close()
-	
+
 	if err != nil {
 		return fmt.Errorf("load engine: %w", err)
 	}

main.go

@@ -1,13 +1,77 @@
 package main
 
 import (
+	"context"
+	"io"
+	"io/ioutil"
+	"log"
 	"os"
+	"path/filepath"
+	"strings"
 
+	"github.com/open-policy-agent/conftest/downloader"
 	"github.com/policy-hub/policy-hub-cli/internal/commands"
+	"github.com/policy-hub/policy-hub-cli/pkg/helpers"
 )
 
+func init() {
+	checkCreateDir()
+	checkCreateRegistriesList()
+	checkCreateRegistriesMetadata()
+}
+
 func main() {
 	if err := commands.NewRootCommand().Execute(); err != nil {
 		os.Exit(1)
 	}
 }
+
+func checkCreateRegistriesMetadata() {
+	metadataPath := helpers.MetadataConfigPath()
+	_, err := os.Stat(metadataPath)
+	if os.IsNotExist(err) {
+
+		registriesListPath := filepath.ToSlash(
+			filepath.Join(helpers.ConfigPath(), helpers.PolicyRegistryFilename))
+		content, err := ioutil.ReadFile(registriesListPath)
+		if err != nil {
+			log.Panic("could not read registries list file: %w", err)
+		}
+
+		lines := strings.Split(string(content), "\n")
+		err = downloader.Download(context.Background(), metadataPath, lines)
+		if err != nil {
+			log.Panic("could not fetch metadata file: %w", err)
+		}
+	}
+}
+
+func checkCreateRegistriesList() {
+	configPath := helpers.ConfigPath()
+	registriesListPath := filepath.ToSlash(
+		filepath.Join(configPath, helpers.PolicyRegistryFilename))
+
+	_, err := os.Stat(registriesListPath)
+	if os.IsNotExist(err) {
+		file, err := os.Create(registriesListPath)
+		defer file.Close()
+		if err != nil {
+			log.Panic("could not create file: %w", err)
+		}
+
+		_, err = io.WriteString(file, helpers.RegistryListDefault)
+		if err != nil {
+			log.Panic("could not write to file: %w", err)
+		}
+
+		file.Sync()
+	}
+}
+
+func checkCreateDir() {
+	configPath := helpers.ConfigPath()
+	_, err := os.Stat(configPath)
+	if os.IsNotExist(err) {
+		os.MkdirAll(configPath, os.ModePerm)
+	}
+}

main_test.go

@@ -4,11 +4,13 @@ import (
 	"bytes"
 	"os"
 	"os/exec"
+	"path/filepath"
 	"testing"
 	"time"
 
 	"github.com/onsi/gomega"
 	"github.com/onsi/gomega/gexec"
+	"github.com/policy-hub/policy-hub-cli/pkg/helpers"
 	"github.com/stretchr/testify/assert"
 )
 
@@ -17,9 +19,11 @@ func TestMainCLI(t *testing.T) {
 	pathToMainCLI, err := gexec.Build("main.go")
 	assert.NoError(t, err, "Error creating binary")
 	defer gexec.CleanupBuildArtifacts()
+	defer os.RemoveAll(helpers.IndexPath())
+	defer os.RemoveAll(helpers.ConfigPath())
 
 	t.Run("we can search registries", func(t *testing.T) {
-		command := exec.Command(pathToMainCLI, "search", "k8s", "-r", "./metadata/registries.yml")
+		command := exec.Command(pathToMainCLI, "search", "k8s")
 		session, err := gexec.Start(command, os.Stdout, os.Stdout)
 		session.Wait()
 		assert.NoError(t, err, "Error running search command")
@@ -37,4 +41,28 @@ func TestMainCLI(t *testing.T) {
 		_, err = os.Stat(policyPackageName)
 		assert.False(t, os.IsNotExist(err), "could not find the directory of policies")
 	})
+
+	t.Run("cli should create registry information", func(t *testing.T) {
+		configPath := helpers.ConfigPath()
+		os.RemoveAll(configPath)
+		defer os.RemoveAll(configPath)
+		command := exec.Command(pathToMainCLI, "help")
+		session, _ := gexec.Start(command, os.Stdout, os.Stdout)
+		session.Wait(10 * time.Second)
+
+		t.Run("should create the config directory", func(t *testing.T) {
+			_, err := os.Stat(configPath)
+			assert.False(t, os.IsNotExist(err), "could not find the directory config directory")
+		})
+
+		t.Run("should create the registries list", func(t *testing.T) {
+			_, err = os.Stat(filepath.ToSlash(filepath.Join(helpers.ConfigPath(), helpers.PolicyRegistryFilename)))
+			assert.False(t, os.IsNotExist(err), "could not find the registries list file")
+		})
+
+		t.Run("should create the registries file from the list", func(t *testing.T) {
+			_, err = os.Stat(helpers.MetadataConfigPath())
+			assert.False(t, os.IsNotExist(err), "could not find the registries metadata files")
+		})
+	})
 }

pkg/helpers/configpath.go

@@ -0,0 +1,39 @@
+package helpers
+
+import (
+	"os"
+	"path/filepath"
+)
+
+const PolicyIndexDirName = ".policy-hub-index"
+const PolicyDirName = ".policy-hub"
+const PolicyRegistryFilename = "registries.list"
+const RegistryMetadataDir = "metadata"
+const DefaultRegistriesFilename = "registries.yml"
+const RegistryListDefault = "https://raw.githubusercontent.com/policy-hub/policy-hub-cli/main/metadata/" + DefaultRegistriesFilename
+
+//DefaultMetaDataFile  Returns the path to the default registries yaml for the current system
+func DefaultMetaDataFile() string {
+	return filepath.ToSlash(filepath.Join(
+		MetadataConfigPath(), DefaultRegistriesFilename))
+}
+
+//MetadataConfigPath  Returns the path to the metadata directory for the current system
+func MetadataConfigPath() string {
+	directory := filepath.ToSlash(filepath.Join(ConfigPath(), RegistryMetadataDir))
+	return directory
+}
+
+//ConfigPath  Returns the path to the config directory for the current system
+func ConfigPath() string {
+	homeDir, _ := os.UserHomeDir()
+	directory := filepath.ToSlash(filepath.Join(homeDir, PolicyDirName))
+	return directory
+}
+
+//IndexPath  Returns the path to the search index directory for the current system
+func IndexPath() string {
+	homeDir, _ := os.UserHomeDir()
+	directory := filepath.ToSlash(filepath.Join(homeDir, PolicyIndexDirName))
+	return directory
+}

pkg/search/index.go

@@ -10,14 +10,14 @@ import (
 
 	"github.com/blevesearch/bleve"
 	"github.com/olekukonko/tablewriter"
+	"github.com/policy-hub/policy-hub-cli/pkg/helpers"
 	"github.com/policy-hub/policy-hub-cli/pkg/metaconfig"
 )
 
 // IndexVersion indicates the version of the search index.
 // This is used to migrate between index versions.
 const IndexVersion = "v1"
 
-
 type Engine struct {
 	index bleve.Index
 }
@@ -81,7 +81,7 @@ func (e *Engine) Close() error {
 
 // constructIndex builds a search index
 func constructIndex() (bleve.Index, error) {
-	cacheDir := cacheDirectory()
+	cacheDir := helpers.IndexPath()
 	if _, err := os.Stat(cacheDir); os.IsNotExist(err) {
 		return setupIndexDirectory()
 	}
@@ -93,7 +93,7 @@ func constructIndex() (bleve.Index, error) {
 func loadIndex() (bleve.Index, error) {
 	index, err := bleve.Open(indexDirectory())
 	if err == bleve.ErrorIndexPathDoesNotExist {
-		index, err := constructIndex(); 
+		index, err := constructIndex()
 		if err != nil {
 			return nil, fmt.Errorf("construct index: %w", err)
 		}
@@ -108,7 +108,7 @@ func loadIndex() (bleve.Index, error) {
 
 // setupIndexDirectory setups the index directory
 func setupIndexDirectory() (bleve.Index, error) {
-	cacheDir := cacheDirectory()
+	cacheDir := helpers.IndexPath()
 	if err := os.MkdirAll(cacheDir, os.ModePerm); err != nil {
 		return nil, fmt.Errorf("make search dir: %w", err)
 	}
@@ -117,7 +117,7 @@ func setupIndexDirectory() (bleve.Index, error) {
 	if err != nil {
 		return nil, fmt.Errorf("create version file: %w", err)
 	}
- 
+
 	mapping := bleve.NewIndexMapping()
 	index, err := bleve.New(indexDirectory(), mapping)
 	if err != nil {
@@ -127,19 +127,7 @@ func setupIndexDirectory() (bleve.Index, error) {
 	return index, nil
 }
 
-// cacheDirectory returns the directory to cache policy-cli configs
-func cacheDirectory() string {
-	const cacheDir = ".policy-hub"
-
-	homeDir, _ := os.UserHomeDir()
-
-	directory := filepath.Join(homeDir, cacheDir)
-	directory = filepath.ToSlash(directory)
-
-	return directory
-}
-
 // indexDirectory returns the directory to store the search index in
 func indexDirectory() string {
-	return filepath.Join(cacheDirectory(), "index")
-}
+	return filepath.Join(helpers.IndexPath(), "index")
+}