From 581ab4f24fe94af8dc70dabb31eab580d2cdb0f8 Mon Sep 17 00:00:00 2001 From: Parth Dhanjal Date: Tue, 16 Aug 2022 18:40:31 -0700 Subject: [PATCH 1/3] WIP: Allow customizations on the scorecard config checks --- pkg/actions/run_scorecard.go | 107 ++++++++++-------- .../scorecardDefaultConfigFragment.yaml | 49 ++++++++ 2 files changed, 106 insertions(+), 50 deletions(-) create mode 100644 pkg/actions/scorecardDefaultConfigFragment.yaml diff --git a/pkg/actions/run_scorecard.go b/pkg/actions/run_scorecard.go index 7cb3e197..a1213f6f 100644 --- a/pkg/actions/run_scorecard.go +++ b/pkg/actions/run_scorecard.go @@ -33,14 +33,18 @@ import ( const defaultSDKScorecardImageName = "quay.io/operator-framework/scorecard-test" const scorecardAnnotation = "operators.operatorframework.io.test.config.v1" +const scorecardDefaultConfigFragment = "github.com/operator-framework/audit/pkg/actions" type BundleAnnotations struct { Annotations map[string]string `yaml:"annotations,omitempty"` } func RunScorecard(bundleDir string, auditBundle *models.AuditBundle) *models.AuditBundle { + log.Info("\n----bundleDir----\n", bundleDir) scorecardTestsPath := filepath.Join(bundleDir, "tests", "scorecard") + log.Info("\n----scorecardTestsPath----\n", scorecardTestsPath) annotationsPath := filepath.Join(bundleDir, "metadata", "annotations.yaml") + log.Info("\n----annotationsPath----\n", annotationsPath) // If find the annotations file then, check for the scorecard path on it. if _, err := os.Stat(annotationsPath); err == nil && !os.IsNotExist(err) { @@ -116,8 +120,11 @@ func RunScorecard(bundleDir string, auditBundle *models.AuditBundle) *models.Aud return auditBundle } + scorecardConfig := false + // Add Logic to update scorecardConfig + // run scorecard against bundle - cmd := exec.Command("operator-sdk", "scorecard", bundleDir, "--wait-time=120s", "--output=json") + cmd := exec.Command("operator-sdk", "scorecard", bundleDir, "--wait-time=120s", "--output=json", "--scorecard-config", scorecardDefaultConfigFragment, "--scorecard-config", scorecardConfig) output, _ := pkg.RunCommand(cmd) if len(output) < 1 { log.Errorf("unable to get scorecard output: %s", output) @@ -158,52 +165,52 @@ func writeScorecardConfig(scorecardConfigPath string) error { return nil } -const scorecardDefaultConfigFragment = `apiVersion: scorecard.operatorframework.io/v1alpha3 -kind: Configuration -metadata: - name: config -stages: -- parallel: true - tests: - - entrypoint: - - scorecard-test - - basic-check-spec - image: quay.io/operator-framework/scorecard-test:v1.22.0 - labels: - suite: basic - test: basic-check-spec-test - - entrypoint: - - scorecard-test - - olm-bundle-validation - image: quay.io/operator-framework/scorecard-test:v1.22.0 - labels: - suite: olm - test: olm-bundle-validation-test - - entrypoint: - - scorecard-test - - olm-crds-have-validation - image: quay.io/operator-framework/scorecard-test:v1.22.0 - labels: - suite: olm - test: olm-crds-have-validation-test - - entrypoint: - - scorecard-test - - olm-crds-have-resources - image: quay.io/operator-framework/scorecard-test:v1.22.0 - labels: - suite: olm - test: olm-crds-have-resources-test - - entrypoint: - - scorecard-test - - olm-spec-descriptors - image: quay.io/operator-framework/scorecard-test:v1.22.0 - labels: - suite: olm - test: olm-spec-descriptors-test - - entrypoint: - - scorecard-test - - olm-status-descriptors - image: quay.io/operator-framework/scorecard-test:v1.22.0 - labels: - suite: olm - test: olm-status-descriptors-test` +// const scorecardDefaultConfigFragment = `apiVersion: scorecard.operatorframework.io/v1alpha3 +// kind: Configuration +// metadata: +// name: config +// stages: +// - parallel: true +// tests: +// - entrypoint: +// - scorecard-test +// - basic-check-spec +// image: quay.io/operator-framework/scorecard-test:v1.22.0 +// labels: +// suite: basic +// test: basic-check-spec-test +// - entrypoint: +// - scorecard-test +// - olm-bundle-validation +// image: quay.io/operator-framework/scorecard-test:v1.22.0 +// labels: +// suite: olm +// test: olm-bundle-validation-test +// - entrypoint: +// - scorecard-test +// - olm-crds-have-validation +// image: quay.io/operator-framework/scorecard-test:v1.22.0 +// labels: +// suite: olm +// test: olm-crds-have-validation-test +// - entrypoint: +// - scorecard-test +// - olm-crds-have-resources +// image: quay.io/operator-framework/scorecard-test:v1.22.0 +// labels: +// suite: olm +// test: olm-crds-have-resources-test +// - entrypoint: +// - scorecard-test +// - olm-spec-descriptors +// image: quay.io/operator-framework/scorecard-test:v1.22.0 +// labels: +// suite: olm +// test: olm-spec-descriptors-test +// - entrypoint: +// - scorecard-test +// - olm-status-descriptors +// image: quay.io/operator-framework/scorecard-test:v1.22.0 +// labels: +// suite: olm +// test: olm-status-descriptors-test` diff --git a/pkg/actions/scorecardDefaultConfigFragment.yaml b/pkg/actions/scorecardDefaultConfigFragment.yaml new file mode 100644 index 00000000..3d529d5e --- /dev/null +++ b/pkg/actions/scorecardDefaultConfigFragment.yaml @@ -0,0 +1,49 @@ +apiVersion: scorecard.operatorframework.io/v1alpha3 +kind: Configuration +metadata: + name: config +stages: +- parallel: true + tests: + - entrypoint: + - scorecard-test + - basic-check-spec + image: quay.io/operator-framework/scorecard-test:v1.22.0 + labels: + suite: basic + test: basic-check-spec-test + - entrypoint: + - scorecard-test + - olm-bundle-validation + image: quay.io/operator-framework/scorecard-test:v1.22.0 + labels: + suite: olm + test: olm-bundle-validation-test + - entrypoint: + - scorecard-test + - olm-crds-have-validation + image: quay.io/operator-framework/scorecard-test:v1.22.0 + labels: + suite: olm + test: olm-crds-have-validation-test + - entrypoint: + - scorecard-test + - olm-crds-have-resources + image: quay.io/operator-framework/scorecard-test:v1.22.0 + labels: + suite: olm + test: olm-crds-have-resources-test + - entrypoint: + - scorecard-test + - olm-spec-descriptors + image: quay.io/operator-framework/scorecard-test:v1.22.0 + labels: + suite: olm + test: olm-spec-descriptors-test + - entrypoint: + - scorecard-test + - olm-status-descriptors + image: quay.io/operator-framework/scorecard-test:v1.22.0 + labels: + suite: olm + test: olm-status-descriptors-test \ No newline at end of file From 8b42bf752ad9427b7209a1f49cd4ff5c7e335cdc Mon Sep 17 00:00:00 2001 From: parthdhanjal Date: Wed, 17 Aug 2022 17:13:17 -0700 Subject: [PATCH 2/3] WIP: Allow customizations on the scorecard config checks --- pkg/actions/run_scorecard.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/actions/run_scorecard.go b/pkg/actions/run_scorecard.go index a1213f6f..d24b40c8 100644 --- a/pkg/actions/run_scorecard.go +++ b/pkg/actions/run_scorecard.go @@ -33,7 +33,6 @@ import ( const defaultSDKScorecardImageName = "quay.io/operator-framework/scorecard-test" const scorecardAnnotation = "operators.operatorframework.io.test.config.v1" -const scorecardDefaultConfigFragment = "github.com/operator-framework/audit/pkg/actions" type BundleAnnotations struct { Annotations map[string]string `yaml:"annotations,omitempty"` @@ -121,10 +120,11 @@ func RunScorecard(bundleDir string, auditBundle *models.AuditBundle) *models.Aud } scorecardConfig := false + scorecardFilePath := "github.com/operator-framework/audit/pkg/actions/scorecardDefaultConfigFragment.yaml" // Add Logic to update scorecardConfig // run scorecard against bundle - cmd := exec.Command("operator-sdk", "scorecard", bundleDir, "--wait-time=120s", "--output=json", "--scorecard-config", scorecardDefaultConfigFragment, "--scorecard-config", scorecardConfig) + cmd := exec.Command("operator-sdk", "scorecard", bundleDir, "--wait-time=120s", "--output=json", "--scorecard-config", scorecardFilePath, "--scorecard-config", scorecardConfig) output, _ := pkg.RunCommand(cmd) if len(output) < 1 { log.Errorf("unable to get scorecard output: %s", output) From 1fa32f8764e2190c96fd044f4317ae18353fac34 Mon Sep 17 00:00:00 2001 From: parthdhanjal Date: Wed, 17 Aug 2022 17:24:29 -0700 Subject: [PATCH 3/3] WIP: Allow customizations on the scorecard config checks --- pkg/actions/run_scorecard.go | 102 +++++++++++++++++------------------ 1 file changed, 51 insertions(+), 51 deletions(-) diff --git a/pkg/actions/run_scorecard.go b/pkg/actions/run_scorecard.go index d24b40c8..d97a6af7 100644 --- a/pkg/actions/run_scorecard.go +++ b/pkg/actions/run_scorecard.go @@ -119,12 +119,12 @@ func RunScorecard(bundleDir string, auditBundle *models.AuditBundle) *models.Aud return auditBundle } - scorecardConfig := false + scorecardConfig := "false" scorecardFilePath := "github.com/operator-framework/audit/pkg/actions/scorecardDefaultConfigFragment.yaml" // Add Logic to update scorecardConfig // run scorecard against bundle - cmd := exec.Command("operator-sdk", "scorecard", bundleDir, "--wait-time=120s", "--output=json", "--scorecard-config", scorecardFilePath, "--scorecard-config", scorecardConfig) + cmd := exec.Command("operator-sdk", "scorecard", bundleDir, "--wait-time=120s", "--output=json", "--scorecard-config", scorecardFilePath, "--scorecard-custom-tests", scorecardConfig) output, _ := pkg.RunCommand(cmd) if len(output) < 1 { log.Errorf("unable to get scorecard output: %s", output) @@ -165,52 +165,52 @@ func writeScorecardConfig(scorecardConfigPath string) error { return nil } -// const scorecardDefaultConfigFragment = `apiVersion: scorecard.operatorframework.io/v1alpha3 -// kind: Configuration -// metadata: -// name: config -// stages: -// - parallel: true -// tests: -// - entrypoint: -// - scorecard-test -// - basic-check-spec -// image: quay.io/operator-framework/scorecard-test:v1.22.0 -// labels: -// suite: basic -// test: basic-check-spec-test -// - entrypoint: -// - scorecard-test -// - olm-bundle-validation -// image: quay.io/operator-framework/scorecard-test:v1.22.0 -// labels: -// suite: olm -// test: olm-bundle-validation-test -// - entrypoint: -// - scorecard-test -// - olm-crds-have-validation -// image: quay.io/operator-framework/scorecard-test:v1.22.0 -// labels: -// suite: olm -// test: olm-crds-have-validation-test -// - entrypoint: -// - scorecard-test -// - olm-crds-have-resources -// image: quay.io/operator-framework/scorecard-test:v1.22.0 -// labels: -// suite: olm -// test: olm-crds-have-resources-test -// - entrypoint: -// - scorecard-test -// - olm-spec-descriptors -// image: quay.io/operator-framework/scorecard-test:v1.22.0 -// labels: -// suite: olm -// test: olm-spec-descriptors-test -// - entrypoint: -// - scorecard-test -// - olm-status-descriptors -// image: quay.io/operator-framework/scorecard-test:v1.22.0 -// labels: -// suite: olm -// test: olm-status-descriptors-test` +const scorecardDefaultConfigFragment = `apiVersion: scorecard.operatorframework.io/v1alpha3 +kind: Configuration +metadata: + name: config +stages: +- parallel: true + tests: + - entrypoint: + - scorecard-test + - basic-check-spec + image: quay.io/operator-framework/scorecard-test:v1.22.0 + labels: + suite: basic + test: basic-check-spec-test + - entrypoint: + - scorecard-test + - olm-bundle-validation + image: quay.io/operator-framework/scorecard-test:v1.22.0 + labels: + suite: olm + test: olm-bundle-validation-test + - entrypoint: + - scorecard-test + - olm-crds-have-validation + image: quay.io/operator-framework/scorecard-test:v1.22.0 + labels: + suite: olm + test: olm-crds-have-validation-test + - entrypoint: + - scorecard-test + - olm-crds-have-resources + image: quay.io/operator-framework/scorecard-test:v1.22.0 + labels: + suite: olm + test: olm-crds-have-resources-test + - entrypoint: + - scorecard-test + - olm-spec-descriptors + image: quay.io/operator-framework/scorecard-test:v1.22.0 + labels: + suite: olm + test: olm-spec-descriptors-test + - entrypoint: + - scorecard-test + - olm-status-descriptors + image: quay.io/operator-framework/scorecard-test:v1.22.0 + labels: + suite: olm + test: olm-status-descriptors-test`