From 288e6202915c72d1fb1fc71a8c362eb7490c97c2 Mon Sep 17 00:00:00 2001 From: nicholasham Date: Thu, 20 Aug 2020 13:37:25 +0100 Subject: [PATCH 1/3] Added run-name command line which is the value that gets outputted in the test label of a metric --- pkg/f1/metrics/labels/labels.go | 6 ++++++ pkg/f1/metrics/metrics.go | 17 +++++++++-------- pkg/f1/options/run_options.go | 1 + pkg/f1/run/run_cmd.go | 12 ++++++++++++ pkg/f1/run/test_runner.go | 4 ++-- pkg/f1/testing/active_scenario.go | 12 ++++++------ 6 files changed, 36 insertions(+), 16 deletions(-) create mode 100644 pkg/f1/metrics/labels/labels.go diff --git a/pkg/f1/metrics/labels/labels.go b/pkg/f1/metrics/labels/labels.go new file mode 100644 index 00000000..14ff03b2 --- /dev/null +++ b/pkg/f1/metrics/labels/labels.go @@ -0,0 +1,6 @@ +package labels +const ( + Test = "test" + Stage = "stage" + Result = "result" +) diff --git a/pkg/f1/metrics/metrics.go b/pkg/f1/metrics/metrics.go index d39bcba4..e103e2db 100644 --- a/pkg/f1/metrics/metrics.go +++ b/pkg/f1/metrics/metrics.go @@ -1,6 +1,7 @@ package metrics import ( + "github.com/form3tech-oss/f1/pkg/f1/metrics/labels" "sync" "github.com/prometheus/client_golang/prometheus" @@ -37,14 +38,14 @@ func Instance() *Metrics { Name: "setup", Help: "Duration of setup functions.", Objectives: percentileObjectives, - }, []string{"test", "result"}), + }, []string{labels.Test, labels.Result}), Iteration: prometheus.NewSummaryVec(prometheus.SummaryOpts{ Namespace: "form3", Subsystem: "loadtest", Name: "iteration", Help: "Duration of iteration functions.", Objectives: percentileObjectives, - }, []string{"test", "stage", "result"}), + }, []string{labels.Test, labels.Stage, labels.Result}), Progress: prometheus.NewSummaryVec(prometheus.SummaryOpts{ Namespace: "form3", Subsystem: "loadtest", @@ -58,7 +59,7 @@ func Instance() *Metrics { Name: "teardown", Help: "Duration of teardown functions.", Objectives: percentileObjectives, - }, []string{"test", "result"}), + }, []string{labels.Test, labels.Result}), } prometheus.MustRegister( m.Setup, @@ -85,14 +86,14 @@ func (metrics *Metrics) Reset() { metrics.Teardown.Reset() } -func (metrics *Metrics) Record(metric MetricType, name string, stage string, result string, nanoseconds int64) { +func (metrics *Metrics) Record(metric MetricType, test string, stage string, result string, nanoseconds int64) { switch metric { case SetupResult: - metrics.Setup.WithLabelValues(name, result).Observe(float64(nanoseconds)) + metrics.Setup.WithLabelValues(test, result).Observe(float64(nanoseconds)) case IterationResult: - metrics.Iteration.WithLabelValues(name, stage, result).Observe(float64(nanoseconds)) - metrics.Progress.WithLabelValues(name, stage, result).Observe(float64(nanoseconds)) + metrics.Iteration.WithLabelValues(test, stage, result).Observe(float64(nanoseconds)) + metrics.Progress.WithLabelValues(test, stage, result).Observe(float64(nanoseconds)) case TeardownResult: - metrics.Teardown.WithLabelValues(name, result).Observe(float64(nanoseconds)) + metrics.Teardown.WithLabelValues(test, result).Observe(float64(nanoseconds)) } } diff --git a/pkg/f1/options/run_options.go b/pkg/f1/options/run_options.go index f593e704..7454bf23 100644 --- a/pkg/f1/options/run_options.go +++ b/pkg/f1/options/run_options.go @@ -7,6 +7,7 @@ import ( ) type RunOptions struct { + RunName string Scenario string MaxDuration time.Duration Concurrency int diff --git a/pkg/f1/run/run_cmd.go b/pkg/f1/run/run_cmd.go index 8d1abcce..94d50a2c 100644 --- a/pkg/f1/run/run_cmd.go +++ b/pkg/f1/run/run_cmd.go @@ -39,6 +39,7 @@ func Cmd(builders []api.Builder, hookFunc logging.RegisterLogHookFunc) *cobra.Co triggerCmd.Flags().IntP("concurrency", "c", 100, "--concurrency 2 (allow at most 2 groups of iterations to run concurrently)") triggerCmd.Flags().Int32P("max-iterations", "i", 0, "--max-iterations 100 (stop after 100 iterations, regardless of remaining duration)") triggerCmd.Flags().Bool("ignore-dropped", false, "dropped requests will not fail the run") + triggerCmd.Flags().String("run-name", "", "Sets the name of the run that appears in metrics") triggerCmd.Flags().AddFlagSet(t.Flags) runCmd.AddCommand(triggerCmd) } @@ -55,6 +56,16 @@ func runCmdExecute(t api.Builder, hookFunc logging.RegisterLogHookFunc) func(cmd if err != nil { return errors.New(fmt.Sprintf("Invalid duration value: %s", err)) } + + runName, err := cmd.Flags().GetString("run-name") + if err != nil { + return errors.New(fmt.Sprintf("Invalid run name value: %s", err)) + } + + if runName == "" { + runName = scenarioName + } + concurrency, err := cmd.Flags().GetInt("concurrency") if err != nil || concurrency < 1 { return errors.New(fmt.Sprintf("Invalid concurrency value: %s", err)) @@ -84,6 +95,7 @@ func runCmdExecute(t api.Builder, hookFunc logging.RegisterLogHookFunc) func(cmd } run, err := NewRun(options.RunOptions{ + RunName: runName, Scenario: scenarioName, MaxDuration: duration, Concurrency: concurrency, diff --git a/pkg/f1/run/test_runner.go b/pkg/f1/run/test_runner.go index 026bba65..bef9ca5c 100644 --- a/pkg/f1/run/test_runner.go +++ b/pkg/f1/run/test_runner.go @@ -39,7 +39,7 @@ func NewRun(options options.RunOptions, t *api.Trigger) (*Run, error) { } prometheusUrl := os.Getenv("PROMETHEUS_PUSH_GATEWAY") if prometheusUrl != "" { - run.pusher = push.New(prometheusUrl, "f1-"+options.Scenario).Gatherer(prometheus.DefaultGatherer) + run.pusher = push.New(prometheusUrl, "f1").Gatherer(prometheus.DefaultGatherer) } if run.Options.RegisterLogHookFunc == nil { run.Options.RegisterLogHookFunc = logging.NoneRegisterLogHookFunc @@ -90,7 +90,7 @@ func (r *Run) Do() *RunResult { metrics.Instance().Reset() var err error - r.activeScenario, err = testing.NewActiveScenarios(r.Options.Scenario, r.Options.Env, testing.GetScenario(r.Options.Scenario), 0) + r.activeScenario, err = testing.NewActiveScenarios(r.Options.RunName, r.Options.Env, testing.GetScenario(r.Options.Scenario), 0) r.pushMetrics() fmt.Println(r.result.Setup()) diff --git a/pkg/f1/testing/active_scenario.go b/pkg/f1/testing/active_scenario.go index b4bf2f5b..95cebc28 100644 --- a/pkg/f1/testing/active_scenario.go +++ b/pkg/f1/testing/active_scenario.go @@ -15,7 +15,7 @@ type ActiveScenario struct { Stages []Stage TeardownFn TeardownFn env map[string]string - Name string + TestRun string id string autoTeardownTimer *CancellableTimer autoTeardownTimerMu sync.RWMutex @@ -23,10 +23,10 @@ type ActiveScenario struct { m *metrics.Metrics } -func NewActiveScenarios(name string, env map[string]string, fn MultiStageSetupFn, autoTeardownIdleDuration time.Duration) (*ActiveScenario, error) { +func NewActiveScenarios(testRun string, env map[string]string, fn MultiStageSetupFn, autoTeardownIdleDuration time.Duration) (*ActiveScenario, error) { s := &ActiveScenario{ - Name: name, + TestRun: testRun, id: uuid.New().String(), env: env, AutoTeardownAfter: autoTeardownIdleDuration, @@ -68,7 +68,7 @@ func (s *ActiveScenario) SetAutoTeardown(timer *CancellableTimer) { } func (s *ActiveScenario) Run(metric metrics.MetricType, stage, vu, iter string, f func(t *T)) error { - t := NewT(s.env, vu, iter, s.Name) + t := NewT(s.env, vu, iter, s.TestRun) start := time.Now() done := make(chan struct{}) go func() { @@ -80,7 +80,7 @@ func (s *ActiveScenario) Run(metric metrics.MetricType, stage, vu, iter string, } // wait for completion <-done - s.m.Record(metric, s.Name, stage, metrics.Result(t.HasFailed()), time.Since(start).Nanoseconds()) + s.m.Record(metric, s.TestRun, stage, metrics.Result(t.HasFailed()), time.Since(start).Nanoseconds()) if t.HasFailed() { return errors.New("failed") } @@ -114,5 +114,5 @@ func (s *ActiveScenario) autoTeardown() { } func (s *ActiveScenario) RecordDroppedIteration() { - s.m.Record(metrics.IterationResult, s.Name, "single", "dropped", 0) + s.m.Record(metrics.IterationResult, s.TestRun, "single", "dropped", 0) } From 55f1ca151f69d12cebb5e013cdedd7575fb35e2a Mon Sep 17 00:00:00 2001 From: nicholasham Date: Thu, 20 Aug 2020 13:38:17 +0100 Subject: [PATCH 2/3] Go imports --- pkg/f1/metrics/labels/labels.go | 5 +++-- pkg/f1/metrics/metrics.go | 3 ++- pkg/f1/run/run_cmd.go | 2 +- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/pkg/f1/metrics/labels/labels.go b/pkg/f1/metrics/labels/labels.go index 14ff03b2..c4bd95e0 100644 --- a/pkg/f1/metrics/labels/labels.go +++ b/pkg/f1/metrics/labels/labels.go @@ -1,6 +1,7 @@ package labels + const ( - Test = "test" - Stage = "stage" + Test = "test" + Stage = "stage" Result = "result" ) diff --git a/pkg/f1/metrics/metrics.go b/pkg/f1/metrics/metrics.go index e103e2db..0bfcf6a9 100644 --- a/pkg/f1/metrics/metrics.go +++ b/pkg/f1/metrics/metrics.go @@ -1,9 +1,10 @@ package metrics import ( - "github.com/form3tech-oss/f1/pkg/f1/metrics/labels" "sync" + "github.com/form3tech-oss/f1/pkg/f1/metrics/labels" + "github.com/prometheus/client_golang/prometheus" ) diff --git a/pkg/f1/run/run_cmd.go b/pkg/f1/run/run_cmd.go index 94d50a2c..18ea1a3a 100644 --- a/pkg/f1/run/run_cmd.go +++ b/pkg/f1/run/run_cmd.go @@ -95,7 +95,7 @@ func runCmdExecute(t api.Builder, hookFunc logging.RegisterLogHookFunc) func(cmd } run, err := NewRun(options.RunOptions{ - RunName: runName, + RunName: runName, Scenario: scenarioName, MaxDuration: duration, Concurrency: concurrency, From 2af10fb7418b6603654df772cf46d25081db8a58 Mon Sep 17 00:00:00 2001 From: nicholasham Date: Thu, 20 Aug 2020 14:13:57 +0100 Subject: [PATCH 3/3] Added scenario name and test run to active scenario --- pkg/f1/options/run_options.go | 2 +- pkg/f1/run/run_cmd.go | 2 +- pkg/f1/run/run_stage_test.go | 4 ++-- pkg/f1/run/test_runner.go | 10 +++++----- pkg/f1/testing/active_scenario.go | 14 ++++++++------ 5 files changed, 17 insertions(+), 15 deletions(-) diff --git a/pkg/f1/options/run_options.go b/pkg/f1/options/run_options.go index 7454bf23..bd57994b 100644 --- a/pkg/f1/options/run_options.go +++ b/pkg/f1/options/run_options.go @@ -8,7 +8,7 @@ import ( type RunOptions struct { RunName string - Scenario string + ScenarioName string MaxDuration time.Duration Concurrency int Env map[string]string diff --git a/pkg/f1/run/run_cmd.go b/pkg/f1/run/run_cmd.go index 18ea1a3a..6ce9bd53 100644 --- a/pkg/f1/run/run_cmd.go +++ b/pkg/f1/run/run_cmd.go @@ -96,7 +96,7 @@ func runCmdExecute(t api.Builder, hookFunc logging.RegisterLogHookFunc) func(cmd run, err := NewRun(options.RunOptions{ RunName: runName, - Scenario: scenarioName, + ScenarioName: scenarioName, MaxDuration: duration, Concurrency: concurrency, Env: loadEnvironment(), diff --git a/pkg/f1/run/run_stage_test.go b/pkg/f1/run/run_stage_test.go index 0bfcec55..05964723 100644 --- a/pkg/f1/run/run_stage_test.go +++ b/pkg/f1/run/run_stage_test.go @@ -82,7 +82,7 @@ func (s *RunTestStage) a_concurrency_of(concurrency int) *RunTestStage { func (s *RunTestStage) i_execute_the_run_command() *RunTestStage { run, err := NewRun( options.RunOptions{ - Scenario: s.scenario, + ScenarioName: s.scenario, MaxDuration: s.duration, Concurrency: s.concurrency, MaxIterations: s.maxIterations, @@ -244,7 +244,7 @@ func (s *RunTestStage) an_iteration_limit_of(iterations int32) *RunTestStage { func (s *RunTestStage) the_test_run_is_started() *RunTestStage { go func() { run, err := NewRun(options.RunOptions{ - Scenario: s.scenario, + ScenarioName: s.scenario, MaxDuration: s.duration, Concurrency: s.concurrency, MaxIterations: s.maxIterations, diff --git a/pkg/f1/run/test_runner.go b/pkg/f1/run/test_runner.go index bef9ca5c..f410cd05 100644 --- a/pkg/f1/run/test_runner.go +++ b/pkg/f1/run/test_runner.go @@ -78,7 +78,7 @@ type Run struct { var startTemplate = template.Must(template.New("result parse"). Funcs(templateFunctions). Parse(`{u}{bold}{intensive_blue}F1 Load Tester{-} -Running {yellow}{{.Options.Scenario}}{-} scenario for {{if .Options.MaxIterations}}up to {{.Options.MaxIterations}} iterations or up to {{end}}{{duration .Options.MaxDuration}} at a rate of {{.RateDescription}}. +Running {yellow}{{.Options.ScenarioName}}{-} scenario for {{if .Options.MaxIterations}}up to {{.Options.MaxIterations}} iterations or up to {{end}}{{duration .Options.MaxDuration}} at a rate of {{.RateDescription}}. `)) func (r *Run) Do() *RunResult { @@ -90,7 +90,7 @@ func (r *Run) Do() *RunResult { metrics.Instance().Reset() var err error - r.activeScenario, err = testing.NewActiveScenarios(r.Options.RunName, r.Options.Env, testing.GetScenario(r.Options.Scenario), 0) + r.activeScenario, err = testing.NewActiveScenarios(r.Options.ScenarioName, r.Options.RunName, r.Options.Env, testing.GetScenario(r.Options.ScenarioName), 0) r.pushMetrics() fmt.Println(r.result.Setup()) @@ -120,9 +120,9 @@ func (r *Run) Do() *RunResult { } func (r *Run) configureLogging() { - r.Options.RegisterLogHookFunc(r.Options.Scenario) + r.Options.RegisterLogHookFunc(r.Options.ScenarioName) if !r.Options.Verbose { - r.result.LogFile = redirectLoggingToFile(r.Options.Scenario) + r.result.LogFile = redirectLoggingToFile(r.Options.ScenarioName) welcomeMessage := renderTemplate(startTemplate, r) log.Info(welcomeMessage) fmt.Printf("Saving logs to %s\n\n", r.result.LogFile) @@ -140,7 +140,7 @@ func (r *Run) teardown() { r.fail(err, "teardown failed") } } else { - log.Infof("nil teardown function for scenario %s", r.Options.Scenario) + log.Infof("nil teardown function for scenario %s", r.Options.ScenarioName) } } diff --git a/pkg/f1/testing/active_scenario.go b/pkg/f1/testing/active_scenario.go index 95cebc28..43ec39b7 100644 --- a/pkg/f1/testing/active_scenario.go +++ b/pkg/f1/testing/active_scenario.go @@ -15,7 +15,8 @@ type ActiveScenario struct { Stages []Stage TeardownFn TeardownFn env map[string]string - TestRun string + ScenarioName string + RunName string id string autoTeardownTimer *CancellableTimer autoTeardownTimerMu sync.RWMutex @@ -23,10 +24,11 @@ type ActiveScenario struct { m *metrics.Metrics } -func NewActiveScenarios(testRun string, env map[string]string, fn MultiStageSetupFn, autoTeardownIdleDuration time.Duration) (*ActiveScenario, error) { +func NewActiveScenarios(scenarioName string, runName string, env map[string]string, fn MultiStageSetupFn, autoTeardownIdleDuration time.Duration) (*ActiveScenario, error) { s := &ActiveScenario{ - TestRun: testRun, + ScenarioName: scenarioName, + RunName: runName, id: uuid.New().String(), env: env, AutoTeardownAfter: autoTeardownIdleDuration, @@ -68,7 +70,7 @@ func (s *ActiveScenario) SetAutoTeardown(timer *CancellableTimer) { } func (s *ActiveScenario) Run(metric metrics.MetricType, stage, vu, iter string, f func(t *T)) error { - t := NewT(s.env, vu, iter, s.TestRun) + t := NewT(s.env, vu, iter, s.ScenarioName) start := time.Now() done := make(chan struct{}) go func() { @@ -80,7 +82,7 @@ func (s *ActiveScenario) Run(metric metrics.MetricType, stage, vu, iter string, } // wait for completion <-done - s.m.Record(metric, s.TestRun, stage, metrics.Result(t.HasFailed()), time.Since(start).Nanoseconds()) + s.m.Record(metric, s.RunName, stage, metrics.Result(t.HasFailed()), time.Since(start).Nanoseconds()) if t.HasFailed() { return errors.New("failed") } @@ -114,5 +116,5 @@ func (s *ActiveScenario) autoTeardown() { } func (s *ActiveScenario) RecordDroppedIteration() { - s.m.Record(metrics.IterationResult, s.TestRun, "single", "dropped", 0) + s.m.Record(metrics.IterationResult, s.RunName, "single", "dropped", 0) }