diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index c86a130..1166e9a 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -12,6 +12,13 @@ on: - main schedule: - cron: "12 0 * * *" + workflow_dispatch: + inputs: + run_benchmarks: + description: 'Run benchmarks' + required: false + default: false + type: boolean env: GOPATH: /home/runner/go @@ -142,3 +149,38 @@ jobs: - name: Run fuzzers run: bin/task fuzz + + benchmark: + name: Benchmark + runs-on: ubuntu-24.04 + timeout-minutes: 30 + + # Do not run this job in parallel for any PR change or branch push. + concurrency: + group: ${{ github.workflow }}-benchmark-${{ github.head_ref || github.ref_name }} + cancel-in-progress: true + + # Only run on main branch, scheduled runs, or when triggered manually + if: github.ref == 'refs/heads/main' || github.event_name == 'schedule' || (github.event_name == 'workflow_dispatch' && inputs.run_benchmarks) + + permissions: + contents: read + + steps: + # TODO https://github.com/FerretDB/github-actions/issues/211 + - name: Checkout code + uses: actions/checkout@v4 + + - name: Setup Go + uses: FerretDB/github-actions/setup-go@main + with: + cache-key: benchmark + + - name: Install Task and tools + run: go generate -x + working-directory: tools + + - name: Run benchmarks and push results + run: bin/task bench-ci + env: + BENCHMARK_MONGODB_URI: ${{ secrets.BENCHMARK_MONGODB_URI }} diff --git a/.gitignore b/.gitignore index a72b787..6aadef7 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ /bin/ *.txt +benchpush diff --git a/Taskfile.yml b/Taskfile.yml index 6c716b6..43eeed8 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -112,6 +112,11 @@ tasks: - go test -bench='{{.BENCH}}' -count={{.BENCH_COUNT}} -benchtime={{.BENCH_TIME}} -timeout=60m ./wirebson | tee new.txt - bin/benchstat old.txt new.txt + bench-ci: + desc: "Run benchmarks and push results to MongoDB (for CI)" + cmds: + - go run ./cmd/benchpush -bench='{{.BENCH}}' -count={{.BENCH_COUNT}} -benchtime={{.BENCH_TIME}} -uri='{{.BENCHMARK_MONGODB_URI}}' + fuzz: desc: "Run fuzzers for about 1 minute (with default FUZZ_TIME)" cmds: diff --git a/cmd/benchpush/main.go b/cmd/benchpush/main.go new file mode 100644 index 0000000..bc1ba17 --- /dev/null +++ b/cmd/benchpush/main.go @@ -0,0 +1,116 @@ +// Copyright 2021 FerretDB Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package main provides a command-line tool for running benchmarks and pushing results to MongoDB. +package main + +import ( + "context" + "flag" + "log/slog" + "os" + "os/exec" + "time" + + "github.com/FerretDB/wire/internal/benchpusher" +) + +func main() { + var ( + mongoURI = flag.String("uri", "", "MongoDB URI for pushing results (if empty, only parse and print)") + benchRegex = flag.String("bench", "Benchmark.*", "Benchmark regex pattern") + benchTime = flag.String("benchtime", "1s", "Benchmark time") + benchCount = flag.String("count", "5", "Benchmark count") + pkg = flag.String("pkg", "./wirebson", "Package to benchmark") + timeout = flag.Duration("timeout", 10*time.Minute, "Benchmark timeout") + ) + flag.Parse() + + logger := slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: slog.LevelInfo})) + + // Run the benchmarks + logger.Info("Running benchmarks...", + slog.String("package", *pkg), + slog.String("pattern", *benchRegex), + slog.String("benchtime", *benchTime), + slog.String("count", *benchCount)) + + ctx, cancel := context.WithTimeout(context.Background(), *timeout) + defer cancel() + + cmd := exec.CommandContext(ctx, "go", "test", + "-bench="+*benchRegex, + "-count="+*benchCount, + "-benchtime="+*benchTime, + "-timeout=60m", + *pkg) + + output, err := cmd.Output() + if err != nil { + if exitErr, ok := err.(*exec.ExitError); ok { + logger.Error("Benchmark command failed", + slog.String("error", err.Error()), + slog.String("stderr", string(exitErr.Stderr))) + } else { + logger.Error("Failed to run benchmark command", slog.String("error", err.Error())) + } + os.Exit(1) + } + + outputStr := string(output) + logger.Info("Benchmark completed", slog.Int("output_length", len(outputStr))) + + // Parse the benchmark output + var client *benchpusher.Client + if *mongoURI != "" { + var err error + client, err = benchpusher.New(*mongoURI, logger) + if err != nil { + logger.Error("Failed to create MongoDB client", slog.String("error", err.Error())) + os.Exit(1) + } + defer client.Close() + } + + results, err := benchpusher.ParseBenchmarkOutput(outputStr) + if err != nil { + logger.Error("Failed to parse benchmark output", slog.String("error", err.Error())) + os.Exit(1) + } + + logger.Info("Parsed benchmark results", slog.Int("count", len(results))) + + // Print results summary + for _, result := range results { + logger.Info("Benchmark result", + slog.String("name", result.Name), + slog.Int("iterations", result.Iterations), + slog.Float64("ns_per_op", result.NsPerOp), + slog.Any("metrics", result.Metrics)) + } + + // Push to MongoDB if URI is provided + if *mongoURI != "" && len(results) > 0 { + logger.Info("Pushing results to MongoDB...") + if err := client.Push(context.Background(), results); err != nil { + logger.Error("Failed to push results to MongoDB", slog.String("error", err.Error())) + os.Exit(1) + } + logger.Info("Successfully pushed results to MongoDB") + } else if *mongoURI == "" { + logger.Info("No MongoDB URI provided, skipping push to database") + } else { + logger.Info("No benchmark results to push") + } +} \ No newline at end of file diff --git a/docs/benchmark-ci.md b/docs/benchmark-ci.md new file mode 100644 index 0000000..3c8379a --- /dev/null +++ b/docs/benchmark-ci.md @@ -0,0 +1,121 @@ +# Benchmark CI + +This document describes the benchmark CI setup for the FerretDB Wire repository. + +## Overview + +The repository includes automated benchmark running that: +1. Runs benchmarks using Go's built-in benchmark framework +2. Parses benchmark results using `golang.org/x/perf/benchfmt` +3. Converts results to BSON format +4. Pushes results to a MongoDB-compatible database for visualization in Grafana + +## Components + +### Internal Package: `internal/benchpusher` + +This package provides: +- Benchmark result parsing using the modern `golang.org/x/perf/benchfmt` library +- MongoDB client for pushing structured benchmark data +- BSON document creation compatible with FerretDB + +### CLI Tool: `cmd/benchpush` + +Command-line tool that: +- Runs `go test -bench` with configurable parameters +- Parses the output and extracts benchmark metrics +- Optionally pushes results to MongoDB if URI is provided + +Usage: +```bash +# Run benchmarks without pushing to database +go run ./cmd/benchpush -bench=BenchmarkDocumentDecode -count=1 -benchtime=100ms + +# Run benchmarks and push to MongoDB +go run ./cmd/benchpush -bench=BenchmarkDocumentDecode -count=5 -benchtime=1s -uri="mongodb://..." +``` + +### Task Integration + +Added `bench-ci` task to `Taskfile.yml`: +```yaml +bench-ci: + desc: "Run benchmarks and push results to MongoDB (for CI)" + cmds: + - go run ./cmd/benchpush -bench='{{.BENCH}}' -count={{.BENCH_COUNT}} -benchtime={{.BENCH_TIME}} -uri='{{.BENCHMARK_MONGODB_URI}}' +``` + +### GitHub Actions Workflow + +Added `benchmark` job to `.github/workflows/go.yml` that: +- Runs on main branch pushes, scheduled runs, and manual workflow dispatch +- Uses the `bench-ci` task to run benchmarks and push results +- Requires `BENCHMARK_MONGODB_URI` secret to be configured + +## Configuration + +### Environment Variables + +- `BENCHMARK_MONGODB_URI`: MongoDB URI for pushing benchmark results +- `RUNNER_NAME`: GitHub Actions runner name (automatically set) +- `GITHUB_REPOSITORY`: Repository name (automatically set) + +### Benchmark Parameters + +Configurable via Taskfile.yml variables: +- `BENCH`: Benchmark regex pattern (default: `Benchmark.*`) +- `BENCH_TIME`: Benchmark duration (default: `1s`) +- `BENCH_COUNT`: Number of benchmark runs (default: `5`) + +## Data Format + +Benchmark results are stored in MongoDB with this structure: + +```json +{ + "time": "2025-07-18T14:30:12.077Z", + "env": { + "runner": "runner-name", + "hostname": "hostname", + "repository": "FerretDB/wire" + }, + "benchmarks": { + "DocumentDecode_handshake1-4": { + "iterations": 3148209, + "ns_per_op": 381.6, + "metrics": { + "B/op": "352.00", + "allocs/op": "10.00" + } + } + } +} +``` + +## Manual Execution + +To manually trigger benchmark runs: + +1. Go to the repository's Actions tab +2. Select "Go" workflow +3. Click "Run workflow" +4. Check "Run benchmarks" option +5. Click "Run workflow" + +## Local Testing + +To test locally without MongoDB: +```bash +# Install task +go generate -x tools/tools.go + +# Run short benchmarks +BENCHMARK_MONGODB_URI="" bin/task bench-ci +``` + +## Integration with Grafana + +The benchmark results pushed to MongoDB can be visualized in Grafana by: +1. Configuring MongoDB as a data source +2. Creating dashboards that query the `benchmarks` collection +3. Displaying metrics like ns/op, memory allocations, and throughput over time \ No newline at end of file diff --git a/go.mod b/go.mod index 213bf5c..c9dedad 100644 --- a/go.mod +++ b/go.mod @@ -10,17 +10,20 @@ require ( github.com/xdg-go/scram v1.1.2 go.mongodb.org/mongo-driver v1.17.4 go.mongodb.org/mongo-driver/v2 v2.2.2 + golang.org/x/perf v0.0.0-20250710210952-7b7c2de18447 ) require ( + github.com/aclements/go-moremath v0.0.0-20210112150236-f10218a38794 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/golang/snappy v1.0.0 // indirect github.com/klauspost/compress v1.16.7 // indirect + github.com/montanaflynn/stats v0.7.1 // indirect github.com/xdg-go/pbkdf2 v1.0.0 // indirect github.com/xdg-go/stringprep v1.0.4 // indirect github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 // indirect - golang.org/x/crypto v0.38.0 // indirect - golang.org/x/sync v0.14.0 // indirect - golang.org/x/text v0.25.0 // indirect + golang.org/x/crypto v0.40.0 // indirect + golang.org/x/sync v0.16.0 // indirect + golang.org/x/text v0.27.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index 73f5136..3f41f12 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,5 @@ +github.com/aclements/go-moremath v0.0.0-20210112150236-f10218a38794 h1:xlwdaKcTNVW4PtpQb8aKA4Pjy0CdJHEqvFbAnvR5m2g= +github.com/aclements/go-moremath v0.0.0-20210112150236-f10218a38794/go.mod h1:7e+I0LQFUI9AXWxOfsQROs9xPhoJtbsyWcjJqDd4KPY= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/golang/snappy v1.0.0 h1:Oy607GVXHs7RtbggtPBnr2RmDArIsAefDwvrdWvRhGs= @@ -6,6 +8,8 @@ github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/klauspost/compress v1.16.7 h1:2mk3MPGNzKyxErAw8YaohYh69+pa4sIQSC0fPGCFR9I= github.com/klauspost/compress v1.16.7/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= +github.com/montanaflynn/stats v0.7.1 h1:etflOAAHORrCC44V+aR6Ftzort912ZU+YLiSTuV8eaE= +github.com/montanaflynn/stats v0.7.1/go.mod h1:etXPPgVO6n31NxCd9KQUMvCM+ve0ruNzt6R8Bnaayow= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= @@ -25,16 +29,18 @@ go.mongodb.org/mongo-driver/v2 v2.2.2 h1:9cYuS3fl1Xhqwpfazso10V7BHQD58kCgtzhfAmJ go.mongodb.org/mongo-driver/v2 v2.2.2/go.mod h1:qQkDMhCGWl3FN509DfdPd4GRBLU/41zqF/k8eTRceps= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.38.0 h1:jt+WWG8IZlBnVbomuhg2Mdq0+BBQaHbtqHEFEigjUV8= -golang.org/x/crypto v0.38.0/go.mod h1:MvrbAqul58NNYPKnOra203SB9vpuZW0e+RRZV+Ggqjw= +golang.org/x/crypto v0.40.0 h1:r4x+VvoG5Fm+eJcxMaY8CQM7Lb0l1lsmjGBQ6s8BfKM= +golang.org/x/crypto v0.40.0/go.mod h1:Qr1vMER5WyS2dfPHAlsOj01wgLbsyWtFn/aY+5+ZdxY= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/perf v0.0.0-20250710210952-7b7c2de18447 h1:rW8zamu+F63omx9uT7NEv45cj82FinwZqCQxB6YpXaY= +golang.org/x/perf v0.0.0-20250710210952-7b7c2de18447/go.mod h1:/3Q08NGeGk1jEFl72e1LrFtPIrpY6mfw8iqURBiwZ2E= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.14.0 h1:woo0S4Yywslg6hp4eUFjTVOyKt0RookbpAHG4c1HmhQ= -golang.org/x/sync v0.14.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= +golang.org/x/sync v0.16.0 h1:ycBJEhp9p4vXvUZNszeOq0kGTPghopOL8q0fq3vstxw= +golang.org/x/sync v0.16.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -46,8 +52,8 @@ golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= -golang.org/x/text v0.25.0 h1:qVyWApTSYLk/drJRO5mDlNYskwQznZmkpV2c8q9zls4= -golang.org/x/text v0.25.0/go.mod h1:WEdwpYrmk1qmdHvhkSTNPm3app7v4rsT8F2UD6+VHIA= +golang.org/x/text v0.27.0 h1:4fGWRpyh641NLlecmyl4LOe6yDdfaYNrGb2zdfo4JV4= +golang.org/x/text v0.27.0/go.mod h1:1D28KMCvyooCX9hBiosv5Tz/+YLxj0j7XhWjpSUF7CU= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= diff --git a/internal/benchpusher/benchpusher.go b/internal/benchpusher/benchpusher.go new file mode 100644 index 0000000..a3d92f2 --- /dev/null +++ b/internal/benchpusher/benchpusher.go @@ -0,0 +1,237 @@ +// Copyright 2021 FerretDB Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package benchpusher provides functionality to parse benchmark results and push them to MongoDB. +package benchpusher + +import ( + "context" + "fmt" + "log/slog" + "net/url" + "os" + "strings" + "time" + + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" + "golang.org/x/perf/benchfmt" +) + +// Client represents a MongoDB client for pushing benchmark results. +type Client struct { + l *slog.Logger + c *mongo.Client + pingerCancel context.CancelFunc + pingerDone chan struct{} + database string + hostname string + runner string + repository string +} + +// BenchmarkResult represents a parsed benchmark result. +type BenchmarkResult struct { + Name string `bson:"name"` + Iterations int `bson:"iterations"` + NsPerOp float64 `bson:"ns_per_op"` + Metrics map[string]string `bson:"metrics"` +} + +// New creates a new MongoDB client for pushing benchmark results. +func New(uri string, l *slog.Logger) (*Client, error) { + if uri == "" { + return nil, fmt.Errorf("MongoDB URI is required") + } + + u, err := url.Parse(uri) + if err != nil { + return nil, err + } + + database := strings.TrimPrefix(u.Path, "/") + if database == "" { + return nil, fmt.Errorf("database name is empty in the URL") + } + + hostname, err := os.Hostname() + if err != nil { + return nil, err + } + + ctx := context.Background() + + opts := options.Client().ApplyURI(uri) + opts.SetDirect(true) + opts.SetConnectTimeout(3 * time.Second) + opts.SetHeartbeatInterval(3 * time.Second) + opts.SetMaxConnIdleTime(0) + opts.SetMinPoolSize(1) + opts.SetMaxPoolSize(1) + opts.SetMaxConnecting(1) + + l.InfoContext(ctx, "Connecting to MongoDB URI to push benchmark results...", slog.String("uri", u.Redacted())) + + c, err := mongo.Connect(ctx, opts) + if err != nil { + return nil, err + } + + pingerCtx, pingerCancel := context.WithCancel(ctx) + + res := &Client{ + l: l, + c: c, + pingerCancel: pingerCancel, + pingerDone: make(chan struct{}), + database: database, + hostname: hostname, + runner: os.Getenv("RUNNER_NAME"), + repository: os.Getenv("GITHUB_REPOSITORY"), + } + + go func() { + res.ping(pingerCtx) + close(res.pingerDone) + }() + + return res, nil +} + +// ping pings the database until connection is established or ctx is canceled. +func (c *Client) ping(ctx context.Context) { + for ctx.Err() == nil { + pingCtx, pingCancel := context.WithTimeout(ctx, 5*time.Second) + + err := c.c.Ping(pingCtx, nil) + if err == nil { + c.l.InfoContext(pingCtx, "Ping successful") + pingCancel() + return + } + + c.l.WarnContext(pingCtx, "Ping failed", slog.String("error", err.Error())) + + // always wait, even if ping returns immediately + <-pingCtx.Done() + pingCancel() + } +} + +// ParseBenchmarkOutput parses benchmark output and returns structured results. +func ParseBenchmarkOutput(output string) ([]BenchmarkResult, error) { + var results []BenchmarkResult + + reader := benchfmt.NewReader(strings.NewReader(output), "") + for reader.Scan() { + record := reader.Result() + if record == nil { + continue + } + + // Only process Result records, not Config records + result, ok := record.(*benchfmt.Result) + if !ok { + continue + } + + benchResult := BenchmarkResult{ + Name: string(result.Name), + Iterations: result.Iters, + Metrics: make(map[string]string), + } + + // Extract standard benchmark metrics + for _, metric := range result.Values { + switch metric.Unit { + case "ns/op", "sec/op": + if metric.Unit == "sec/op" { + // Convert seconds to nanoseconds + benchResult.NsPerOp = metric.Value * 1e9 + } else { + benchResult.NsPerOp = metric.Value + } + case "B/op", "allocs/op", "MB/s": + benchResult.Metrics[metric.Unit] = fmt.Sprintf("%.2f", metric.Value) + default: + benchResult.Metrics[metric.Unit] = fmt.Sprintf("%.2f", metric.Value) + } + } + + results = append(results, benchResult) + } + + if err := reader.Err(); err != nil { + return nil, fmt.Errorf("failed to parse benchmark output: %w", err) + } + + return results, nil +} + +// ParseBenchmarkOutput parses benchmark output and returns structured results (method version). +func (c *Client) ParseBenchmarkOutput(output string) ([]BenchmarkResult, error) { + return ParseBenchmarkOutput(output) +} + +// Push pushes benchmark results to MongoDB. +func (c *Client) Push(ctx context.Context, results []BenchmarkResult) error { + if len(results) == 0 { + c.l.InfoContext(ctx, "No benchmark results to push") + return nil + } + + var benchmarks bson.D + for _, result := range results { + // Replace dots with underscores to make it compatible with FerretDB v1 + name := strings.ReplaceAll(result.Name, ".", "_") + benchmarks = append(benchmarks, bson.E{Key: name, Value: bson.D{ + {"iterations", result.Iterations}, + {"ns_per_op", result.NsPerOp}, + {"metrics", result.Metrics}, + }}) + } + + doc := bson.D{ + {"time", time.Now()}, + {"env", bson.D{ + {"runner", c.runner}, + {"hostname", c.hostname}, + {"repository", c.repository}, + }}, + {"benchmarks", benchmarks}, + } + + c.l.InfoContext(ctx, "Pushing benchmark results to MongoDB...", slog.Int("count", len(results))) + + c.ping(ctx) + + _, err := c.c.Database(c.database).Collection("benchmarks").InsertOne(ctx, doc) + if err != nil { + return fmt.Errorf("failed to insert benchmark results: %w", err) + } + + return nil +} + +// Close closes all connections. +func (c *Client) Close() { + c.pingerCancel() + <-c.pingerDone + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + c.c.Disconnect(ctx) +} \ No newline at end of file diff --git a/internal/benchpusher/benchpusher_test.go b/internal/benchpusher/benchpusher_test.go new file mode 100644 index 0000000..a5378ac --- /dev/null +++ b/internal/benchpusher/benchpusher_test.go @@ -0,0 +1,94 @@ +// Copyright 2021 FerretDB Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package benchpusher + +import ( + "log/slog" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestParseBenchmarkOutput(t *testing.T) { + // Sample benchmark output that matches the format from wire benchmarks + sampleOutput := `goos: linux +goarch: amd64 +pkg: github.com/FerretDB/wire/wirebson +cpu: AMD EPYC 7763 64-Core Processor +BenchmarkDocumentDecode/handshake1-4 3148209 381.6 ns/op 352 B/op 10 allocs/op +BenchmarkDocumentDecode/handshake2-4 3153933 381.2 ns/op 352 B/op 10 allocs/op +BenchmarkDocumentDecode/nested-4 10174455 116.6 ns/op 88 B/op 3 allocs/op +PASS +ok github.com/FerretDB/wire/wirebson 65.233s` + + logger := slog.Default() + client := &Client{l: logger} + + results, err := client.ParseBenchmarkOutput(sampleOutput) + require.NoError(t, err) + + // Debug: print what we got + t.Logf("Parsed %d results", len(results)) + for i, result := range results { + t.Logf("Result %d: Name=%s, Iterations=%d, NsPerOp=%f, Metrics=%v", + i, result.Name, result.Iterations, result.NsPerOp, result.Metrics) + } + + // Should parse the benchmark lines + assert.Greater(t, len(results), 0, "Should parse at least one benchmark result") + + // Check that we can find specific benchmarks + var handshake1Found, nestedFound bool + for _, result := range results { + if strings.Contains(result.Name, "handshake1") { + handshake1Found = true + assert.Greater(t, result.NsPerOp, 0.0, "Should have positive ns/op") + assert.Greater(t, result.Iterations, 0, "Should have positive iterations") + assert.Contains(t, result.Metrics, "B/op", "Should have B/op metric") + assert.Contains(t, result.Metrics, "allocs/op", "Should have allocs/op metric") + } + if strings.Contains(result.Name, "nested") { + nestedFound = true + } + } + + assert.True(t, handshake1Found, "Should find handshake1 benchmark") + assert.True(t, nestedFound, "Should find nested benchmark") +} + +func TestParseBenchmarkOutput_Empty(t *testing.T) { + logger := slog.Default() + client := &Client{l: logger} + + results, err := client.ParseBenchmarkOutput("") + require.NoError(t, err) + assert.Empty(t, results, "Empty input should return empty results") +} + +func TestParseBenchmarkOutput_InvalidFormat(t *testing.T) { + logger := slog.Default() + client := &Client{l: logger} + + // Test with non-benchmark output + invalidOutput := `This is not benchmark output +Just some random text +No benchmarks here` + + results, err := client.ParseBenchmarkOutput(invalidOutput) + require.NoError(t, err) + assert.Empty(t, results, "Invalid format should return empty results") +} \ No newline at end of file diff --git a/internal/benchpusher/integration_test.go b/internal/benchpusher/integration_test.go new file mode 100644 index 0000000..180b13c --- /dev/null +++ b/internal/benchpusher/integration_test.go @@ -0,0 +1,84 @@ +// Copyright 2021 FerretDB Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package benchpusher + +import ( + "log/slog" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func testLogger() *slog.Logger { + return slog.Default() +} + +// TestPushWithoutMongoDB tests that pushing without MongoDB client doesn't crash +func TestPushWithoutMongoDB(t *testing.T) { + results := []BenchmarkResult{ + { + Name: "TestBenchmark", + Iterations: 1000, + NsPerOp: 100.5, + Metrics: map[string]string{"B/op": "64.00", "allocs/op": "2.00"}, + }, + } + + // This should not panic and should be gracefully handled + // when no MongoDB client is available + // (actual implementation would need MongoDB client to push) + assert.NotEmpty(t, results) +} + +// TestBenchmarkResultStructure tests the structure of benchmark results +func TestBenchmarkResultStructure(t *testing.T) { + result := BenchmarkResult{ + Name: "BenchmarkExample", + Iterations: 1000000, + NsPerOp: 150.5, + Metrics: map[string]string{ + "B/op": "128.00", + "allocs/op": "4.00", + "MB/s": "25.50", + }, + } + + assert.Equal(t, "BenchmarkExample", result.Name) + assert.Equal(t, 1000000, result.Iterations) + assert.Equal(t, 150.5, result.NsPerOp) + assert.Equal(t, "128.00", result.Metrics["B/op"]) + assert.Equal(t, "4.00", result.Metrics["allocs/op"]) + assert.Equal(t, "25.50", result.Metrics["MB/s"]) +} + +// TestNewClientWithInvalidURI tests client creation with invalid URI +func TestNewClientWithInvalidURI(t *testing.T) { + logger := testLogger() + + // Test with empty URI + _, err := New("", logger) + require.Error(t, err) + assert.Contains(t, err.Error(), "MongoDB URI is required") + + // Test with invalid URI + _, err = New("invalid-uri", logger) + require.Error(t, err) + + // Test with URI without database + _, err = New("mongodb://localhost:27017", logger) + require.Error(t, err) + assert.Contains(t, err.Error(), "database name is empty") +} \ No newline at end of file