From 6a2529f08fe50ef8f7ee82aa3b503a8ef11d5f60 Mon Sep 17 00:00:00 2001 From: Ramkumar Chinchani Date: Thu, 13 Jan 2022 22:06:35 +0000 Subject: [PATCH] ci/cd: add continuous benchmark action Signed-off-by: Ramkumar Chinchani --- .github/workflows/benchmark.yaml | 40 +++++++++++++++++++++++++++++++ Makefile | 8 +++++++ cmd/zb/main.go | 8 +++---- cmd/zb/perf.go | 41 +++++++++++++++++++++++++++++--- 4 files changed, 90 insertions(+), 7 deletions(-) create mode 100644 .github/workflows/benchmark.yaml diff --git a/.github/workflows/benchmark.yaml b/.github/workflows/benchmark.yaml new file mode 100644 index 00000000..e09fae59 --- /dev/null +++ b/.github/workflows/benchmark.yaml @@ -0,0 +1,40 @@ +name: "Minimal Continuous Benchmark" +on: + push: + branches: + - main + pull_request: + # The branches below must be a subset of the branches above + branches: [main] + +jobs: + benchmark: + name: Performance regression check + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-go@v2 + with: + go-version: 1.17.x + # Run benchmark with `go test -bench` and stores the output to a file + - name: Run benchmark + run: make BENCH_OUTPUT=ci-cd run-bench + # Download previous benchmark result from cache (if exists) + - name: Download previous benchmark data + uses: actions/cache@v1 + with: + path: ./cache + key: ${{ runner.os }}-benchmark + # Run `github-action-benchmark` action + - name: Store benchmark result + uses: benchmark-action/github-action-benchmark@v1 + with: + # What benchmark tool the output.txt came from + tool: 'customBiggerIsBetter' + # Where the output from the benchmark tool is stored + output-file-path: ci-cd.json + # Where the previous data file is stored + external-data-json-path: ./cache/benchmark-data.json + # Workflow will fail when an alert happens + fail-on-alert: true + # Upload the updated cache file for the next job by actions/cache diff --git a/Makefile b/Makefile index 20302685..11b6e78b 100644 --- a/Makefile +++ b/Makefile @@ -12,6 +12,7 @@ GOLINTER := $(TOOLSDIR)/bin/golangci-lint NOTATION := $(TOOLSDIR)/bin/notation OS ?= linux ARCH ?= amd64 +BENCH_OUTPUT ?= stdout .PHONY: all all: swagger binary binary-minimal binary-debug binary-arch binary-arch-minimal cli cli-arch bench bench-arch exporter-minimal verify-config test test-clean check @@ -64,6 +65,13 @@ test: check-skopeo $(NOTATION) go test -tags extended,containers_image_openpgp -v -trimpath -race -timeout 15m -cover -coverpkg ./... -coverprofile=coverage-extended.txt -covermode=atomic ./... go test -tags minimal,containers_image_openpgp -v -trimpath -race -cover -coverpkg ./... -coverprofile=coverage-minimal.txt -covermode=atomic ./... +.PHONY: run-bench +run-bench: binary bench + bin/zot serve examples/config-minimal.json & + sleep 5 + bin/zb -c 10 -n 100 -o $(BENCH_OUTPUT) http://localhost:8080 + killall zot + .PHONY: test-clean test-clean: $(shell sudo rm -rf /etc/containers/certs.d/127.0.0.1:8089/) diff --git a/cmd/zb/main.go b/cmd/zb/main.go index d97d75ac..651004f1 100644 --- a/cmd/zb/main.go +++ b/cmd/zb/main.go @@ -13,7 +13,7 @@ import ( func NewPerfRootCmd() *cobra.Command { showVersion := false - var auth, workdir, repo, output string + var auth, workdir, repo, outFmt string var concurrency, requests int @@ -45,7 +45,7 @@ func NewPerfRootCmd() *cobra.Command { requests = concurrency * (requests / concurrency) - Perf(workdir, url, auth, repo, concurrency, requests) + Perf(workdir, url, auth, repo, concurrency, requests, outFmt) }, } @@ -59,8 +59,8 @@ func NewPerfRootCmd() *cobra.Command { "Number of multiple requests to make at a time") rootCmd.Flags().IntVarP(&requests, "requests", "n", 1, "Number of requests to perform") - rootCmd.Flags().StringVarP(&output, "output-format", "o", "", - "Output format of test results [default: stdout]") + rootCmd.Flags().StringVarP(&outFmt, "output-format", "o", "", + "Output format of test results: stdout (default), json, ci-cd") // "version" rootCmd.Flags().BoolVarP(&showVersion, "version", "v", false, "show the version and exit") diff --git a/cmd/zb/perf.go b/cmd/zb/perf.go index 82aa55fd..3ca305c8 100644 --- a/cmd/zb/perf.go +++ b/cmd/zb/perf.go @@ -4,6 +4,7 @@ import ( "crypto/rand" "encoding/json" "fmt" + "io/ioutil" "log" "net/http" "net/url" @@ -33,6 +34,7 @@ const ( smallBlob = 1 * MiB mediumBlob = 10 * MiB largeBlob = 100 * MiB + cicdFmt = "ci-cd" ) // helper routines @@ -198,7 +200,17 @@ func updateStats(summary *statsSummary, record statsRecord) { summary.latencies = append(summary.latencies, record.latency) } -func printStats(requests int, summary *statsSummary) { +type cicdTestSummary struct { + Name string `json:"name"` + Unit string `json:"unit"` + Value interface{} `json:"value"` + Range string `json:"range,omitempty"` +} + +//nolint:gochecknoglobals // used only in this test +var cicdSummary = []cicdTestSummary{} + +func printStats(requests int, summary *statsSummary, outFmt string) { log.Printf("============\n") log.Printf("Test name:\t%s", summary.name) log.Printf("Time taken for tests:\t%v", summary.total) @@ -219,6 +231,18 @@ func printStats(requests int, summary *statsSummary) { log.Printf("%s:\t%v", "p90", summary.latencies[requests*9/10]) log.Printf("%s:\t%v", "p99", summary.latencies[requests*99/100]) log.Printf("\n") + + // ci/cd + if outFmt == cicdFmt { + cicdSummary = append(cicdSummary, + cicdTestSummary{ + Name: summary.name, + Unit: "requests per sec", + Value: summary.rps, + Range: "3", + }, + ) + } } // test suites/funcs. @@ -644,7 +668,7 @@ var testSuite = []testConfig{ // nolint:gochecknoglobals // used only in this te }, } -func Perf(workdir, url, auth, repo string, concurrency int, requests int) { +func Perf(workdir, url, auth, repo string, concurrency int, requests int, outFmt string) { // logging log.SetFlags(0) log.SetOutput(tabwriter.NewWriter(os.Stdout, 0, 0, 1, ' ', tabwriter.TabIndent)) @@ -691,6 +715,17 @@ func Perf(workdir, url, auth, repo string, concurrency int, requests int) { sort.Sort(Durations(summary.latencies)) - printStats(requests, &summary) + printStats(requests, &summary, outFmt) + } + + if outFmt == cicdFmt { + jsonOut, err := json.Marshal(cicdSummary) + if err != nil { + log.Fatal(err) //nolint:gocritic // file closed on exit + } + + if err := ioutil.WriteFile(fmt.Sprintf("%s.json", outFmt), jsonOut, defaultFilePerms); err != nil { + log.Fatal(err) + } } }