From 9c7e77e12a26cb1d9fdd74330d386544450cda55 Mon Sep 17 00:00:00 2001 From: Vishwas Rajashekar Date: Sat, 21 Mar 2026 15:13:17 +0530 Subject: [PATCH] feat(zb): list tests, test regex filter, docs update (#3884) feat(zb): list tests and test regex filter + misc This change introduces the following changes to zb. Test Filtering =============== Allows users to selectively run tests by specifying a standard regex that matches on the name of the test. Test Listing =============== Allows users to list out the available tests as well as the matched tests when using the regex filter. The documentation README has also been updated with examples and the command help. The documentation for skip cleanup has been updated. Signed-off-by: Vishwas Rajashekar --- cmd/zb/README.md | 142 ++++++++++++++++++++++++++++++++++++++++++++++- cmd/zb/main.go | 30 ++++++++-- cmd/zb/perf.go | 23 ++++++++ 3 files changed, 189 insertions(+), 6 deletions(-) diff --git a/cmd/zb/README.md b/cmd/zb/README.md index 15ab788e..61eaa686 100644 --- a/cmd/zb/README.md +++ b/cmd/zb/README.md @@ -5,20 +5,23 @@ ``` Usage: - zb [options] [flags] + zb [flags] Flags: -A, --auth-creds string Use colon-separated BASIC auth creds -c, --concurrency int Number of multiple requests to make at a time (default 1) -h, --help help for zb + -l, --list-tests Print a list of all available tests. When used together with test regex, lists the tests that match the regex. -o, --output-format string Output format of test results: stdout (default), json, ci-cd -r, --repo string Use specified repo on remote registry for test data -n, --requests int Number of requests to perform (default 1) + --skip-cleanup Skip clean up of pushed repos from remote registry after running benchmark (default false) -s, --src-cidr string Use specified cidr to obtain ips to make requests from, src-ips and src-cidr are mutually exclusive -i, --src-ips string Use colon-separated ips to make requests from, src-ips and src-cidr are mutually exclusive + -t, --test-regex string Optional regex for selectively running tests. If blank, all tests are run by default. -v, --version Show the version and exit -d, --working-dir string Use specified directory to store test data - ``` +``` ## Command example ``` @@ -75,6 +78,141 @@ p99: 26.375356ms ... ``` +## List tests + +``` +$ zb -l http://localhost:9000 +Get Catalog +Push Monolith 1MB +Push Monolith 10MB +Push Monolith 100MB +Push Chunk Streamed 1MB +Push Chunk Streamed 10MB +Push Chunk Streamed 100MB +Pull 1MB +Pull 10MB +Pull 100MB +Pull Mixed 20% 1MB, 70% 10MB, 10% 100MB +Push Monolith Mixed 20% 1MB, 70% 10MB, 10% 100MB +Push Chunk Mixed 33% 1MB, 33% 10MB, 33% 100MB +Pull 75% and Push 25% Mixed 1MB +Pull 75% and Push 25% Mixed 10MB +Pull 75% and Push 25% Mixed 100MB +``` + +## List tests with Regex + +``` +$ zb -l --test-regex "^(Push Monolith|Pull) 1MB$" http://localhost:9000 +Push Monolith 1MB +Pull 1MB +``` + +## Selective test run example with only push + +``` +$ zb --src-cidr 127.0.0.0/8 --test-regex "^Push Monolith 1MB$" http://localhost:9000 +Registry URL: http://localhost:9000 + +Concurrency Level: 1 +Total requests: 1 +Working dir: /home/darkaether/projects/github/zot + +Preparing test data ... +Starting tests ... +Skipping test Get Catalog +============ +Test name: Push Monolith 1MB +Time taken for tests: 18.700779ms +Requests per second: 53.47371 +Complete requests: 1 +Failed requests: 0 + +2xx responses: 1 + +min: 15.970773ms +max: 15.970773ms +p50: 15.970773ms +p75: 15.970773ms +p90: 15.970773ms +p99: 15.970773ms + +Skipping test Push Monolith 10MB +Skipping test Push Monolith 100MB +Skipping test Push Chunk Streamed 1MB +Skipping test Push Chunk Streamed 10MB +Skipping test Push Chunk Streamed 100MB +Skipping test Pull 1MB +Skipping test Pull 10MB +Skipping test Pull 100MB +Skipping test Pull Mixed 20% 1MB, 70% 10MB, 10% 100MB +Skipping test Push Monolith Mixed 20% 1MB, 70% 10MB, 10% 100MB +Skipping test Push Chunk Mixed 33% 1MB, 33% 10MB, 33% 100MB +Skipping test Pull 75% and Push 25% Mixed 1MB +Skipping test Pull 75% and Push 25% Mixed 10MB +Skipping test Pull 75% and Push 25% Mixed 100MB +``` + +## Selective test run with a push and corresponding pull + +``` +$ zb --src-cidr 127.0.0.0/8 --test-regex "^(Push Monolith|Pull) 1MB$" http://localhost:9000 +Registry URL: http://localhost:9000 + +Concurrency Level: 1 +Total requests: 1 +Working dir: /home/darkaether/projects/github/zot + +Preparing test data ... +Starting tests ... +Skipping test Get Catalog +============ +Test name: Push Monolith 1MB +Time taken for tests: 19.136523ms +Requests per second: 52.256096 +Complete requests: 1 +Failed requests: 0 + +2xx responses: 1 + +min: 16.496555ms +max: 16.496555ms +p50: 16.496555ms +p75: 16.496555ms +p90: 16.496555ms +p99: 16.496555ms + +Skipping test Push Monolith 10MB +Skipping test Push Monolith 100MB +Skipping test Push Chunk Streamed 1MB +Skipping test Push Chunk Streamed 10MB +Skipping test Push Chunk Streamed 100MB +============ +Test name: Pull 1MB +Time taken for tests: 17.836719ms +Requests per second: 56.06412 +Complete requests: 1 +Failed requests: 0 + +2xx responses: 1 + +min: 3.774833ms +max: 3.774833ms +p50: 3.774833ms +p75: 3.774833ms +p90: 3.774833ms +p99: 3.774833ms + +Skipping test Pull 10MB +Skipping test Pull 100MB +Skipping test Pull Mixed 20% 1MB, 70% 10MB, 10% 100MB +Skipping test Push Monolith Mixed 20% 1MB, 70% 10MB, 10% 100MB +Skipping test Push Chunk Mixed 33% 1MB, 33% 10MB, 33% 100MB +Skipping test Pull 75% and Push 25% Mixed 1MB +Skipping test Pull 75% and Push 25% Mixed 10MB +Skipping test Pull 75% and Push 25% Mixed 100MB +``` + # References [1] [https://github.com/opencontainers/distribution-spec/tree/main/conformance](https://github.com/opencontainers/distribution-spec/tree/main/conformance) diff --git a/cmd/zb/main.go b/cmd/zb/main.go index d0840d03..5f84d48f 100644 --- a/cmd/zb/main.go +++ b/cmd/zb/main.go @@ -2,6 +2,7 @@ package main import ( "os" + "regexp" distspec "github.com/opencontainers/distribution-spec/specs-go" "github.com/spf13/cobra" @@ -14,11 +15,11 @@ import ( func NewPerfRootCmd() *cobra.Command { showVersion := false - var auth, workdir, repo, outFmt, srcIPs, srcCIDR string + var auth, workdir, repo, outFmt, srcIPs, srcCIDR, testRegexStr string var concurrency, requests int - var skipCleanup bool + var skipCleanup, listTests bool rootCmd := &cobra.Command{ Use: "zb ", @@ -43,13 +44,30 @@ func NewPerfRootCmd() *cobra.Command { url = args[0] } + var err error + if requests < concurrency { panic("requests cannot be less than concurrency") } + var testRegex *regexp.Regexp + + if testRegexStr != "" { + testRegex, err = regexp.Compile(testRegexStr) + if err != nil { + panic("Test filter regex was invalid: " + err.Error()) + } + } + + if listTests { + ListTests(testRegex) + + return + } + requests = concurrency * (requests / concurrency) - Perf(workdir, url, auth, repo, concurrency, requests, outFmt, srcIPs, srcCIDR, skipCleanup) + Perf(workdir, url, auth, repo, concurrency, requests, outFmt, srcIPs, srcCIDR, skipCleanup, testRegex) }, } @@ -70,7 +88,11 @@ func NewPerfRootCmd() *cobra.Command { rootCmd.Flags().StringVarP(&outFmt, "output-format", "o", "", "Output format of test results: stdout (default), json, ci-cd") rootCmd.Flags().BoolVar(&skipCleanup, "skip-cleanup", false, - "Clean up pushed repos from remote registry after running benchmark (default true)") + "Skip clean up of pushed repos from remote registry after running benchmark (default false)") + rootCmd.Flags().StringVarP(&testRegexStr, "test-regex", "t", "", + "Optional regex for selectively running tests. If blank, all tests are run by default.") + rootCmd.Flags().BoolVarP(&listTests, "list-tests", "l", false, + "Print a list of all available tests. When used together with test regex, lists the tests that match the regex.") // "version" rootCmd.Flags().BoolVarP(&showVersion, "version", "v", false, "Show the version and exit") diff --git a/cmd/zb/perf.go b/cmd/zb/perf.go index c0d2511a..4f537f58 100644 --- a/cmd/zb/perf.go +++ b/cmd/zb/perf.go @@ -11,6 +11,7 @@ import ( urlparser "net/url" "os" "path" + "regexp" "sort" "strings" "sync" @@ -661,10 +662,26 @@ var testSuite = []testConfig{ //nolint:gochecknoglobals // used only in this tes }, } +// ListTests logs the available test names with one on each line. +// When testRegex is not nil, only the tests that match the regex are listed. +func ListTests(testRegex *regexp.Regexp) { + log.SetFlags(0) + log.SetOutput(tabwriter.NewWriter(os.Stdout, 0, 0, 1, ' ', tabwriter.TabIndent)) + + for _, tconfig := range testSuite { + if testRegex != nil && !testRegex.MatchString(tconfig.name) { + continue + } + + log.Println(tconfig.name) + } +} + func Perf( workdir, url, auth, repo string, concurrency int, requests int, outFmt string, srcIPs string, srcCIDR string, skipCleanup bool, + testRegex *regexp.Regexp, ) { json := jsoniter.ConfigCompatibleWithStandardLibrary @@ -723,6 +740,12 @@ func Perf( } for _, tconfig := range testSuite { + if testRegex != nil && !testRegex.MatchString(tconfig.name) { + log.Printf("Skipping test %s\n", tconfig.name) + + continue + } + statsCh := make(chan statsRecord, requests) var wg sync.WaitGroup