diff --git a/cmd/zb/README.md b/cmd/zb/README.md index 15ab788e..61eaa686 100644 --- a/cmd/zb/README.md +++ b/cmd/zb/README.md @@ -5,20 +5,23 @@ ``` Usage: - zb [options] [flags] + zb [flags] Flags: -A, --auth-creds string Use colon-separated BASIC auth creds -c, --concurrency int Number of multiple requests to make at a time (default 1) -h, --help help for zb + -l, --list-tests Print a list of all available tests. When used together with test regex, lists the tests that match the regex. -o, --output-format string Output format of test results: stdout (default), json, ci-cd -r, --repo string Use specified repo on remote registry for test data -n, --requests int Number of requests to perform (default 1) + --skip-cleanup Skip clean up of pushed repos from remote registry after running benchmark (default false) -s, --src-cidr string Use specified cidr to obtain ips to make requests from, src-ips and src-cidr are mutually exclusive -i, --src-ips string Use colon-separated ips to make requests from, src-ips and src-cidr are mutually exclusive + -t, --test-regex string Optional regex for selectively running tests. If blank, all tests are run by default. -v, --version Show the version and exit -d, --working-dir string Use specified directory to store test data - ``` +``` ## Command example ``` @@ -75,6 +78,141 @@ p99: 26.375356ms ... ``` +## List tests + +``` +$ zb -l http://localhost:9000 +Get Catalog +Push Monolith 1MB +Push Monolith 10MB +Push Monolith 100MB +Push Chunk Streamed 1MB +Push Chunk Streamed 10MB +Push Chunk Streamed 100MB +Pull 1MB +Pull 10MB +Pull 100MB +Pull Mixed 20% 1MB, 70% 10MB, 10% 100MB +Push Monolith Mixed 20% 1MB, 70% 10MB, 10% 100MB +Push Chunk Mixed 33% 1MB, 33% 10MB, 33% 100MB +Pull 75% and Push 25% Mixed 1MB +Pull 75% and Push 25% Mixed 10MB +Pull 75% and Push 25% Mixed 100MB +``` + +## List tests with Regex + +``` +$ zb -l --test-regex "^(Push Monolith|Pull) 1MB$" http://localhost:9000 +Push Monolith 1MB +Pull 1MB +``` + +## Selective test run example with only push + +``` +$ zb --src-cidr 127.0.0.0/8 --test-regex "^Push Monolith 1MB$" http://localhost:9000 +Registry URL: http://localhost:9000 + +Concurrency Level: 1 +Total requests: 1 +Working dir: /home/darkaether/projects/github/zot + +Preparing test data ... +Starting tests ... +Skipping test Get Catalog +============ +Test name: Push Monolith 1MB +Time taken for tests: 18.700779ms +Requests per second: 53.47371 +Complete requests: 1 +Failed requests: 0 + +2xx responses: 1 + +min: 15.970773ms +max: 15.970773ms +p50: 15.970773ms +p75: 15.970773ms +p90: 15.970773ms +p99: 15.970773ms + +Skipping test Push Monolith 10MB +Skipping test Push Monolith 100MB +Skipping test Push Chunk Streamed 1MB +Skipping test Push Chunk Streamed 10MB +Skipping test Push Chunk Streamed 100MB +Skipping test Pull 1MB +Skipping test Pull 10MB +Skipping test Pull 100MB +Skipping test Pull Mixed 20% 1MB, 70% 10MB, 10% 100MB +Skipping test Push Monolith Mixed 20% 1MB, 70% 10MB, 10% 100MB +Skipping test Push Chunk Mixed 33% 1MB, 33% 10MB, 33% 100MB +Skipping test Pull 75% and Push 25% Mixed 1MB +Skipping test Pull 75% and Push 25% Mixed 10MB +Skipping test Pull 75% and Push 25% Mixed 100MB +``` + +## Selective test run with a push and corresponding pull + +``` +$ zb --src-cidr 127.0.0.0/8 --test-regex "^(Push Monolith|Pull) 1MB$" http://localhost:9000 +Registry URL: http://localhost:9000 + +Concurrency Level: 1 +Total requests: 1 +Working dir: /home/darkaether/projects/github/zot + +Preparing test data ... +Starting tests ... +Skipping test Get Catalog +============ +Test name: Push Monolith 1MB +Time taken for tests: 19.136523ms +Requests per second: 52.256096 +Complete requests: 1 +Failed requests: 0 + +2xx responses: 1 + +min: 16.496555ms +max: 16.496555ms +p50: 16.496555ms +p75: 16.496555ms +p90: 16.496555ms +p99: 16.496555ms + +Skipping test Push Monolith 10MB +Skipping test Push Monolith 100MB +Skipping test Push Chunk Streamed 1MB +Skipping test Push Chunk Streamed 10MB +Skipping test Push Chunk Streamed 100MB +============ +Test name: Pull 1MB +Time taken for tests: 17.836719ms +Requests per second: 56.06412 +Complete requests: 1 +Failed requests: 0 + +2xx responses: 1 + +min: 3.774833ms +max: 3.774833ms +p50: 3.774833ms +p75: 3.774833ms +p90: 3.774833ms +p99: 3.774833ms + +Skipping test Pull 10MB +Skipping test Pull 100MB +Skipping test Pull Mixed 20% 1MB, 70% 10MB, 10% 100MB +Skipping test Push Monolith Mixed 20% 1MB, 70% 10MB, 10% 100MB +Skipping test Push Chunk Mixed 33% 1MB, 33% 10MB, 33% 100MB +Skipping test Pull 75% and Push 25% Mixed 1MB +Skipping test Pull 75% and Push 25% Mixed 10MB +Skipping test Pull 75% and Push 25% Mixed 100MB +``` + # References [1] [https://github.com/opencontainers/distribution-spec/tree/main/conformance](https://github.com/opencontainers/distribution-spec/tree/main/conformance) diff --git a/cmd/zb/main.go b/cmd/zb/main.go index d0840d03..5f84d48f 100644 --- a/cmd/zb/main.go +++ b/cmd/zb/main.go @@ -2,6 +2,7 @@ package main import ( "os" + "regexp" distspec "github.com/opencontainers/distribution-spec/specs-go" "github.com/spf13/cobra" @@ -14,11 +15,11 @@ import ( func NewPerfRootCmd() *cobra.Command { showVersion := false - var auth, workdir, repo, outFmt, srcIPs, srcCIDR string + var auth, workdir, repo, outFmt, srcIPs, srcCIDR, testRegexStr string var concurrency, requests int - var skipCleanup bool + var skipCleanup, listTests bool rootCmd := &cobra.Command{ Use: "zb ", @@ -43,13 +44,30 @@ func NewPerfRootCmd() *cobra.Command { url = args[0] } + var err error + if requests < concurrency { panic("requests cannot be less than concurrency") } + var testRegex *regexp.Regexp + + if testRegexStr != "" { + testRegex, err = regexp.Compile(testRegexStr) + if err != nil { + panic("Test filter regex was invalid: " + err.Error()) + } + } + + if listTests { + ListTests(testRegex) + + return + } + requests = concurrency * (requests / concurrency) - Perf(workdir, url, auth, repo, concurrency, requests, outFmt, srcIPs, srcCIDR, skipCleanup) + Perf(workdir, url, auth, repo, concurrency, requests, outFmt, srcIPs, srcCIDR, skipCleanup, testRegex) }, } @@ -70,7 +88,11 @@ func NewPerfRootCmd() *cobra.Command { rootCmd.Flags().StringVarP(&outFmt, "output-format", "o", "", "Output format of test results: stdout (default), json, ci-cd") rootCmd.Flags().BoolVar(&skipCleanup, "skip-cleanup", false, - "Clean up pushed repos from remote registry after running benchmark (default true)") + "Skip clean up of pushed repos from remote registry after running benchmark (default false)") + rootCmd.Flags().StringVarP(&testRegexStr, "test-regex", "t", "", + "Optional regex for selectively running tests. If blank, all tests are run by default.") + rootCmd.Flags().BoolVarP(&listTests, "list-tests", "l", false, + "Print a list of all available tests. When used together with test regex, lists the tests that match the regex.") // "version" rootCmd.Flags().BoolVarP(&showVersion, "version", "v", false, "Show the version and exit") diff --git a/cmd/zb/perf.go b/cmd/zb/perf.go index c0d2511a..4f537f58 100644 --- a/cmd/zb/perf.go +++ b/cmd/zb/perf.go @@ -11,6 +11,7 @@ import ( urlparser "net/url" "os" "path" + "regexp" "sort" "strings" "sync" @@ -661,10 +662,26 @@ var testSuite = []testConfig{ //nolint:gochecknoglobals // used only in this tes }, } +// ListTests logs the available test names with one on each line. +// When testRegex is not nil, only the tests that match the regex are listed. +func ListTests(testRegex *regexp.Regexp) { + log.SetFlags(0) + log.SetOutput(tabwriter.NewWriter(os.Stdout, 0, 0, 1, ' ', tabwriter.TabIndent)) + + for _, tconfig := range testSuite { + if testRegex != nil && !testRegex.MatchString(tconfig.name) { + continue + } + + log.Println(tconfig.name) + } +} + func Perf( workdir, url, auth, repo string, concurrency int, requests int, outFmt string, srcIPs string, srcCIDR string, skipCleanup bool, + testRegex *regexp.Regexp, ) { json := jsoniter.ConfigCompatibleWithStandardLibrary @@ -723,6 +740,12 @@ func Perf( } for _, tconfig := range testSuite { + if testRegex != nil && !testRegex.MatchString(tconfig.name) { + log.Printf("Skipping test %s\n", tconfig.name) + + continue + } + statsCh := make(chan statsRecord, requests) var wg sync.WaitGroup