diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index da7d2587..a8bb2fa5 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -62,7 +62,7 @@ jobs: sudo apt-get update sudo apt-get install rpm sudo apt-get install snapd - sudo apt-get install libgpgme-dev libassuan-dev libbtrfs-dev libdevmapper-dev pkg-config + sudo apt-get install libgpgme-dev libassuan-dev libbtrfs-dev libdevmapper-dev pkg-config python3-jsonschema git clone https://github.com/containers/skopeo -b v1.12.0 $GITHUB_WORKSPACE/src/github.com/containers/skopeo cd $GITHUB_WORKSPACE/src/github.com/containers/skopeo && \ make bin/skopeo && \ @@ -90,6 +90,11 @@ jobs: curl -X POST -H "Content-Type: application/json" -d @.pkg/debug/githubWorkflows/introspection-query.json http://localhost:5000/v2/_zot/ext/search | jq > bin/zot-gql-introspection-result.json pkill zot + - name: Generate zot config schema on Release + if: github.event_name == 'release' && github.event.action == 'published' && matrix.os == 'linux' && matrix.arch == 'amd64' + run: | + make verify-config-schema + - if: github.event_name == 'release' && github.event.action == 'published' name: Publish artifacts on releases uses: svenstaro/upload-release-action@v2 diff --git a/Makefile b/Makefile index 726f293c..fa6891a5 100644 --- a/Makefile +++ b/Makefile @@ -425,7 +425,7 @@ run: binary ./bin/zot-$(OS)-$(ARCH) serve examples/config-test.json .PHONY: verify-config -verify-config: _verify-config verify-config-warnings verify-config-commited +verify-config: _verify-config verify-config-warnings verify-config-commited verify-config-schema .PHONY: _verify-config _verify-config: binary @@ -451,6 +451,15 @@ verify-config-commited: _verify-config exit 1;\ fi; \ +.PHONY: check-jsonschema +check-jsonschema: + jsonschema --version || (echo "You need python3-jsonschema to validate config examples against generated schema"; exit 1) + +.PHONY: verify-config-schema +verify-config-schema: binary check-jsonschema + ./bin/zot-$(OS)-$(ARCH) schema > bin/zot-schema.json + for i in $(filter-out $(wildcard examples/config-*-credentials.json), $(wildcard examples/config-*.json)); do echo $$i; jsonschema bin/zot-schema.json -i "$$i" -o pretty; done + .PHONY: gqlgen gqlgen: cd pkg/extensions/search;\ diff --git a/go.mod b/go.mod index 850c3939..96dac39f 100644 --- a/go.mod +++ b/go.mod @@ -64,6 +64,7 @@ require ( github.com/redis/go-redis/v9 v9.18.0 github.com/regclient/regclient v0.11.2 github.com/santhosh-tekuri/jsonschema/v5 v5.3.1 + github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 github.com/sigstore/cosign/v3 v3.0.5 github.com/sigstore/sigstore v1.10.4 github.com/smartystreets/goconvey v1.8.1 @@ -442,7 +443,6 @@ require ( github.com/sagikazarmark/locafero v0.12.0 // indirect github.com/samber/lo v1.52.0 // indirect github.com/samber/oops v1.18.1 // indirect - github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 // indirect github.com/sassoftware/go-rpmutils v0.4.0 // indirect github.com/sassoftware/relic v7.2.1+incompatible // indirect github.com/secure-systems-lab/go-securesystemslib v0.10.0 // indirect diff --git a/pkg/cli/server/root.go b/pkg/cli/server/root.go index 91f02cb5..fc4696da 100644 --- a/pkg/cli/server/root.go +++ b/pkg/cli/server/root.go @@ -234,6 +234,8 @@ func NewServerRootCmd() *cobra.Command { rootCmd.AddCommand(newVerifyCmd(conf)) // "scrub" rootCmd.AddCommand(newScrubCmd(conf)) + // "schema" + rootCmd.AddCommand(newSchemaCmd()) // "verify-feature" rootCmd.AddCommand(newVerifyFeatureCmd(conf)) // "version" diff --git a/pkg/cli/server/root_test.go b/pkg/cli/server/root_test.go index f5aa8641..3037ff99 100644 --- a/pkg/cli/server/root_test.go +++ b/pkg/cli/server/root_test.go @@ -1,6 +1,7 @@ package server_test import ( + "bytes" "encoding/json" "fmt" "os" @@ -86,6 +87,138 @@ func TestServerUsage(t *testing.T) { }) } +func TestSchema(t *testing.T) { + Convey("Test schema command", t, func(c C) { + cmd := cli.NewServerRootCmd() + buf := bytes.NewBuffer(nil) + + cmd.SetArgs([]string{"schema"}) + cmd.SetOut(buf) + + err := cmd.Execute() + So(err, ShouldBeNil) + + var schemaDoc map[string]any + err = json.Unmarshal(buf.Bytes(), &schemaDoc) + So(err, ShouldBeNil) + + So(schemaDoc["$schema"], ShouldEqual, "http://json-schema.org/draft-07/schema#") + So(schemaDoc["title"], ShouldEqual, "zot config schema") + + defs, ok := schemaDoc["definitions"].(map[string]any) + So(ok, ShouldBeTrue) + So(defs, ShouldContainKey, "zotregistry.dev~1zot~1v2~1pkg~1api~1config.Config") + So(defs, ShouldContainKey, "zotregistry.dev~1zot~1v2~1pkg~1extensions~1config.ExtensionConfig") + + rootRef, ok := schemaDoc["$ref"].(string) + So(ok, ShouldBeTrue) + So(rootRef, ShouldEqual, "#/definitions/zotregistry.dev~01zot~01v2~01pkg~01api~01config.Config") + + configSchema, ok := mustResolveSchemaRef(rootRef, defs) + So(ok, ShouldBeTrue) + configProperties, ok := configSchema["properties"].(map[string]any) + So(ok, ShouldBeTrue) + So(configProperties, ShouldContainKey, "distSpecVersion") + So(configProperties, ShouldContainKey, "storage") + So(configProperties, ShouldContainKey, "http") + So(configProperties, ShouldContainKey, "extensions") + So(configProperties, ShouldNotContainKey, "hTTP") + + storageSchema, ok := mustResolvePropertySchema(configProperties, "storage", defs) + So(ok, ShouldBeTrue) + storageProperties, ok := storageSchema["properties"].(map[string]any) + So(ok, ShouldBeTrue) + So(storageProperties, ShouldContainKey, "rootDirectory") + So(storageProperties, ShouldContainKey, "gcDelay") + So(storageProperties, ShouldContainKey, "gcInterval") + So(storageProperties, ShouldContainKey, "subPaths") + So(storageProperties, ShouldNotContainKey, "gcMaxSchedulerDelay") + So(storageProperties, ShouldNotContainKey, "gCDelay") + So(storageProperties, ShouldNotContainKey, "gCInterval") + + httpSchema, ok := mustResolvePropertySchema(configProperties, "http", defs) + So(ok, ShouldBeTrue) + httpProperties, ok := httpSchema["properties"].(map[string]any) + So(ok, ShouldBeTrue) + So(httpProperties, ShouldContainKey, "address") + So(httpProperties, ShouldContainKey, "port") + So(httpProperties, ShouldContainKey, "accessControl") + + extensionsSchema, ok := mustResolvePropertySchema(configProperties, "extensions", defs) + So(ok, ShouldBeTrue) + extensionsProperties, ok := extensionsSchema["properties"].(map[string]any) + So(ok, ShouldBeTrue) + So(extensionsProperties, ShouldContainKey, "search") + So(extensionsProperties, ShouldContainKey, "sync") + So(extensionsProperties, ShouldContainKey, "metrics") + + syncSchema, ok := mustResolvePropertySchema(extensionsProperties, "sync", defs) + So(ok, ShouldBeTrue) + syncProperties, ok := syncSchema["properties"].(map[string]any) + So(ok, ShouldBeTrue) + So(syncProperties, ShouldContainKey, "registries") + + registriesSchema, ok := mustResolvePropertySchema(syncProperties, "registries", defs) + So(ok, ShouldBeTrue) + registriesItemSchema, ok := registriesSchema["items"].(map[string]any) + So(ok, ShouldBeTrue) + + registrySchema, ok := mustResolveSchema(registriesItemSchema, defs) + So(ok, ShouldBeTrue) + registryProperties, ok := registrySchema["properties"].(map[string]any) + So(ok, ShouldBeTrue) + So(registryProperties, ShouldNotContainKey, "responseHeaderTimeout") + }) +} + +func mustResolvePropertySchema(properties map[string]any, name string, defs map[string]any) (map[string]any, bool) { + schema, ok := properties[name].(map[string]any) + if !ok { + return nil, false + } + + return mustResolveSchema(schema, defs) +} + +func mustResolveSchema(schema map[string]any, defs map[string]any) (map[string]any, bool) { + if schema == nil { + return nil, false + } + + if ref, ok := schema["$ref"].(string); ok { + return mustResolveSchemaRef(ref, defs) + } + + if anyOf, ok := schema["anyOf"].([]any); ok { + for _, option := range anyOf { + optionSchema, ok := option.(map[string]any) + if !ok { + continue + } + + if optionType, ok := optionSchema["type"].(string); ok && optionType == "null" { + continue + } + + return mustResolveSchema(optionSchema, defs) + } + } + + return schema, true +} + +func mustResolveSchemaRef(ref string, defs map[string]any) (map[string]any, bool) { + defKey := strings.TrimPrefix(ref, "#/definitions/") + // Decode one JSON Pointer token level to the stored definition key. + defKey = strings.NewReplacer("~1", "/", "~0", "~").Replace(defKey) + schema, ok := defs[defKey].(map[string]any) + if !ok { + return nil, false + } + + return schema, true +} + func TestServe(t *testing.T) { oldArgs := os.Args diff --git a/pkg/cli/server/schema.go b/pkg/cli/server/schema.go new file mode 100644 index 00000000..f95714bc --- /dev/null +++ b/pkg/cli/server/schema.go @@ -0,0 +1,385 @@ +package server + +import ( + "encoding/json" + "fmt" + "maps" + "reflect" + "slices" + "sort" + "strings" + + "github.com/santhosh-tekuri/jsonschema/v6" + "github.com/spf13/cobra" + + "zotregistry.dev/zot/v2/pkg/api/config" +) + +func newSchemaCmd() *cobra.Command { + schemaCmd := &cobra.Command{ + Use: "schema", + Short: "`schema` dumps JSON Schema for zot config", + Long: "`schema` dumps JSON Schema for zot config", + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, _ []string) error { + schemaDoc := buildConfigSchemaDocument() + + schemaJSON, err := json.MarshalIndent(schemaDoc, "", " ") + if err != nil { + return err + } + + compiler := jsonschema.NewCompiler() + compiler.DefaultDraft(jsonschema.Draft7) + + if err := compiler.AddResource("zot://config-schema.json", schemaDoc); err != nil { + return err + } + + if _, err := compiler.Compile("zot://config-schema.json"); err != nil { + return fmt.Errorf("generated schema is invalid: %w", err) + } + + if _, err := cmd.OutOrStdout().Write(schemaJSON); err != nil { + return err + } + + _, err = cmd.OutOrStdout().Write([]byte("\n")) + + return err + }, + } + + return schemaCmd +} + +func buildConfigSchemaDocument() map[string]any { + gen := newSchemaGenerator() + + configSchema := gen.schemaForType(reflect.TypeFor[config.Config]()) + + doc := map[string]any{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "zot config schema", + "$ref": configSchema["$ref"], + "definitions": gen.defs, + } + + return doc +} + +type schemaGenerator struct { + defs map[string]any +} + +func newSchemaGenerator() *schemaGenerator { + return &schemaGenerator{defs: map[string]any{}} +} + +func (g *schemaGenerator) schemaForType(reflectType reflect.Type) map[string]any { + if reflectType.Kind() == reflect.Pointer { + return nullableSchema(g.schemaForType(derefPointerType(reflectType))) + } + + if reflectType.PkgPath() == "time" && reflectType.Name() == "Duration" { + return map[string]any{"type": "string"} + } + + switch reflectType.Kind() { + case reflect.Bool: + return map[string]any{"type": "boolean"} + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64, + reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64: + return map[string]any{"type": "integer"} + case reflect.Float32, reflect.Float64: + return map[string]any{"type": "number"} + case reflect.String: + return map[string]any{"type": "string"} + case reflect.Slice, reflect.Array: + return map[string]any{ + "type": "array", + "items": g.schemaForType(reflectType.Elem()), + } + case reflect.Map: + return map[string]any{ + "type": "object", + "additionalProperties": g.schemaForType(reflectType.Elem()), + } + case reflect.Interface: + return map[string]any{} + case reflect.Struct: + return g.schemaForStruct(reflectType) + default: + return map[string]any{} + } +} + +func (g *schemaGenerator) schemaForStruct(reflectType reflect.Type) map[string]any { + defName := schemaDefName(reflectType) + if defName != "." { + if _, ok := g.defs[defName]; ok { + return map[string]any{"$ref": "#/definitions/" + schemaDefRefToken(defName)} + } + + g.defs[defName] = map[string]any{} + } + + properties := map[string]any{} + + for i := range reflectType.NumField() { + field := reflectType.Field(i) + + if !field.IsExported() { + continue + } + + fieldName, squash, skip := schemaFieldName(field) + if skip { + continue + } + + fieldSchema := g.schemaForType(field.Type) + + if squash { + if ref, ok := fieldSchema["$ref"].(string); ok { + defKey := schemaDefFromRefToken(strings.TrimPrefix(ref, "#/definitions/")) + if def, ok := g.defs[defKey].(map[string]any); ok { + mergeProperties(properties, def) + } + } else { + mergeProperties(properties, fieldSchema) + } + + continue + } + + properties[fieldName] = fieldSchema + for _, alias := range schemaFieldAliases(fieldName) { + if _, exists := properties[alias]; !exists { + properties[alias] = fieldSchema + } + } + } + + schema := map[string]any{ + "type": "object", + "properties": properties, + "additionalProperties": false, + } + + if defName == "." { + return schema + } + + g.defs[defName] = schema + + return map[string]any{"$ref": "#/definitions/" + schemaDefRefToken(defName)} +} + +func schemaFieldName(field reflect.StructField) (string, bool, bool) { + jsonName, _, jsonSkip := parseStructTag(field.Tag.Get("json")) + if jsonSkip { + return "", false, true + } + + mapstructureName, mapstructureFlags, mapstructureSkip := parseStructTag(field.Tag.Get("mapstructure")) + if mapstructureSkip { + return "", false, true + } + + yamlName, yamlFlags, yamlSkip := parseStructTag(field.Tag.Get("yaml")) + if yamlSkip { + return "", false, true + } + + if hasFlag(mapstructureFlags, "squash") || hasFlag(yamlFlags, "inline") { + return "", true, false + } + + if jsonName != "" { + return jsonName, false, false + } + + if mapstructureName != "" { + return mapstructureName, false, false + } + + if yamlName != "" { + return yamlName, false, false + } + + if field.Anonymous { + return "", true, false + } + + return lowerCamelCase(field.Name), false, false +} + +func parseStructTag(tag string) (string, []string, bool) { + if tag == "" { + return "", nil, false + } + + parts := strings.Split(tag, ",") + flags := []string(nil) + if parts[0] == "-" { + return "", nil, true + } + + if len(parts) > 1 { + flags = parts[1:] + } + + return parts[0], flags, false +} + +func hasFlag(flags []string, target string) bool { + return slices.Contains(flags, target) +} + +func mergeProperties(dst map[string]any, schema map[string]any) { + props, ok := schema["properties"].(map[string]any) + if !ok { + return + } + + keys := make([]string, 0, len(props)) + for k := range props { + keys = append(keys, k) + } + sort.Strings(keys) + + for _, key := range keys { + dst[key] = props[key] + } +} + +func lowerCamelCase(input string) string { + if input == "" { + return "" + } + + upperPrefixLen := 0 + for upperPrefixLen < len(input) && input[upperPrefixLen] >= 'A' && input[upperPrefixLen] <= 'Z' { + upperPrefixLen++ + } + + if upperPrefixLen == 0 { + return input + } + + if upperPrefixLen == len(input) { + return strings.ToLower(input) + } + + if upperPrefixLen == 1 { + return strings.ToLower(input[:1]) + input[1:] + } + + return strings.ToLower(input[:upperPrefixLen-1]) + input[upperPrefixLen-1:] +} + +func schemaFieldAliases(fieldName string) []string { + aliases := []string{} + + lower := strings.ToLower(fieldName) + if lower != fieldName { + aliases = append(aliases, lower) + } + + collapsed := collapseUppercaseRuns(fieldName) + if collapsed != fieldName && collapsed != lower { + aliases = append(aliases, collapsed) + } + + return aliases +} + +func collapseUppercaseRuns(input string) string { + if input == "" { + return "" + } + + builder := strings.Builder{} + builder.Grow(len(input)) + + for i := 0; i < len(input); { + uppercaseEnd := i + for uppercaseEnd < len(input) && input[uppercaseEnd] >= 'A' && input[uppercaseEnd] <= 'Z' { + uppercaseEnd++ + } + + if uppercaseEnd-i >= 2 { + builder.WriteByte(input[i]) + builder.WriteString(strings.ToLower(input[i+1 : uppercaseEnd])) + i = uppercaseEnd + + continue + } + + builder.WriteByte(input[i]) + i++ + } + + return builder.String() +} + +func schemaDefName(reflectType reflect.Type) string { + if reflectType.Name() == "" { + return "." + } + + name := reflectType.PkgPath() + "." + reflectType.Name() + + // Keep definition keys safe/collision-free using RFC 6901 escaping. + // "~" becomes "~0" and "/" becomes "~1". + replacer := strings.NewReplacer("~", "~0", "/", "~1") + + return replacer.Replace(name) +} + +func schemaDefRefToken(defName string) string { + // $ref uses JSON Pointer tokens, which decode "~1"->"/" and "~0"->"~". + // Since defName is already RFC 6901-escaped for storage, escape it once more for pointer usage. + replacer := strings.NewReplacer("~", "~0", "/", "~1") + + return replacer.Replace(defName) +} + +func schemaDefFromRefToken(token string) string { + // Decode one JSON Pointer token level back to the stored definition key. + replacer := strings.NewReplacer("~1", "/", "~0", "~") + + return replacer.Replace(token) +} + +func derefPointerType(reflectType reflect.Type) reflect.Type { + for reflectType.Kind() == reflect.Pointer { + reflectType = reflectType.Elem() + } + + return reflectType +} + +func nullableSchema(schema map[string]any) map[string]any { + if schemaType, ok := schema["type"].(string); ok { + nullable := mapsClone(schema) + nullable["type"] = []any{schemaType, "null"} + + return nullable + } + + return map[string]any{ + "anyOf": []any{ + schema, + map[string]any{"type": "null"}, + }, + } +} + +func mapsClone(src map[string]any) map[string]any { + dst := make(map[string]any, len(src)) + maps.Copy(dst, src) + + return dst +} diff --git a/pkg/cli/server/schema_test.go b/pkg/cli/server/schema_test.go new file mode 100644 index 00000000..ced6dc3e --- /dev/null +++ b/pkg/cli/server/schema_test.go @@ -0,0 +1,90 @@ +package server_test + +import ( + "bytes" + "encoding/json" + "testing" + + "github.com/santhosh-tekuri/jsonschema/v6" + . "github.com/smartystreets/goconvey/convey" + + cli "zotregistry.dev/zot/v2/pkg/cli/server" +) + +func TestSchemaAllowsNullForPointerFields(t *testing.T) { + Convey("generated schema allows explicit null for pointer-backed config fields", t, func() { + cmd := cli.NewServerRootCmd() + buf := bytes.NewBuffer(nil) + + cmd.SetArgs([]string{"schema"}) + cmd.SetOut(buf) + + err := cmd.Execute() + So(err, ShouldBeNil) + + var schemaDoc map[string]any + err = json.Unmarshal(buf.Bytes(), &schemaDoc) + So(err, ShouldBeNil) + + compiler := jsonschema.NewCompiler() + compiler.DefaultDraft(jsonschema.Draft7) + + err = compiler.AddResource("zot://config-schema.json", schemaDoc) + So(err, ShouldBeNil) + + compiledSchema, err := compiler.Compile("zot://config-schema.json") + So(err, ShouldBeNil) + + configWithNullPointers := map[string]any{ + "http": map[string]any{ + "auth": map[string]any{ + "secureSession": nil, + "mtls": nil, + }, + }, + "storage": map[string]any{ + "retention": map[string]any{ + "policies": []any{ + map[string]any{ + "deleteUntagged": nil, + "keepTags": []any{ + map[string]any{ + "pulledWithin": nil, + "pushedWithin": nil, + }, + }, + }, + }, + }, + }, + "extensions": map[string]any{ + "sync": map[string]any{ + "enable": nil, + "registries": []any{ + map[string]any{ + "tlsVerify": nil, + "retryDelay": nil, + "onlySigned": nil, + "syncLegacyCosignTags": nil, + }, + }, + }, + }, + "cluster": nil, + } + + err = compiledSchema.Validate(configWithNullPointers) + So(err, ShouldBeNil) + + err = compiledSchema.Validate(map[string]any{ + "storage": map[string]any{ + "rootDirectory": "/tmp/zot", + }, + "http": map[string]any{ + "address": "127.0.0.1", + "port": "8080", + }, + }) + So(err, ShouldBeNil) + }) +} diff --git a/pkg/storage/common/common.go b/pkg/storage/common/common.go index 4f751aaa..a6635537 100644 --- a/pkg/storage/common/common.go +++ b/pkg/storage/common/common.go @@ -19,7 +19,7 @@ import ( "github.com/opencontainers/image-spec/schema" imeta "github.com/opencontainers/image-spec/specs-go" ispec "github.com/opencontainers/image-spec/specs-go/v1" - "github.com/santhosh-tekuri/jsonschema/v5" + jsonschemaV5 "github.com/santhosh-tekuri/jsonschema/v5" zerr "zotregistry.dev/zot/v2/errors" zcommon "zotregistry.dev/zot/v2/pkg/common" @@ -890,16 +890,12 @@ func ValidateImageIndexSchema(buf []byte) error { } func IsEmptyLayersError(err error) bool { - var validationErr *jsonschema.ValidationError - if errors.As(err, &validationErr) { - if len(validationErr.Causes) == 1 && strings.Contains(err.Error(), manifestWithEmptyLayersErrMsg) { - return true - } else { - return false - } + var validationErr *jsonschemaV5.ValidationError + if !errors.As(err, &validationErr) { + return false } - return false + return len(validationErr.Causes) == 1 && strings.Contains(validationErr.Error(), manifestWithEmptyLayersErrMsg) } // DedupeTaskGenerator takes all blobs paths found in the storage.imagestore and groups them by digest.