From bc5fd1a35778ff74a2fb71efd7eaa6812fb88ad5 Mon Sep 17 00:00:00 2001 From: Piaras Hoban <4415593+phoban01@users.noreply.github.com> Date: Fri, 2 May 2025 20:30:06 +0100 Subject: [PATCH] feat(events): add events extension (#3045) * feat: add events config Signed-off-by: Piaras Hoban * feat: implement event support with log sink Signed-off-by: Piaras Hoban * feat: integrate events and update tests Signed-off-by: Piaras Hoban * refactor: update event config Signed-off-by: Piaras Hoban * feat: implement http and nats sinks. remove log sink Signed-off-by: Piaras Hoban * refactor: events extension setup Signed-off-by: Piaras Hoban * chore: cleanup tests to use nil event recorder Signed-off-by: Piaras Hoban * chore: update events config example and add more logging Signed-off-by: Piaras Hoban * refactor: better use of build tags for minimal binary Signed-off-by: Piaras Hoban * fix: missing store param in evelated privileges tests Signed-off-by: Piaras Hoban * fix: regression in config decoding Signed-off-by: Piaras Hoban * chore: update check logs script to enable cross-platform usage via GREP_BIN_PATH envvar Signed-off-by: Piaras Hoban * chore: fix log lint issue for events Signed-off-by: Piaras Hoban * chore: fix failing events disabled test Signed-off-by: Piaras Hoban * test: add blackbox tests for events Signed-off-by: Piaras Hoban * chore: specify architecture when downloading binaries in Makefile Signed-off-by: Piaras Hoban * chore: improve failure handling when no valid sinks are provided Signed-off-by: Piaras Hoban * test: fix data race in events test Signed-off-by: Piaras Hoban * chore: cleanup event decoding Signed-off-by: Piaras Hoban * test: fix logging tests Signed-off-by: Piaras Hoban * test: make nats server test more reliable Signed-off-by: Piaras Hoban * chore: go mod cleanup Signed-off-by: Piaras Hoban * test: add sleep when setting up nats client Signed-off-by: Piaras Hoban * fix: ensure event sink errors do not propogate Signed-off-by: Piaras Hoban * test: increase coverage for events Signed-off-by: Piaras Hoban * feat(events): Refactor events to be non-blocking from caller. Signed-off-by: Asgeir Nilsen Signed-off-by: Piaras Hoban * chore: remove harded-coded linux Co-authored-by: Andrei Aaron Signed-off-by: Piaras Hoban * feat(events): fail to start if incorrect event sink is configured Signed-off-by: Piaras Hoban * test: allow cli tests to return errors instead of panic Signed-off-by: Piaras Hoban * chore: bump nats server to v2.11.3 Signed-off-by: Piaras Hoban --------- Signed-off-by: Piaras Hoban Signed-off-by: Asgeir Nilsen Co-authored-by: Asgeir Nilsen Co-authored-by: Andrei Aaron --- Makefile | 22 +- errors/errors.go | 272 +++++++------- examples/config-events.json | 24 ++ go.mod | 11 +- go.sum | 23 ++ pkg/api/authn_test.go | 4 +- pkg/api/config/config.go | 18 + pkg/api/config/config_test.go | 28 ++ pkg/api/controller.go | 20 +- pkg/cli/client/cve_cmd_test.go | 2 +- pkg/cli/server/extensions_test.go | 81 +++- pkg/cli/server/root.go | 16 +- pkg/cli/server/root_test.go | 15 +- pkg/extensions/config/config.go | 2 + pkg/extensions/config/events/config.go | 55 +++ pkg/extensions/config/events/decoder.go | 52 +++ pkg/extensions/events/builder.go | 55 +++ pkg/extensions/events/common.go | 32 ++ pkg/extensions/events/events.go | 150 ++++++++ pkg/extensions/events/events_test.go | 350 ++++++++++++++++++ pkg/extensions/events/extension.go | 27 ++ pkg/extensions/events/http_sink.go | 130 +++++++ pkg/extensions/events/http_sink_test.go | 140 +++++++ pkg/extensions/events/nats_sink.go | 96 +++++ pkg/extensions/events/nats_sink_test.go | 130 +++++++ pkg/extensions/extension_events.go | 61 +++ pkg/extensions/extension_events_disabled.go | 24 ++ .../extension_events_disabled_test.go | 54 +++ pkg/extensions/extension_image_trust_test.go | 10 +- pkg/extensions/lint/lint_test.go | 14 +- pkg/extensions/scrub/scrub_test.go | 6 +- pkg/extensions/search/cve/cve_test.go | 2 +- pkg/extensions/search/cve/scan_test.go | 2 +- .../search/cve/trivy/scanner_internal_test.go | 12 +- .../search/cve/trivy/scanner_test.go | 4 +- pkg/extensions/search/search_test.go | 12 +- pkg/extensions/sync/destination.go | 2 +- pkg/extensions/sync/sync_internal_test.go | 4 +- pkg/meta/hooks_test.go | 2 +- pkg/meta/parse_test.go | 8 +- pkg/storage/common/common_test.go | 6 +- pkg/storage/gc/gc_internal_test.go | 4 +- pkg/storage/gc/gc_test.go | 6 +- pkg/storage/imagestore/imagestore.go | 21 +- pkg/storage/local/local.go | 4 +- pkg/storage/local/local_elevated_test.go | 2 +- pkg/storage/local/local_test.go | 122 +++--- pkg/storage/s3/s3.go | 4 +- pkg/storage/s3/s3_test.go | 13 +- pkg/storage/scrub_test.go | 4 +- pkg/storage/storage.go | 15 +- pkg/storage/storage_test.go | 20 +- pkg/test/oci-utils/oci_layout_test.go | 6 +- pkg/test/oci-utils/store.go | 2 +- scripts/check_logs.sh | 19 +- test/blackbox/ci.sh | 3 +- test/blackbox/events_config_decoding.bats | 122 ++++++ test/blackbox/events_http.bats | 167 +++++++++ test/blackbox/events_http_lint_failure.bats | 162 ++++++++ test/blackbox/events_nats.bats | 158 ++++++++ test/blackbox/events_nats_lint_failure.bats | 161 ++++++++ test/blackbox/events_sink_failure.bats | 98 +++++ test/blackbox/helpers_events.bash | 122 ++++++ 63 files changed, 2907 insertions(+), 306 deletions(-) create mode 100644 examples/config-events.json create mode 100644 pkg/extensions/config/events/config.go create mode 100644 pkg/extensions/config/events/decoder.go create mode 100644 pkg/extensions/events/builder.go create mode 100644 pkg/extensions/events/common.go create mode 100644 pkg/extensions/events/events.go create mode 100644 pkg/extensions/events/events_test.go create mode 100644 pkg/extensions/events/extension.go create mode 100644 pkg/extensions/events/http_sink.go create mode 100644 pkg/extensions/events/http_sink_test.go create mode 100644 pkg/extensions/events/nats_sink.go create mode 100644 pkg/extensions/events/nats_sink_test.go create mode 100644 pkg/extensions/extension_events.go create mode 100644 pkg/extensions/extension_events_disabled.go create mode 100644 pkg/extensions/extension_events_disabled_test.go create mode 100644 test/blackbox/events_config_decoding.bats create mode 100644 test/blackbox/events_http.bats create mode 100644 test/blackbox/events_http_lint_failure.bats create mode 100644 test/blackbox/events_nats.bats create mode 100644 test/blackbox/events_nats_lint_failure.bats create mode 100644 test/blackbox/events_sink_failure.bats create mode 100644 test/blackbox/helpers_events.bash diff --git a/Makefile b/Makefile index 19d13c8e..5a038621 100644 --- a/Makefile +++ b/Makefile @@ -18,6 +18,7 @@ COSIGN_VERSION := 2.2.0 HELM := $(TOOLSDIR)/bin/helm ORAS := $(TOOLSDIR)/bin/oras ORAS_VERSION := 1.2.1 +HELM_VERSION := v3.9.1 REGCLIENT := $(TOOLSDIR)/bin/regctl REGCLIENT_VERSION := v0.5.7 CRICTL := $(TOOLSDIR)/bin/crictl @@ -35,6 +36,7 @@ BATS := $(TOOLSDIR)/bin/bats TESTDATA := $(TOP_LEVEL)/test/data OS ?= $(shell go env GOOS) ARCH ?= $(shell go env GOARCH) +GREP_BIN_PATH ?= $(shell which grep) PROTOC := $(TOOLSDIR)/bin/protoc PROTOC_VERSION := 24.4 @@ -53,8 +55,8 @@ else ifeq ($(HOST_ARCH),arm64) endif BENCH_OUTPUT ?= stdout -ALL_EXTENSIONS = debug,imagetrust,lint,metrics,mgmt,profile,scrub,search,sync,ui,userprefs -EXTENSIONS ?= sync,search,scrub,metrics,lint,ui,mgmt,profile,userprefs,imagetrust +ALL_EXTENSIONS = debug,imagetrust,lint,metrics,mgmt,profile,scrub,search,sync,ui,userprefs,events +EXTENSIONS ?= sync,search,scrub,metrics,lint,ui,mgmt,profile,userprefs,imagetrust,events UI_DEPENDENCIES := search,mgmt,userprefs # freebsd is not supported for pie builds if CGO is disabled # see supported platforms at https://cs.opensource.google/go/go/+/master:src/internal/platform/supported.go;l=222-231;drc=d7fcb5cf80953f1d63246f1ae9defa60c5ce2d76 @@ -258,31 +260,31 @@ check-awslocal: $(NOTATION): mkdir -p $(TOOLSDIR)/bin - curl -Lo notation.tar.gz https://github.com/notaryproject/notation/releases/download/v$(NOTATION_VERSION)/notation_$(NOTATION_VERSION)_linux_amd64.tar.gz + curl -Lo notation.tar.gz https://github.com/notaryproject/notation/releases/download/v$(NOTATION_VERSION)/notation_$(NOTATION_VERSION)_$(OS)_$(ARCH).tar.gz tar xvzf notation.tar.gz -C $(TOOLSDIR)/bin notation rm notation.tar.gz $(ORAS): mkdir -p $(TOOLSDIR)/bin - curl -Lo oras.tar.gz https://github.com/oras-project/oras/releases/download/v$(ORAS_VERSION)/oras_$(ORAS_VERSION)_linux_amd64.tar.gz + curl -Lo oras.tar.gz https://github.com/oras-project/oras/releases/download/v$(ORAS_VERSION)/oras_$(ORAS_VERSION)_$(OS)_$(ARCH).tar.gz tar xvzf oras.tar.gz -C $(TOOLSDIR)/bin oras rm oras.tar.gz $(HELM): mkdir -p $(TOOLSDIR)/bin - curl -Lo helm.tar.gz https://get.helm.sh/helm-v3.9.1-linux-amd64.tar.gz - tar xvzf helm.tar.gz -C $(TOOLSDIR)/bin linux-amd64/helm --strip-components=1 + curl -Lo helm.tar.gz https://get.helm.sh/helm-$(HELM_VERSION)-$(OS)-$(ARCH).tar.gz + tar xvzf helm.tar.gz --strip-components=1 -C $(TOOLSDIR)/bin $(OS)-$(ARCH)/helm rm helm.tar.gz $(REGCLIENT): mkdir -p $(TOOLSDIR)/bin - curl -Lo regctl https://github.com/regclient/regclient/releases/download/$(REGCLIENT_VERSION)/regctl-linux-amd64 + curl -Lo regctl https://github.com/regclient/regclient/releases/download/$(REGCLIENT_VERSION)/regctl-$(OS)-$(ARCH) mv regctl $(TOOLSDIR)/bin/regctl chmod +x $(TOOLSDIR)/bin/regctl $(CRICTL): mkdir -p $(TOOLSDIR)/bin - curl -Lo crictl.tar.gz https://github.com/kubernetes-sigs/cri-tools/releases/download/$(CRICTL_VERSION)/crictl-$(CRICTL_VERSION)-linux-amd64.tar.gz + curl -Lo crictl.tar.gz https://github.com/kubernetes-sigs/cri-tools/releases/download/$(CRICTL_VERSION)/crictl-$(CRICTL_VERSION)-$(OS)-$(ARCH).tar.gz tar xvzf crictl.tar.gz && rm crictl.tar.gz mv crictl $(TOOLSDIR)/bin/crictl chmod +x $(TOOLSDIR)/bin/crictl @@ -521,7 +523,7 @@ run-cloud-scale-out-redis-high-scale-tests: check-blackbox-prerequisites check-a .PHONY: run-blackbox-ci run-blackbox-ci: check-blackbox-prerequisites binary binary-minimal cli - echo running CI bats tests concurently + echo running CI bats tests concurrently BATS_FLAGS="$(BATS_FLAGS)" test/blackbox/ci.sh .PHONY: run-blackbox-cloud-ci @@ -560,7 +562,7 @@ $(STACKER): check-linux $(COSIGN): mkdir -p $(TOOLSDIR)/bin - curl -fsSL https://github.com/sigstore/cosign/releases/download/v$(COSIGN_VERSION)/cosign-linux-amd64 -o $@; \ + curl -fsSL https://github.com/sigstore/cosign/releases/download/v$(COSIGN_VERSION)/cosign-$(OS)-$(ARCH) -o $@; \ chmod +x $@ # set ZUI_VERSION to empty string in order to clone zui locally and build default branch diff --git a/errors/errors.go b/errors/errors.go index 21413267..37312244 100644 --- a/errors/errors.go +++ b/errors/errors.go @@ -47,137 +47,143 @@ func GetDetails(err error) map[string]string { } var ( - ErrBadConfig = errors.New("invalid server config") - ErrCliBadConfig = errors.New("invalid cli config") - ErrRepoNotFound = errors.New("repository not found") - ErrRepoBadVersion = errors.New("unsupported repository layout version") - ErrRepoBadLayout = errors.New("invalid repository layout") - ErrManifestNotFound = errors.New("manifest not found") - ErrBadManifest = errors.New("invalid manifest content") - ErrUploadNotFound = errors.New("upload destination not found") - ErrBadUploadRange = errors.New("bad upload content-length") - ErrBlobNotFound = errors.New("blob not found") - ErrBadBlob = errors.New("bad blob") - ErrBadBlobDigest = errors.New("bad blob digest") - ErrBlobReferenced = errors.New("blob referenced by manifest") - ErrManifestReferenced = errors.New("manifest referenced by index image") - ErrUnknownCode = errors.New("unknown error code") - ErrBadCACert = errors.New("invalid tls ca cert") - ErrBadUser = errors.New("non-existent user") - ErrEntriesExceeded = errors.New("too many entries returned") - ErrLDAPEmptyPassphrase = errors.New("empty ldap passphrase") - ErrLDAPBadConn = errors.New("bad ldap connection") - ErrLDAPConfig = errors.New("invalid LDAP configuration") - ErrCacheRootBucket = errors.New("unable to create/update root cache bucket") - ErrCacheNoBucket = errors.New("unable to find cache bucket") - ErrCacheMiss = errors.New("cache miss") - ErrRequireCred = errors.New("bind ldap credentials required") - ErrInvalidCred = errors.New("invalid ldap credentials") - ErrEmptyJSON = errors.New("cli config json is empty") - ErrInvalidArgs = errors.New("invalid cli arguments") - ErrInvalidFlagsCombination = errors.New("invalid cli combination of flags") - ErrInvalidURL = errors.New("invalid URL format") - ErrExtensionNotEnabled = errors.New("functionality is not built/configured in the current server") - ErrUnauthorizedAccess = errors.New("unauthorized access. check credentials") - ErrCannotResetConfigKey = errors.New("cannot reset given config key") - ErrConfigNotFound = errors.New("config with the given name does not exist") - ErrNoURLProvided = errors.New("no URL provided") - ErrIllegalConfigKey = errors.New("given config key is not allowed") - ErrScanNotSupported = errors.New("scanning is not supported for given image media type") - ErrCLITimeout = errors.New("query timed out while waiting for results") - ErrDuplicateConfigName = errors.New("cli config name already added") - ErrInvalidRoute = errors.New("invalid route prefix") - ErrImgStoreNotFound = errors.New("image store not found corresponding to given route") - ErrLocalImgStoreNotFound = errors.New("local image store not found corresponding to given route") - ErrEmptyValue = errors.New("empty cache value") - ErrEmptyRepoList = errors.New("no repository found") - ErrCVESearchDisabled = errors.New("cve search is disabled") - ErrCVEDBNotFound = errors.New("cve-db is not present") - ErrInvalidRepositoryName = errors.New("not a valid repository name") - ErrSyncMissingCatalog = errors.New("couldn't fetch upstream registry's catalog") - ErrInvalidMetric = errors.New("invalid metric func") - ErrInjected = errors.New("injected failure") - ErrSyncInvalidUpstreamURL = errors.New("upstream url not found in sync config") - ErrRegistryNoContent = errors.New("could not find a Content that matches localRepo") - ErrSyncReferrerNotFound = errors.New("couldn't find upstream referrer") - ErrImageLintAnnotations = errors.New("lint checks failed") - ErrParsingAuthHeader = errors.New("failed parsing authorization header") - ErrBadType = errors.New("invalid type") - ErrParsingHTTPHeader = errors.New("invalid HTTP header") - ErrBadRange = errors.New("bad range for streaming blob") - ErrBadLayerCount = errors.New("manifest layers count doesn't correspond to config history") - ErrManifestConflict = errors.New("multiple manifests found") - ErrImageMetaNotFound = errors.New("image meta not found") - ErrUnexpectedMediaType = errors.New("unexpected media type") - ErrRepoMetaNotFound = errors.New("repo metadata not found for given repo name") - ErrTagMetaNotFound = errors.New("tag metadata not found for given repo and tag names") - ErrTypeAssertionFailed = errors.New("failed DatabaseDriver type assertion") - ErrInvalidRequestParams = errors.New("request parameter has invalid value") - ErrBadCtxFormat = errors.New("type assertion failed") - ErrEmptyRepoName = errors.New("repo name can't be empty string") - ErrEmptyTag = errors.New("tag can't be empty string") - ErrEmptyDigest = errors.New("digest can't be empty string") - ErrInvalidRepoRefFormat = errors.New("invalid image reference format, use [repo:tag] or [repo@digest]") - ErrLimitIsNegative = errors.New("pagination limit has negative value") - ErrLimitIsExcessive = errors.New("pagination limit has excessive value") - ErrOffsetIsNegative = errors.New("pagination offset has negative value") - ErrSortCriteriaNotSupported = errors.New("the pagination sort criteria is not supported") - ErrMediaTypeNotSupported = errors.New("media type is not supported") - ErrTimeout = errors.New("operation timeout") - ErrNotImplemented = errors.New("not implemented") - ErrDedupeRebuild = errors.New("couldn't rebuild dedupe index") - ErrMissingAuthHeader = errors.New("required authorization header is missing") - ErrUserAPIKeyNotFound = errors.New("user info for given API key hash not found") - ErrUserSessionNotFound = errors.New("user session for given ID not found") - ErrInvalidMetaDBVersion = errors.New("unrecognized version meta") - ErrBucketDoesNotExist = errors.New("bucket does not exist") - ErrOpenIDProviderDoesNotExist = errors.New("openid provider does not exist in given config") - ErrHashKeyNotCreated = errors.New("cookiestore generated random hash key is nil, aborting") - ErrFailedTypeAssertion = errors.New("type assertion failed") - ErrInvalidOldUserStarredRepos = errors.New("invalid old entry for user starred repos") - ErrUnmarshalledRepoListIsNil = errors.New("list of repos is still nil") - ErrCouldNotMarshalStarredRepos = errors.New("could not repack entry for user starred repos") - ErrInvalidOldUserBookmarkedRepos = errors.New("invalid old entry for user bookmarked repos") - ErrCouldNotMarshalBookmarkedRepos = errors.New("could not repack entry for user bookmarked repos") - ErrUserDataNotFound = errors.New("user data not found for given user identifier") - ErrUserDataNotAllowed = errors.New("user data operations are not allowed") - ErrCouldNotPersistData = errors.New("could not persist to db") - ErrSignConfigDirNotSet = errors.New("signature config dir not set") - ErrBadSignatureManifestDigest = errors.New("bad signature manifest digest") - ErrInvalidSignatureType = errors.New("invalid signature type") - ErrSyncPingRegistry = errors.New("unable to ping any registry URLs") - ErrSyncImageNotSigned = errors.New("synced image is not signed") - ErrSyncImageFilteredOut = errors.New("image is filtered out by sync config") - ErrSyncParseRemoteRepo = errors.New("failed to parse remote repo") - ErrInvalidTruststoreType = errors.New("invalid signature truststore type") - ErrInvalidTruststoreName = errors.New("invalid signature truststore name") - ErrInvalidCertificateContent = errors.New("invalid signature certificate content") - ErrInvalidPublicKeyContent = errors.New("invalid signature public key content") - ErrInvalidStateCookie = errors.New("auth state cookie not present or differs from original state") - ErrSyncNoURLsLeft = errors.New("no valid registry urls left after filtering local ones") - ErrInvalidCLIParameter = errors.New("invalid cli parameter") - ErrGQLEndpointNotFound = errors.New("the server doesn't have a gql endpoint") - ErrGQLQueryNotSupported = errors.New("query is not supported or has different arguments") - ErrBadHTTPStatusCode = errors.New("the response doesn't contain the expected status code") - ErrFileAlreadyCancelled = errors.New("storageDriver file already cancelled") - ErrFileAlreadyClosed = errors.New("storageDriver file already closed") - ErrFileAlreadyCommitted = errors.New("storageDriver file already committed") - ErrInvalidOutputFormat = errors.New("invalid cli output format") - ErrServerIsRunning = errors.New("server is running") - ErrDatabaseFileAlreadyInUse = errors.New("boltdb file is already in use") - ErrFlagValueUnsupported = errors.New("supported values ") - ErrUnknownSubcommand = errors.New("unknown cli subcommand") - ErrMultipleReposSameName = errors.New("can't have multiple repos with the same name") - ErrRetentionPolicyNotFound = errors.New("retention repo or tag policy not found") - ErrFormatNotSupported = errors.New("the given output format is not supported") - ErrAPINotSupported = errors.New("registry at the given address doesn't implement the correct API") - ErrURLNotFound = errors.New("url not found") - ErrInvalidSearchQuery = errors.New("invalid search query") - ErrImageNotFound = errors.New("image not found") - ErrAmbiguousInput = errors.New("input is not specific enough") - ErrReceivedUnexpectedAuthHeader = errors.New("received unexpected www-authenticate header") - ErrNoBearerToken = errors.New("no bearer token given") - ErrInvalidBearerToken = errors.New("invalid bearer token given") - ErrInsufficientScope = errors.New("bearer token does not have sufficient scope") - ErrCouldNotLoadCertificate = errors.New("failed to load certificate") + ErrBadConfig = errors.New("invalid server config") + ErrCliBadConfig = errors.New("invalid cli config") + ErrRepoNotFound = errors.New("repository not found") + ErrRepoBadVersion = errors.New("unsupported repository layout version") + ErrRepoBadLayout = errors.New("invalid repository layout") + ErrManifestNotFound = errors.New("manifest not found") + ErrBadManifest = errors.New("invalid manifest content") + ErrUploadNotFound = errors.New("upload destination not found") + ErrBadUploadRange = errors.New("bad upload content-length") + ErrBlobNotFound = errors.New("blob not found") + ErrBadBlob = errors.New("bad blob") + ErrBadBlobDigest = errors.New("bad blob digest") + ErrBlobReferenced = errors.New("blob referenced by manifest") + ErrManifestReferenced = errors.New("manifest referenced by index image") + ErrUnknownCode = errors.New("unknown error code") + ErrBadCACert = errors.New("invalid tls ca cert") + ErrBadUser = errors.New("non-existent user") + ErrEntriesExceeded = errors.New("too many entries returned") + ErrLDAPEmptyPassphrase = errors.New("empty ldap passphrase") + ErrLDAPBadConn = errors.New("bad ldap connection") + ErrLDAPConfig = errors.New("invalid LDAP configuration") + ErrCacheRootBucket = errors.New("unable to create/update root cache bucket") + ErrCacheNoBucket = errors.New("unable to find cache bucket") + ErrCacheMiss = errors.New("cache miss") + ErrRequireCred = errors.New("bind ldap credentials required") + ErrInvalidCred = errors.New("invalid ldap credentials") + ErrEmptyJSON = errors.New("cli config json is empty") + ErrInvalidArgs = errors.New("invalid cli arguments") + ErrInvalidFlagsCombination = errors.New("invalid cli combination of flags") + ErrInvalidURL = errors.New("invalid URL format") + ErrExtensionNotEnabled = errors.New("functionality is not built/configured in the current server") + ErrUnauthorizedAccess = errors.New("unauthorized access. check credentials") + ErrCannotResetConfigKey = errors.New("cannot reset given config key") + ErrConfigNotFound = errors.New("config with the given name does not exist") + ErrNoURLProvided = errors.New("no URL provided") + ErrIllegalConfigKey = errors.New("given config key is not allowed") + ErrScanNotSupported = errors.New("scanning is not supported for given image media type") + ErrCLITimeout = errors.New("query timed out while waiting for results") + ErrDuplicateConfigName = errors.New("cli config name already added") + ErrInvalidRoute = errors.New("invalid route prefix") + ErrImgStoreNotFound = errors.New("image store not found corresponding to given route") + ErrLocalImgStoreNotFound = errors.New("local image store not found corresponding to given route") + ErrEmptyValue = errors.New("empty cache value") + ErrEmptyRepoList = errors.New("no repository found") + ErrCVESearchDisabled = errors.New("cve search is disabled") + ErrCVEDBNotFound = errors.New("cve-db is not present") + ErrInvalidRepositoryName = errors.New("not a valid repository name") + ErrSyncMissingCatalog = errors.New("couldn't fetch upstream registry's catalog") + ErrInvalidMetric = errors.New("invalid metric func") + ErrInjected = errors.New("injected failure") + ErrSyncInvalidUpstreamURL = errors.New("upstream url not found in sync config") + ErrRegistryNoContent = errors.New("could not find a Content that matches localRepo") + ErrSyncReferrerNotFound = errors.New("couldn't find upstream referrer") + ErrImageLintAnnotations = errors.New("lint checks failed") + ErrParsingAuthHeader = errors.New("failed parsing authorization header") + ErrBadType = errors.New("invalid type") + ErrParsingHTTPHeader = errors.New("invalid HTTP header") + ErrBadRange = errors.New("bad range for streaming blob") + ErrBadLayerCount = errors.New("manifest layers count doesn't correspond to config history") + ErrManifestConflict = errors.New("multiple manifests found") + ErrImageMetaNotFound = errors.New("image meta not found") + ErrUnexpectedMediaType = errors.New("unexpected media type") + ErrRepoMetaNotFound = errors.New("repo metadata not found for given repo name") + ErrTagMetaNotFound = errors.New("tag metadata not found for given repo and tag names") + ErrTypeAssertionFailed = errors.New("failed DatabaseDriver type assertion") + ErrInvalidRequestParams = errors.New("request parameter has invalid value") + ErrBadCtxFormat = errors.New("type assertion failed") + ErrEmptyRepoName = errors.New("repo name can't be empty string") + ErrEmptyTag = errors.New("tag can't be empty string") + ErrEmptyDigest = errors.New("digest can't be empty string") + ErrInvalidRepoRefFormat = errors.New("invalid image reference format, use [repo:tag] or [repo@digest]") + ErrLimitIsNegative = errors.New("pagination limit has negative value") + ErrLimitIsExcessive = errors.New("pagination limit has excessive value") + ErrOffsetIsNegative = errors.New("pagination offset has negative value") + ErrSortCriteriaNotSupported = errors.New("the pagination sort criteria is not supported") + ErrMediaTypeNotSupported = errors.New("media type is not supported") + ErrTimeout = errors.New("operation timeout") + ErrNotImplemented = errors.New("not implemented") + ErrDedupeRebuild = errors.New("couldn't rebuild dedupe index") + ErrMissingAuthHeader = errors.New("required authorization header is missing") + ErrUserAPIKeyNotFound = errors.New("user info for given API key hash not found") + ErrUserSessionNotFound = errors.New("user session for given ID not found") + ErrInvalidMetaDBVersion = errors.New("unrecognized version meta") + ErrBucketDoesNotExist = errors.New("bucket does not exist") + ErrOpenIDProviderDoesNotExist = errors.New("openid provider does not exist in given config") + ErrHashKeyNotCreated = errors.New("cookiestore generated random hash key is nil, aborting") + ErrFailedTypeAssertion = errors.New("type assertion failed") + ErrInvalidOldUserStarredRepos = errors.New("invalid old entry for user starred repos") + ErrUnmarshalledRepoListIsNil = errors.New("list of repos is still nil") + ErrCouldNotMarshalStarredRepos = errors.New("could not repack entry for user starred repos") + ErrInvalidOldUserBookmarkedRepos = errors.New("invalid old entry for user bookmarked repos") + ErrCouldNotMarshalBookmarkedRepos = errors.New("could not repack entry for user bookmarked repos") + ErrUserDataNotFound = errors.New("user data not found for given user identifier") + ErrUserDataNotAllowed = errors.New("user data operations are not allowed") + ErrCouldNotPersistData = errors.New("could not persist to db") + ErrSignConfigDirNotSet = errors.New("signature config dir not set") + ErrBadSignatureManifestDigest = errors.New("bad signature manifest digest") + ErrInvalidSignatureType = errors.New("invalid signature type") + ErrSyncPingRegistry = errors.New("unable to ping any registry URLs") + ErrSyncImageNotSigned = errors.New("synced image is not signed") + ErrSyncImageFilteredOut = errors.New("image is filtered out by sync config") + ErrSyncParseRemoteRepo = errors.New("failed to parse remote repo") + ErrInvalidTruststoreType = errors.New("invalid signature truststore type") + ErrInvalidTruststoreName = errors.New("invalid signature truststore name") + ErrInvalidCertificateContent = errors.New("invalid signature certificate content") + ErrInvalidPublicKeyContent = errors.New("invalid signature public key content") + ErrInvalidStateCookie = errors.New("auth state cookie not present or differs from original state") + ErrSyncNoURLsLeft = errors.New("no valid registry urls left after filtering local ones") + ErrInvalidCLIParameter = errors.New("invalid cli parameter") + ErrGQLEndpointNotFound = errors.New("the server doesn't have a gql endpoint") + ErrGQLQueryNotSupported = errors.New("query is not supported or has different arguments") + ErrBadHTTPStatusCode = errors.New("the response doesn't contain the expected status code") + ErrFileAlreadyCancelled = errors.New("storageDriver file already cancelled") + ErrFileAlreadyClosed = errors.New("storageDriver file already closed") + ErrFileAlreadyCommitted = errors.New("storageDriver file already committed") + ErrInvalidOutputFormat = errors.New("invalid cli output format") + ErrServerIsRunning = errors.New("server is running") + ErrDatabaseFileAlreadyInUse = errors.New("boltdb file is already in use") + ErrFlagValueUnsupported = errors.New("supported values ") + ErrUnknownSubcommand = errors.New("unknown cli subcommand") + ErrMultipleReposSameName = errors.New("can't have multiple repos with the same name") + ErrRetentionPolicyNotFound = errors.New("retention repo or tag policy not found") + ErrFormatNotSupported = errors.New("the given output format is not supported") + ErrAPINotSupported = errors.New("registry at the given address doesn't implement the correct API") + ErrURLNotFound = errors.New("url not found") + ErrInvalidSearchQuery = errors.New("invalid search query") + ErrImageNotFound = errors.New("image not found") + ErrAmbiguousInput = errors.New("input is not specific enough") + ErrReceivedUnexpectedAuthHeader = errors.New("received unexpected www-authenticate header") + ErrNoBearerToken = errors.New("no bearer token given") + ErrInvalidBearerToken = errors.New("invalid bearer token given") + ErrInsufficientScope = errors.New("bearer token does not have sufficient scope") + ErrCouldNotLoadCertificate = errors.New("failed to load certificate") + ErrEventTypeEmpty = errors.New("event type empty") + ErrEventSinkIsNil = errors.New("event sink is nil") + ErrUnsupportedEventSink = errors.New("event sink is not supported") + ErrInvalidEventSinkType = errors.New("invalid sink type") + ErrEventSinkAddressEmpty = errors.New("address field cannot be empty") + ErrCouldNotCreateHTTPEventTransport = errors.New("default transport is not *http.Transport") ) diff --git a/examples/config-events.json b/examples/config-events.json new file mode 100644 index 00000000..9d787553 --- /dev/null +++ b/examples/config-events.json @@ -0,0 +1,24 @@ +{ + "distSpecVersion": "1.1.1", + "storage": { + "rootDirectory": "/tmp/zot" + }, + "http": { + "address": "127.0.0.1", + "port": "8080" + }, + "log": { + "level": "debug" + }, + "extensions": { + "events": { + "enable": true, + "sinks": [{ + "type": "nats", + "address": "nats://127.0.0.1:4222", + "timeout": "10s", + "channel": "alerts" + }] + } + } +} \ No newline at end of file diff --git a/go.mod b/go.mod index 812e14ec..cbcbc10c 100644 --- a/go.mod +++ b/go.mod @@ -20,6 +20,8 @@ require ( github.com/bmatcuk/doublestar/v4 v4.8.1 github.com/briandowns/spinner v1.23.2 github.com/chartmuseum/auth v0.5.0 + github.com/cloudevents/sdk-go/protocol/nats/v2 v2.15.2 + github.com/cloudevents/sdk-go/v2 v2.15.2 github.com/containers/image/v5 v5.34.3 github.com/dchest/siphash v1.2.3 github.com/didip/tollbooth/v7 v7.0.2 @@ -42,6 +44,8 @@ require ( github.com/json-iterator/go v1.1.12 github.com/migueleliasweb/go-github-mock v1.1.0 github.com/mitchellh/mapstructure v1.5.1-0.20231216201459-8508981c8b6c + github.com/nats-io/nats-server/v2 v2.11.3 + github.com/nats-io/nats.go v1.41.2 github.com/nmcclain/ldap v0.0.0-20210720162743-7f8d1e44eeba github.com/notaryproject/notation-core-go v1.3.0 github.com/notaryproject/notation-go v1.3.2 @@ -74,6 +78,7 @@ require ( google.golang.org/protobuf v1.36.6 gopkg.in/resty.v1 v1.12.0 gopkg.in/yaml.v3 v3.0.1 + k8s.io/apimachinery v0.32.3 modernc.org/sqlite v1.37.0 oras.land/oras-go/v2 v2.5.0 ) @@ -280,6 +285,7 @@ require ( github.com/google/go-github/v55 v55.0.0 // indirect github.com/google/go-github/v64 v64.0.0 // indirect github.com/google/go-querystring v1.1.0 // indirect + github.com/google/go-tpm v0.9.4 // indirect github.com/google/gofuzz v1.2.0 // indirect github.com/google/licenseclassifier/v2 v2.0.0 // indirect github.com/google/s2a-go v0.1.9 // indirect @@ -339,6 +345,7 @@ require ( github.com/mattn/go-runewidth v0.0.16 // indirect github.com/mattn/go-shellwords v1.0.12 // indirect github.com/miekg/pkcs11 v1.1.1 // indirect + github.com/minio/highwayhash v1.0.3 // indirect github.com/mitchellh/copystructure v1.2.0 // indirect github.com/mitchellh/go-homedir v1.1.0 // indirect github.com/mitchellh/go-testing-interface v1.14.1 // indirect @@ -362,6 +369,9 @@ require ( github.com/muhlemmer/gu v0.3.1 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f // indirect + github.com/nats-io/jwt/v2 v2.7.4 // indirect + github.com/nats-io/nkeys v0.4.11 // indirect + github.com/nats-io/nuid v1.0.1 // indirect github.com/ncruces/go-strftime v0.1.9 // indirect github.com/nmcclain/asn1-ber v0.0.0-20170104154839-2661553a0484 // indirect github.com/notaryproject/notation-plugin-framework-go v1.0.0 // indirect @@ -512,7 +522,6 @@ require ( helm.sh/helm/v3 v3.17.3 // indirect k8s.io/api v0.32.3 // indirect k8s.io/apiextensions-apiserver v0.32.2 // indirect - k8s.io/apimachinery v0.32.3 // indirect k8s.io/apiserver v0.32.2 // indirect k8s.io/cli-runtime v0.32.3 // indirect k8s.io/client-go v0.32.3 // indirect diff --git a/go.sum b/go.sum index d58fd793..1d513203 100644 --- a/go.sum +++ b/go.sum @@ -806,6 +806,8 @@ github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHG github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be h1:9AeTilPcZAjCFIImctFaOjnTIavg87rW78vTPkQqLI8= github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be/go.mod h1:ySMOLuWl6zY27l47sB3qLNK6tF2fkHG55UZxx8oIVo4= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= +github.com/antithesishq/antithesis-sdk-go v0.4.3-default-no-op h1:+OSa/t11TFhqfrX0EOSqQBDJ0YlpmK0rDSiB19dg9M0= +github.com/antithesishq/antithesis-sdk-go v0.4.3-default-no-op/go.mod h1:IUpT2DPAKh6i/YhSbt6Gl3v2yvUZjmKncl7U91fup7E= github.com/apache/arrow/go/v10 v10.0.1/go.mod h1:YvhnlEePVnBS4+0z3fhPfUy7W1Ikj0Ih0vcRo/gZ1M0= github.com/apache/arrow/go/v11 v11.0.0/go.mod h1:Eg5OsL5H+e299f7u5ssuXsuHQVEGC4xei5aX110hRiI= github.com/apache/thrift v0.16.0/go.mod h1:PHK3hniurgQaNMZYaCLEqXKsYK8upmhPbmdP2FXSqgU= @@ -967,6 +969,10 @@ github.com/clbanning/mxj/v2 v2.5.5/go.mod h1:hNiWqW14h+kc+MdF9C6/YoRfjEJoR3ou6tn github.com/clbanning/mxj/v2 v2.7.0 h1:WA/La7UGCanFe5NpHF0Q3DNtnCsVoxbPKuyBNHWRyME= github.com/clbanning/mxj/v2 v2.7.0/go.mod h1:hNiWqW14h+kc+MdF9C6/YoRfjEJoR3ou6tn/Qo+ve2s= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cloudevents/sdk-go/protocol/nats/v2 v2.15.2 h1:grQPId+rXCeR5RcmK5uBlissnlot7kBlHd8YJ7iZOPg= +github.com/cloudevents/sdk-go/protocol/nats/v2 v2.15.2/go.mod h1:KQA5rf2uSgtCnXsAFyFXtwiDboL/pB6gsg4VTErhfLA= +github.com/cloudevents/sdk-go/v2 v2.15.2 h1:54+I5xQEnI73RBhWHxbI1XJcqOFOVJN85vb41+8mHUc= +github.com/cloudevents/sdk-go/v2 v2.15.2/go.mod h1:lL7kSWAE/V8VI4Wh0jbL2v/jvqsm6tjmaQBSvxcv4uE= github.com/cloudflare/circl v1.6.0 h1:cr5JKic4HI+LkINy2lg3W2jF8sHCVTBncJr5gIIq7qk= github.com/cloudflare/circl v1.6.0/go.mod h1:uddAzsPgqdMAYatqJ0lsjX1oECcQLIlRpzZh3pJrofs= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= @@ -1361,6 +1367,8 @@ github.com/google/go-github/v64 v64.0.0 h1:4G61sozmY3eiPAjjoOHponXDBONm+utovTKby github.com/google/go-github/v64 v64.0.0/go.mod h1:xB3vqMQNdHzilXBiO2I+M7iEFtHf+DP/omBOv6tQzVo= github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU= +github.com/google/go-tpm v0.9.4 h1:awZRf9FwOeTunQmHoDYSHJps3ie6f1UlhS1fOdPEt1I= +github.com/google/go-tpm v0.9.4/go.mod h1:h9jEsEECg7gtLis0upRBQU+GhYVH6jMjrFxI8u6bVUY= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= @@ -1663,6 +1671,8 @@ github.com/migueleliasweb/go-github-mock v1.1.0 h1:GKaOBPsrPGkAKgtfuWY8MclS1xR6M github.com/migueleliasweb/go-github-mock v1.1.0/go.mod h1:pYe/XlGs4BGMfRY4vmeixVsODHnVDDhJ9zoi0qzSMHc= github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8/go.mod h1:mC1jAcsrzbxHt8iiaC+zU4b1ylILSosueou12R++wfY= github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3/go.mod h1:RagcQ7I8IeTMnF8JTXieKnO4Z6JCsikNEzj0DwauVzE= +github.com/minio/highwayhash v1.0.3 h1:kbnuUMoHYyVl7szWjSxJnxw11k2U709jqFPPmIUyD6Q= +github.com/minio/highwayhash v1.0.3/go.mod h1:GGYsuwP/fPD6Y9hMiXuapVvlIUEhFhMTh0rxU3ik1LQ= github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa15WveJJGw= github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HKCj9FbZEVFJRxO9s= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= @@ -1721,6 +1731,16 @@ github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8m github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f h1:y5//uYreIhSUg3J1GEMiLbxo1LJaP8RfCpH6pymGZus= github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= +github.com/nats-io/jwt/v2 v2.7.4 h1:jXFuDDxs/GQjGDZGhNgH4tXzSUK6WQi2rsj4xmsNOtI= +github.com/nats-io/jwt/v2 v2.7.4/go.mod h1:me11pOkwObtcBNR8AiMrUbtVOUGkqYjMQZ6jnSdVUIA= +github.com/nats-io/nats-server/v2 v2.11.3 h1:AbGtXxuwjo0gBroLGGr/dE0vf24kTKdRnBq/3z/Fdoc= +github.com/nats-io/nats-server/v2 v2.11.3/go.mod h1:6Z6Fd+JgckqzKig7DYwhgrE7bJ6fypPHnGPND+DqgMY= +github.com/nats-io/nats.go v1.41.2 h1:5UkfLAtu/036s99AhFRlyNDI1Ieylb36qbGjJzHixos= +github.com/nats-io/nats.go v1.41.2/go.mod h1:iRWIPokVIFbVijxuMQq4y9ttaBTMe0SFdlZfMDd+33g= +github.com/nats-io/nkeys v0.4.11 h1:q44qGV008kYd9W1b1nEBkNzvnWxtRSQ7A8BoqRrcfa0= +github.com/nats-io/nkeys v0.4.11/go.mod h1:szDimtgmfOi9n25JpfIdGw12tZFYXqhGxjhVxsatHVE= +github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw= +github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= github.com/ncruces/go-strftime v0.1.9 h1:bY0MQC28UADQmHmaF5dgpLmImcShSi2kHU9XLdhx/f4= github.com/ncruces/go-strftime v0.1.9/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= @@ -2062,6 +2082,8 @@ github.com/ulikunitz/xz v0.5.12/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0o github.com/urfave/cli/v2 v2.3.0/go.mod h1:LJmUH05zAU44vOAcrfzZQKsZbVcdbOG8rtL3/XcUArI= github.com/urfave/cli/v2 v2.27.6 h1:VdRdS98FNhKZ8/Az8B7MTyGQmpIr36O1EHybx/LaZ4g= github.com/urfave/cli/v2 v2.27.6/go.mod h1:3Sevf16NykTbInEnD0yKkjDAeZDS0A6bzhBH5hrMvTQ= +github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= +github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= github.com/vbatts/tar-split v0.12.1 h1:CqKoORW7BUWBe7UL/iqTVvkTBOF8UvOMKOIZykxnnbo= github.com/vbatts/tar-split v0.12.1/go.mod h1:eF6B6i6ftWQcDqEn3/iGFRFRo8cBIMSJVOpnNdfTMFA= github.com/vektah/gqlparser/v2 v2.5.25 h1:FmWtFEa+invTIzWlWK6Vk7BVEZU/97QBzeI8Z1JjGt8= @@ -2527,6 +2549,7 @@ golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.29.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.32.0 h1:s77OFDvIQeibCmezSnk/q6iAfkdiQaJi4VzroCFrN20= golang.org/x/sys v0.32.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= diff --git a/pkg/api/authn_test.go b/pkg/api/authn_test.go index ad64e755..3aecfe7d 100644 --- a/pkg/api/authn_test.go +++ b/pkg/api/authn_test.go @@ -954,7 +954,7 @@ func TestCookiestoreCleanup(t *testing.T) { err = os.Chtimes(sessionPath, changeTime, changeTime) So(err, ShouldBeNil) - imgStore := local.NewImageStore(rootDir, false, false, log, metrics, nil, nil, nil) + imgStore := local.NewImageStore(rootDir, false, false, log, metrics, nil, nil, nil, nil) storeController := storage.StoreController{ DefaultStore: imgStore, @@ -989,7 +989,7 @@ func TestCookiestoreCleanup(t *testing.T) { err = os.WriteFile(sessionPath, []byte("session"), storageConstants.DefaultFilePerms) So(err, ShouldBeNil) - imgStore := local.NewImageStore(rootDir, false, false, log, metrics, nil, nil, nil) + imgStore := local.NewImageStore(rootDir, false, false, log, metrics, nil, nil, nil, nil) storeController := storage.StoreController{ DefaultStore: imgStore, diff --git a/pkg/api/config/config.go b/pkg/api/config/config.go index af0b593b..39f86639 100644 --- a/pkg/api/config/config.go +++ b/pkg/api/config/config.go @@ -341,6 +341,20 @@ func (c *Config) Sanitize() *Config { sanitizedConfig.HTTP.Auth.LDAP.bindPassword = "******" } + if c.IsEventRecorderEnabled() { + for i, sink := range c.Extensions.Events.Sinks { + if sink.Credentials == nil { + continue + } + + if err := DeepCopy(&c.Extensions.Events.Sinks[i], &sanitizedConfig.Extensions.Events.Sinks[i]); err != nil { + panic(err) + } + + sanitizedConfig.Extensions.Events.Sinks[i].Credentials.Password = "******" + } + } + return sanitizedConfig } @@ -516,6 +530,10 @@ func (c *Config) IsCompatEnabled() bool { return len(c.HTTP.Compat) > 0 } +func (c *Config) IsEventRecorderEnabled() bool { + return c.Extensions != nil && c.Extensions.Events != nil && *c.Extensions.Events.Enable +} + func IsOpenIDSupported(provider string) bool { for _, supportedProvider := range openIDSupportedProviders { if supportedProvider == provider { diff --git a/pkg/api/config/config_test.go b/pkg/api/config/config_test.go index 61e4d629..5701ccda 100644 --- a/pkg/api/config/config_test.go +++ b/pkg/api/config/config_test.go @@ -7,6 +7,8 @@ import ( . "github.com/smartystreets/goconvey/convey" "zotregistry.dev/zot/pkg/api/config" + extconf "zotregistry.dev/zot/pkg/extensions/config" + "zotregistry.dev/zot/pkg/extensions/config/events" ) func TestConfig(t *testing.T) { @@ -129,4 +131,30 @@ func TestConfig(t *testing.T) { So(conf.IsRetentionEnabled(), ShouldBeTrue) }) + + Convey("Test IsEventRecorderEnabled()", t, func() { + conf := config.New() + So(conf.IsEventRecorderEnabled(), ShouldBeFalse) + + // Enable the event recorder + enable := true + conf.Extensions = &extconf.ExtensionConfig{} + conf.Extensions.Events = &events.Config{ + Enable: &enable, + } + + So(conf.IsEventRecorderEnabled(), ShouldBeTrue) + + // Disabled scenario + disable := false + conf.Extensions.Events.Enable = &disable + So(conf.IsEventRecorderEnabled(), ShouldBeFalse) + + // nil pointers + conf.Extensions.Events = nil + So(conf.IsEventRecorderEnabled(), ShouldBeFalse) + + conf.Extensions = nil + So(conf.IsEventRecorderEnabled(), ShouldBeFalse) + }) } diff --git a/pkg/api/controller.go b/pkg/api/controller.go index e3cc8b9c..46c88842 100644 --- a/pkg/api/controller.go +++ b/pkg/api/controller.go @@ -4,6 +4,7 @@ import ( "context" "crypto/tls" "crypto/x509" + goerrors "errors" "fmt" "net" "net/http" @@ -21,6 +22,7 @@ import ( "zotregistry.dev/zot/pkg/common" ext "zotregistry.dev/zot/pkg/extensions" extconf "zotregistry.dev/zot/pkg/extensions/config" + "zotregistry.dev/zot/pkg/extensions/events" "zotregistry.dev/zot/pkg/extensions/monitoring" "zotregistry.dev/zot/pkg/log" "zotregistry.dev/zot/pkg/meta" @@ -44,6 +46,7 @@ type Controller struct { Audit *log.Logger Server *http.Server Metrics monitoring.MetricServer + EventRecorder events.Recorder CveScanner ext.CveScanner SyncOnDemand SyncOnDemand RelyingParties map[string]rp.RelyingParty @@ -261,6 +264,10 @@ func (c *Controller) Init() error { c.Metrics = monitoring.NewMetricsServer(enabled, c.Log) + if err := c.InitEventRecorder(); err != nil { + return err + } + if err := c.InitImageStore(); err != nil { //nolint:contextcheck return err } @@ -291,7 +298,7 @@ func (c *Controller) InitCVEInfo() { func (c *Controller) InitImageStore() error { linter := ext.GetLinter(c.Config, c.Log) - storeController, err := storage.New(c.Config, linter, c.Metrics, c.Log) + storeController, err := storage.New(c.Config, linter, c.Metrics, c.Log, c.EventRecorder) if err != nil { return err } @@ -352,6 +359,17 @@ func (c *Controller) InitMetaDB() error { return nil } +func (c *Controller) InitEventRecorder() error { + eventRecorder, err := ext.NewEventRecorder(c.Config, c.Log) + if err != nil && !goerrors.Is(err, errors.ErrExtensionNotEnabled) { + return err + } + + c.EventRecorder = eventRecorder + + return nil +} + func (c *Controller) LoadNewConfig(newConfig *config.Config) { // reload access control config c.Config.HTTP.AccessControl = newConfig.HTTP.AccessControl diff --git a/pkg/cli/client/cve_cmd_test.go b/pkg/cli/client/cve_cmd_test.go index addfff95..adad8cb3 100644 --- a/pkg/cli/client/cve_cmd_test.go +++ b/pkg/cli/client/cve_cmd_test.go @@ -119,7 +119,7 @@ func TestNegativeServerResponse(t *testing.T) { dir := t.TempDir() imageStore := local.NewImageStore(dir, false, false, - log.NewLogger("debug", ""), monitoring.NewMetricsServer(false, log.NewLogger("debug", "")), nil, nil, nil) + log.NewLogger("debug", ""), monitoring.NewMetricsServer(false, log.NewLogger("debug", "")), nil, nil, nil, nil) storeController := storage.StoreController{ DefaultStore: imageStore, diff --git a/pkg/cli/server/extensions_test.go b/pkg/cli/server/extensions_test.go index 0d27cd18..6e41d7cd 100644 --- a/pkg/cli/server/extensions_test.go +++ b/pkg/cli/server/extensions_test.go @@ -1,5 +1,5 @@ -//go:build sync && scrub && metrics && search && userprefs && mgmt && imagetrust -// +build sync,scrub,metrics,search,userprefs,mgmt,imagetrust +//go:build sync && scrub && metrics && search && userprefs && mgmt && imagetrust && events +// +build sync,scrub,metrics,search,userprefs,mgmt,imagetrust,events package server_test @@ -13,6 +13,7 @@ import ( . "github.com/smartystreets/goconvey/convey" "gopkg.in/resty.v1" + zerr "zotregistry.dev/zot/errors" "zotregistry.dev/zot/pkg/api/config" cli "zotregistry.dev/zot/pkg/cli/server" . "zotregistry.dev/zot/pkg/test/common" @@ -1946,3 +1947,79 @@ func TestSyncWithRemoteStorageConfig(t *testing.T) { "using both sync and remote storage features needs config.Extensions.Sync.DownloadDir to be specified") }) } + +func TestEventsExtension(t *testing.T) { + oldArgs := os.Args + + defer func() { os.Args = oldArgs }() + + Convey("Events explicitly disabled", t, func(c C) { + content := `{ + "storage": { + "rootDirectory": "%s" + }, + "http": { + "address": "127.0.0.1", + "port": "%s" + }, + "log": { + "level": "debug", + "output": "%s" + }, + "extensions": { + "events": { + "enable": false + } + } + }` + + logPath, err := runCLIWithConfig(t.TempDir(), content) + So(err, ShouldBeNil) + defer os.Remove(logPath) // clean up + + found, err := ReadLogFileAndSearchString(logPath, + "events disabled in configuration", 10*time.Second) + + if !found { + data, err := os.ReadFile(logPath) + So(err, ShouldBeNil) + t.Log(string(data)) + } + + So(err, ShouldBeNil) + So(found, ShouldBeTrue) + }) + + Convey("Unsupported event sink", t, func(c C) { + content := `{ + "storage": { + "rootDirectory": "%s" + }, + "http": { + "address": "127.0.0.1", + "port": "%s" + }, + "log": { + "level": "debug", + "output": "%s" + }, + "extensions": { + "events": { + "enable": true, + "sinks": [{ + "type": "unsupported" + }] + } + } + }` + + logPath, err := runCLIWithConfig(t.TempDir(), content) + defer func(p string) { + if p != "" { + os.Remove(p) + } + }(logPath) // clean up + So(err, ShouldNotBeNil) + So(err.Error(), ShouldContainSubstring, zerr.ErrUnsupportedEventSink.Error()) + }) +} diff --git a/pkg/cli/server/root.go b/pkg/cli/server/root.go index 1201a912..13a1f42e 100644 --- a/pkg/cli/server/root.go +++ b/pkg/cli/server/root.go @@ -27,6 +27,7 @@ import ( "zotregistry.dev/zot/pkg/api/constants" "zotregistry.dev/zot/pkg/common" extconf "zotregistry.dev/zot/pkg/extensions/config" + eventsconf "zotregistry.dev/zot/pkg/extensions/config/events" "zotregistry.dev/zot/pkg/extensions/monitoring" zlog "zotregistry.dev/zot/pkg/log" storageConstants "zotregistry.dev/zot/pkg/storage/constants" @@ -820,8 +821,19 @@ func LoadConfiguration(config *config.Config, configPath string) error { } metaData := &mapstructure.Metadata{} - if err := viperInstance.UnmarshalExact(&config, metadataConfig(metaData)); err != nil { - log.Error().Err(err).Msg("failed to unmarshaling new config") + + decoderOpts := []viper.DecoderConfigOption{ + metadataConfig(metaData), + viper.DecodeHook( + mapstructure.ComposeDecodeHookFunc( + mapstructure.StringToTimeDurationHookFunc(), + eventsconf.SinkConfigDecoderHook(), + ), + ), + } + + if err := viperInstance.UnmarshalExact(&config, decoderOpts...); err != nil { + log.Error().Err(err).Msg("failed to unmarshal new config") return err } diff --git a/pkg/cli/server/root_test.go b/pkg/cli/server/root_test.go index 5f6038c1..39d9a4c3 100644 --- a/pkg/cli/server/root_test.go +++ b/pkg/cli/server/root_test.go @@ -2413,13 +2413,20 @@ func runCLIWithConfig(tempDir string, config string) (string, error) { os.Args = []string{"cli_test", "serve", cfgfile.Name()} + // Run CLI in a goroutine, but handle errors via a channel + errCh := make(chan error, 1) go func() { - err = cli.NewServerRootCmd().Execute() - if err != nil { - panic(err) - } + errCh <- cli.NewServerRootCmd().Execute() }() + select { + case err := <-errCh: + if err != nil { + return "", err + } + case <-time.After(250 * time.Millisecond): // No startup error + } + WaitTillServerReady(baseURL) return logFile.Name(), nil diff --git a/pkg/extensions/config/config.go b/pkg/extensions/config/config.go index 3137adf6..b1f422fe 100644 --- a/pkg/extensions/config/config.go +++ b/pkg/extensions/config/config.go @@ -3,6 +3,7 @@ package config import ( "time" + "zotregistry.dev/zot/pkg/extensions/config/events" "zotregistry.dev/zot/pkg/extensions/config/sync" ) @@ -21,6 +22,7 @@ type ExtensionConfig struct { Mgmt *MgmtConfig APIKey *APIKeyConfig Trust *ImageTrustConfig + Events *events.Config } type ImageTrustConfig struct { diff --git a/pkg/extensions/config/events/config.go b/pkg/extensions/config/events/config.go new file mode 100644 index 00000000..a8c7e495 --- /dev/null +++ b/pkg/extensions/config/events/config.go @@ -0,0 +1,55 @@ +package events + +import ( + "time" +) + +type SinkType string + +func (s SinkType) String() string { + return string(s) +} + +const ( + HTTP SinkType = "http" + NATS SinkType = "nats" +) + +func IsSupportedSink(sinkType SinkType) bool { + supportedSinks := map[SinkType]struct{}{ + HTTP: {}, + NATS: {}, + } + + _, ok := supportedSinks[sinkType] + + return ok +} + +// Config holds configuration for the events extension. +type Config struct { + Enable *bool + Sinks []SinkConfig +} + +type SinkConfig struct { + *Credentials + *TLSConfig + Type SinkType + Address string + Channel string + Timeout time.Duration + Proxy *string +} + +type Credentials struct { + Username string + Password string + File *string +} + +type TLSConfig struct { + CACertFile string + CertFile string + KeyFile string +} diff --git a/pkg/extensions/config/events/decoder.go b/pkg/extensions/config/events/decoder.go new file mode 100644 index 00000000..6a6c2fe5 --- /dev/null +++ b/pkg/extensions/config/events/decoder.go @@ -0,0 +1,52 @@ +package events + +import ( + "reflect" + + "github.com/mitchellh/mapstructure" + + zerr "zotregistry.dev/zot/errors" +) + +// SinkConfigDecoderHook provides a mapstructure hook for decoding SinkConfig interfaces. +func SinkConfigDecoderHook() mapstructure.DecodeHookFunc { + return func(_ reflect.Type, target reflect.Type, data interface{}) (interface{}, error) { + // Only apply this hook when converting to SinkConfig + if target.Name() != "SinkConfig" { + return data, nil + } + + if target != reflect.TypeOf((*SinkConfig)(nil)).Elem() { + return data, nil + } + + dataMap, ok := data.(map[string]interface{}) + if !ok { + return data, nil + } + + config := &SinkConfig{} + + decoderConfig := &mapstructure.DecoderConfig{ + DecodeHook: mapstructure.StringToTimeDurationHookFunc(), + Result: config, + WeaklyTypedInput: true, + TagName: "mapstructure", + } + + decoder, err := mapstructure.NewDecoder(decoderConfig) + if err != nil { + return nil, err + } + + if err := decoder.Decode(dataMap); err != nil { + return nil, err + } + + if !IsSupportedSink(config.Type) { + return nil, zerr.ErrUnsupportedEventSink + } + + return config, nil + } +} diff --git a/pkg/extensions/events/builder.go b/pkg/extensions/events/builder.go new file mode 100644 index 00000000..1bd66f62 --- /dev/null +++ b/pkg/extensions/events/builder.go @@ -0,0 +1,55 @@ +//go:build events +// +build events + +package events + +import ( + "time" + + cloudevents "github.com/cloudevents/sdk-go/v2" + "github.com/google/uuid" + + zerr "zotregistry.dev/zot/errors" +) + +type eventBuilder struct { + data map[string]any + eventType EventType +} + +func newEventBuilder() *eventBuilder { + return &eventBuilder{ + data: make(map[string]any), + } +} + +func (b *eventBuilder) WithDataField(name string, value any) *eventBuilder { + b.data[name] = value + + return b +} + +func (b *eventBuilder) WithEventType(eventType EventType) *eventBuilder { + b.eventType = eventType + + return b +} + +func (b *eventBuilder) Build() (*cloudevents.Event, error) { + if b.eventType == "" { + return nil, zerr.ErrEventTypeEmpty + } + event := cloudevents.NewEvent() + event.SetType(b.eventType.String()) + event.SetID(uuid.New().String()) + event.SetTime(time.Now()) + event.SetSource(EventSource) + + if b.data != nil { + if err := event.SetData(cloudevents.ApplicationJSON, b.data); err != nil { + return nil, err + } + } + + return &event, nil +} diff --git a/pkg/extensions/events/common.go b/pkg/extensions/events/common.go new file mode 100644 index 00000000..702e82f0 --- /dev/null +++ b/pkg/extensions/events/common.go @@ -0,0 +1,32 @@ +package events + +import ( + "time" +) + +const ( + DefaultHTTPTimeout = 30 * time.Second + EventSource = "zotregistry.dev" +) + +type EventType string + +const ( + ImageUpdatedEventType EventType = "zotregistry.image.updated" + ImageDeletedEventType EventType = "zotregistry.image.deleted" + ImageLintFailedEventType EventType = "zotregistry.image.lint_failed" + RepositoryCreatedEventType EventType = "zotregistry.repository.created" +) + +func (e EventType) String() string { + return string(e) +} + +type Recorder interface { + Close() + + RepositoryCreated(name string) + ImageUpdated(name, reference, digest, mediaType, manifest string) + ImageDeleted(name, reference, digest, mediaType string) + ImageLintFailed(name, reference, digest, mediaType, manifest string) +} diff --git a/pkg/extensions/events/events.go b/pkg/extensions/events/events.go new file mode 100644 index 00000000..d5b86abb --- /dev/null +++ b/pkg/extensions/events/events.go @@ -0,0 +1,150 @@ +//go:build events +// +build events + +package events + +import ( + "crypto/tls" + "crypto/x509" + "errors" + "os" + + cloudevents "github.com/cloudevents/sdk-go/v2" + + eventsconf "zotregistry.dev/zot/pkg/extensions/config/events" + "zotregistry.dev/zot/pkg/log" +) + +type eventRecorder struct { + log log.Logger + sinks []Sink +} + +var _ Recorder = (*eventRecorder)(nil) + +func (r eventRecorder) Close() { + err := r.closeSinks() + if err != nil { + r.log.Error().Err(err).Msg("failed to close sinks") + } +} + +func (r eventRecorder) closeSinks() error { + var retErr error + + for _, sink := range r.sinks { + if err := sink.Close(); err != nil { + retErr = errors.Join(retErr, err) + } + } + + return retErr +} + +func (r eventRecorder) publish(event *cloudevents.Event) { + go func() { + for _, sink := range r.sinks { + if response := sink.Emit(event); cloudevents.IsNACK(response) || cloudevents.IsUndelivered(response) { + r.log.Error().Err(response).Msg("failed to publish event") + } + } + + r.log.Info().Msgf("event published successfully: %s", event.Type()) + }() +} + +func (r eventRecorder) RepositoryCreated(name string) { + event, err := newEventBuilder(). + WithEventType(RepositoryCreatedEventType). + WithDataField("name", name). + Build() + if err != nil { + r.log.Warn().Err(err).Msg("failed to create event") + + return + } + + r.publish(event) +} + +func (r eventRecorder) ImageUpdated(name, reference, digest, mediaType, manifest string) { + event, err := newEventBuilder(). + WithEventType(ImageUpdatedEventType). + WithDataField("name", name). + WithDataField("reference", reference). + WithDataField("digest", digest). + WithDataField("mediaType", mediaType). + WithDataField("manifest", manifest). + Build() + if err != nil { + r.log.Warn().Err(err).Msg("failed to create event") + + return + } + + r.publish(event) +} + +func (r eventRecorder) ImageDeleted(name, reference, digest, mediaType string) { + event, err := newEventBuilder(). + WithEventType(ImageDeletedEventType). + WithDataField("name", name). + WithDataField("reference", reference). + WithDataField("digest", digest). + WithDataField("mediaType", mediaType). + Build() + if err != nil { + r.log.Warn().Err(err).Msg("failed to create event") + + return + } + + r.publish(event) +} + +func (r eventRecorder) ImageLintFailed(name, reference, digest, mediaType, manifest string) { + event, err := newEventBuilder(). + WithEventType(ImageLintFailedEventType). + WithDataField("name", name). + WithDataField("reference", reference). + WithDataField("digest", digest). + WithDataField("mediaType", mediaType). + WithDataField("manifest", manifest). + Build() + if err != nil { + r.log.Warn().Err(err).Msg("failed to create event") + + return + } + + r.publish(event) +} + +func getTLSConfig(config eventsconf.SinkConfig) (*tls.Config, error) { + tlsConfig := &tls.Config{ + MinVersion: tls.VersionTLS12, + } + + if config.TLSConfig.CACertFile != "" { + caCert, err := os.ReadFile(config.TLSConfig.CACertFile) + if err != nil { + return nil, err + } + + caCertPool := x509.NewCertPool() + if !caCertPool.AppendCertsFromPEM(caCert) { + return nil, err + } + tlsConfig.RootCAs = caCertPool + } + + if config.TLSConfig.CertFile != "" && config.TLSConfig.KeyFile != "" { + cert, err := tls.LoadX509KeyPair(config.TLSConfig.CertFile, config.TLSConfig.KeyFile) + if err != nil { + return nil, err + } + tlsConfig.Certificates = []tls.Certificate{cert} + } + + return tlsConfig, nil +} diff --git a/pkg/extensions/events/events_test.go b/pkg/extensions/events/events_test.go new file mode 100644 index 00000000..2f358d36 --- /dev/null +++ b/pkg/extensions/events/events_test.go @@ -0,0 +1,350 @@ +//go:build events +// +build events + +package events_test + +import ( + "fmt" + "net/http" + "net/http/httptest" + "testing" + "time" + + cloudevents "github.com/cloudevents/sdk-go/v2" + cehttp "github.com/cloudevents/sdk-go/v2/protocol/http" + "github.com/google/go-containerregistry/pkg/v1/types" + "github.com/nats-io/nats-server/v2/server" + "github.com/nats-io/nats.go" + . "github.com/smartystreets/goconvey/convey" + "k8s.io/apimachinery/pkg/util/rand" + + zerr "zotregistry.dev/zot/errors" + eventsconf "zotregistry.dev/zot/pkg/extensions/config/events" + "zotregistry.dev/zot/pkg/extensions/events" + "zotregistry.dev/zot/pkg/log" +) + +type mockSink struct { + store chan *cloudevents.Event +} + +func (s *mockSink) Emit(e *cloudevents.Event) cloudevents.Result { + s.store <- e + + return nil +} + +func (s *mockSink) Close() error { + return nil +} + +var _ events.Sink = (*mockSink)(nil) + +func newMockSink() *mockSink { + return &mockSink{ + store: make(chan *cloudevents.Event), + } +} + +func TestEventSinkMissing(t *testing.T) { + Convey("missing sink", t, func() { + _, err := events.NewRecorder(log.NewLogger("debug", "")) + So(err, ShouldNotBeNil) + So(err, ShouldEqual, zerr.ErrEventSinkIsNil) + }) +} + +func TestEvents(t *testing.T) { + Convey("emits events", t, func() { + sink := newMockSink() + recorder, err := events.NewRecorder(log.NewLogger("debug", ""), sink) + So(err, ShouldBeNil) + Convey("repository created", func() { + recorder.RepositoryCreated("test") + ev := <-sink.store + So(ev.Type(), ShouldEqual, events.RepositoryCreatedEventType.String()) + }) + Convey("image updated", func() { + recorder.ImageUpdated("test", "v1", "", string(types.OCIManifestSchema1), "") + ev := <-sink.store + So(ev.Type(), ShouldEqual, events.ImageUpdatedEventType.String()) + }) + Convey("image deleted", func() { + recorder.ImageDeleted("test", "v1", "", string(types.OCIManifestSchema1)) + ev := <-sink.store + So(ev.Type(), ShouldEqual, events.ImageDeletedEventType.String()) + }) + Convey("image lint failed", func() { + recorder.ImageLintFailed("test", "v1", "", string(types.OCIManifestSchema1), "") + ev := <-sink.store + So(ev.Type(), ShouldEqual, events.ImageLintFailedEventType.String()) + }) + }) +} + +func TestHTTPSinkEvents(t *testing.T) { + Convey("emits events to http sink", t, func() { + eventChan := make(chan *cloudevents.Event, 1) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + event, err := cehttp.NewEventFromHTTPRequest(r) + if err != nil { + w.WriteHeader(http.StatusBadRequest) + + return + } + + eventChan <- event + + w.WriteHeader(http.StatusOK) + })) + + defer server.Close() + + config := eventsconf.SinkConfig{ + Type: eventsconf.HTTP, + Address: server.URL, + Timeout: 5 * time.Second, + } + sink, err := events.NewHTTPSink(config) + So(err, ShouldBeNil) + + recorder, err := events.NewRecorder(log.NewLogger("debug", ""), sink) + So(err, ShouldBeNil) + + Convey("repository created", func() { + recorder.RepositoryCreated("test") + e := getEvent(t, eventChan) + So(e, ShouldNotBeNil) + So(e.Type(), ShouldEqual, events.RepositoryCreatedEventType.String()) + }) + + Convey("image updated", func() { + recorder.ImageUpdated("test", "v1", "", string(types.OCIManifestSchema1), "") + e := getEvent(t, eventChan) + So(e, ShouldNotBeNil) + So(e.Type(), ShouldEqual, events.ImageUpdatedEventType.String()) + }) + + Convey("image deleted", func() { + recorder.ImageDeleted("test", "v1", "", string(types.OCIManifestSchema1)) + e := getEvent(t, eventChan) + So(e, ShouldNotBeNil) + So(e.Type(), ShouldEqual, events.ImageDeletedEventType.String()) + }) + + Convey("image lint failed", func() { + recorder.ImageLintFailed("test", "v1", "", string(types.OCIManifestSchema1), "") + e := getEvent(t, eventChan) + So(e, ShouldNotBeNil) + So(e.Type(), ShouldEqual, events.ImageLintFailedEventType.String()) + }) + }) +} + +func TestNATSSinkEvents(t *testing.T) { + Convey("emits events to nats sink", t, func() { + Convey("repository created", func() { + natsServer, natsURL := setupTestNATSServer(t) + defer natsServer.Shutdown() + + testChannel := "test-events-" + randomString() + + recorder, err := createRecorder(t, natsURL, testChannel) + defer recorder.Close() + So(err, ShouldBeNil) + + eventChan := make(chan *cloudevents.Event, 1) + + nc, err := createSubscription(t, natsURL, testChannel, eventChan) + defer nc.Close() + So(err, ShouldBeNil) + + recorder.RepositoryCreated("test") + + e := getEvent(t, eventChan) + So(e, ShouldNotBeNil) + So(e.Type(), ShouldEqual, events.RepositoryCreatedEventType.String()) + }) + + Convey("image updated", func() { + natsServer, natsURL := setupTestNATSServer(t) + defer natsServer.Shutdown() + + testChannel := "test-events-" + randomString() + + recorder, err := createRecorder(t, natsURL, testChannel) + So(err, ShouldBeNil) + defer recorder.Close() + + eventChan := make(chan *cloudevents.Event, 1) + + nc, err := createSubscription(t, natsURL, testChannel, eventChan) + defer nc.Close() + So(err, ShouldBeNil) + + recorder.ImageUpdated("test", "v1", "", string(types.OCIManifestSchema1), "") + + e := getEvent(t, eventChan) + So(e, ShouldNotBeNil) + So(e.Type(), ShouldEqual, events.ImageUpdatedEventType.String()) + }) + + Convey("image deleted", func() { + natsServer, natsURL := setupTestNATSServer(t) + defer natsServer.Shutdown() + + testChannel := "test-events-" + randomString() + + eventChan := make(chan *cloudevents.Event, 1) + + nc, err := createSubscription(t, natsURL, testChannel, eventChan) + defer nc.Close() + So(err, ShouldBeNil) + + recorder, err := createRecorder(t, natsURL, testChannel) + defer recorder.Close() + So(err, ShouldBeNil) + + recorder.ImageDeleted("test", "v1", "", string(types.OCIManifestSchema1)) + + e := getEvent(t, eventChan) + So(e, ShouldNotBeNil) + So(e.Type(), ShouldEqual, events.ImageDeletedEventType.String()) + }) + + Convey("image lint failed", func() { + natsServer, natsURL := setupTestNATSServer(t) + defer natsServer.Shutdown() + + testChannel := "test-events-" + randomString() + + recorder, err := createRecorder(t, natsURL, testChannel) + defer recorder.Close() + So(err, ShouldBeNil) + + eventChan := make(chan *cloudevents.Event, 1) + + nc, err := createSubscription(t, natsURL, testChannel, eventChan) + defer nc.Close() + So(err, ShouldBeNil) + + recorder.ImageLintFailed("test", "v1", "", string(types.OCIManifestSchema1), "") + + e := getEvent(t, eventChan) + So(e, ShouldNotBeNil) + So(e.Type(), ShouldEqual, events.ImageLintFailedEventType.String()) + }) + }) +} + +func setupTestNATSServer(t *testing.T) (*server.Server, string) { + t.Helper() + + opts := server.Options{ + Host: "127.0.0.1", + Port: -1, // Use random available port + NoLog: true, + NoSigs: true, + MaxControlLine: 4096, + } + + natsServer, err := server.NewServer(&opts) + if err != nil { + panic(err) + } + + go natsServer.Start() + + if !natsServer.ReadyForConnections(5 * time.Second) { + panic("NATS server failed to start") + } + + return natsServer, natsServer.ClientURL() +} + +func createRecorder(t *testing.T, natsURL, testChannel string) (events.Recorder, error) { + t.Helper() + config := eventsconf.SinkConfig{ + Type: eventsconf.NATS, + Address: natsURL, + Channel: testChannel, + Timeout: 15 * time.Second, + } + + sink, err := events.NewNATSSink(config) + if err != nil { + return nil, err + } + + recorder, err := events.NewRecorder(log.NewLogger("debug", ""), sink) + if err != nil { + return nil, err + } + + return recorder, nil +} + +func createSubscription(t *testing.T, natsURL, channelName string, bus chan *cloudevents.Event) (*nats.Conn, error) { + t.Helper() + + natsConnection, err := nats.Connect(natsURL) + if err != nil { + return nil, err + } + + _, err = natsConnection.Subscribe(channelName, func(msg *nats.Msg) { + event := cloudevents.NewEvent() + + headers := msg.Header + event.SetID(headers.Get("ce-id")) + event.SetSource(headers.Get("ce-source")) + event.SetType(headers.Get("ce-type")) + + if subj := headers.Get("ce-subject"); subj != "" { + event.SetSubject(subj) + } + + if err := event.UnmarshalJSON(msg.Data); err == nil { + bus <- &event + } + + _ = msg.Respond([]byte("OK")) + }) + if err != nil { + return nil, err + } + + err = natsConnection.FlushTimeout(2 * time.Second) + if err != nil { + return nil, fmt.Errorf("flush failed: %w", err) + } + + return natsConnection, nil +} + +func getEvent(t *testing.T, c chan *cloudevents.Event) *cloudevents.Event { + t.Helper() + + var evt *cloudevents.Event + select { + case evt = <-c: + case <-time.After(time.Second * 2): + t.Fatal("timed out waiting for event") + } + + return evt +} + +const charset = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789" + +func randomString() string { + rand.Seed(time.Now().UnixNano()) + + buf := make([]byte, 5) + + for i := range buf { + buf[i] = charset[rand.Intn(len(charset))] + } + + return string(buf) +} diff --git a/pkg/extensions/events/extension.go b/pkg/extensions/events/extension.go new file mode 100644 index 00000000..976a50c3 --- /dev/null +++ b/pkg/extensions/events/extension.go @@ -0,0 +1,27 @@ +//go:build events +// +build events + +package events + +import ( + cloudevents "github.com/cloudevents/sdk-go/v2" + + zerr "zotregistry.dev/zot/errors" + "zotregistry.dev/zot/pkg/log" +) + +type Sink interface { + Emit(*cloudevents.Event) cloudevents.Result + Close() error +} + +func NewRecorder(logger log.Logger, sinks ...Sink) (Recorder, error) { + if sinks == nil { + return nil, zerr.ErrEventSinkIsNil + } + + return &eventRecorder{ + sinks: sinks, + log: logger, + }, nil +} diff --git a/pkg/extensions/events/http_sink.go b/pkg/extensions/events/http_sink.go new file mode 100644 index 00000000..dcfc79cb --- /dev/null +++ b/pkg/extensions/events/http_sink.go @@ -0,0 +1,130 @@ +//go:build events +// +build events + +package events + +import ( + "context" + "encoding/base64" + "net/http" + "net/url" + + cloudevents "github.com/cloudevents/sdk-go/v2" + cehttp "github.com/cloudevents/sdk-go/v2/protocol/http" + + zerr "zotregistry.dev/zot/errors" + eventsconf "zotregistry.dev/zot/pkg/extensions/config/events" +) + +type HTTPSink struct { + cloudevents.Client + config eventsconf.SinkConfig +} + +func NewHTTPSink(config eventsconf.SinkConfig) (*HTTPSink, error) { + if config.Type != eventsconf.HTTP { + return nil, zerr.ErrInvalidEventSinkType + } + + if config.Address == "" { + return nil, zerr.ErrEventSinkAddressEmpty + } + + // Create the basic http client + httpClient, err := GetHTTPClientForConfig(config) + if err != nil { + return nil, err + } + + opts := []cehttp.Option{ + cehttp.WithTarget(config.Address), + cehttp.WithClient(*httpClient), + } + + if config.Credentials != nil && config.Credentials.Username != "" { + opts = append(opts, cehttp.WithHeader("Authorization", + "Basic "+BasicAuth(config.Credentials.Username, config.Credentials.Password))) + } + + // Create CloudEvents HTTP protocol + provider, err := cehttp.New(opts...) + if err != nil { + return nil, err + } + + // Create CloudEvents client + ceClient, err := cloudevents.NewClient(provider) + if err != nil { + return nil, err + } + + return &HTTPSink{ + Client: ceClient, + config: config, + }, nil +} + +// Emit sends the event to the sink. +func (s *HTTPSink) Emit(event *cloudevents.Event) cloudevents.Result { + ctx, cancel := context.WithTimeout(context.Background(), s.config.Timeout) + defer cancel() + + if err := event.Validate(); err != nil { + return err + } + + if s.config.Channel != "" { + event.SetExtension("channel", s.config.Channel) + } + + // Send the event + return s.Send(ctx, *event) +} + +// Close implements a method to clean up resources. +func (s *HTTPSink) Close() error { + // For HTTP clients, typically no specific cleanup is needed + // We could cancel any in-flight requests if we tracked them + return nil +} + +func GetHTTPClientForConfig(config eventsconf.SinkConfig) (*http.Client, error) { + transport, ok := http.DefaultTransport.(*http.Transport) + if !ok { + return nil, zerr.ErrCouldNotCreateHTTPEventTransport + } + transport = transport.Clone() + + if config.Proxy != nil && *config.Proxy != "" { + proxyURL, err := url.Parse(*config.Proxy) + if err != nil { + return nil, err + } + transport.Proxy = http.ProxyURL(proxyURL) + } + + if config.TLSConfig != nil && (config.TLSConfig.CACertFile != "" || config.TLSConfig.CertFile != "") { + tlsConfig, err := getTLSConfig(config) + if err != nil { + return nil, err + } + transport.TLSClientConfig = tlsConfig + } + + timeout := config.Timeout + if timeout == 0 { + timeout = DefaultHTTPTimeout + } + + return &http.Client{ + Transport: transport, + Timeout: timeout, + }, nil +} + +// Helper function for basic auth encoding. +func BasicAuth(username, password string) string { + auth := username + ":" + password + + return base64.StdEncoding.EncodeToString([]byte(auth)) +} diff --git a/pkg/extensions/events/http_sink_test.go b/pkg/extensions/events/http_sink_test.go new file mode 100644 index 00000000..9c49713e --- /dev/null +++ b/pkg/extensions/events/http_sink_test.go @@ -0,0 +1,140 @@ +//go:build events +// +build events + +package events_test + +import ( + "testing" + "time" + + cloudevents "github.com/cloudevents/sdk-go/v2" + . "github.com/smartystreets/goconvey/convey" + + zerr "zotregistry.dev/zot/errors" + eventsconf "zotregistry.dev/zot/pkg/extensions/config/events" + "zotregistry.dev/zot/pkg/extensions/events" +) + +func TestHTTPSink(t *testing.T) { + Convey("NewHTTPSink returns error for invalid type", t, func() { + cfg := eventsconf.SinkConfig{ + Type: "invalid", + Address: "http://localhost", + } + + sink, err := events.NewHTTPSink(cfg) + So(sink, ShouldBeNil) + So(err, ShouldEqual, zerr.ErrInvalidEventSinkType) + }) + + Convey("NewHTTPSink returns error for empty address", t, func() { + cfg := eventsconf.SinkConfig{ + Type: eventsconf.HTTP, + } + + sink, err := events.NewHTTPSink(cfg) + So(sink, ShouldBeNil) + So(err, ShouldEqual, zerr.ErrEventSinkAddressEmpty) + }) + + Convey("NewHTTPSink returns sink for valid config", t, func() { + cfg := eventsconf.SinkConfig{ + Type: eventsconf.HTTP, + Address: "http://localhost", + } + + sink, err := events.NewHTTPSink(cfg) + So(err, ShouldBeNil) + So(sink, ShouldNotBeNil) + }) + + Convey("NewHTTPSink handles basic auth config", t, func() { + cfg := eventsconf.SinkConfig{ + Type: eventsconf.HTTP, + Address: "http://localhost", + Credentials: &eventsconf.Credentials{ + Username: "user", + Password: "pass", + }, + } + + sink, err := events.NewHTTPSink(cfg) + So(err, ShouldBeNil) + So(sink, ShouldNotBeNil) + }) + + Convey("GetHTTPClientForConfig returns error for invalid proxy", t, func() { + badProxy := "://bad-url" + cfg := eventsconf.SinkConfig{ + Proxy: &badProxy, + } + + client, err := events.GetHTTPClientForConfig(cfg) + So(client, ShouldBeNil) + So(err, ShouldNotBeNil) + }) + + Convey("GetHTTPClientForConfig returns client with default transport", t, func() { + cfg := eventsconf.SinkConfig{ + Timeout: 2 * time.Second, + } + + client, err := events.GetHTTPClientForConfig(cfg) + So(err, ShouldBeNil) + So(client, ShouldNotBeNil) + }) + + Convey("BasicAuth encodes credentials", t, func() { + auth := events.BasicAuth("foo", "bar") + So(auth, ShouldEqual, "Zm9vOmJhcg==") + }) + + Convey("HTTPSink emits event and sets channel extension", t, func() { + event := cloudevents.NewEvent() + event.SetID("1234") + event.SetType("test.event") + event.SetSource("unit.test") + + cfg := eventsconf.SinkConfig{ + Type: eventsconf.HTTP, + Address: "http://localhost", + Timeout: 1 * time.Second, + Channel: "test-channel", + } + + sink, err := events.NewHTTPSink(cfg) + So(err, ShouldBeNil) + + _ = sink.Emit(&event) + So(event.Extensions()["channel"], ShouldEqual, "test-channel") + }) + + Convey("HTTPSink.Emit returns error for invalid event", t, func() { + event := cloudevents.NewEvent() // invalid + + cfg := eventsconf.SinkConfig{ + Type: eventsconf.HTTP, + Address: "http://localhost", + Timeout: 1 * time.Second, + } + + sink, err := events.NewHTTPSink(cfg) + So(err, ShouldBeNil) + + err = sink.Emit(&event) + So(err, ShouldNotBeNil) + }) + + Convey("HTTPSink.Close completes successfully", t, func() { + cfg := eventsconf.SinkConfig{ + Type: eventsconf.HTTP, + Address: "http://localhost", + } + + sink, err := events.NewHTTPSink(cfg) + So(err, ShouldBeNil) + + err = sink.Close() + So(err, ShouldBeNil) + }) +} diff --git a/pkg/extensions/events/nats_sink.go b/pkg/extensions/events/nats_sink.go new file mode 100644 index 00000000..24cc8ed6 --- /dev/null +++ b/pkg/extensions/events/nats_sink.go @@ -0,0 +1,96 @@ +//go:build events +// +build events + +package events + +import ( + "context" + "fmt" + + cenats "github.com/cloudevents/sdk-go/protocol/nats/v2" + cloudevents "github.com/cloudevents/sdk-go/v2" + "github.com/nats-io/nats.go" + + zerr "zotregistry.dev/zot/errors" + eventsconf "zotregistry.dev/zot/pkg/extensions/config/events" +) + +// NATSSink implements a CloudEvents sink that publishes to NATS. +type NATSSink struct { + cloudevents.Client + conn *nats.Conn + config eventsconf.SinkConfig +} + +// NewNATSSink creates a new NATS sink. +func NewNATSSink(config eventsconf.SinkConfig) (*NATSSink, error) { + if config.Type != eventsconf.NATS { + return nil, zerr.ErrInvalidEventSinkType + } + + if config.Address == "" { + return nil, zerr.ErrEventSinkAddressEmpty + } + + opts := []nats.Option{ + nats.Name(EventSource), + nats.Timeout(config.Timeout), + } + + if config.Credentials != nil { + if config.Credentials.File != nil && *config.Credentials.File != "" { + opts = append(opts, nats.UserCredentials(*config.Credentials.File)) + } else if config.Credentials.Username != "" { + opts = append(opts, nats.UserInfo( + config.Credentials.Username, + config.Credentials.Password, + )) + } + } + + if config.TLSConfig != nil && (config.TLSConfig.CACertFile != "" || config.TLSConfig.CertFile != "") { + tlsConfig, err := getTLSConfig(config) + if err != nil { + return nil, err + } + + opts = append(opts, nats.Secure(tlsConfig)) + } + + sender, err := cenats.NewSender(config.Address, config.Channel, opts) + if err != nil { + return nil, fmt.Errorf("failed to create NATS protocol: %w", err) + } + + ceClient, err := cloudevents.NewClient(sender) + if err != nil { + return nil, fmt.Errorf("failed to create CloudEvents client: %w", err) + } + + return &NATSSink{ + Client: ceClient, + conn: sender.Conn, + config: config, + }, nil +} + +// Emit sends a CloudEvent to NATS. +func (s *NATSSink) Emit(event *cloudevents.Event) cloudevents.Result { + if err := event.Validate(); err != nil { + return err + } + + ctx, cancel := context.WithTimeout(context.Background(), s.config.Timeout) + defer cancel() + + return s.Send(ctx, *event) +} + +// Close closes the NATS connection. +func (s *NATSSink) Close() error { + if s.conn != nil { + s.conn.Close() + } + + return nil +} diff --git a/pkg/extensions/events/nats_sink_test.go b/pkg/extensions/events/nats_sink_test.go new file mode 100644 index 00000000..1492287a --- /dev/null +++ b/pkg/extensions/events/nats_sink_test.go @@ -0,0 +1,130 @@ +//go:build events +// +build events + +package events_test + +import ( + "testing" + "time" + + cloudevents "github.com/cloudevents/sdk-go/v2" + . "github.com/smartystreets/goconvey/convey" + + zerr "zotregistry.dev/zot/errors" + eventsconf "zotregistry.dev/zot/pkg/extensions/config/events" + "zotregistry.dev/zot/pkg/extensions/events" +) + +func TestNATSSink(t *testing.T) { + Convey("NewNATSSink returns error for invalid type", t, func() { + cfg := eventsconf.SinkConfig{ + Type: "invalid", + Address: "nats://localhost", + } + + sink, err := events.NewNATSSink(cfg) + So(sink, ShouldBeNil) + So(err, ShouldEqual, zerr.ErrInvalidEventSinkType) + }) + + Convey("NewNATSSink returns error for empty address", t, func() { + cfg := eventsconf.SinkConfig{ + Type: eventsconf.NATS, + } + + sink, err := events.NewNATSSink(cfg) + So(sink, ShouldBeNil) + So(err, ShouldEqual, zerr.ErrEventSinkAddressEmpty) + }) + + Convey("NewNATSSink with username/password credentials", t, func() { + natsServer, natsURL := setupTestNATSServer(t) + defer natsServer.Shutdown() + + cfg := eventsconf.SinkConfig{ + Type: eventsconf.NATS, + Address: natsURL, + Timeout: 2 * time.Second, + Credentials: &eventsconf.Credentials{ + Username: "user", + Password: "pass", + }, + } + + sink, err := events.NewNATSSink(cfg) + So(err, ShouldBeNil) + So(sink, ShouldNotBeNil) + }) + + Convey("NewNATSSink with nonexistent credentials file", t, func() { + natsServer, natsURL := setupTestNATSServer(t) + defer natsServer.Shutdown() + + credsFile := "nonexistent.creds" + cfg := eventsconf.SinkConfig{ + Type: eventsconf.NATS, + Address: natsURL, + Timeout: 1 * time.Second, + Credentials: &eventsconf.Credentials{ + File: &credsFile, + }, + } + + sink, err := events.NewNATSSink(cfg) + So(sink, ShouldBeNil) + So(err, ShouldNotBeNil) + }) + + Convey("NewNATSSink fails with invalid TLS config", t, func() { + natsServer, natsURL := setupTestNATSServer(t) + defer natsServer.Shutdown() + + cfg := eventsconf.SinkConfig{ + Type: eventsconf.NATS, + Address: natsURL, + TLSConfig: &eventsconf.TLSConfig{ + CACertFile: "invalid", + CertFile: "invalid", + }, + } + + sink, err := events.NewNATSSink(cfg) + So(sink, ShouldBeNil) + So(err, ShouldNotBeNil) + }) + + Convey("Emit returns error for invalid event", t, func() { + natsServer, natsURL := setupTestNATSServer(t) + defer natsServer.Shutdown() + + cfg := eventsconf.SinkConfig{ + Type: eventsconf.NATS, + Address: natsURL, + Timeout: 1 * time.Second, + } + + sink, err := events.NewNATSSink(cfg) + So(err, ShouldBeNil) + + event := cloudevents.NewEvent() // invalid: no ID/type/source + err = sink.Emit(&event) + So(err, ShouldNotBeNil) + }) + + Convey("Close succeeds even without Emit", t, func() { + natsServer, natsURL := setupTestNATSServer(t) + defer natsServer.Shutdown() + + cfg := eventsconf.SinkConfig{ + Type: eventsconf.NATS, + Address: natsURL, + Timeout: 1 * time.Second, + } + + sink, err := events.NewNATSSink(cfg) + So(err, ShouldBeNil) + + err = sink.Close() + So(err, ShouldBeNil) + }) +} diff --git a/pkg/extensions/extension_events.go b/pkg/extensions/extension_events.go new file mode 100644 index 00000000..8b56786b --- /dev/null +++ b/pkg/extensions/extension_events.go @@ -0,0 +1,61 @@ +//go:build events +// +build events + +package extensions + +import ( + zerr "zotregistry.dev/zot/errors" + "zotregistry.dev/zot/pkg/api/config" + eventsconfig "zotregistry.dev/zot/pkg/extensions/config/events" + "zotregistry.dev/zot/pkg/extensions/events" + "zotregistry.dev/zot/pkg/log" +) + +func NewEventRecorder(config *config.Config, log log.Logger) (events.Recorder, error) { + if !config.IsEventRecorderEnabled() { + log.Info().Msg("events disabled in configuration") + + return nil, zerr.ErrExtensionNotEnabled + } + + eventConfig := config.Extensions.Events + + if eventConfig.Sinks == nil || len(eventConfig.Sinks) == 0 { + log.Info().Msg("no sinks provided, skipping events extension setup") + + return nil, zerr.ErrExtensionNotEnabled + } + + var sinks []events.Sink + + log.Info().Msg("setting up event sinks") + + for _, sinkConfig := range eventConfig.Sinks { + switch sinkConfig.Type { + case eventsconfig.HTTP: + sink, err := events.NewHTTPSink(sinkConfig) + if err != nil { + return nil, err + } + + sinks = append(sinks, sink) + case eventsconfig.NATS: + sink, err := events.NewNATSSink(sinkConfig) + if err != nil { + return nil, err + } + + sinks = append(sinks, sink) + default: + log.Warn().Msgf("skipping unsupported sink type: %s", sinkConfig.Type) + } + } + + if len(sinks) == 0 { + log.Warn().Msg("no sinks provided, skipping events extension setup") + + return nil, zerr.ErrExtensionNotEnabled + } + + return events.NewRecorder(log, sinks...) +} diff --git a/pkg/extensions/extension_events_disabled.go b/pkg/extensions/extension_events_disabled.go new file mode 100644 index 00000000..407b86b1 --- /dev/null +++ b/pkg/extensions/extension_events_disabled.go @@ -0,0 +1,24 @@ +//go:build !events +// +build !events + +package extensions + +import ( + zerr "zotregistry.dev/zot/errors" + "zotregistry.dev/zot/pkg/api/config" + "zotregistry.dev/zot/pkg/extensions/events" + "zotregistry.dev/zot/pkg/log" +) + +func NewEventRecorder(config *config.Config, log log.Logger) (events.Recorder, error) { + if !config.IsEventRecorderEnabled() { + log.Info().Msg("events disabled in configuration") + + return nil, zerr.ErrExtensionNotEnabled + } + + log.Warn().Msg("skipping setting up events because given zot binary doesn't include this feature, " + + "please build a binary that does so") + + return nil, zerr.ErrExtensionNotEnabled +} diff --git a/pkg/extensions/extension_events_disabled_test.go b/pkg/extensions/extension_events_disabled_test.go new file mode 100644 index 00000000..acaa700a --- /dev/null +++ b/pkg/extensions/extension_events_disabled_test.go @@ -0,0 +1,54 @@ +//go:build !events +// +build !events + +package extensions_test + +import ( + "os" + "testing" + + . "github.com/smartystreets/goconvey/convey" + + "zotregistry.dev/zot/pkg/api" + "zotregistry.dev/zot/pkg/api/config" + extconf "zotregistry.dev/zot/pkg/extensions/config" + eventsconf "zotregistry.dev/zot/pkg/extensions/config/events" + test "zotregistry.dev/zot/pkg/test/common" +) + +func TestEventsExtension(t *testing.T) { + Convey("event generation is skipped when extension is disabled", t, func() { + conf := config.New() + port := test.GetFreePort() + + globalDir := t.TempDir() + defaultValue := true + + logFile, err := os.CreateTemp(globalDir, "zot-log*.txt") + So(err, ShouldBeNil) + defer os.Remove(logFile.Name()) + + conf.HTTP.Port = port + conf.Storage.RootDirectory = globalDir + conf.Storage.Commit = true + conf.Extensions = &extconf.ExtensionConfig{} + conf.Extensions.Events = &eventsconf.Config{ + Enable: &defaultValue, + } + conf.Log.Level = "warn" + conf.Log.Output = logFile.Name() + + ctlr := api.NewController(conf) + ctlrManager := test.NewControllerManager(ctlr) + + ctlrManager.StartAndWait(port) + defer ctlrManager.StopServer() + + data, err := os.ReadFile(logFile.Name()) + So(err, ShouldBeNil) + + So(string(data), ShouldContainSubstring, + "skipping setting up events because given zot binary doesn't include this feature, "+ + "please build a binary that does so") + }) +} diff --git a/pkg/extensions/extension_image_trust_test.go b/pkg/extensions/extension_image_trust_test.go index 59b83252..16a0bd4e 100644 --- a/pkg/extensions/extension_image_trust_test.go +++ b/pkg/extensions/extension_image_trust_test.go @@ -214,7 +214,7 @@ func RunSignatureUploadAndVerificationTests(t *testing.T, cacheDriverParams map[ logger.Logger = logger.Output(writers) imageStore := local.NewImageStore(globalDir, false, false, - logger, monitoring.NewMetricsServer(false, logger), nil, nil, nil) + logger, monitoring.NewMetricsServer(false, logger), nil, nil, nil, nil) storeController := storage.StoreController{ DefaultStore: imageStore, @@ -335,7 +335,7 @@ func RunSignatureUploadAndVerificationTests(t *testing.T, cacheDriverParams map[ logger.Logger = logger.Output(writers) imageStore := local.NewImageStore(globalDir, false, false, - logger, monitoring.NewMetricsServer(false, logger), nil, nil, nil) + logger, monitoring.NewMetricsServer(false, logger), nil, nil, nil, nil) storeController := storage.StoreController{ DefaultStore: imageStore, @@ -443,7 +443,7 @@ func RunSignatureUploadAndVerificationTests(t *testing.T, cacheDriverParams map[ logger.Logger = logger.Output(writers) imageStore := local.NewImageStore(globalDir, false, false, - logger, monitoring.NewMetricsServer(false, logger), nil, nil, nil) + logger, monitoring.NewMetricsServer(false, logger), nil, nil, nil, nil) storeController := storage.StoreController{ DefaultStore: imageStore, @@ -606,7 +606,7 @@ func RunSignatureUploadAndVerificationTests(t *testing.T, cacheDriverParams map[ logger.Logger = logger.Output(writers) imageStore := local.NewImageStore(globalDir, false, false, - logger, monitoring.NewMetricsServer(false, logger), nil, nil, nil) + logger, monitoring.NewMetricsServer(false, logger), nil, nil, nil, nil) storeController := storage.StoreController{ DefaultStore: imageStore, @@ -870,7 +870,7 @@ func RunSignatureUploadAndVerificationTests(t *testing.T, cacheDriverParams map[ logger.Logger = logger.Output(writers) imageStore := local.NewImageStore(globalDir, false, false, - logger, monitoring.NewMetricsServer(false, logger), nil, nil, nil) + logger, monitoring.NewMetricsServer(false, logger), nil, nil, nil, nil) storeController := storage.StoreController{ DefaultStore: imageStore, diff --git a/pkg/extensions/lint/lint_test.go b/pkg/extensions/lint/lint_test.go index d29ed76b..5309a3af 100644 --- a/pkg/extensions/lint/lint_test.go +++ b/pkg/extensions/lint/lint_test.go @@ -489,7 +489,7 @@ func TestVerifyMandatoryAnnotationsFunction(t *testing.T) { linter := lint.NewLinter(lintConfig, log.NewLogger("debug", "")) imgStore := local.NewImageStore(dir, false, false, - log.NewLogger("debug", ""), monitoring.NewMetricsServer(false, log.NewLogger("debug", "")), linter, nil, nil) + log.NewLogger("debug", ""), monitoring.NewMetricsServer(false, log.NewLogger("debug", "")), linter, nil, nil, nil) indexContent, err := imgStore.GetIndexContent("zot-test") So(err, ShouldBeNil) @@ -521,7 +521,7 @@ func TestVerifyMandatoryAnnotationsFunction(t *testing.T) { linter := lint.NewLinter(lintConfig, log.NewLogger("debug", "")) imgStore := local.NewImageStore(dir, false, false, - log.NewLogger("debug", ""), monitoring.NewMetricsServer(false, log.NewLogger("debug", "")), linter, nil, nil) + log.NewLogger("debug", ""), monitoring.NewMetricsServer(false, log.NewLogger("debug", "")), linter, nil, nil, nil) indexContent, err := imgStore.GetIndexContent("zot-test") So(err, ShouldBeNil) @@ -591,7 +591,7 @@ func TestVerifyMandatoryAnnotationsFunction(t *testing.T) { linter := lint.NewLinter(lintConfig, log.NewLogger("debug", "")) imgStore := local.NewImageStore(dir, false, false, - log.NewLogger("debug", ""), monitoring.NewMetricsServer(false, log.NewLogger("debug", "")), linter, nil, nil) + log.NewLogger("debug", ""), monitoring.NewMetricsServer(false, log.NewLogger("debug", "")), linter, nil, nil, nil) pass, err := linter.CheckMandatoryAnnotations("zot-test", digest, imgStore) So(err, ShouldBeNil) @@ -653,7 +653,7 @@ func TestVerifyMandatoryAnnotationsFunction(t *testing.T) { linter := lint.NewLinter(lintConfig, log.NewLogger("debug", "")) imgStore := local.NewImageStore(dir, false, false, - log.NewLogger("debug", ""), monitoring.NewMetricsServer(false, log.NewLogger("debug", "")), linter, nil, nil) + log.NewLogger("debug", ""), monitoring.NewMetricsServer(false, log.NewLogger("debug", "")), linter, nil, nil, nil) pass, err := linter.CheckMandatoryAnnotations("zot-test", digest, imgStore) So(err, ShouldNotBeNil) @@ -717,7 +717,7 @@ func TestVerifyMandatoryAnnotationsFunction(t *testing.T) { linter := lint.NewLinter(lintConfig, log.NewLogger("debug", "")) imgStore := local.NewImageStore(dir, false, false, - log.NewLogger("debug", ""), monitoring.NewMetricsServer(false, log.NewLogger("debug", "")), linter, nil, nil) + log.NewLogger("debug", ""), monitoring.NewMetricsServer(false, log.NewLogger("debug", "")), linter, nil, nil, nil) pass, err := linter.CheckMandatoryAnnotations("zot-test", digest, imgStore) So(err, ShouldBeNil) @@ -780,7 +780,7 @@ func TestVerifyMandatoryAnnotationsFunction(t *testing.T) { linter := lint.NewLinter(lintConfig, log.NewLogger("debug", "")) imgStore := local.NewImageStore(dir, false, false, - log.NewLogger("debug", ""), monitoring.NewMetricsServer(false, log.NewLogger("debug", "")), linter, nil, nil) + log.NewLogger("debug", ""), monitoring.NewMetricsServer(false, log.NewLogger("debug", "")), linter, nil, nil, nil) err = os.Chmod(path.Join(dir, "zot-test", "blobs"), 0o000) if err != nil { @@ -878,7 +878,7 @@ func TestVerifyMandatoryAnnotationsFunction(t *testing.T) { linter := lint.NewLinter(lintConfig, log.NewLogger("debug", "")) imgStore := local.NewImageStore(dir, false, false, - log.NewLogger("debug", ""), monitoring.NewMetricsServer(false, log.NewLogger("debug", "")), linter, nil, nil) + log.NewLogger("debug", ""), monitoring.NewMetricsServer(false, log.NewLogger("debug", "")), linter, nil, nil, nil) err = os.Chmod(path.Join(dir, "zot-test", "blobs", "sha256", manifest.Config.Digest.Encoded()), 0o000) if err != nil { diff --git a/pkg/extensions/scrub/scrub_test.go b/pkg/extensions/scrub/scrub_test.go index 90f87e2c..f9d0e137 100644 --- a/pkg/extensions/scrub/scrub_test.go +++ b/pkg/extensions/scrub/scrub_test.go @@ -195,7 +195,7 @@ func TestRunScrubRepo(t *testing.T) { UseRelPaths: true, }, log) imgStore := local.NewImageStore(dir, true, - true, log, metrics, nil, cacheDriver, nil) + true, log, metrics, nil, cacheDriver, nil, nil) srcStorageCtlr := ociutils.GetDefaultStoreController(dir, log) image := CreateDefaultVulnerableImage() @@ -231,7 +231,7 @@ func TestRunScrubRepo(t *testing.T) { UseRelPaths: true, }, log) imgStore := local.NewImageStore(dir, true, - true, log, metrics, nil, cacheDriver, nil) + true, log, metrics, nil, cacheDriver, nil, nil) srcStorageCtlr := ociutils.GetDefaultStoreController(dir, log) image := CreateDefaultVulnerableImage() @@ -272,7 +272,7 @@ func TestRunScrubRepo(t *testing.T) { Name: "cache", UseRelPaths: true, }, log) - imgStore := local.NewImageStore(dir, true, true, log, metrics, nil, cacheDriver, nil) + imgStore := local.NewImageStore(dir, true, true, log, metrics, nil, cacheDriver, nil, nil) srcStorageCtlr := ociutils.GetDefaultStoreController(dir, log) image := CreateDefaultVulnerableImage() diff --git a/pkg/extensions/search/cve/cve_test.go b/pkg/extensions/search/cve/cve_test.go index d74c6cdc..e49394b1 100644 --- a/pkg/extensions/search/cve/cve_test.go +++ b/pkg/extensions/search/cve/cve_test.go @@ -319,7 +319,7 @@ func TestImageFormat(t *testing.T) { dbDir := t.TempDir() metrics := monitoring.NewMetricsServer(false, log) - defaultStore := local.NewImageStore(imgDir, false, false, log, metrics, nil, nil, nil) + defaultStore := local.NewImageStore(imgDir, false, false, log, metrics, nil, nil, nil, nil) storeController := storage.StoreController{DefaultStore: defaultStore} params := boltdb.DBParameters{ diff --git a/pkg/extensions/search/cve/scan_test.go b/pkg/extensions/search/cve/scan_test.go index b2404b7f..bcdb8c3b 100644 --- a/pkg/extensions/search/cve/scan_test.go +++ b/pkg/extensions/search/cve/scan_test.go @@ -505,7 +505,7 @@ func TestScanGeneratorWithRealData(t *testing.T) { metrics := monitoring.NewMetricsServer(true, logger) imageStore := local.NewImageStore(rootDir, false, false, - logger, metrics, nil, nil, nil) + logger, metrics, nil, nil, nil, nil) storeController := storage.StoreController{DefaultStore: imageStore} image := CreateRandomVulnerableImage() diff --git a/pkg/extensions/search/cve/trivy/scanner_internal_test.go b/pkg/extensions/search/cve/trivy/scanner_internal_test.go index 91d5a9fa..883bf9ff 100644 --- a/pkg/extensions/search/cve/trivy/scanner_internal_test.go +++ b/pkg/extensions/search/cve/trivy/scanner_internal_test.go @@ -54,11 +54,11 @@ func TestMultipleStoragePath(t *testing.T) { // Create ImageStore - firstStore := local.NewImageStore(firstRootDir, false, false, log, metrics, nil, nil, nil) + firstStore := local.NewImageStore(firstRootDir, false, false, log, metrics, nil, nil, nil, nil) - secondStore := local.NewImageStore(secondRootDir, false, false, log, metrics, nil, nil, nil) + secondStore := local.NewImageStore(secondRootDir, false, false, log, metrics, nil, nil, nil, nil) - thirdStore := local.NewImageStore(thirdRootDir, false, false, log, metrics, nil, nil, nil) + thirdStore := local.NewImageStore(thirdRootDir, false, false, log, metrics, nil, nil, nil, nil) storeController := storage.StoreController{} @@ -172,7 +172,7 @@ func TestTrivyLibraryErrors(t *testing.T) { metrics := monitoring.NewMetricsServer(false, log) // Create ImageStore - store := local.NewImageStore(rootDir, false, false, log, metrics, nil, nil, nil) + store := local.NewImageStore(rootDir, false, false, log, metrics, nil, nil, nil, nil) storeController := storage.StoreController{} storeController.DefaultStore = store @@ -313,7 +313,7 @@ func TestImageScannable(t *testing.T) { // Continue with initializing the objects the scanner depends on metrics := monitoring.NewMetricsServer(false, log) - store := local.NewImageStore(rootDir, false, false, log, metrics, nil, nil, nil) + store := local.NewImageStore(rootDir, false, false, log, metrics, nil, nil, nil, nil) storeController := storage.StoreController{} storeController.DefaultStore = store @@ -367,7 +367,7 @@ func TestTrivyDBUrl(t *testing.T) { metrics := monitoring.NewMetricsServer(false, log) // Create ImageStore - store := local.NewImageStore(rootDir, false, false, log, metrics, nil, nil, nil) + store := local.NewImageStore(rootDir, false, false, log, metrics, nil, nil, nil, nil) storeController := storage.StoreController{} storeController.DefaultStore = store diff --git a/pkg/extensions/search/cve/trivy/scanner_test.go b/pkg/extensions/search/cve/trivy/scanner_test.go index 9e2b37a4..3475a8b8 100644 --- a/pkg/extensions/search/cve/trivy/scanner_test.go +++ b/pkg/extensions/search/cve/trivy/scanner_test.go @@ -168,7 +168,7 @@ func TestVulnerableLayer(t *testing.T) { log := log.NewLogger("debug", "") imageStore := local.NewImageStore(tempDir, false, false, - log, monitoring.NewMetricsServer(false, log), nil, nil, nil) + log, monitoring.NewMetricsServer(false, log), nil, nil, nil, nil) storeController := storage.StoreController{ DefaultStore: imageStore, @@ -239,7 +239,7 @@ func TestVulnerableLayer(t *testing.T) { log := log.NewLogger("debug", "") imageStore := local.NewImageStore(tempDir, false, false, - log, monitoring.NewMetricsServer(false, log), nil, nil, nil) + log, monitoring.NewMetricsServer(false, log), nil, nil, nil, nil) storeController := storage.StoreController{ DefaultStore: imageStore, diff --git a/pkg/extensions/search/search_test.go b/pkg/extensions/search/search_test.go index 6d8197b0..1033217e 100644 --- a/pkg/extensions/search/search_test.go +++ b/pkg/extensions/search/search_test.go @@ -1159,7 +1159,7 @@ func TestExpandedRepoInfo(t *testing.T) { ctlr := api.NewController(conf) imageStore := local.NewImageStore(tempDir, false, false, - log.NewLogger("debug", ""), monitoring.NewMetricsServer(false, log.NewLogger("debug", "")), nil, nil, nil) + log.NewLogger("debug", ""), monitoring.NewMetricsServer(false, log.NewLogger("debug", "")), nil, nil, nil, nil) storeController := storage.StoreController{ DefaultStore: imageStore, @@ -1281,7 +1281,7 @@ func TestExpandedRepoInfo(t *testing.T) { log := log.NewLogger("debug", "") metrics := monitoring.NewMetricsServer(false, log) - testStorage := local.NewImageStore(rootDir, false, false, log, metrics, nil, nil, nil) + testStorage := local.NewImageStore(rootDir, false, false, log, metrics, nil, nil, nil, nil) resp, err := resty.R().Get(baseURL + "/v2/") So(resp, ShouldNotBeNil) @@ -1637,7 +1637,7 @@ func TestExpandedRepoInfo(t *testing.T) { ctlr := api.NewController(conf) imageStore := local.NewImageStore(conf.Storage.RootDirectory, false, false, - log.NewLogger("debug", ""), monitoring.NewMetricsServer(false, log.NewLogger("debug", "")), nil, nil, nil) + log.NewLogger("debug", ""), monitoring.NewMetricsServer(false, log.NewLogger("debug", "")), nil, nil, nil, nil) storeController := storage.StoreController{ DefaultStore: imageStore, @@ -1799,7 +1799,7 @@ func TestExpandedRepoInfo(t *testing.T) { log := log.NewLogger("debug", "") metrics := monitoring.NewMetricsServer(false, log) - testStorage := local.NewImageStore(rootDir, false, false, log, metrics, nil, nil, nil) + testStorage := local.NewImageStore(rootDir, false, false, log, metrics, nil, nil, nil, nil) resp, err := resty.R().Get(baseURL + "/v2/") So(resp, ShouldNotBeNil) @@ -6054,7 +6054,7 @@ func TestMetaDBWhenDeletingImages(t *testing.T) { // get signatur digest log := log.NewLogger("debug", "") metrics := monitoring.NewMetricsServer(false, log) - storage := local.NewImageStore(dir, false, false, log, metrics, nil, nil, nil) + storage := local.NewImageStore(dir, false, false, log, metrics, nil, nil, nil, nil) indexBlob, err := storage.GetIndexContent(repo) So(err, ShouldBeNil) @@ -6128,7 +6128,7 @@ func TestMetaDBWhenDeletingImages(t *testing.T) { // get signatur digest log := log.NewLogger("debug", "") metrics := monitoring.NewMetricsServer(false, log) - storage := local.NewImageStore(dir, false, false, log, metrics, nil, nil, nil) + storage := local.NewImageStore(dir, false, false, log, metrics, nil, nil, nil, nil) indexBlob, err := storage.GetIndexContent(repo) So(err, ShouldBeNil) diff --git a/pkg/extensions/sync/destination.go b/pkg/extensions/sync/destination.go index 160e47a6..057c07f5 100644 --- a/pkg/extensions/sync/destination.go +++ b/pkg/extensions/sync/destination.go @@ -318,5 +318,5 @@ func getImageStoreFromImageReference(repo string, imageReference ref.Ref, log lo func getImageStore(rootDir string, log log.Logger) storageTypes.ImageStore { metrics := monitoring.NewMetricsServer(false, log) - return local.NewImageStore(rootDir, false, false, log, metrics, nil, nil, nil) + return local.NewImageStore(rootDir, false, false, log, metrics, nil, nil, nil, nil) } diff --git a/pkg/extensions/sync/sync_internal_test.go b/pkg/extensions/sync/sync_internal_test.go index 2de5abb6..381c549f 100644 --- a/pkg/extensions/sync/sync_internal_test.go +++ b/pkg/extensions/sync/sync_internal_test.go @@ -55,7 +55,7 @@ func TestDestinationRegistry(t *testing.T) { UseRelPaths: true, }, log) - syncImgStore := local.NewImageStore(dir, true, true, log, metrics, nil, cacheDriver, nil) + syncImgStore := local.NewImageStore(dir, true, true, log, metrics, nil, cacheDriver, nil, nil) repoName := "repo" storeController := storage.StoreController{DefaultStore: syncImgStore} @@ -185,7 +185,7 @@ func TestDestinationRegistry(t *testing.T) { MandatoryAnnotations: []string{"annot1"}, }, log) - syncImgStore := local.NewImageStore(dir, true, true, log, metrics, linter, cacheDriver, nil) + syncImgStore := local.NewImageStore(dir, true, true, log, metrics, linter, cacheDriver, nil, nil) repoName := "repo" storeController := storage.StoreController{DefaultStore: syncImgStore} diff --git a/pkg/meta/hooks_test.go b/pkg/meta/hooks_test.go index 48e8b021..ef40d246 100644 --- a/pkg/meta/hooks_test.go +++ b/pkg/meta/hooks_test.go @@ -26,7 +26,7 @@ func TestOnUpdateManifest(t *testing.T) { storeController := storage.StoreController{} log := log.NewLogger("debug", "") metrics := monitoring.NewMetricsServer(false, log) - storeController.DefaultStore = local.NewImageStore(rootDir, true, true, log, metrics, nil, nil, nil) + storeController.DefaultStore = local.NewImageStore(rootDir, true, true, log, metrics, nil, nil, nil, nil) params := boltdb.DBParameters{ RootDir: rootDir, diff --git a/pkg/meta/parse_test.go b/pkg/meta/parse_test.go index ad8409f7..9f9b2c10 100644 --- a/pkg/meta/parse_test.go +++ b/pkg/meta/parse_test.go @@ -368,7 +368,7 @@ func RunParseStorageTests(rootDir string, metaDB mTypes.MetaDB, log log.Logger) Convey("Test with simple case", func() { imageStore := local.NewImageStore(rootDir, false, false, - log, monitoring.NewMetricsServer(false, log), nil, nil, nil) + log, monitoring.NewMetricsServer(false, log), nil, nil, nil, nil) storeController := storage.StoreController{DefaultStore: imageStore} manifests := []ispec.Manifest{} @@ -443,7 +443,7 @@ func RunParseStorageTests(rootDir string, metaDB mTypes.MetaDB, log log.Logger) Convey("Accept orphan signatures", func() { imageStore := local.NewImageStore(rootDir, false, false, - log, monitoring.NewMetricsServer(false, log), nil, nil, nil) + log, monitoring.NewMetricsServer(false, log), nil, nil, nil, nil) storeController := storage.StoreController{DefaultStore: imageStore} @@ -488,7 +488,7 @@ func RunParseStorageTests(rootDir string, metaDB mTypes.MetaDB, log log.Logger) Convey("Check statistics after load", func() { imageStore := local.NewImageStore(rootDir, false, false, - log, monitoring.NewMetricsServer(false, log), nil, nil, nil) + log, monitoring.NewMetricsServer(false, log), nil, nil, nil, nil) storeController := storage.StoreController{DefaultStore: imageStore} // add an image @@ -529,7 +529,7 @@ func RunParseStorageTests(rootDir string, metaDB mTypes.MetaDB, log log.Logger) // make sure pushTimestamp is always populated to not interfere with retention logic Convey("Always update pushTimestamp if its value is 0(time.Time{})", func() { imageStore := local.NewImageStore(rootDir, false, false, - log, monitoring.NewMetricsServer(false, log), nil, nil, nil) + log, monitoring.NewMetricsServer(false, log), nil, nil, nil, nil) storeController := storage.StoreController{DefaultStore: imageStore} // add an image diff --git a/pkg/storage/common/common_test.go b/pkg/storage/common/common_test.go index f7ec668b..f20aff1a 100644 --- a/pkg/storage/common/common_test.go +++ b/pkg/storage/common/common_test.go @@ -37,7 +37,7 @@ func TestValidateManifest(t *testing.T) { Name: "cache", UseRelPaths: true, }, log) - imgStore := local.NewImageStore(dir, true, true, log, metrics, nil, cacheDriver, nil) + imgStore := local.NewImageStore(dir, true, true, log, metrics, nil, cacheDriver, nil, nil) content := []byte("this is a blob") digest := godigest.FromBytes(content) @@ -199,7 +199,7 @@ func TestGetReferrersErrors(t *testing.T) { UseRelPaths: true, }, log) - imgStore := local.NewImageStore(dir, false, true, log, metrics, nil, cacheDriver, nil) + imgStore := local.NewImageStore(dir, false, true, log, metrics, nil, cacheDriver, nil, nil) artifactType := "application/vnd.example.icecream.v1" validDigest := godigest.FromBytes([]byte("blob")) @@ -420,7 +420,7 @@ func TestGetBlobDescriptorFromRepo(t *testing.T) { driver := local.New(true) imgStore := imagestore.NewImageStore(tdir, tdir, true, - true, log, metrics, nil, driver, cacheDriver, nil) + true, log, metrics, nil, driver, cacheDriver, nil, nil) repoName := "zot-test" diff --git a/pkg/storage/gc/gc_internal_test.go b/pkg/storage/gc/gc_internal_test.go index 59ef0d76..3ed1b359 100644 --- a/pkg/storage/gc/gc_internal_test.go +++ b/pkg/storage/gc/gc_internal_test.go @@ -47,7 +47,7 @@ func TestGarbageCollectManifestErrors(t *testing.T) { Name: "cache", UseRelPaths: true, }, log) - imgStore := local.NewImageStore(dir, true, true, log, metrics, nil, cacheDriver, nil) + imgStore := local.NewImageStore(dir, true, true, log, metrics, nil, cacheDriver, nil, nil) gc := NewGarbageCollect(imgStore, mocks.MetaDBMock{}, Options{ Delay: storageConstants.DefaultGCDelay, @@ -171,7 +171,7 @@ func TestGarbageCollectIndexErrors(t *testing.T) { Name: "cache", UseRelPaths: true, }, log) - imgStore := local.NewImageStore(dir, true, true, log, metrics, nil, cacheDriver, nil) + imgStore := local.NewImageStore(dir, true, true, log, metrics, nil, cacheDriver, nil, nil) gc := NewGarbageCollect(imgStore, mocks.MetaDBMock{}, Options{ Delay: storageConstants.DefaultGCDelay, diff --git a/pkg/storage/gc/gc_test.go b/pkg/storage/gc/gc_test.go index bcec4a67..c0c2661d 100644 --- a/pkg/storage/gc/gc_test.go +++ b/pkg/storage/gc/gc_test.go @@ -140,13 +140,13 @@ func TestGarbageCollectAndRetention(t *testing.T) { panic(err) } - imgStore = s3.NewImageStore(rootDir, cacheDir, true, false, log, metrics, nil, store, nil, nil) + imgStore = s3.NewImageStore(rootDir, cacheDir, true, false, log, metrics, nil, store, nil, nil, nil) } else { // Create temporary directory rootDir := t.TempDir() // Create ImageStore - imgStore = local.NewImageStore(rootDir, false, false, log, metrics, nil, nil, nil) + imgStore = local.NewImageStore(rootDir, false, false, log, metrics, nil, nil, nil, nil) // init metaDB params := boltdb.DBParameters{ @@ -1105,7 +1105,7 @@ func TestGarbageCollectDeletion(t *testing.T) { rootDir := t.TempDir() // Create ImageStore - imgStore := local.NewImageStore(rootDir, false, false, log, metrics, nil, nil, nil) + imgStore := local.NewImageStore(rootDir, false, false, log, metrics, nil, nil, nil, nil) // init metaDB params := boltdb.DBParameters{ diff --git a/pkg/storage/imagestore/imagestore.go b/pkg/storage/imagestore/imagestore.go index ab753f18..a2b93ab5 100644 --- a/pkg/storage/imagestore/imagestore.go +++ b/pkg/storage/imagestore/imagestore.go @@ -22,6 +22,7 @@ import ( zerr "zotregistry.dev/zot/errors" zcommon "zotregistry.dev/zot/pkg/common" "zotregistry.dev/zot/pkg/compat" + "zotregistry.dev/zot/pkg/extensions/events" "zotregistry.dev/zot/pkg/extensions/monitoring" syncConstants "zotregistry.dev/zot/pkg/extensions/sync/constants" zlog "zotregistry.dev/zot/pkg/log" @@ -45,6 +46,7 @@ type ImageStore struct { lock *sync.RWMutex log zlog.Logger metrics monitoring.MetricServer + events events.Recorder cache storageTypes.Cache dedupe bool linter common.Lint @@ -69,7 +71,7 @@ func (is *ImageStore) DirExists(d string) bool { // Use the last argument to properly set a cache database, or it will default to boltDB local storage. func NewImageStore(rootDir string, cacheDir string, dedupe, commit bool, log zlog.Logger, metrics monitoring.MetricServer, linter common.Lint, storeDriver storageTypes.Driver, - cacheDriver storageTypes.Cache, compat []compat.MediaCompatibility, + cacheDriver storageTypes.Cache, compat []compat.MediaCompatibility, recorder events.Recorder, ) storageTypes.ImageStore { if err := storeDriver.EnsureDir(rootDir); err != nil { log.Error().Err(err).Str("rootDir", rootDir).Msg("failed to create root dir") @@ -88,6 +90,7 @@ func NewImageStore(rootDir string, cacheDir string, dedupe, commit bool, log zlo commit: commit, cache: cacheDriver, compat: compat, + events: recorder, } return imgStore @@ -194,6 +197,10 @@ func (is *ImageStore) initRepo(name string) error { return err } + + if is.events != nil { + is.events.RepositoryCreated(name) + } } return nil @@ -675,6 +682,10 @@ func (is *ImageStore) PutImageManifest(repo, reference, mediaType string, //noli is.log.Error().Err(err).Str("repository", repo).Str("reference", reference). Msg("linter didn't pass") + if is.events != nil { + is.events.ImageLintFailed(repo, reference, mDigest.String(), mediaType, string(body)) + } + return "", "", err } @@ -682,6 +693,10 @@ func (is *ImageStore) PutImageManifest(repo, reference, mediaType string, //noli return "", "", err } + if is.events != nil { + is.events.ImageUpdated(repo, reference, mDigest.String(), mediaType, string(body)) + } + return mDigest, subjectDigest, nil } @@ -779,6 +794,10 @@ func (is *ImageStore) deleteImageManifest(repo, reference string, detectCollisio } } + if is.events != nil { + is.events.ImageDeleted(repo, reference, manifestDesc.Digest.String(), manifestDesc.MediaType) + } + return nil } diff --git a/pkg/storage/local/local.go b/pkg/storage/local/local.go index 3c2e372d..2267b7eb 100644 --- a/pkg/storage/local/local.go +++ b/pkg/storage/local/local.go @@ -2,6 +2,7 @@ package local import ( "zotregistry.dev/zot/pkg/compat" + "zotregistry.dev/zot/pkg/extensions/events" "zotregistry.dev/zot/pkg/extensions/monitoring" zlog "zotregistry.dev/zot/pkg/log" common "zotregistry.dev/zot/pkg/storage/common" @@ -13,7 +14,7 @@ import ( // Use the last argument to properly set a cache database, or it will default to boltDB local storage. func NewImageStore(rootDir string, dedupe, commit bool, log zlog.Logger, metrics monitoring.MetricServer, linter common.Lint, cacheDriver storageTypes.Cache, - compat []compat.MediaCompatibility, + compat []compat.MediaCompatibility, recorder events.Recorder, ) storageTypes.ImageStore { return imagestore.NewImageStore( rootDir, @@ -26,5 +27,6 @@ func NewImageStore(rootDir string, dedupe, commit bool, log zlog.Logger, New(commit), cacheDriver, compat, + recorder, ) } diff --git a/pkg/storage/local/local_elevated_test.go b/pkg/storage/local/local_elevated_test.go index 9c75ac61..c4ef5d4b 100644 --- a/pkg/storage/local/local_elevated_test.go +++ b/pkg/storage/local/local_elevated_test.go @@ -35,7 +35,7 @@ func TestElevatedPrivilegesInvalidDedupe(t *testing.T) { Name: "cache", UseRelPaths: true, }, log) - imgStore := local.NewImageStore(dir, true, true, log, metrics, nil, cacheDriver, nil) + imgStore := local.NewImageStore(dir, true, true, log, metrics, nil, cacheDriver, nil, nil) upload, err := imgStore.NewBlobUpload("dedupe1") So(err, ShouldBeNil) diff --git a/pkg/storage/local/local_test.go b/pkg/storage/local/local_test.go index e38c9848..5a02cffc 100644 --- a/pkg/storage/local/local_test.go +++ b/pkg/storage/local/local_test.go @@ -83,7 +83,7 @@ func TestStorageFSAPIs(t *testing.T) { UseRelPaths: true, }, log) - imgStore := local.NewImageStore(dir, true, true, log, metrics, nil, cacheDriver, nil) + imgStore := local.NewImageStore(dir, true, true, log, metrics, nil, cacheDriver, nil, nil) Convey("Repo layout", t, func(c C) { Convey("Bad image manifest", func() { @@ -217,7 +217,7 @@ func FuzzNewBlobUpload(f *testing.F) { UseRelPaths: true, }, log) - imgStore := local.NewImageStore(dir, true, true, log, metrics, nil, cacheDriver, nil) + imgStore := local.NewImageStore(dir, true, true, log, metrics, nil, cacheDriver, nil, nil) _, err := imgStore.NewBlobUpload(data) if err != nil { @@ -244,7 +244,7 @@ func FuzzPutBlobChunk(f *testing.F) { Name: "cache", UseRelPaths: true, }, log) - imgStore := local.NewImageStore(dir, true, true, log, metrics, nil, cacheDriver, nil) + imgStore := local.NewImageStore(dir, true, true, log, metrics, nil, cacheDriver, nil, nil) repoName := data @@ -280,7 +280,7 @@ func FuzzPutBlobChunkStreamed(f *testing.F) { Name: "cache", UseRelPaths: true, }, log) - imgStore := local.NewImageStore(dir, true, true, log, metrics, nil, cacheDriver, nil) + imgStore := local.NewImageStore(dir, true, true, log, metrics, nil, cacheDriver, nil, nil) repoName := data @@ -314,7 +314,7 @@ func FuzzGetBlobUpload(f *testing.F) { UseRelPaths: true, }, log) imgStore := local.NewImageStore(dir, true, true, log, metrics, nil, - cacheDriver, nil) + cacheDriver, nil, nil) _, err := imgStore.GetBlobUpload(data1, data2) if err != nil { @@ -340,7 +340,7 @@ func FuzzTestPutGetImageManifest(f *testing.F) { Name: "cache", UseRelPaths: true, }, *log) - imgStore := local.NewImageStore(dir, true, true, *log, metrics, nil, cacheDriver, nil) + imgStore := local.NewImageStore(dir, true, true, *log, metrics, nil, cacheDriver, nil, nil) cblob, cdigest := GetRandomImageConfig() @@ -396,7 +396,7 @@ func FuzzTestPutDeleteImageManifest(f *testing.F) { Name: "cache", UseRelPaths: true, }, *log) - imgStore := local.NewImageStore(dir, true, true, *log, metrics, nil, cacheDriver, nil) + imgStore := local.NewImageStore(dir, true, true, *log, metrics, nil, cacheDriver, nil, nil) cblob, cdigest := GetRandomImageConfig() @@ -457,7 +457,7 @@ func FuzzTestDeleteImageManifest(f *testing.F) { Name: "cache", UseRelPaths: true, }, *log) - imgStore := local.NewImageStore(dir, true, true, *log, metrics, nil, cacheDriver, nil) + imgStore := local.NewImageStore(dir, true, true, *log, metrics, nil, cacheDriver, nil, nil) digest, _, err := newRandomBlobForFuzz(data) if err != nil { @@ -494,7 +494,7 @@ func FuzzInitRepo(f *testing.F) { Name: "cache", UseRelPaths: true, }, *log) - imgStore := local.NewImageStore(dir, true, true, *log, metrics, nil, cacheDriver, nil) + imgStore := local.NewImageStore(dir, true, true, *log, metrics, nil, cacheDriver, nil, nil) err := imgStore.InitRepo(data) if err != nil { @@ -520,7 +520,7 @@ func FuzzInitValidateRepo(f *testing.F) { Name: "cache", UseRelPaths: true, }, *log) - imgStore := local.NewImageStore(dir, true, true, *log, metrics, nil, cacheDriver, nil) + imgStore := local.NewImageStore(dir, true, true, *log, metrics, nil, cacheDriver, nil, nil) err := imgStore.InitRepo(data) if err != nil { @@ -555,7 +555,7 @@ func FuzzGetImageTags(f *testing.F) { Name: "cache", UseRelPaths: true, }, *log) - imgStore := local.NewImageStore(dir, true, true, *log, metrics, nil, cacheDriver, nil) + imgStore := local.NewImageStore(dir, true, true, *log, metrics, nil, cacheDriver, nil, nil) _, err := imgStore.GetImageTags(data) if err != nil { @@ -581,7 +581,7 @@ func FuzzBlobUploadPath(f *testing.F) { Name: "cache", UseRelPaths: true, }, *log) - imgStore := local.NewImageStore(dir, true, true, *log, metrics, nil, cacheDriver, nil) + imgStore := local.NewImageStore(dir, true, true, *log, metrics, nil, cacheDriver, nil, nil) _ = imgStore.BlobUploadPath(repo, uuid) }) @@ -600,7 +600,7 @@ func FuzzBlobUploadInfo(f *testing.F) { Name: "cache", UseRelPaths: true, }, *log) - imgStore := local.NewImageStore(dir, true, true, *log, metrics, nil, cacheDriver, nil) + imgStore := local.NewImageStore(dir, true, true, *log, metrics, nil, cacheDriver, nil, nil) repo := data _, err := imgStore.BlobUploadInfo(repo, uuid) @@ -626,7 +626,7 @@ func FuzzTestGetImageManifest(f *testing.F) { Name: "cache", UseRelPaths: true, }, log) - imgStore := local.NewImageStore(dir, true, true, log, metrics, nil, cacheDriver, nil) + imgStore := local.NewImageStore(dir, true, true, log, metrics, nil, cacheDriver, nil, nil) repoName := data @@ -655,7 +655,7 @@ func FuzzFinishBlobUpload(f *testing.F) { Name: "cache", UseRelPaths: true, }, log) - imgStore := local.NewImageStore(dir, true, true, log, metrics, nil, cacheDriver, nil) + imgStore := local.NewImageStore(dir, true, true, log, metrics, nil, cacheDriver, nil, nil) repoName := data @@ -707,7 +707,7 @@ func FuzzFullBlobUpload(f *testing.F) { Name: "cache", UseRelPaths: true, }, *log) - imgStore := local.NewImageStore(dir, true, true, *log, metrics, nil, cacheDriver, nil) + imgStore := local.NewImageStore(dir, true, true, *log, metrics, nil, cacheDriver, nil, nil) ldigest, lblob, err := newRandomBlobForFuzz(data) if err != nil { @@ -750,7 +750,7 @@ func TestStorageCacheErrors(t *testing.T) { GetBlobFn: func(digest godigest.Digest) (string, error) { return getBlobPath, nil }, - }, nil) + }, nil, nil) err := imgStore.InitRepo(originRepo) So(err, ShouldBeNil) @@ -780,7 +780,7 @@ func FuzzDedupeBlob(f *testing.F) { Name: "cache", UseRelPaths: true, }, *log) - imgStore := local.NewImageStore(dir, true, true, *log, metrics, nil, cacheDriver, nil) + imgStore := local.NewImageStore(dir, true, true, *log, metrics, nil, cacheDriver, nil, nil) blobDigest := godigest.FromString(data) @@ -821,7 +821,7 @@ func FuzzDeleteBlobUpload(f *testing.F) { Name: "cache", UseRelPaths: true, }, *log) - imgStore := local.NewImageStore(dir, true, true, *log, metrics, nil, cacheDriver, nil) + imgStore := local.NewImageStore(dir, true, true, *log, metrics, nil, cacheDriver, nil, nil) uuid, err := imgStore.NewBlobUpload(repoName) if err != nil { @@ -853,7 +853,7 @@ func FuzzBlobPath(f *testing.F) { Name: "cache", UseRelPaths: true, }, *log) - imgStore := local.NewImageStore(dir, true, true, *log, metrics, nil, cacheDriver, nil) + imgStore := local.NewImageStore(dir, true, true, *log, metrics, nil, cacheDriver, nil, nil) digest := godigest.FromString(data) _ = imgStore.BlobPath(repoName, digest) @@ -874,7 +874,7 @@ func FuzzCheckBlob(f *testing.F) { Name: "cache", UseRelPaths: true, }, *log) - imgStore := local.NewImageStore(dir, true, true, *log, metrics, nil, cacheDriver, nil) + imgStore := local.NewImageStore(dir, true, true, *log, metrics, nil, cacheDriver, nil, nil) digest := godigest.FromString(data) _, _, err := imgStore.FullBlobUpload(repoName, bytes.NewReader([]byte(data)), digest) @@ -907,7 +907,7 @@ func FuzzGetBlob(f *testing.F) { Name: "cache", UseRelPaths: true, }, *log) - imgStore := local.NewImageStore(dir, true, true, *log, metrics, nil, cacheDriver, nil) + imgStore := local.NewImageStore(dir, true, true, *log, metrics, nil, cacheDriver, nil, nil) digest := godigest.FromString(data) _, _, err := imgStore.FullBlobUpload(repoName, bytes.NewReader([]byte(data)), digest) @@ -948,7 +948,7 @@ func FuzzDeleteBlob(f *testing.F) { Name: "cache", UseRelPaths: true, }, *log) - imgStore := local.NewImageStore(dir, true, true, *log, metrics, nil, cacheDriver, nil) + imgStore := local.NewImageStore(dir, true, true, *log, metrics, nil, cacheDriver, nil, nil) digest := godigest.FromString(data) _, _, err := imgStore.FullBlobUpload(repoName, bytes.NewReader([]byte(data)), digest) @@ -985,7 +985,7 @@ func FuzzGetIndexContent(f *testing.F) { Name: "cache", UseRelPaths: true, }, *log) - imgStore := local.NewImageStore(dir, true, true, *log, metrics, nil, cacheDriver, nil) + imgStore := local.NewImageStore(dir, true, true, *log, metrics, nil, cacheDriver, nil, nil) digest := godigest.FromString(data) _, _, err := imgStore.FullBlobUpload(repoName, bytes.NewReader([]byte(data)), digest) @@ -1022,7 +1022,7 @@ func FuzzGetBlobContent(f *testing.F) { Name: "cache", UseRelPaths: true, }, *log) - imgStore := local.NewImageStore(dir, true, true, *log, metrics, nil, cacheDriver, nil) + imgStore := local.NewImageStore(dir, true, true, *log, metrics, nil, cacheDriver, nil, nil) digest := godigest.FromString(data) _, _, err := imgStore.FullBlobUpload(repoName, bytes.NewReader([]byte(data)), digest) @@ -1060,7 +1060,7 @@ func FuzzRunGCRepo(f *testing.F) { Name: "cache", UseRelPaths: true, }, log) - imgStore := local.NewImageStore(dir, true, true, log, metrics, nil, cacheDriver, nil) + imgStore := local.NewImageStore(dir, true, true, log, metrics, nil, cacheDriver, nil, nil) gc := gc.NewGarbageCollect(imgStore, mocks.MetaDBMock{}, gc.Options{ Delay: storageConstants.DefaultGCDelay, @@ -1104,9 +1104,9 @@ func TestDedupeLinks(t *testing.T) { var imgStore storageTypes.ImageStore if testCase.dedupe { - imgStore = local.NewImageStore(dir, testCase.dedupe, true, log, metrics, nil, cacheDriver, nil) + imgStore = local.NewImageStore(dir, testCase.dedupe, true, log, metrics, nil, cacheDriver, nil, nil) } else { - imgStore = local.NewImageStore(dir, testCase.dedupe, true, log, metrics, nil, nil, nil) + imgStore = local.NewImageStore(dir, testCase.dedupe, true, log, metrics, nil, nil, nil, nil) } // run on empty image store @@ -1282,7 +1282,7 @@ func TestDedupeLinks(t *testing.T) { Convey("test RunDedupeForDigest directly, trigger stat error on original blob", func() { // rebuild with dedupe true - imgStore := local.NewImageStore(dir, true, true, log, metrics, nil, cacheDriver, nil) + imgStore := local.NewImageStore(dir, true, true, log, metrics, nil, cacheDriver, nil, nil) duplicateBlobs := []string{ path.Join(dir, "dedupe1", "blobs", "sha256", blobDigest1), @@ -1303,7 +1303,7 @@ func TestDedupeLinks(t *testing.T) { defer taskScheduler.Shutdown() // rebuild with dedupe true - imgStore := local.NewImageStore(dir, true, true, log, metrics, nil, cacheDriver, nil) + imgStore := local.NewImageStore(dir, true, true, log, metrics, nil, cacheDriver, nil, nil) imgStore.RunDedupeBlobs(time.Duration(0), taskScheduler) @@ -1317,7 +1317,7 @@ func TestDedupeLinks(t *testing.T) { defer taskScheduler.Shutdown() // rebuild with dedupe true - imgStore := local.NewImageStore(dir, true, true, log, metrics, nil, cacheDriver, nil) + imgStore := local.NewImageStore(dir, true, true, log, metrics, nil, cacheDriver, nil, nil) imgStore.RunDedupeBlobs(time.Duration(0), taskScheduler) // wait until rebuild finishes @@ -1337,8 +1337,7 @@ func TestDedupeLinks(t *testing.T) { taskScheduler := runAndGetScheduler() defer taskScheduler.Shutdown() - imgStore := local.NewImageStore(dir, true, true, log, metrics, nil, nil, nil) - + imgStore := local.NewImageStore(dir, true, true, log, metrics, nil, nil, nil, nil) // rebuild with dedupe true imgStore.RunDedupeBlobs(time.Duration(0), taskScheduler) // wait until rebuild finishes @@ -1367,8 +1366,7 @@ func TestDedupeLinks(t *testing.T) { PutBlobFn: func(digest godigest.Digest, path string) error { return errCache }, - }, nil) - // rebuild with dedupe true, should have samefile blobs + }, nil, nil) // rebuild with dedupe true, should have samefile blobs imgStore.RunDedupeBlobs(time.Duration(0), taskScheduler) // wait until rebuild finishes @@ -1400,7 +1398,7 @@ func TestDedupeLinks(t *testing.T) { return nil }, - }, nil) + }, nil, nil) // rebuild with dedupe true, should have samefile blobs imgStore.RunDedupeBlobs(time.Duration(0), taskScheduler) // wait until rebuild finishes @@ -1495,7 +1493,7 @@ func TestDedupe(t *testing.T) { UseRelPaths: true, }, log) - il := local.NewImageStore(dir, true, true, log, metrics, nil, cacheDriver, nil) + il := local.NewImageStore(dir, true, true, log, metrics, nil, cacheDriver, nil, nil) So(il.DedupeBlob("", "", "", ""), ShouldNotBeNil) }) @@ -1516,7 +1514,7 @@ func TestNegativeCases(t *testing.T) { }, log) So(local.NewImageStore(dir, true, - true, log, metrics, nil, cacheDriver, nil), ShouldNotBeNil) + true, log, metrics, nil, cacheDriver, nil, nil), ShouldNotBeNil) if os.Geteuid() != 0 { cacheDriver, _ := storage.Create("boltdb", cache.BoltDBDriverParameters{ @@ -1524,7 +1522,7 @@ func TestNegativeCases(t *testing.T) { Name: "cache", UseRelPaths: true, }, log) - So(local.NewImageStore("/deadBEEF", true, true, log, metrics, nil, cacheDriver, nil), ShouldBeNil) + So(local.NewImageStore("/deadBEEF", true, true, log, metrics, nil, cacheDriver, nil, nil), ShouldBeNil) } }) @@ -1539,7 +1537,7 @@ func TestNegativeCases(t *testing.T) { UseRelPaths: true, }, log) - imgStore := local.NewImageStore(dir, true, true, log, metrics, nil, cacheDriver, nil) + imgStore := local.NewImageStore(dir, true, true, log, metrics, nil, cacheDriver, nil, nil) err := os.Chmod(dir, 0o000) // remove all perms if err != nil { @@ -1589,7 +1587,7 @@ func TestNegativeCases(t *testing.T) { UseRelPaths: true, }, log) - imgStore := local.NewImageStore(dir, true, true, log, metrics, nil, cacheDriver, nil) + imgStore := local.NewImageStore(dir, true, true, log, metrics, nil, cacheDriver, nil, nil) So(imgStore, ShouldNotBeNil) So(imgStore.InitRepo("test"), ShouldBeNil) @@ -1703,7 +1701,7 @@ func TestNegativeCases(t *testing.T) { UseRelPaths: true, }, log) - imgStore := local.NewImageStore(dir, true, true, log, metrics, nil, cacheDriver, nil) + imgStore := local.NewImageStore(dir, true, true, log, metrics, nil, cacheDriver, nil, nil) So(imgStore, ShouldNotBeNil) So(imgStore.InitRepo("test"), ShouldBeNil) @@ -1730,7 +1728,7 @@ func TestNegativeCases(t *testing.T) { UseRelPaths: true, }, log) - imgStore := local.NewImageStore(dir, true, true, log, metrics, nil, cacheDriver, nil) + imgStore := local.NewImageStore(dir, true, true, log, metrics, nil, cacheDriver, nil, nil) So(imgStore, ShouldNotBeNil) So(imgStore.InitRepo("test"), ShouldBeNil) @@ -1778,7 +1776,7 @@ func TestNegativeCases(t *testing.T) { UseRelPaths: true, }, log) - imgStore := local.NewImageStore(dir, true, true, log, metrics, nil, cacheDriver, nil) + imgStore := local.NewImageStore(dir, true, true, log, metrics, nil, cacheDriver, nil, nil) So(imgStore, ShouldNotBeNil) So(imgStore.InitRepo("test"), ShouldBeNil) @@ -1956,7 +1954,7 @@ func TestInjectWriteFile(t *testing.T) { UseRelPaths: true, }, log) - imgStore := local.NewImageStore(dir, true, false, log, metrics, nil, cacheDriver, nil) + imgStore := local.NewImageStore(dir, true, false, log, metrics, nil, cacheDriver, nil, nil) Convey("Failure path not reached", func() { err := imgStore.InitRepo("repo1") @@ -1987,7 +1985,7 @@ func TestGarbageCollectForImageStore(t *testing.T) { UseRelPaths: true, }, log) - imgStore := local.NewImageStore(dir, true, true, log, metrics, nil, cacheDriver, nil) + imgStore := local.NewImageStore(dir, true, true, log, metrics, nil, cacheDriver, nil, nil) repoName := "gc-all-repos-short" //nolint:goconst // test data gc := gc.NewGarbageCollect(imgStore, mocks.MetaDBMock{}, gc.Options{ @@ -2035,7 +2033,7 @@ func TestGarbageCollectForImageStore(t *testing.T) { UseRelPaths: true, }, log) - imgStore := local.NewImageStore(dir, true, true, log, metrics, nil, cacheDriver, nil) + imgStore := local.NewImageStore(dir, true, true, log, metrics, nil, cacheDriver, nil, nil) repoName := "gc-all-repos-short" gc := gc.NewGarbageCollect(imgStore, mocks.MetaDBMock{}, gc.Options{ @@ -2073,7 +2071,7 @@ func TestGarbageCollectForImageStore(t *testing.T) { Name: "cache", UseRelPaths: true, }, log) - imgStore := local.NewImageStore(dir, true, true, log, metrics, nil, cacheDriver, nil) + imgStore := local.NewImageStore(dir, true, true, log, metrics, nil, cacheDriver, nil, nil) repoName := "gc-sig" gc := gc.NewGarbageCollect(imgStore, mocks.MetaDBMock{}, gc.Options{ @@ -2151,7 +2149,7 @@ func TestGarbageCollectForImageStore(t *testing.T) { UseRelPaths: true, }, log) - imgStore := local.NewImageStore(dir, true, true, log, metrics, nil, cacheDriver, nil) + imgStore := local.NewImageStore(dir, true, true, log, metrics, nil, cacheDriver, nil, nil) repoName := "gc-all-repos-short" gc := gc.NewGarbageCollect(imgStore, mocks.MetaDBMock{}, gc.Options{ @@ -2226,7 +2224,7 @@ func TestGarbageCollectImageUnknownManifest(t *testing.T) { UseRelPaths: true, }, log) - imgStore := local.NewImageStore(dir, true, true, log, metrics, nil, cacheDriver, nil) + imgStore := local.NewImageStore(dir, true, true, log, metrics, nil, cacheDriver, nil, nil) storeController := storage.StoreController{ DefaultStore: imgStore, @@ -2409,7 +2407,7 @@ func TestGarbageCollectErrors(t *testing.T) { UseRelPaths: true, }, log) - imgStore := local.NewImageStore(dir, true, true, log, metrics, nil, cacheDriver, nil) + imgStore := local.NewImageStore(dir, true, true, log, metrics, nil, cacheDriver, nil, nil) repoName := "gc-index" gc := gc.NewGarbageCollect(imgStore, mocks.MetaDBMock{}, gc.Options{ @@ -2656,7 +2654,7 @@ func TestInitRepo(t *testing.T) { UseRelPaths: true, }, log) - imgStore := local.NewImageStore(dir, true, true, log, metrics, nil, cacheDriver, nil) + imgStore := local.NewImageStore(dir, true, true, log, metrics, nil, cacheDriver, nil, nil) err := os.Mkdir(path.Join(dir, "test-dir"), 0o000) So(err, ShouldBeNil) @@ -2678,7 +2676,7 @@ func TestValidateRepo(t *testing.T) { UseRelPaths: true, }, log) - imgStore := local.NewImageStore(dir, true, true, log, metrics, nil, cacheDriver, nil) + imgStore := local.NewImageStore(dir, true, true, log, metrics, nil, cacheDriver, nil, nil) err := os.Mkdir(path.Join(dir, "test-dir"), 0o000) So(err, ShouldBeNil) @@ -2698,7 +2696,7 @@ func TestValidateRepo(t *testing.T) { UseRelPaths: true, }, log) - imgStore := local.NewImageStore(dir, true, true, log, metrics, nil, cacheDriver, nil) + imgStore := local.NewImageStore(dir, true, true, log, metrics, nil, cacheDriver, nil, nil) _, err := imgStore.ValidateRepo(".") So(err, ShouldNotBeNil) @@ -2743,7 +2741,7 @@ func TestGetRepositories(t *testing.T) { UseRelPaths: true, }, log) - imgStore := local.NewImageStore(dir, true, true, log, metrics, nil, cacheDriver, nil) + imgStore := local.NewImageStore(dir, true, true, log, metrics, nil, cacheDriver, nil, nil) // Create valid directory with permissions err := os.Mkdir(path.Join(dir, "test-dir"), 0o755) //nolint: gosec @@ -2838,7 +2836,7 @@ func TestGetRepositories(t *testing.T) { UseRelPaths: true, }, log) - imgStore := local.NewImageStore(dir, true, true, log, metrics, nil, cacheDriver, nil) + imgStore := local.NewImageStore(dir, true, true, log, metrics, nil, cacheDriver, nil, nil) // Root dir does not contain repos repos, err := imgStore.GetRepositories() @@ -2885,7 +2883,7 @@ func TestGetRepositories(t *testing.T) { }, log) imgStore := local.NewImageStore(rootDir, - true, true, log, metrics, nil, cacheDriver, nil, + true, true, log, metrics, nil, cacheDriver, nil, nil, ) // Root dir does not contain repos @@ -2928,7 +2926,7 @@ func TestGetNextRepository(t *testing.T) { UseRelPaths: true, }, log) - imgStore := local.NewImageStore(dir, true, true, log, metrics, nil, cacheDriver, nil) + imgStore := local.NewImageStore(dir, true, true, log, metrics, nil, cacheDriver, nil, nil) firstRepoName := "repo1" secondRepoName := "repo2" @@ -2981,7 +2979,7 @@ func TestPutBlobChunkStreamed(t *testing.T) { UseRelPaths: true, }, log) - imgStore := local.NewImageStore(dir, true, true, log, metrics, nil, cacheDriver, nil) + imgStore := local.NewImageStore(dir, true, true, log, metrics, nil, cacheDriver, nil, nil) uuid, err := imgStore.NewBlobUpload("test") So(err, ShouldBeNil) @@ -3011,7 +3009,7 @@ func TestPullRange(t *testing.T) { UseRelPaths: true, }, log) - imgStore := local.NewImageStore(dir, true, true, log, metrics, nil, cacheDriver, nil) + imgStore := local.NewImageStore(dir, true, true, log, metrics, nil, cacheDriver, nil, nil) repoName := "pull-range" upload, err := imgStore.NewBlobUpload(repoName) @@ -3053,7 +3051,7 @@ func TestStatIndex(t *testing.T) { dir := t.TempDir() log := zlog.Logger{Logger: zerolog.New(os.Stdout)} metrics := monitoring.NewMetricsServer(false, log) - imgStore := local.NewImageStore(dir, true, true, log, metrics, nil, nil, nil) + imgStore := local.NewImageStore(dir, true, true, log, metrics, nil, nil, nil, nil) err := WriteImageToFileSystem(CreateRandomImage(), "repo", "tag", storage.StoreController{DefaultStore: imgStore}) @@ -3077,7 +3075,7 @@ func TestStorageDriverErr(t *testing.T) { UseRelPaths: true, }, log) - imgStore := local.NewImageStore(dir, true, true, log, metrics, nil, cacheDriver, nil) + imgStore := local.NewImageStore(dir, true, true, log, metrics, nil, cacheDriver, nil, nil) Convey("Init repo", t, func() { err := imgStore.InitRepo(repoName) diff --git a/pkg/storage/s3/s3.go b/pkg/storage/s3/s3.go index 685e2a3a..20188aec 100644 --- a/pkg/storage/s3/s3.go +++ b/pkg/storage/s3/s3.go @@ -7,6 +7,7 @@ import ( _ "github.com/distribution/distribution/v3/registry/storage/driver/s3-aws" "zotregistry.dev/zot/pkg/compat" + "zotregistry.dev/zot/pkg/extensions/events" "zotregistry.dev/zot/pkg/extensions/monitoring" zlog "zotregistry.dev/zot/pkg/log" common "zotregistry.dev/zot/pkg/storage/common" @@ -19,7 +20,7 @@ import ( // Use the last argument to properly set a cache database, or it will default to boltDB local storage. func NewImageStore(rootDir string, cacheDir string, dedupe, commit bool, log zlog.Logger, metrics monitoring.MetricServer, linter common.Lint, store driver.StorageDriver, - cacheDriver storageTypes.Cache, compat []compat.MediaCompatibility, + cacheDriver storageTypes.Cache, compat []compat.MediaCompatibility, recorder events.Recorder, ) storageTypes.ImageStore { return imagestore.NewImageStore( rootDir, @@ -32,5 +33,6 @@ func NewImageStore(rootDir string, cacheDir string, dedupe, commit bool, log zlo New(store), cacheDriver, compat, + recorder, ) } diff --git a/pkg/storage/s3/s3_test.go b/pkg/storage/s3/s3_test.go index b602349f..185b7ee9 100644 --- a/pkg/storage/s3/s3_test.go +++ b/pkg/storage/s3/s3_test.go @@ -75,7 +75,7 @@ func createMockStorage(rootDir string, cacheDir string, dedupe bool, store drive }, log) } - il := s3.NewImageStore(rootDir, cacheDir, dedupe, false, log, metrics, nil, store, cacheDriver, nil) + il := s3.NewImageStore(rootDir, cacheDir, dedupe, false, log, metrics, nil, store, cacheDriver, nil, nil) return il } @@ -86,7 +86,7 @@ func createMockStorageWithMockCache(rootDir string, dedupe bool, store driver.St log := log.Logger{Logger: zerolog.New(os.Stdout)} metrics := monitoring.NewMetricsServer(false, log) - il := s3.NewImageStore(rootDir, "", dedupe, false, log, metrics, nil, store, cacheDriver, nil) + il := s3.NewImageStore(rootDir, "", dedupe, false, log, metrics, nil, store, cacheDriver, nil, nil) return il } @@ -135,10 +135,11 @@ func createObjectsStore(rootDir string, cacheDir string, dedupe bool) ( var cacheDriver storageTypes.Cache - var err error - // from pkg/cli/server/root.go/applyDefaultValues, s3 magic s3CacheDBPath := path.Join(cacheDir, storageConstants.BoltdbName+storageConstants.DBExtensionName) + + var err error + if _, err = os.Stat(s3CacheDBPath); dedupe || (!dedupe && err == nil) { cacheDriver, _ = storage.Create("boltdb", cache.BoltDBDriverParameters{ RootDir: cacheDir, @@ -147,7 +148,7 @@ func createObjectsStore(rootDir string, cacheDir string, dedupe bool) ( }, log) } - il := s3.NewImageStore(rootDir, cacheDir, dedupe, false, log, metrics, nil, store, cacheDriver, nil) + il := s3.NewImageStore(rootDir, cacheDir, dedupe, false, log, metrics, nil, store, cacheDriver, nil, nil) return store, il, err } @@ -181,7 +182,7 @@ func createObjectsStoreDynamo(rootDir string, cacheDir string, dedupe bool, tabl panic(err) } - il := s3.NewImageStore(rootDir, cacheDir, dedupe, false, log, metrics, nil, store, cacheDriver, nil) + il := s3.NewImageStore(rootDir, cacheDir, dedupe, false, log, metrics, nil, store, cacheDriver, nil, nil) return store, il, err } diff --git a/pkg/storage/scrub_test.go b/pkg/storage/scrub_test.go index d7e1a895..fbe8d59f 100644 --- a/pkg/storage/scrub_test.go +++ b/pkg/storage/scrub_test.go @@ -50,7 +50,7 @@ func TestLocalCheckAllBlobsIntegrity(t *testing.T) { UseRelPaths: true, }, log) driver := local.New(true) - imgStore := local.NewImageStore(tdir, true, true, log, metrics, nil, cacheDriver, nil) + imgStore := local.NewImageStore(tdir, true, true, log, metrics, nil, cacheDriver, nil, nil) RunCheckAllBlobsIntegrityTests(t, imgStore, driver, log) }) @@ -73,7 +73,7 @@ func TestRedisCheckAllBlobsIntegrity(t *testing.T) { UseRelPaths: false, }, log) driver := local.New(true) - imgStore := local.NewImageStore(tdir, true, true, log, metrics, nil, cacheDriver, nil) + imgStore := local.NewImageStore(tdir, true, true, log, metrics, nil, cacheDriver, nil, nil) RunCheckAllBlobsIntegrityTests(t, imgStore, driver, log) }) diff --git a/pkg/storage/storage.go b/pkg/storage/storage.go index f0694110..5a3f109a 100644 --- a/pkg/storage/storage.go +++ b/pkg/storage/storage.go @@ -13,6 +13,7 @@ import ( zerr "zotregistry.dev/zot/errors" "zotregistry.dev/zot/pkg/api/config" zcommon "zotregistry.dev/zot/pkg/common" + "zotregistry.dev/zot/pkg/extensions/events" "zotregistry.dev/zot/pkg/extensions/monitoring" "zotregistry.dev/zot/pkg/log" common "zotregistry.dev/zot/pkg/storage/common" @@ -23,7 +24,7 @@ import ( ) func New(config *config.Config, linter common.Lint, metrics monitoring.MetricServer, - log log.Logger, + log log.Logger, recorder events.Recorder, ) (StoreController, error) { storeController := StoreController{} @@ -58,7 +59,7 @@ func New(config *config.Config, linter common.Lint, metrics monitoring.MetricSer //nolint:typecheck,contextcheck rootDir := config.Storage.RootDirectory defaultStore = local.NewImageStore(rootDir, - config.Storage.Dedupe, config.Storage.Commit, log, metrics, linter, cacheDriver, config.HTTP.Compat, + config.Storage.Dedupe, config.Storage.Commit, log, metrics, linter, cacheDriver, config.HTTP.Compat, recorder, ) } else { storeName := fmt.Sprintf("%v", config.Storage.StorageDriver["name"]) @@ -92,7 +93,7 @@ func New(config *config.Config, linter common.Lint, metrics monitoring.MetricSer // false positive lint - linter does not implement Lint method //nolint: typecheck,contextcheck defaultStore = s3.NewImageStore(rootDir, config.Storage.RootDirectory, - config.Storage.Dedupe, config.Storage.Commit, log, metrics, linter, store, cacheDriver, config.HTTP.Compat) + config.Storage.Dedupe, config.Storage.Commit, log, metrics, linter, store, cacheDriver, config.HTTP.Compat, recorder) } storeController.DefaultStore = defaultStore @@ -102,7 +103,7 @@ func New(config *config.Config, linter common.Lint, metrics monitoring.MetricSer subPaths := config.Storage.SubPaths //nolint: contextcheck - subImageStore, err := getSubStore(config, subPaths, linter, metrics, log) + subImageStore, err := getSubStore(config, subPaths, linter, metrics, log, recorder) if err != nil { log.Error().Err(err).Str("component", "controller").Msg("failed to get sub image store") @@ -117,7 +118,7 @@ func New(config *config.Config, linter common.Lint, metrics monitoring.MetricSer } func getSubStore(cfg *config.Config, subPaths map[string]config.StorageConfig, - linter common.Lint, metrics monitoring.MetricServer, log log.Logger, + linter common.Lint, metrics monitoring.MetricServer, log log.Logger, recorder events.Recorder, ) (map[string]storageTypes.ImageStore, error) { imgStoreMap := make(map[string]storageTypes.ImageStore, 0) @@ -170,7 +171,7 @@ func getSubStore(cfg *config.Config, subPaths map[string]config.StorageConfig, rootDir := storageConfig.RootDirectory imgStoreMap[storageConfig.RootDirectory] = local.NewImageStore(rootDir, - storageConfig.Dedupe, storageConfig.Commit, log, metrics, linter, cacheDriver, cfg.HTTP.Compat, + storageConfig.Dedupe, storageConfig.Commit, log, metrics, linter, cacheDriver, cfg.HTTP.Compat, recorder, ) subImageStore[route] = imgStoreMap[storageConfig.RootDirectory] @@ -210,7 +211,7 @@ func getSubStore(cfg *config.Config, subPaths map[string]config.StorageConfig, // false positive lint - linter does not implement Lint method //nolint: typecheck subImageStore[route] = s3.NewImageStore(rootDir, storageConfig.RootDirectory, - storageConfig.Dedupe, storageConfig.Commit, log, metrics, linter, store, cacheDriver, cfg.HTTP.Compat, + storageConfig.Dedupe, storageConfig.Commit, log, metrics, linter, store, cacheDriver, cfg.HTTP.Compat, recorder, ) } } diff --git a/pkg/storage/storage_test.go b/pkg/storage/storage_test.go index effd5070..caff8017 100644 --- a/pkg/storage/storage_test.go +++ b/pkg/storage/storage_test.go @@ -109,7 +109,7 @@ func createObjectsStore(options createObjectStoreOpts) ( storeDriver := local.New(true) imgStore := imagestore.NewImageStore(options.rootDir, options.cacheDir, true, - true, log, metrics, nil, storeDriver, cacheDriver, nil) + true, log, metrics, nil, storeDriver, cacheDriver, nil, nil) return storeDriver, imgStore, cacheDriver, nil } @@ -143,7 +143,7 @@ func createObjectsStore(options createObjectStoreOpts) ( } imgStore := s3.NewImageStore(options.rootDir, options.cacheDir, true, false, log, - metrics, nil, s3Driver, cacheDriver, nil) + metrics, nil, s3Driver, cacheDriver, nil, nil) return s3.New(s3Driver), imgStore, cacheDriver, err } @@ -183,7 +183,7 @@ func TestStorageNew(t *testing.T) { conf.Storage.RootDirectory = "dir" conf.Storage.StorageDriver = map[string]interface{}{} - _, err := storage.New(conf, nil, nil, zlog.NewLogger("debug", "")) + _, err := storage.New(conf, nil, nil, zlog.NewLogger("debug", ""), nil) So(err, ShouldNotBeNil) }) } @@ -1051,7 +1051,7 @@ func TestMandatoryAnnotations(t *testing.T) { LintFn: func(repo string, manifestDigest godigest.Digest, imageStore storageTypes.ImageStore) (bool, error) { return false, nil }, - }, store, cacheDriver, nil) + }, store, cacheDriver, nil, nil) defer cleanupStorage(store, testDir) } else { @@ -1063,7 +1063,7 @@ func TestMandatoryAnnotations(t *testing.T) { LintFn: func(repo string, manifestDigest godigest.Digest, imageStore storageTypes.ImageStore) (bool, error) { return false, nil }, - }, store, cacheDriver, nil) + }, store, cacheDriver, nil, nil) } Convey("Setup manifest", t, func() { @@ -1116,7 +1116,7 @@ func TestMandatoryAnnotations(t *testing.T) { //nolint: err113 return false, errors.New("linter error") }, - }, store, nil, nil) + }, store, nil, nil, nil) } else { var cacheDriver storageTypes.Cache store, _, cacheDriver, _ = createObjectsStore(opts) @@ -1127,7 +1127,7 @@ func TestMandatoryAnnotations(t *testing.T) { //nolint: err113 return false, errors.New("linter error") }, - }, store, cacheDriver, nil) + }, store, cacheDriver, nil, nil) } _, _, err = imgStore.PutImageManifest("test", "1.0.0", ispec.MediaTypeImageManifest, manifestBuf) @@ -1151,7 +1151,7 @@ func TestStorageSubpaths(t *testing.T) { }, } - _, err := storage.New(config, nil, nil, zlog.NewLogger("debug", "")) + _, err := storage.New(config, nil, nil, zlog.NewLogger("debug", ""), nil) So(err, ShouldBeNil) }) @@ -1176,7 +1176,7 @@ func TestStorageSubpaths(t *testing.T) { err := os.WriteFile(dbPath, []byte(""), 0o000) So(err, ShouldBeNil) - _, err = storage.New(config, nil, nil, zlog.NewLogger("debug", "")) + _, err = storage.New(config, nil, nil, zlog.NewLogger("debug", ""), nil) So(err, ShouldNotBeNil) err = os.Chmod(dbPath, 0o600) @@ -1200,7 +1200,7 @@ func TestStorageSubpaths(t *testing.T) { }, } - _, err := storage.New(config, nil, nil, zlog.NewLogger("debug", "")) + _, err := storage.New(config, nil, nil, zlog.NewLogger("debug", ""), nil) So(err, ShouldNotBeNil) }) } diff --git a/pkg/test/oci-utils/oci_layout_test.go b/pkg/test/oci-utils/oci_layout_test.go index 293822ab..d68f4889 100644 --- a/pkg/test/oci-utils/oci_layout_test.go +++ b/pkg/test/oci-utils/oci_layout_test.go @@ -433,7 +433,7 @@ func TestExtractImageDetails(t *testing.T) { dir := t.TempDir() testLogger := log.NewLogger("debug", "") imageStore := local.NewImageStore(dir, false, false, - testLogger, monitoring.NewMetricsServer(false, testLogger), nil, nil, nil) + testLogger, monitoring.NewMetricsServer(false, testLogger), nil, nil, nil, nil) storeController := storage.StoreController{ DefaultStore: imageStore, @@ -457,7 +457,7 @@ func TestExtractImageDetails(t *testing.T) { dir := t.TempDir() testLogger := log.NewLogger("debug", "") imageStore := local.NewImageStore(dir, false, false, - testLogger, monitoring.NewMetricsServer(false, testLogger), nil, nil, nil) + testLogger, monitoring.NewMetricsServer(false, testLogger), nil, nil, nil, nil) storeController := storage.StoreController{ DefaultStore: imageStore, @@ -477,7 +477,7 @@ func TestExtractImageDetails(t *testing.T) { dir := t.TempDir() testLogger := log.NewLogger("debug", "") imageStore := local.NewImageStore(dir, false, false, - testLogger, monitoring.NewMetricsServer(false, testLogger), nil, nil, nil) + testLogger, monitoring.NewMetricsServer(false, testLogger), nil, nil, nil, nil) storeController := storage.StoreController{ DefaultStore: imageStore, diff --git a/pkg/test/oci-utils/store.go b/pkg/test/oci-utils/store.go index 2bb15e78..6f6c01bd 100644 --- a/pkg/test/oci-utils/store.go +++ b/pkg/test/oci-utils/store.go @@ -19,7 +19,7 @@ func GetDefaultImageStore(rootDir string, log zLog.Logger) stypes.ImageStore { return true, nil }, }, - mocks.CacheMock{}, nil, + mocks.CacheMock{}, nil, nil, ) } diff --git a/scripts/check_logs.sh b/scripts/check_logs.sh index 56b5fad7..b3bb826c 100755 --- a/scripts/check_logs.sh +++ b/scripts/check_logs.sh @@ -1,5 +1,10 @@ #!/bin/bash +GREP_BIN=grep +if [ ! -z "$GREP_BIN_PATH" ]; then + GREP_BIN=$GREP_BIN_PATH +fi + # Colors for terminal if test -t 1; then # check if it supports colors @@ -20,25 +25,25 @@ function lintLogContainingUpperCase { word_char="[\.-0-9a-z ]" capital_word="([a-z]*[A-Z][a-zA-Z]*)" - grep --with-filename -n -P "Msg[f]?\(\"(($word_char|$exception)*)(?!$exception)($capital_word)($exclude_linter)" $1 + $GREP_BIN --with-filename -n -P "Msg[f]?\(\"(($word_char|$exception)*)(?!$exception)($capital_word)($exclude_linter)" $1 } # lintLogStartingWithUpperCase searched for log messages that start with an upper case letter function lintLogStartingWithUpperCase { - grep --with-filename -n "Msg[f]\?(\"[A-Z]" $1 | grep -v -P "Msg[f]?\(\"$exception($exclude_linter)" + $GREP_BIN --with-filename -n "Msg[f]\?(\"[A-Z]" $1 | $GREP_BIN -v -P "Msg[f]?\(\"$exception($exclude_linter)" } # lintLogStartingWithComponent searches for log messages that starts with a component "component:" function lintLogStartingWithComponent { - # We'll check for different functions that can generate errors or logs. If they start with + # We'll check for different functions that can generate errors or logs. If they start with # a number words followed by ":", it's considered as starting with a component. # Examples: '.Msgf("component:")', '.Errorf("com ponent:")', '.Msg("com-ponent:")' - grep --with-filename -n -E "(Errorf|errors.New|Msg[f]?)\(\"[a-zA-Z-]+( [a-zA-Z-]+){0,1}:($exclude_linter)" $1 + $GREP_BIN --with-filename -n -E "(Errorf|errors.New|Msg[f]?)\(\"[a-zA-Z-]+( [a-zA-Z-]+){0,1}:($exclude_linter)" $1 } # lintErrorLogsBeggining searches for log messages that don't start with "failed to" function lintErrorLogsBeggining { - grep --with-filename -n -P "Error\(\)(?:.*)\n?.(?:.*)Msg[f]?\(\"(?!(failed to|failed due|invalid|unexpected|unsupported))($exclude_linter)" $1 + $GREP_BIN --with-filename -n -P "Error\(\)(?:.*)\n?.(?:.*)Msg[f]?\(\"(?!(failed to|failed due|invalid|unexpected|unsupported))($exclude_linter)" $1 } function printLintError { @@ -55,7 +60,7 @@ function printLintError { fi } -files=$(find . -name '*.go' | grep -v '_test.go') +files=$(find . -name '*.go' | $GREP_BIN -v '_test.go') found_linting_error=false @@ -67,7 +72,7 @@ do while IFS= read -r line; do printLintError "Log message should not start with a CAPITAL letter" "$(echo $line | tr -s [:space:])" done <<< "$lintOutput" - fi + fi lintOutput=$(lintLogStartingWithComponent "$file") if [ $? -eq 0 ]; then diff --git a/test/blackbox/ci.sh b/test/blackbox/ci.sh index ae97a81c..20efcb08 100755 --- a/test/blackbox/ci.sh +++ b/test/blackbox/ci.sh @@ -9,7 +9,8 @@ PATH=$PATH:${SCRIPTPATH}/../../hack/tools/bin tests=("pushpull" "pushpull_authn" "delete_images" "referrers" "metadata" "anonymous_policy" "annotations" "detect_manifest_collision" "cve" "sync" "sync_docker" "sync_replica_cluster" - "scrub" "garbage_collect" "metrics" "metrics_minimal" "multiarch_index" "docker_compat" "redis_local") + "scrub" "garbage_collect" "metrics" "metrics_minimal" "multiarch_index" "docker_compat" "redis_local" + "events_nats" "events_http" "events_nats_lint_failure" "events_http_lint_failure" "events_sink_failure" "events_config_decoding") for test in ${tests[*]}; do ${BATS} ${BATS_FLAGS} ${SCRIPTPATH}/${test}.bats > ${test}.log & pids+=($!) diff --git a/test/blackbox/events_config_decoding.bats b/test/blackbox/events_config_decoding.bats new file mode 100644 index 00000000..2f609457 --- /dev/null +++ b/test/blackbox/events_config_decoding.bats @@ -0,0 +1,122 @@ +# Note: Intended to be run as "make run-blackbox-tests" or "make run-blackbox-ci" +# Makefile target installs & checks all necessary tooling +# Extra tools that are not covered in Makefile target needs to be added in verify_prerequisites() + +load helpers_zot +load helpers_events + +function verify_prerequisites() { + if [ ! $(command -v curl) ]; then + echo "you need to install curl as a prerequisite to running the tests" >&3 + return 1 + fi + + if [ ! $(command -v jq) ]; then + echo "you need to install jq as a prerequisite to running the tests" >&3 + return 1 + fi + + if [ ! $(command -v docker) ]; then + echo "you need to install docker as a prerequisite to running the tests" >&3 + return 1 + fi +} + +function setup_file() { + # verify prerequisites are available + if ! $(verify_prerequisites); then + exit 1 + fi +} + +@test "startup error when invalid sink is specified" { + # Setup zot server + local zot_root_dir=${BATS_FILE_TMPDIR}/zot + local zot_config_file=${BATS_FILE_TMPDIR}/zot_config.json + mkdir -p ${zot_root_dir} + zot_port=$(get_free_port) + cat > ${zot_config_file}< ${zot_config_file}<&3 + return 1 + fi + + if [ ! $(command -v jq) ]; then + echo "you need to install jq as a prerequisite to running the tests" >&3 + return 1 + fi + + if [ ! $(command -v docker) ]; then + echo "you need to install docker as a prerequisite to running the tests" >&3 + return 1 + fi +} + +function setup_file() { + # verify prerequisites are available + if ! $(verify_prerequisites); then + exit 1 + fi + + # Setup http server + http_server_port=$(get_free_port) + http_event_dir="${BATS_FILE_TMPDIR}/http_events" + http_server_start http_receiver "${http_server_port}" "${http_event_dir}" + echo ${http_server_port} > ${BATS_FILE_TMPDIR}/http_server.port + wait_for_http_server $http_server_port + + skopeo --insecure-policy copy --format=oci docker://ghcr.io/project-zot/golang:1.20 oci:${TEST_DATA_DIR}/golang:1.20 + + # Setup zot server + local zot_root_dir=${BATS_FILE_TMPDIR}/zot + local zot_config_file=${BATS_FILE_TMPDIR}/zot_config.json + local oci_data_dir=${BATS_FILE_TMPDIR}/oci + mkdir -p ${zot_root_dir} + mkdir -p ${oci_data_dir} + zot_port=$(get_free_port) + echo ${zot_port} > ${BATS_FILE_TMPDIR}/zot.port + cat > ${zot_config_file}<&3 + return 1 + fi + + if [ ! $(command -v jq) ]; then + echo "you need to install jq as a prerequisite to running the tests" >&3 + return 1 + fi + + if [ ! $(command -v docker) ]; then + echo "you need to install docker as a prerequisite to running the tests" >&3 + return 1 + fi + + if [ ! $(command -v oras) ]; then + echo "you need to install oras as a prerequisite to running the tests" >&3 + return 1 + fi +} + +function setup_file() { + # verify prerequisites are available + if ! $(verify_prerequisites); then + exit 1 + fi + + # Setup http server + http_server_port=$(get_free_port) + http_event_dir="${BATS_FILE_TMPDIR}/http_events" + http_server_start http_receiver_lint "${http_server_port}" "${http_event_dir}" + echo ${http_server_port} > ${BATS_FILE_TMPDIR}/http_server.port + wait_for_http_server $http_server_port + + skopeo --insecure-policy copy --format=oci docker://ghcr.io/project-zot/golang:1.20 oci:${TEST_DATA_DIR}/golang:1.20 + + # Setup zot server + local zot_root_dir=${BATS_FILE_TMPDIR}/zot + local zot_config_file=${BATS_FILE_TMPDIR}/zot_config.json + local oci_data_dir=${BATS_FILE_TMPDIR}/oci + mkdir -p ${zot_root_dir} + mkdir -p ${oci_data_dir} + zot_port=$(get_free_port) + echo ${zot_port} > ${BATS_FILE_TMPDIR}/zot.port + cat > ${zot_config_file}< config.json + + # Create dummy layer + echo "this is a bogus artifact" > artifact.txt + + # Push using oras with intentionally broken config + type + run oras push --plain-http 127.0.0.1:${zot_port}/test-artifact:v0 \ + --config config.json:application/vnd.oci.image.config.v1+json \ + artifact.txt:text/plain -d -v + + rm -f artifact.txt config.json + + # Check the correct number of events were generated + count=$(find "${output_path}" -type f | wc -l) + [ "$count" -eq 2 ] + + # Validate the event + result=$(jq '.' ${output_path}/2.json) + echo $result + [ $(echo "${result}" | jq -r '.headers["Ce-Type"]') = "zotregistry.image.lint_failed" ] + [ $(echo "${result}" | jq -r '.body.name') = "test-artifact" ] + [ $(echo "${result}" | jq -r '.body.reference') = "v0" ] +} + +@test "http/publish image with annotations" { + http_server_port=$(cat ${BATS_FILE_TMPDIR}/http_server.port) + zot_port=$(cat ${BATS_FILE_TMPDIR}/zot.port) + output_path=${BATS_FILE_TMPDIR}/http_events + + run curl -XGET http://127.0.0.1:${http_server_port}/reset + [ "$status" -eq 0 ] + [ -d "${output_path}" ] && rm -f "${output_path}"/*.json + + # Create dummy config + echo '{}' > config.json + + # Create dummy layer + echo "this is a bogus artifact" > artifact.txt + + # Push using oras with intentionally broken config + type + run oras push --plain-http 127.0.0.1:${zot_port}/test-artifact:v1 \ + --annotation "event-test=true" \ + --config config.json:application/vnd.oci.image.config.v1+json \ + artifact.txt:text/plain -d -v + + rm -f artifact.txt config.json + + # Check the correct number of events were generated + count=$(find "${output_path}" -type f | wc -l) + [ "$count" -eq 1 ] + + # Validate the event + result=$(jq '.' ${output_path}/1.json) + [ $(echo "${result}" | jq -r '.headers["Ce-Type"]') = "zotregistry.image.updated" ] + [ $(echo "${result}" | jq -r '.body.name') = "test-artifact" ] + [ $(echo "${result}" | jq -r '.body.reference') = "v1" ] +} diff --git a/test/blackbox/events_nats.bats b/test/blackbox/events_nats.bats new file mode 100644 index 00000000..cf0417c3 --- /dev/null +++ b/test/blackbox/events_nats.bats @@ -0,0 +1,158 @@ +# Note: Intended to be run as "make run-blackbox-tests" or "make run-blackbox-ci" +# Makefile target installs & checks all necessary tooling +# Extra tools that are not covered in Makefile target needs to be added in verify_prerequisites() + +load helpers_zot +load helpers_events + +function verify_prerequisites() { + if [ ! $(command -v curl) ]; then + echo "you need to install curl as a prerequisite to running the tests" >&3 + return 1 + fi + + if [ ! $(command -v jq) ]; then + echo "you need to install jq as a prerequisite to running the tests" >&3 + return 1 + fi + + if [ ! $(command -v docker) ]; then + echo "you need to install docker as a prerequisite to running the tests" >&3 + return 1 + fi +} + +function setup_file() { + # verify prerequisites are available + if ! $(verify_prerequisites); then + exit 1 + fi + + # Setup nats server + nats_server_port=$(get_free_port) + nats_server_start nats_server_local ${nats_server_port} + echo ${nats_server_port} > ${BATS_FILE_TMPDIR}/nats_server.port + + skopeo --insecure-policy copy --format=oci docker://ghcr.io/project-zot/golang:1.20 oci:${TEST_DATA_DIR}/golang:1.20 + + # Setup zot server + local zot_root_dir=${BATS_FILE_TMPDIR}/zot + local zot_config_file=${BATS_FILE_TMPDIR}/zot_config.json + local oci_data_dir=${BATS_FILE_TMPDIR}/oci + mkdir -p ${zot_root_dir} + mkdir -p ${oci_data_dir} + zot_port=$(get_free_port) + echo ${zot_port} > ${BATS_FILE_TMPDIR}/zot.port + cat > ${zot_config_file}<&3 + return 1 + fi + + if [ ! $(command -v jq) ]; then + echo "you need to install jq as a prerequisite to running the tests" >&3 + return 1 + fi + + if [ ! $(command -v docker) ]; then + echo "you need to install docker as a prerequisite to running the tests" >&3 + return 1 + fi + + if [ ! $(command -v oras) ]; then + echo "you need to install oras as a prerequisite to running the tests" >&3 + return 1 + fi +} + +function setup_file() { + # verify prerequisites are available + if ! $(verify_prerequisites); then + exit 1 + fi + + # Setup nats server + nats_server_port=$(get_free_port) + nats_server_start nats_server_local_lint ${nats_server_port} + echo ${nats_server_port} > ${BATS_FILE_TMPDIR}/nats_server.port + + skopeo --insecure-policy copy --format=oci docker://ghcr.io/project-zot/golang:1.20 oci:${TEST_DATA_DIR}/golang:1.20 + + # Setup zot server + local zot_root_dir=${BATS_FILE_TMPDIR}/zot + local zot_config_file=${BATS_FILE_TMPDIR}/zot_config.json + local oci_data_dir=${BATS_FILE_TMPDIR}/oci + mkdir -p ${zot_root_dir} + mkdir -p ${oci_data_dir} + zot_port=$(get_free_port) + echo ${zot_port} > ${BATS_FILE_TMPDIR}/zot.port + cat > ${zot_config_file}< config.json + + # Create dummy layer + echo "this is a bogus artifact" > artifact.txt + + # Push using oras with intentionally broken config + type + run oras push --plain-http 127.0.0.1:${zot_port}/test-artifact:v0 \ + --config config.json:application/vnd.oci.image.config.v1+json \ + artifact.txt:text/plain -d -v + + rm -f artifact.txt config.json + + # Check the correct number of events were generated + count=$(find "${output_path}" -type f | wc -l) + [ "$count" -eq 2 ] + + # Validate the event + result=$(jq '.Data | @base64d | fromjson' ${output_path}/2.json) + echo $result + [ $(echo "${result}" | jq -r '.type') = "zotregistry.image.lint_failed" ] + [ $(echo "${result}" | jq -r '.data.name') = "test-artifact" ] + [ $(echo "${result}" | jq -r '.data.reference') = "v0" ] +} + +@test "nats/publish image with annotations" { + nats_server_port=$(cat ${BATS_FILE_TMPDIR}/nats_server.port) + zot_port=$(cat ${BATS_FILE_TMPDIR}/zot.port) + output_path=${BATS_FILE_TMPDIR}/events/lint_success + + # Wait for event + run wait_event_on_subject "zot.test" ${nats_server_port} ${output_path} 1 + [ "$status" -eq 0 ] + + # Create dummy config + echo '{}' > config.json + + # Create dummy layer + echo "this is a bogus artifact" > artifact.txt + + # Push using oras with intentionally broken config + type + run oras push --plain-http 127.0.0.1:${zot_port}/test-artifact:v1 \ + --annotation "event-test=true" \ + --config config.json:application/vnd.oci.image.config.v1+json \ + artifact.txt:text/plain -d -v + + rm -f artifact.txt config.json + + # Check the correct number of events were generated + count=$(find "${output_path}" -type f | wc -l) + [ "$count" -eq 1 ] + + # Validate the event + result=$(jq '.Data | @base64d | fromjson' ${output_path}/1.json) + [ $(echo "${result}" | jq -r '.type') = "zotregistry.image.updated" ] + [ $(echo "${result}" | jq -r '.data.name') = "test-artifact" ] + [ $(echo "${result}" | jq -r '.data.reference') = "v1" ] +} diff --git a/test/blackbox/events_sink_failure.bats b/test/blackbox/events_sink_failure.bats new file mode 100644 index 00000000..c0b3b245 --- /dev/null +++ b/test/blackbox/events_sink_failure.bats @@ -0,0 +1,98 @@ +# Note: Intended to be run as "make run-blackbox-tests" or "make run-blackbox-ci" +# Makefile target installs & checks all necessary tooling +# Extra tools that are not covered in Makefile target needs to be added in verify_prerequisites() + +load helpers_zot +load helpers_events + +function verify_prerequisites() { + if [ ! $(command -v curl) ]; then + echo "you need to install curl as a prerequisite to running the tests" >&3 + return 1 + fi + + if [ ! $(command -v jq) ]; then + echo "you need to install jq as a prerequisite to running the tests" >&3 + return 1 + fi + + if [ ! $(command -v docker) ]; then + echo "you need to install docker as a prerequisite to running the tests" >&3 + return 1 + fi +} + +function setup_file() { + # verify prerequisites are available + if ! $(verify_prerequisites); then + exit 1 + fi + + # Setup http server + http_server_port=$(get_free_port) + http_event_dir="${BATS_FILE_TMPDIR}/http_events" + http_server_start http_receiver_failure "${http_server_port}" "${http_event_dir}" + wait_for_http_server $http_server_port + + skopeo --insecure-policy copy --format=oci docker://ghcr.io/project-zot/golang:1.20 oci:${TEST_DATA_DIR}/golang:1.20 + + # Setup zot server + local zot_root_dir=${BATS_FILE_TMPDIR}/zot + local zot_config_file=${BATS_FILE_TMPDIR}/zot_config.json + local oci_data_dir=${BATS_FILE_TMPDIR}/oci + mkdir -p ${zot_root_dir} + mkdir -p ${oci_data_dir} + zot_port=$(get_free_port) + echo ${zot_port} > ${BATS_FILE_TMPDIR}/zot.port + cat > ${zot_config_file}< /dev/null && \ + echo " +import os +import json +from flask import Flask, request, Response + +app = Flask(__name__) +counter = 0 + +USERNAME = \"jane.joe\" +PASSWORD = \"opensesame\" + +def check_auth(auth): + return auth and auth.username == USERNAME and auth.password == PASSWORD + +def authenticate(): + return Response( + \"Unauthorized\", 401, + {\"WWW-Authenticate\": \"Basic realm=\\\"Login Required\\\"\"} + ) + +@app.route(\"/reset\", methods=[\"GET\"]) +def reset_counter(): + global counter + counter = 0 + return \"\", 200 + +@app.route(\"/events\", methods=[\"POST\"]) +def receive_event(): + auth = request.authorization + if not check_auth(auth): + return authenticate + + global counter + counter += 1 + method = request.method + headers = dict(request.headers) + raw_data = request.data.decode(\"utf-8\", errors=\"replace\") + try: + body = json.loads(raw_data) + except Exception: + body = raw_data # fallback to plain text + + event = { + \"method\": method, + \"headers\": headers, + \"body\": body + } + + filename = f\"/data/{counter}.json\" + + with open(filename, \"w\") as f: + json.dump(event, f, indent=2) + + return \"\", 200 + +app.run(host=\"0.0.0.0\", port=8080) + " > app.py && python app.py +' +} + +function http_server_stop() { + local cname="$1" + docker rm -f "${cname}" >/dev/null 2>&1 +} + +function wait_for_http_server() { + local port="$1" + local timeout=10 + local elapsed=0 + + while [ "$elapsed" -lt "$timeout" ]; do + if curl --silent --fail --output /dev/null "http://127.0.0.1:${port}/reset"; then + return 0 + fi + sleep 1 + elapsed=$((elapsed + 1)) + done + + return 1 +} \ No newline at end of file