From 87084f286b67deedb6e16b1a75abca7fa934668e Mon Sep 17 00:00:00 2001 From: Ramkumar Chinchani Date: Wed, 19 Jan 2022 19:54:17 +0000 Subject: [PATCH] storage: improve/fix oci image validation Signed-off-by: Ramkumar Chinchani --- pkg/api/controller_test.go | 255 ++++++++++--------- pkg/cli/client_test.go | 8 +- pkg/cli/cve_cmd_test.go | 8 +- pkg/cli/image_cmd_test.go | 59 +++-- pkg/cli/root_test.go | 2 +- pkg/compliance/v1_0_0/check.go | 130 +++++++--- pkg/compliance/v1_0_0/check_test.go | 2 +- pkg/exporter/api/controller_test.go | 2 +- pkg/extensions/monitoring/monitoring_test.go | 16 +- pkg/extensions/search/common/common_test.go | 2 +- pkg/extensions/search/cve/cve_test.go | 2 +- pkg/extensions/search/digest/digest_test.go | 2 +- pkg/extensions/sync/sync_internal_test.go | 2 +- pkg/extensions/sync/sync_test.go | 75 ++++-- pkg/log/log_test.go | 2 +- pkg/storage/storage_fs.go | 78 ++++-- pkg/storage/storage_fs_test.go | 47 +++- pkg/storage/storage_test.go | 61 ++++- {test => pkg/test}/common.go | 83 +++++- {test => pkg/test}/common_test.go | 14 +- 20 files changed, 586 insertions(+), 264 deletions(-) rename {test => pkg/test}/common.go (61%) rename {test => pkg/test}/common_test.go (82%) diff --git a/pkg/api/controller_test.go b/pkg/api/controller_test.go index d946adb8..29a9a26c 100644 --- a/pkg/api/controller_test.go +++ b/pkg/api/controller_test.go @@ -44,7 +44,6 @@ import ( "zotregistry.io/zot/pkg/api/config" "zotregistry.io/zot/pkg/storage" "zotregistry.io/zot/pkg/test" - . "zotregistry.io/zot/test" ) const ( @@ -101,8 +100,8 @@ func TestNew(t *testing.T) { func TestRunAlreadyRunningServer(t *testing.T) { Convey("Run server on unavailable port", t, func() { - port := GetFreePort() - baseURL := GetBaseURL(port) + port := test.GetFreePort() + baseURL := test.GetBaseURL(port) conf := config.New() conf.HTTP.Port = port @@ -144,7 +143,7 @@ func TestRunAlreadyRunningServer(t *testing.T) { func TestObjectStorageController(t *testing.T) { skipIt(t) Convey("Negative make a new object storage controller", t, func() { - port := GetFreePort() + port := test.GetFreePort() conf := config.New() conf.HTTP.Port = port storageDriverParams := map[string]interface{}{ @@ -162,8 +161,8 @@ func TestObjectStorageController(t *testing.T) { }) Convey("Make a new object storage controller", t, func() { - port := GetFreePort() - baseURL := GetBaseURL(port) + port := test.GetFreePort() + baseURL := test.GetBaseURL(port) conf := config.New() conf.HTTP.Port = port @@ -187,15 +186,15 @@ func TestObjectStorageController(t *testing.T) { go startServer(ctlr) defer stopServer(ctlr) - WaitTillServerReady(baseURL) + test.WaitTillServerReady(baseURL) }) } func TestObjectStorageControllerSubPaths(t *testing.T) { skipIt(t) Convey("Make a new object storage controller", t, func() { - port := GetFreePort() - baseURL := GetBaseURL(port) + port := test.GetFreePort() + baseURL := test.GetBaseURL(port) conf := config.New() conf.HTTP.Port = port @@ -225,14 +224,14 @@ func TestObjectStorageControllerSubPaths(t *testing.T) { go startServer(ctlr) defer stopServer(ctlr) - WaitTillServerReady(baseURL) + test.WaitTillServerReady(baseURL) }) } func TestHtpasswdSingleCred(t *testing.T) { Convey("Single cred", t, func() { - port := GetFreePort() - baseURL := GetBaseURL(port) + port := test.GetFreePort() + baseURL := test.GetBaseURL(port) singleCredtests := []string{} user := ALICE password := ALICE @@ -244,7 +243,7 @@ func TestHtpasswdSingleCred(t *testing.T) { conf := config.New() conf.HTTP.Port = port - htpasswdPath := MakeHtpasswdFileFromString(testString) + htpasswdPath := test.MakeHtpasswdFileFromString(testString) defer os.Remove(htpasswdPath) conf.HTTP.Auth = &config.AuthConfig{ HTPasswd: config.AuthHTPasswd{ @@ -261,7 +260,7 @@ func TestHtpasswdSingleCred(t *testing.T) { go startServer(ctlr) defer stopServer(ctlr) - WaitTillServerReady(baseURL) + test.WaitTillServerReady(baseURL) // with creds, should get expected status code resp, _ := resty.R().SetBasicAuth(user, password).Get(baseURL + "/v2/") @@ -295,11 +294,11 @@ func TestHtpasswdTwoCreds(t *testing.T) { for _, testString := range twoCredTests { func() { - port := GetFreePort() - baseURL := GetBaseURL(port) + port := test.GetFreePort() + baseURL := test.GetBaseURL(port) conf := config.New() conf.HTTP.Port = port - htpasswdPath := MakeHtpasswdFileFromString(testString) + htpasswdPath := test.MakeHtpasswdFileFromString(testString) defer os.Remove(htpasswdPath) conf.HTTP.Auth = &config.AuthConfig{ HTPasswd: config.AuthHTPasswd{ @@ -316,7 +315,7 @@ func TestHtpasswdTwoCreds(t *testing.T) { go startServer(ctlr) defer stopServer(ctlr) - WaitTillServerReady(baseURL) + test.WaitTillServerReady(baseURL) // with creds, should get expected status code resp, _ := resty.R().SetBasicAuth(user1, password1).Get(baseURL + "/v2/") @@ -351,11 +350,11 @@ func TestHtpasswdFiveCreds(t *testing.T) { } func() { - port := GetFreePort() - baseURL := GetBaseURL(port) + port := test.GetFreePort() + baseURL := test.GetBaseURL(port) conf := config.New() conf.HTTP.Port = port - htpasswdPath := MakeHtpasswdFileFromString(credString.String()) + htpasswdPath := test.MakeHtpasswdFileFromString(credString.String()) defer os.Remove(htpasswdPath) conf.HTTP.Auth = &config.AuthConfig{ HTPasswd: config.AuthHTPasswd{ @@ -372,7 +371,7 @@ func TestHtpasswdFiveCreds(t *testing.T) { go startServer(ctlr) defer stopServer(ctlr) - WaitTillServerReady(baseURL) + test.WaitTillServerReady(baseURL) // with creds, should get expected status code for key, val := range tests { @@ -391,8 +390,8 @@ func TestHtpasswdFiveCreds(t *testing.T) { func TestRatelimit(t *testing.T) { Convey("Make a new controller", t, func() { - port := GetFreePort() - baseURL := GetBaseURL(port) + port := test.GetFreePort() + baseURL := test.GetBaseURL(port) conf := config.New() conf.HTTP.Port = port @@ -410,7 +409,7 @@ func TestRatelimit(t *testing.T) { go startServer(ctlr) defer stopServer(ctlr) - WaitTillServerReady(baseURL) + test.WaitTillServerReady(baseURL) Convey("Ratelimit", func() { client := resty.New() @@ -428,8 +427,8 @@ func TestRatelimit(t *testing.T) { }) Convey("Make a new controller", t, func() { - port := GetFreePort() - baseURL := GetBaseURL(port) + port := test.GetFreePort() + baseURL := test.GetBaseURL(port) conf := config.New() conf.HTTP.Port = port @@ -451,7 +450,7 @@ func TestRatelimit(t *testing.T) { go startServer(ctlr) defer stopServer(ctlr) - WaitTillServerReady(baseURL) + test.WaitTillServerReady(baseURL) Convey("Method Ratelimit", func() { client := resty.New() // first request should succeed @@ -468,8 +467,8 @@ func TestRatelimit(t *testing.T) { }) Convey("Make a new controller", t, func() { - port := GetFreePort() - baseURL := GetBaseURL(port) + port := test.GetFreePort() + baseURL := test.GetBaseURL(port) conf := config.New() conf.HTTP.Port = port @@ -493,7 +492,7 @@ func TestRatelimit(t *testing.T) { go startServer(ctlr) defer stopServer(ctlr) - WaitTillServerReady(baseURL) + test.WaitTillServerReady(baseURL) Convey("Global and Method Ratelimit", func() { client := resty.New() // first request should succeed @@ -512,11 +511,11 @@ func TestRatelimit(t *testing.T) { func TestBasicAuth(t *testing.T) { Convey("Make a new controller", t, func() { - port := GetFreePort() - baseURL := GetBaseURL(port) + port := test.GetFreePort() + baseURL := test.GetBaseURL(port) conf := config.New() conf.HTTP.Port = port - htpasswdPath := MakeHtpasswdFile() + htpasswdPath := test.MakeHtpasswdFile() defer os.Remove(htpasswdPath) conf.HTTP.Auth = &config.AuthConfig{ @@ -534,7 +533,7 @@ func TestBasicAuth(t *testing.T) { go startServer(ctlr) defer stopServer(ctlr) - WaitTillServerReady(baseURL) + test.WaitTillServerReady(baseURL) // without creds, should get access error resp, err := resty.R().Get(baseURL + "/v2/") @@ -558,8 +557,8 @@ func TestBasicAuth(t *testing.T) { func TestInterruptedBlobUpload(t *testing.T) { Convey("Successfully cleaning interrupted blob uploads", t, func() { - port := GetFreePort() - baseURL := GetBaseURL(port) + port := test.GetFreePort() + baseURL := test.GetBaseURL(port) conf := config.New() conf.HTTP.Port = port @@ -574,7 +573,7 @@ func TestInterruptedBlobUpload(t *testing.T) { go startServer(ctlr) defer stopServer(ctlr) - WaitTillServerReady(baseURL) + test.WaitTillServerReady(baseURL) client := resty.New() blob := make([]byte, 50*1024*1024) @@ -788,11 +787,11 @@ func TestInterruptedBlobUpload(t *testing.T) { func TestMultipleInstance(t *testing.T) { Convey("Negative test zot multiple instance", t, func() { - port := GetFreePort() - baseURL := GetBaseURL(port) + port := test.GetFreePort() + baseURL := test.GetBaseURL(port) conf := config.New() conf.HTTP.Port = port - htpasswdPath := MakeHtpasswdFile() + htpasswdPath := test.MakeHtpasswdFile() defer os.Remove(htpasswdPath) conf.HTTP.Auth = &config.AuthConfig{ @@ -823,7 +822,7 @@ func TestMultipleInstance(t *testing.T) { go startServer(ctlr) defer stopServer(ctlr) - WaitTillServerReady(baseURL) + test.WaitTillServerReady(baseURL) client := resty.New() @@ -834,11 +833,11 @@ func TestMultipleInstance(t *testing.T) { }) Convey("Test zot multiple instance", t, func() { - port := GetFreePort() - baseURL := GetBaseURL(port) + port := test.GetFreePort() + baseURL := test.GetBaseURL(port) conf := config.New() conf.HTTP.Port = port - htpasswdPath := MakeHtpasswdFile() + htpasswdPath := test.MakeHtpasswdFile() defer os.Remove(htpasswdPath) conf.HTTP.Auth = &config.AuthConfig{ @@ -865,7 +864,7 @@ func TestMultipleInstance(t *testing.T) { go startServer(ctlr) defer stopServer(ctlr) - WaitTillServerReady(baseURL) + test.WaitTillServerReady(baseURL) // without creds, should get access error resp, err := resty.R().Get(baseURL + "/v2/") @@ -893,12 +892,12 @@ func TestTLSWithBasicAuth(t *testing.T) { So(err, ShouldBeNil) caCertPool := x509.NewCertPool() caCertPool.AppendCertsFromPEM(caCert) - htpasswdPath := MakeHtpasswdFile() + htpasswdPath := test.MakeHtpasswdFile() defer os.Remove(htpasswdPath) - port := GetFreePort() - baseURL := GetBaseURL(port) - secureBaseURL := GetSecureBaseURL(port) + port := test.GetFreePort() + baseURL := test.GetBaseURL(port) + secureBaseURL := test.GetSecureBaseURL(port) resty.SetTLSClientConfig(&tls.Config{RootCAs: caCertPool, MinVersion: tls.VersionTLS12}) defer func() { resty.SetTLSClientConfig(nil) }() @@ -924,7 +923,7 @@ func TestTLSWithBasicAuth(t *testing.T) { go startServer(ctlr) defer stopServer(ctlr) - WaitTillServerReady(baseURL) + test.WaitTillServerReady(baseURL) // accessing insecure HTTP site should fail resp, err := resty.R().Get(baseURL) @@ -958,12 +957,12 @@ func TestTLSWithBasicAuthAllowReadAccess(t *testing.T) { So(err, ShouldBeNil) caCertPool := x509.NewCertPool() caCertPool.AppendCertsFromPEM(caCert) - htpasswdPath := MakeHtpasswdFile() + htpasswdPath := test.MakeHtpasswdFile() defer os.Remove(htpasswdPath) - port := GetFreePort() - baseURL := GetBaseURL(port) - secureBaseURL := GetSecureBaseURL(port) + port := test.GetFreePort() + baseURL := test.GetBaseURL(port) + secureBaseURL := test.GetSecureBaseURL(port) resty.SetTLSClientConfig(&tls.Config{RootCAs: caCertPool, MinVersion: tls.VersionTLS12}) defer func() { resty.SetTLSClientConfig(nil) }() @@ -990,7 +989,7 @@ func TestTLSWithBasicAuthAllowReadAccess(t *testing.T) { go startServer(ctlr) defer stopServer(ctlr) - WaitTillServerReady(baseURL) + test.WaitTillServerReady(baseURL) // accessing insecure HTTP site should fail resp, err := resty.R().Get(baseURL) @@ -1026,9 +1025,9 @@ func TestTLSMutualAuth(t *testing.T) { caCertPool := x509.NewCertPool() caCertPool.AppendCertsFromPEM(caCert) - port := GetFreePort() - baseURL := GetBaseURL(port) - secureBaseURL := GetSecureBaseURL(port) + port := test.GetFreePort() + baseURL := test.GetBaseURL(port) + secureBaseURL := test.GetSecureBaseURL(port) resty.SetTLSClientConfig(&tls.Config{RootCAs: caCertPool, MinVersion: tls.VersionTLS12}) defer func() { resty.SetTLSClientConfig(nil) }() @@ -1050,7 +1049,7 @@ func TestTLSMutualAuth(t *testing.T) { go startServer(ctlr) defer stopServer(ctlr) - WaitTillServerReady(baseURL) + test.WaitTillServerReady(baseURL) // accessing insecure HTTP site should fail resp, err := resty.R().Get(baseURL) @@ -1098,9 +1097,9 @@ func TestTLSMutualAuthAllowReadAccess(t *testing.T) { caCertPool := x509.NewCertPool() caCertPool.AppendCertsFromPEM(caCert) - port := GetFreePort() - baseURL := GetBaseURL(port) - secureBaseURL := GetSecureBaseURL(port) + port := test.GetFreePort() + baseURL := test.GetBaseURL(port) + secureBaseURL := test.GetSecureBaseURL(port) resty.SetTLSClientConfig(&tls.Config{RootCAs: caCertPool, MinVersion: tls.VersionTLS12}) defer func() { resty.SetTLSClientConfig(nil) }() @@ -1123,7 +1122,7 @@ func TestTLSMutualAuthAllowReadAccess(t *testing.T) { go startServer(ctlr) defer stopServer(ctlr) - WaitTillServerReady(baseURL) + test.WaitTillServerReady(baseURL) // accessing insecure HTTP site should fail resp, err := resty.R().Get(baseURL) @@ -1177,12 +1176,12 @@ func TestTLSMutualAndBasicAuth(t *testing.T) { So(err, ShouldBeNil) caCertPool := x509.NewCertPool() caCertPool.AppendCertsFromPEM(caCert) - htpasswdPath := MakeHtpasswdFile() + htpasswdPath := test.MakeHtpasswdFile() defer os.Remove(htpasswdPath) - port := GetFreePort() - baseURL := GetBaseURL(port) - secureBaseURL := GetSecureBaseURL(port) + port := test.GetFreePort() + baseURL := test.GetBaseURL(port) + secureBaseURL := test.GetSecureBaseURL(port) resty.SetTLSClientConfig(&tls.Config{RootCAs: caCertPool, MinVersion: tls.VersionTLS12}) defer func() { resty.SetTLSClientConfig(nil) }() @@ -1209,7 +1208,7 @@ func TestTLSMutualAndBasicAuth(t *testing.T) { go startServer(ctlr) defer stopServer(ctlr) - WaitTillServerReady(baseURL) + test.WaitTillServerReady(baseURL) // accessing insecure HTTP site should fail resp, err := resty.R().Get(baseURL) @@ -1259,12 +1258,12 @@ func TestTLSMutualAndBasicAuthAllowReadAccess(t *testing.T) { So(err, ShouldBeNil) caCertPool := x509.NewCertPool() caCertPool.AppendCertsFromPEM(caCert) - htpasswdPath := MakeHtpasswdFile() + htpasswdPath := test.MakeHtpasswdFile() defer os.Remove(htpasswdPath) - port := GetFreePort() - baseURL := GetBaseURL(port) - secureBaseURL := GetSecureBaseURL(port) + port := test.GetFreePort() + baseURL := test.GetBaseURL(port) + secureBaseURL := test.GetSecureBaseURL(port) resty.SetTLSClientConfig(&tls.Config{RootCAs: caCertPool, MinVersion: tls.VersionTLS12}) defer func() { resty.SetTLSClientConfig(nil) }() @@ -1292,7 +1291,7 @@ func TestTLSMutualAndBasicAuthAllowReadAccess(t *testing.T) { go startServer(ctlr) defer stopServer(ctlr) - WaitTillServerReady(baseURL) + test.WaitTillServerReady(baseURL) // accessing insecure HTTP site should fail resp, err := resty.R().Get(baseURL) @@ -1423,8 +1422,8 @@ func TestBasicAuthWithLDAP(t *testing.T) { l.Start() defer l.Stop() - port := GetFreePort() - baseURL := GetBaseURL(port) + port := test.GetFreePort() + baseURL := test.GetBaseURL(port) conf := config.New() conf.HTTP.Port = port @@ -1449,7 +1448,7 @@ func TestBasicAuthWithLDAP(t *testing.T) { go startServer(ctlr) defer stopServer(ctlr) - WaitTillServerReady(baseURL) + test.WaitTillServerReady(baseURL) // without creds, should get access error resp, err := resty.R().Get(baseURL + "/v2/") @@ -1476,8 +1475,8 @@ func TestBearerAuth(t *testing.T) { authTestServer := makeAuthTestServer() defer authTestServer.Close() - port := GetFreePort() - baseURL := GetBaseURL(port) + port := test.GetFreePort() + baseURL := test.GetBaseURL(port) conf := config.New() conf.HTTP.Port = port @@ -1500,7 +1499,7 @@ func TestBearerAuth(t *testing.T) { go startServer(ctlr) defer stopServer(ctlr) - WaitTillServerReady(baseURL) + test.WaitTillServerReady(baseURL) blob := []byte("hello, blob!") digest := godigest.FromBytes(blob).String() @@ -1642,8 +1641,8 @@ func TestBearerAuthWithAllowReadAccess(t *testing.T) { authTestServer := makeAuthTestServer() defer authTestServer.Close() - port := GetFreePort() - baseURL := GetBaseURL(port) + port := test.GetFreePort() + baseURL := test.GetBaseURL(port) conf := config.New() conf.HTTP.Port = port @@ -1667,7 +1666,7 @@ func TestBearerAuthWithAllowReadAccess(t *testing.T) { go startServer(ctlr) defer stopServer(ctlr) - WaitTillServerReady(baseURL) + test.WaitTillServerReady(baseURL) blob := []byte("hello, blob!") digest := godigest.FromBytes(blob).String() @@ -1860,12 +1859,12 @@ func parseBearerAuthHeader(authHeaderRaw string) *authHeader { func TestAuthorizationWithBasicAuth(t *testing.T) { Convey("Make a new controller", t, func() { - port := GetFreePort() - baseURL := GetBaseURL(port) + port := test.GetFreePort() + baseURL := test.GetBaseURL(port) conf := config.New() conf.HTTP.Port = port - htpasswdPath := MakeHtpasswdFile() + htpasswdPath := test.MakeHtpasswdFile() defer os.Remove(htpasswdPath) conf.HTTP.Auth = &config.AuthConfig{ @@ -1897,7 +1896,7 @@ func TestAuthorizationWithBasicAuth(t *testing.T) { panic(err) } defer os.RemoveAll(dir) - err = CopyFiles("../../test/data", dir) + err = test.CopyFiles("../../test/data", dir) if err != nil { panic(err) } @@ -1905,7 +1904,7 @@ func TestAuthorizationWithBasicAuth(t *testing.T) { go startServer(ctlr) defer stopServer(ctlr) - WaitTillServerReady(baseURL) + test.WaitTillServerReady(baseURL) blob := []byte("hello, blob!") digest := godigest.FromBytes(blob).String() @@ -2356,12 +2355,12 @@ func TestAuthorizationWithBasicAuth(t *testing.T) { func TestInvalidCases(t *testing.T) { Convey("Invalid repo dir", t, func() { - port := GetFreePort() - baseURL := GetBaseURL(port) + port := test.GetFreePort() + baseURL := test.GetBaseURL(port) conf := config.New() conf.HTTP.Port = port - htpasswdPath := MakeHtpasswdFileFromString(getCredString(username, passphrase)) + htpasswdPath := test.MakeHtpasswdFileFromString(getCredString(username, passphrase)) defer os.Remove(htpasswdPath) @@ -2392,7 +2391,7 @@ func TestInvalidCases(t *testing.T) { panic(err) } }(ctlr) - WaitTillServerReady(baseURL) + test.WaitTillServerReady(baseURL) digest := "sha256:8dd57e171a61368ffcfde38045ddb6ed74a32950c271c1da93eaddfb66a77e78" name := "zot-c-test" @@ -2419,8 +2418,8 @@ func TestHTTPReadOnly(t *testing.T) { singleCredtests = append(singleCredtests, getCredString(user, password)) singleCredtests = append(singleCredtests, getCredString(user, password)+"\n") - port := GetFreePort() - baseURL := GetBaseURL(port) + port := test.GetFreePort() + baseURL := test.GetBaseURL(port) for _, testString := range singleCredtests { func() { @@ -2429,7 +2428,7 @@ func TestHTTPReadOnly(t *testing.T) { // enable read-only mode conf.HTTP.ReadOnly = true - htpasswdPath := MakeHtpasswdFileFromString(testString) + htpasswdPath := test.MakeHtpasswdFileFromString(testString) defer os.Remove(htpasswdPath) conf.HTTP.Auth = &config.AuthConfig{ HTPasswd: config.AuthHTPasswd{ @@ -2446,7 +2445,7 @@ func TestHTTPReadOnly(t *testing.T) { go startServer(ctlr) defer stopServer(ctlr) - WaitTillServerReady(baseURL) + test.WaitTillServerReady(baseURL) // with creds, should get expected status code resp, _ := resty.R().SetBasicAuth(user, password).Get(baseURL + "/v2/") @@ -2471,12 +2470,12 @@ func TestHTTPReadOnly(t *testing.T) { func TestCrossRepoMount(t *testing.T) { Convey("Cross Repo Mount", t, func() { - port := GetFreePort() - baseURL := GetBaseURL(port) + port := test.GetFreePort() + baseURL := test.GetBaseURL(port) conf := config.New() conf.HTTP.Port = port - htpasswdPath := MakeHtpasswdFileFromString(getCredString(username, passphrase)) + htpasswdPath := test.MakeHtpasswdFileFromString(getCredString(username, passphrase)) defer os.Remove(htpasswdPath) @@ -2493,7 +2492,7 @@ func TestCrossRepoMount(t *testing.T) { panic(err) } - err = CopyFiles("../../test/data", dir) + err = test.CopyFiles("../../test/data", dir) if err != nil { panic(err) } @@ -2502,7 +2501,7 @@ func TestCrossRepoMount(t *testing.T) { go startServer(ctlr) defer stopServer(ctlr) - WaitTillServerReady(baseURL) + test.WaitTillServerReady(baseURL) params := make(map[string]string) digest := "sha256:63a795ca90aa6e7cca60941e826810a4cd0a2e73ea02bf458241df2a5c973e29" @@ -2655,12 +2654,12 @@ func TestCrossRepoMount(t *testing.T) { }) Convey("Disable dedupe and cache", t, func() { - port := GetFreePort() - baseURL := GetBaseURL(port) + port := test.GetFreePort() + baseURL := test.GetBaseURL(port) conf := config.New() conf.HTTP.Port = port - htpasswdPath := MakeHtpasswdFileFromString(getCredString(username, passphrase)) + htpasswdPath := test.MakeHtpasswdFileFromString(getCredString(username, passphrase)) defer os.Remove(htpasswdPath) @@ -2677,7 +2676,7 @@ func TestCrossRepoMount(t *testing.T) { panic(err) } - err = CopyFiles("../../test/data", dir) + err = test.CopyFiles("../../test/data", dir) if err != nil { panic(err) } @@ -2689,7 +2688,7 @@ func TestCrossRepoMount(t *testing.T) { go startServer(ctlr) defer stopServer(ctlr) - WaitTillServerReady(baseURL) + test.WaitTillServerReady(baseURL) digest := "sha256:7a0437f04f83f084b7ed68ad9c4a4947e12fc4e1b006b38129bac89114ec3621" name := "zot-c-test" @@ -2799,12 +2798,12 @@ func TestParallelRequests(t *testing.T) { }, } - port := GetFreePort() - baseURL := GetBaseURL(port) + port := test.GetFreePort() + baseURL := test.GetBaseURL(port) conf := config.New() conf.HTTP.Port = port - htpasswdPath := MakeHtpasswdFileFromString(getCredString(username, passphrase)) + htpasswdPath := test.MakeHtpasswdFileFromString(getCredString(username, passphrase)) conf.HTTP.Auth = &config.AuthConfig{ HTPasswd: config.AuthHTPasswd{ @@ -2838,7 +2837,7 @@ func TestParallelRequests(t *testing.T) { ctlr.Config.Storage.RootDirectory = dir go startServer(ctlr) - WaitTillServerReady(baseURL) + test.WaitTillServerReady(baseURL) // without creds, should get access error for i, testcase := range testCases { @@ -3048,12 +3047,12 @@ func TestParallelRequests(t *testing.T) { func TestHardLink(t *testing.T) { Convey("Validate hard link", t, func() { - port := GetFreePort() - baseURL := GetBaseURL(port) + port := test.GetFreePort() + baseURL := test.GetBaseURL(port) conf := config.New() conf.HTTP.Port = port - htpasswdPath := MakeHtpasswdFileFromString(getCredString(username, passphrase)) + htpasswdPath := test.MakeHtpasswdFileFromString(getCredString(username, passphrase)) conf.HTTP.Auth = &config.AuthConfig{ HTPasswd: config.AuthHTPasswd{ @@ -3093,7 +3092,7 @@ func TestHardLink(t *testing.T) { go startServer(ctlr) defer stopServer(ctlr) - WaitTillServerReady(baseURL) + test.WaitTillServerReady(baseURL) err = os.Chmod(dir, 0o644) if err != nil { @@ -3112,8 +3111,8 @@ func TestHardLink(t *testing.T) { func TestImageSignatures(t *testing.T) { Convey("Validate signatures", t, func() { // start a new server - port := GetFreePort() - baseURL := GetBaseURL(port) + port := test.GetFreePort() + baseURL := test.GetBaseURL(port) conf := config.New() conf.HTTP.Port = port @@ -3150,7 +3149,7 @@ func TestImageSignatures(t *testing.T) { resp, err := resty.R().Post(baseURL + fmt.Sprintf("/v2/%s/blobs/uploads/", repoName)) So(err, ShouldBeNil) So(resp.StatusCode(), ShouldEqual, http.StatusAccepted) - loc := Location(baseURL, resp) + loc := test.Location(baseURL, resp) So(loc, ShouldNotBeEmpty) resp, err = resty.R().Get(loc) @@ -3169,11 +3168,29 @@ func TestImageSignatures(t *testing.T) { So(resp.Header().Get("Content-Length"), ShouldEqual, "0") So(resp.Header().Get(api.DistContentDigestKey), ShouldNotBeEmpty) + // upload image config blob + resp, err = resty.R().Post(baseURL + fmt.Sprintf("/v2/%s/blobs/uploads/", repoName)) + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, http.StatusAccepted) + loc = test.Location(baseURL, resp) + cblob, cdigest := test.GetRandomImageConfig() + + resp, err = resty.R(). + SetContentLength(true). + SetHeader("Content-Length", fmt.Sprintf("%d", len(cblob))). + SetHeader("Content-Type", "application/octet-stream"). + SetQueryParam("digest", cdigest.String()). + SetBody(cblob). + Put(loc) + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, http.StatusCreated) + // create a manifest manifest := ispec.Manifest{ Config: ispec.Descriptor{ - Digest: digest, - Size: int64(len(content)), + MediaType: "application/vnd.oci.image.config.v1+json", + Digest: cdigest, + Size: int64(len(cblob)), }, Layers: []ispec.Descriptor{ { diff --git a/pkg/cli/client_test.go b/pkg/cli/client_test.go index dde7d87c..90b1a3e3 100644 --- a/pkg/cli/client_test.go +++ b/pkg/cli/client_test.go @@ -19,7 +19,7 @@ import ( "gopkg.in/resty.v1" "zotregistry.io/zot/pkg/api" "zotregistry.io/zot/pkg/api/config" - . "zotregistry.io/zot/test" + "zotregistry.io/zot/pkg/test" ) const ( @@ -53,7 +53,7 @@ func TestTLSWithAuth(t *testing.T) { defer func() { resty.SetTLSClientConfig(nil) }() conf := config.New() conf.HTTP.Port = SecurePort1 - htpasswdPath := MakeHtpasswdFile() + htpasswdPath := test.MakeHtpasswdFile() defer os.Remove(htpasswdPath) conf.HTTP.Auth = &config.AuthConfig{ @@ -102,7 +102,7 @@ func TestTLSWithAuth(t *testing.T) { home := os.Getenv("HOME") destCertsDir := filepath.Join(home, certsDir1) - if err = CopyFiles(sourceCertsDir, destCertsDir); err != nil { + if err = test.CopyFiles(sourceCertsDir, destCertsDir); err != nil { panic(err) } defer os.RemoveAll(destCertsDir) @@ -201,7 +201,7 @@ func TestTLSWithoutAuth(t *testing.T) { home := os.Getenv("HOME") destCertsDir := filepath.Join(home, certsDir1) - if err = CopyFiles(sourceCertsDir, destCertsDir); err != nil { + if err = test.CopyFiles(sourceCertsDir, destCertsDir); err != nil { panic(err) } defer os.RemoveAll(destCertsDir) diff --git a/pkg/cli/cve_cmd_test.go b/pkg/cli/cve_cmd_test.go index f3051af3..f3bb655b 100644 --- a/pkg/cli/cve_cmd_test.go +++ b/pkg/cli/cve_cmd_test.go @@ -21,7 +21,7 @@ import ( "zotregistry.io/zot/pkg/api" "zotregistry.io/zot/pkg/api/config" extconf "zotregistry.io/zot/pkg/extensions/config" - . "zotregistry.io/zot/test" + "zotregistry.io/zot/pkg/test" ) func TestSearchCVECmd(t *testing.T) { @@ -286,8 +286,8 @@ func TestSearchCVECmd(t *testing.T) { } func TestServerCVEResponse(t *testing.T) { - port := GetFreePort() - url := GetBaseURL(port) + port := test.GetFreePort() + url := test.GetBaseURL(port) conf := config.New() conf.HTTP.Port = port ctlr := api.NewController(conf) @@ -297,7 +297,7 @@ func TestServerCVEResponse(t *testing.T) { panic(err) } - err = CopyFiles("../../test/data/zot-cve-test", path.Join(dir, "zot-cve-test")) + err = test.CopyFiles("../../test/data/zot-cve-test", path.Join(dir, "zot-cve-test")) if err != nil { panic(err) } diff --git a/pkg/cli/image_cmd_test.go b/pkg/cli/image_cmd_test.go index a83086e4..901a5005 100644 --- a/pkg/cli/image_cmd_test.go +++ b/pkg/cli/image_cmd_test.go @@ -25,7 +25,7 @@ import ( "zotregistry.io/zot/pkg/api" "zotregistry.io/zot/pkg/api/config" extconf "zotregistry.io/zot/pkg/extensions/config" - . "zotregistry.io/zot/test" + "zotregistry.io/zot/pkg/test" ) func TestSearchImageCmd(t *testing.T) { @@ -282,8 +282,8 @@ func TestOutputFormat(t *testing.T) { func TestServerResponse(t *testing.T) { Convey("Test from real server", t, func() { - port := GetFreePort() - url := GetBaseURL(port) + port := test.GetFreePort() + url := test.GetBaseURL(port) conf := config.New() conf.HTTP.Port = port conf.Extensions = &extconf.ExtensionConfig{ @@ -334,8 +334,8 @@ func TestServerResponse(t *testing.T) { str := space.ReplaceAllString(buff.String(), " ") actual := strings.TrimSpace(str) So(actual, ShouldContainSubstring, "IMAGE NAME TAG DIGEST SIZE") - So(actual, ShouldContainSubstring, "repo7 test:2.0 a0ca253b 15B") - So(actual, ShouldContainSubstring, "repo7 test:1.0 a0ca253b 15B") + So(actual, ShouldContainSubstring, "repo7 test:2.0 883fc0c5 15B") + So(actual, ShouldContainSubstring, "repo7 test:1.0 883fc0c5 15B") }) Convey("Test all images verbose", func() { @@ -359,8 +359,8 @@ func TestServerResponse(t *testing.T) { // repo7 test:1.0 a0ca253b b8781e88 15B // b8781e88 15B So(actual, ShouldContainSubstring, "IMAGE NAME TAG DIGEST CONFIG LAYERS SIZE") - So(actual, ShouldContainSubstring, "repo7 test:2.0 a0ca253b b8781e88 15B b8781e88 15B") - So(actual, ShouldContainSubstring, "repo7 test:1.0 a0ca253b b8781e88 15B b8781e88 15B") + So(actual, ShouldContainSubstring, "repo7 test:2.0 883fc0c5 3a1d2d0c 15B b8781e88 15B") + So(actual, ShouldContainSubstring, "repo7 test:1.0 883fc0c5 3a1d2d0c 15B b8781e88 15B") }) Convey("Test image by name config url", func() { @@ -378,8 +378,8 @@ func TestServerResponse(t *testing.T) { str := space.ReplaceAllString(buff.String(), " ") actual := strings.TrimSpace(str) So(actual, ShouldContainSubstring, "IMAGE NAME TAG DIGEST SIZE") - So(actual, ShouldContainSubstring, "repo7 test:2.0 a0ca253b 15B") - So(actual, ShouldContainSubstring, "repo7 test:1.0 a0ca253b 15B") + So(actual, ShouldContainSubstring, "repo7 test:2.0 883fc0c5 15B") + So(actual, ShouldContainSubstring, "repo7 test:1.0 883fc0c5 15B") Convey("with shorthand", func() { args := []string{"imagetest", "-n", "repo7"} @@ -396,13 +396,13 @@ func TestServerResponse(t *testing.T) { str := space.ReplaceAllString(buff.String(), " ") actual := strings.TrimSpace(str) So(actual, ShouldContainSubstring, "IMAGE NAME TAG DIGEST SIZE") - So(actual, ShouldContainSubstring, "repo7 test:2.0 a0ca253b 15B") - So(actual, ShouldContainSubstring, "repo7 test:1.0 a0ca253b 15B") + So(actual, ShouldContainSubstring, "repo7 test:2.0 883fc0c5 15B") + So(actual, ShouldContainSubstring, "repo7 test:1.0 883fc0c5 15B") }) }) Convey("Test image by digest", func() { - args := []string{"imagetest", "--digest", "a0ca253b"} + args := []string{"imagetest", "--digest", "883fc0c5"} configPath := makeConfigFile(fmt.Sprintf(`{"configs":[{"_name":"imagetest","url":"%s","showspinner":false}]}`, url)) defer os.Remove(configPath) cmd := NewImageCommand(new(searchService)) @@ -420,10 +420,10 @@ func TestServerResponse(t *testing.T) { // repo7 test:2.0 a0ca253b 15B // repo7 test:1.0 a0ca253b 15B So(actual, ShouldContainSubstring, "IMAGE NAME TAG DIGEST SIZE") - So(actual, ShouldContainSubstring, "repo7 test:2.0 a0ca253b 15B") - So(actual, ShouldContainSubstring, "repo7 test:1.0 a0ca253b 15B") + So(actual, ShouldContainSubstring, "repo7 test:2.0 883fc0c5 15B") + So(actual, ShouldContainSubstring, "repo7 test:1.0 883fc0c5 15B") Convey("with shorthand", func() { - args := []string{"imagetest", "-d", "a0ca253b"} + args := []string{"imagetest", "-d", "883fc0c5"} configPath := makeConfigFile(fmt.Sprintf(`{"configs":[{"_name":"imagetest","url":"%s","showspinner":false}]}`, url)) defer os.Remove(configPath) cmd := NewImageCommand(new(searchService)) @@ -437,8 +437,8 @@ func TestServerResponse(t *testing.T) { str := space.ReplaceAllString(buff.String(), " ") actual := strings.TrimSpace(str) So(actual, ShouldContainSubstring, "IMAGE NAME TAG DIGEST SIZE") - So(actual, ShouldContainSubstring, "repo7 test:2.0 a0ca253b 15B") - So(actual, ShouldContainSubstring, "repo7 test:1.0 a0ca253b 15B") + So(actual, ShouldContainSubstring, "repo7 test:2.0 883fc0c5 15B") + So(actual, ShouldContainSubstring, "repo7 test:1.0 883fc0c5 15B") }) }) @@ -462,18 +462,32 @@ func TestServerResponse(t *testing.T) { func uploadManifest(url string) { // create a blob/layer resp, _ := resty.R().Post(url + "/v2/repo7/blobs/uploads/") - loc := Location(url, resp) + loc := test.Location(url, resp) content := []byte("this is a blob5") digest := godigest.FromBytes(content) _, _ = resty.R().SetQueryParam("digest", digest.String()). SetHeader("Content-Type", "application/octet-stream").SetBody(content).Put(loc) + // upload image config blob + resp, _ = resty.R().Post(url + "/v2/repo7/blobs/uploads/") + loc = test.Location(url, resp) + cblob, cdigest := test.GetImageConfig() + + _, _ = resty.R(). + SetContentLength(true). + SetHeader("Content-Length", fmt.Sprintf("%d", len(cblob))). + SetHeader("Content-Type", "application/octet-stream"). + SetQueryParam("digest", cdigest.String()). + SetBody(cblob). + Put(loc) + // create a manifest manifest := ispec.Manifest{ Config: ispec.Descriptor{ - Digest: digest, - Size: int64(len(content)), + MediaType: "application/vnd.oci.image.config.v1+json", + Digest: cdigest, + Size: int64(len(cblob)), }, Layers: []ispec.Descriptor{ { @@ -493,8 +507,9 @@ func uploadManifest(url string) { // create a manifest with same blob but a different tag manifest = ispec.Manifest{ Config: ispec.Descriptor{ - Digest: digest, - Size: int64(len(content)), + MediaType: "application/vnd.oci.image.config.v1+json", + Digest: cdigest, + Size: int64(len(cblob)), }, Layers: []ispec.Descriptor{ { diff --git a/pkg/cli/root_test.go b/pkg/cli/root_test.go index 14192364..a7f0e79a 100644 --- a/pkg/cli/root_test.go +++ b/pkg/cli/root_test.go @@ -14,7 +14,7 @@ import ( "zotregistry.io/zot/pkg/api" "zotregistry.io/zot/pkg/api/config" "zotregistry.io/zot/pkg/cli" - . "zotregistry.io/zot/test" + . "zotregistry.io/zot/pkg/test" ) func TestServerUsage(t *testing.T) { diff --git a/pkg/compliance/v1_0_0/check.go b/pkg/compliance/v1_0_0/check.go index fd8209ec..2e359298 100644 --- a/pkg/compliance/v1_0_0/check.go +++ b/pkg/compliance/v1_0_0/check.go @@ -21,9 +21,7 @@ import ( "gopkg.in/resty.v1" "zotregistry.io/zot/pkg/api" "zotregistry.io/zot/pkg/compliance" - - // nolint:golint,stylecheck,revive - . "zotregistry.io/zot/test" + "zotregistry.io/zot/pkg/test" ) func CheckWorkflows(t *testing.T, config *compliance.Config) { @@ -118,7 +116,7 @@ func CheckWorkflows(t *testing.T, config *compliance.Config) { resp, err := resty.R().Post(baseURL + "/v2/repo2/blobs/uploads/") So(err, ShouldBeNil) So(resp.StatusCode(), ShouldEqual, http.StatusAccepted) - loc := Location(baseURL, resp) + loc := test.Location(baseURL, resp) So(loc, ShouldNotBeEmpty) resp, err = resty.R().Get(loc) @@ -154,7 +152,7 @@ func CheckWorkflows(t *testing.T, config *compliance.Config) { SetHeader("Content-Type", "application/octet-stream").SetBody(content).Put(loc) So(err, ShouldBeNil) So(resp.StatusCode(), ShouldEqual, http.StatusCreated) - blobLoc := Location(baseURL, resp) + blobLoc := test.Location(baseURL, resp) So(blobLoc, ShouldNotBeEmpty) So(resp.Header().Get("Content-Length"), ShouldEqual, "0") So(resp.Header().Get(api.DistContentDigestKey), ShouldNotBeEmpty) @@ -198,7 +196,7 @@ func CheckWorkflows(t *testing.T, config *compliance.Config) { Post(baseURL + "/v2/repo2/blobs/uploads/") So(err, ShouldBeNil) So(resp.StatusCode(), ShouldEqual, http.StatusCreated) - loc := Location(baseURL, resp) + loc := test.Location(baseURL, resp) So(loc, ShouldNotBeEmpty) // blob reference should be accessible resp, err = resty.R().Get(loc) @@ -211,7 +209,7 @@ func CheckWorkflows(t *testing.T, config *compliance.Config) { resp, err := resty.R().Post(baseURL + "/v2/repo10/repo20/repo30/blobs/uploads/") So(err, ShouldBeNil) So(resp.StatusCode(), ShouldEqual, http.StatusAccepted) - loc := Location(baseURL, resp) + loc := test.Location(baseURL, resp) So(loc, ShouldNotBeEmpty) resp, err = resty.R().Get(loc) @@ -247,7 +245,7 @@ func CheckWorkflows(t *testing.T, config *compliance.Config) { SetHeader("Content-Type", "application/octet-stream").SetBody(content).Put(loc) So(err, ShouldBeNil) So(resp.StatusCode(), ShouldEqual, http.StatusCreated) - blobLoc := Location(baseURL, resp) + blobLoc := test.Location(baseURL, resp) So(blobLoc, ShouldNotBeEmpty) So(resp.Header().Get("Content-Length"), ShouldEqual, "0") So(resp.Header().Get(api.DistContentDigestKey), ShouldNotBeEmpty) @@ -266,7 +264,7 @@ func CheckWorkflows(t *testing.T, config *compliance.Config) { resp, err := resty.R().Post(baseURL + "/v2/repo3/blobs/uploads/") So(err, ShouldBeNil) So(resp.StatusCode(), ShouldEqual, http.StatusAccepted) - loc := Location(baseURL, resp) + loc := test.Location(baseURL, resp) So(loc, ShouldNotBeEmpty) var buf bytes.Buffer @@ -313,7 +311,7 @@ func CheckWorkflows(t *testing.T, config *compliance.Config) { SetHeader("Content-Type", "application/octet-stream").SetBody(chunk2).Put(loc) So(err, ShouldBeNil) So(resp.StatusCode(), ShouldEqual, http.StatusCreated) - blobLoc := Location(baseURL, resp) + blobLoc := test.Location(baseURL, resp) So(err, ShouldBeNil) So(resp.StatusCode(), ShouldEqual, http.StatusCreated) So(blobLoc, ShouldNotBeEmpty) @@ -334,7 +332,7 @@ func CheckWorkflows(t *testing.T, config *compliance.Config) { resp, err := resty.R().Post(baseURL + "/v2/repo40/repo50/repo60/blobs/uploads/") So(err, ShouldBeNil) So(resp.StatusCode(), ShouldEqual, http.StatusAccepted) - loc := Location(baseURL, resp) + loc := test.Location(baseURL, resp) So(loc, ShouldNotBeEmpty) var buf bytes.Buffer @@ -381,7 +379,7 @@ func CheckWorkflows(t *testing.T, config *compliance.Config) { SetHeader("Content-Type", "application/octet-stream").SetBody(chunk2).Put(loc) So(err, ShouldBeNil) So(resp.StatusCode(), ShouldEqual, http.StatusCreated) - blobLoc := Location(baseURL, resp) + blobLoc := test.Location(baseURL, resp) So(err, ShouldBeNil) So(resp.StatusCode(), ShouldEqual, http.StatusCreated) So(blobLoc, ShouldNotBeEmpty) @@ -403,7 +401,7 @@ func CheckWorkflows(t *testing.T, config *compliance.Config) { resp, err := resty.R().Post(baseURL + "/v2/repo4/blobs/uploads/") So(err, ShouldBeNil) So(resp.StatusCode(), ShouldEqual, http.StatusAccepted) - loc := Location(baseURL, resp) + loc := test.Location(baseURL, resp) So(loc, ShouldNotBeEmpty) // delete this upload @@ -418,7 +416,7 @@ func CheckWorkflows(t *testing.T, config *compliance.Config) { resp, err := resty.R().Post(baseURL + "/v2/repo5/blobs/uploads/") So(err, ShouldBeNil) So(resp.StatusCode(), ShouldEqual, http.StatusAccepted) - loc := Location(baseURL, resp) + loc := test.Location(baseURL, resp) So(loc, ShouldNotBeEmpty) content := []byte("this is a blob4") @@ -429,7 +427,7 @@ func CheckWorkflows(t *testing.T, config *compliance.Config) { SetHeader("Content-Type", "application/octet-stream").SetBody(content).Put(loc) So(err, ShouldBeNil) So(resp.StatusCode(), ShouldEqual, http.StatusCreated) - blobLoc := Location(baseURL, resp) + blobLoc := test.Location(baseURL, resp) So(blobLoc, ShouldNotBeEmpty) So(resp.Header().Get(api.DistContentDigestKey), ShouldNotBeEmpty) @@ -454,7 +452,7 @@ func CheckWorkflows(t *testing.T, config *compliance.Config) { resp, err := resty.R().Post(baseURL + "/v2/repo7/blobs/uploads/") So(err, ShouldBeNil) So(resp.StatusCode(), ShouldEqual, http.StatusAccepted) - loc := Location(baseURL, resp) + loc := test.Location(baseURL, resp) So(loc, ShouldNotBeEmpty) // since we are not specifying any prefix i.e provided in config while starting server, @@ -484,11 +482,29 @@ func CheckWorkflows(t *testing.T, config *compliance.Config) { So(err, ShouldBeNil) So(resp.StatusCode(), ShouldEqual, http.StatusNotFound) + // upload image config blob + resp, err = resty.R().Post(baseURL + "/v2/repo7/blobs/uploads/") + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, http.StatusAccepted) + loc = test.Location(baseURL, resp) + cblob, cdigest := test.GetRandomImageConfig() + + resp, err = resty.R(). + SetContentLength(true). + SetHeader("Content-Length", fmt.Sprintf("%d", len(cblob))). + SetHeader("Content-Type", "application/octet-stream"). + SetQueryParam("digest", cdigest.String()). + SetBody(cblob). + Put(loc) + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, http.StatusCreated) + // create a manifest manifest := ispec.Manifest{ Config: ispec.Descriptor{ - Digest: digest, - Size: int64(len(content)), + MediaType: "application/vnd.oci.image.config.v1+json", + Digest: cdigest, + Size: int64(len(cblob)), }, Layers: []ispec.Descriptor{ { @@ -522,11 +538,13 @@ func CheckWorkflows(t *testing.T, config *compliance.Config) { content = []byte("this is a blob5") digest = godigest.FromBytes(content) So(digest, ShouldNotBeNil) + // create a manifest with same blob but a different tag manifest = ispec.Manifest{ Config: ispec.Descriptor{ - Digest: digest, - Size: int64(len(content)), + MediaType: "application/vnd.oci.image.config.v1+json", + Digest: cdigest, + Size: int64(len(cblob)), }, Layers: []ispec.Descriptor{ { @@ -619,7 +637,7 @@ func CheckWorkflows(t *testing.T, config *compliance.Config) { resp, err := resty.R().Post(baseURL + "/v2/page0/blobs/uploads/") So(err, ShouldBeNil) So(resp.StatusCode(), ShouldEqual, http.StatusAccepted) - loc := Location(baseURL, resp) + loc := test.Location(baseURL, resp) So(loc, ShouldNotBeEmpty) resp, err = resty.R().Get(loc) @@ -638,11 +656,29 @@ func CheckWorkflows(t *testing.T, config *compliance.Config) { So(resp.Header().Get("Content-Length"), ShouldEqual, "0") So(resp.Header().Get(api.DistContentDigestKey), ShouldNotBeEmpty) + // upload image config blob + resp, err = resty.R().Post(baseURL + "/v2/page0/blobs/uploads/") + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, http.StatusAccepted) + loc = test.Location(baseURL, resp) + cblob, cdigest := test.GetRandomImageConfig() + + resp, err = resty.R(). + SetContentLength(true). + SetHeader("Content-Length", fmt.Sprintf("%d", len(cblob))). + SetHeader("Content-Type", "application/octet-stream"). + SetQueryParam("digest", cdigest.String()). + SetBody(cblob). + Put(loc) + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, http.StatusCreated) + // create a manifest manifest := ispec.Manifest{ Config: ispec.Descriptor{ - Digest: digest, - Size: int64(len(content)), + MediaType: "application/vnd.oci.image.config.v1+json", + Digest: cdigest, + Size: int64(len(cblob)), }, Layers: []ispec.Descriptor{ { @@ -722,7 +758,7 @@ func CheckWorkflows(t *testing.T, config *compliance.Config) { resp, err := resty.R().Post(baseURL + "/v2/firsttest/first/blobs/uploads/") So(err, ShouldBeNil) So(resp.StatusCode(), ShouldEqual, http.StatusAccepted) - firstloc := Location(baseURL, resp) + firstloc := test.Location(baseURL, resp) So(firstloc, ShouldNotBeEmpty) resp, err = resty.R().Get(firstloc) @@ -737,7 +773,7 @@ func CheckWorkflows(t *testing.T, config *compliance.Config) { resp, err = resty.R().Post(baseURL + "/v2/secondtest/second/blobs/uploads/") So(err, ShouldBeNil) So(resp.StatusCode(), ShouldEqual, http.StatusAccepted) - secondloc := Location(baseURL, resp) + secondloc := test.Location(baseURL, resp) So(secondloc, ShouldNotBeEmpty) resp, err = resty.R().Get(secondloc) @@ -788,11 +824,45 @@ func CheckWorkflows(t *testing.T, config *compliance.Config) { So(err, ShouldBeNil) So(resp.StatusCode(), ShouldEqual, http.StatusNotFound) + // upload image config blob + resp, err = resty.R().Post(baseURL + "/v2/firsttest/first/blobs/uploads/") + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, http.StatusAccepted) + loc := test.Location(baseURL, resp) + cblob, cdigest := test.GetRandomImageConfig() + + resp, err = resty.R(). + SetContentLength(true). + SetHeader("Content-Length", fmt.Sprintf("%d", len(cblob))). + SetHeader("Content-Type", "application/octet-stream"). + SetQueryParam("digest", cdigest.String()). + SetBody(cblob). + Put(loc) + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, http.StatusCreated) + + // upload image config blob + resp, err = resty.R().Post(baseURL + "/v2/secondtest/second/blobs/uploads/") + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, http.StatusAccepted) + loc = test.Location(baseURL, resp) + + resp, err = resty.R(). + SetContentLength(true). + SetHeader("Content-Length", fmt.Sprintf("%d", len(cblob))). + SetHeader("Content-Type", "application/octet-stream"). + SetQueryParam("digest", cdigest.String()). + SetBody(cblob). + Put(loc) + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, http.StatusCreated) + // create a manifest manifest := ispec.Manifest{ Config: ispec.Descriptor{ - Digest: digest, - Size: int64(len(content)), + MediaType: "application/vnd.oci.image.config.v1+json", + Digest: cdigest, + Size: int64(len(cblob)), }, Layers: []ispec.Descriptor{ { @@ -828,11 +898,13 @@ func CheckWorkflows(t *testing.T, config *compliance.Config) { content = []byte("this is a blob5") digest = godigest.FromBytes(content) So(digest, ShouldNotBeNil) + // create a manifest with same blob but a different tag manifest = ispec.Manifest{ Config: ispec.Descriptor{ - Digest: digest, - Size: int64(len(content)), + MediaType: "application/vnd.oci.image.config.v1+json", + Digest: cdigest, + Size: int64(len(cblob)), }, Layers: []ispec.Descriptor{ { diff --git a/pkg/compliance/v1_0_0/check_test.go b/pkg/compliance/v1_0_0/check_test.go index 7521aaf9..96efeb9f 100644 --- a/pkg/compliance/v1_0_0/check_test.go +++ b/pkg/compliance/v1_0_0/check_test.go @@ -13,7 +13,7 @@ import ( "zotregistry.io/zot/pkg/api/config" "zotregistry.io/zot/pkg/compliance" "zotregistry.io/zot/pkg/compliance/v1_0_0" - . "zotregistry.io/zot/test" + . "zotregistry.io/zot/pkg/test" ) // nolint: gochecknoglobals diff --git a/pkg/exporter/api/controller_test.go b/pkg/exporter/api/controller_test.go index cc76a6a2..58cffe44 100644 --- a/pkg/exporter/api/controller_test.go +++ b/pkg/exporter/api/controller_test.go @@ -26,7 +26,7 @@ import ( zotcfg "zotregistry.io/zot/pkg/api/config" "zotregistry.io/zot/pkg/exporter/api" "zotregistry.io/zot/pkg/extensions/monitoring" - . "zotregistry.io/zot/test" + . "zotregistry.io/zot/pkg/test" ) const ( diff --git a/pkg/extensions/monitoring/monitoring_test.go b/pkg/extensions/monitoring/monitoring_test.go index edaa6ce6..e609e4e4 100644 --- a/pkg/extensions/monitoring/monitoring_test.go +++ b/pkg/extensions/monitoring/monitoring_test.go @@ -18,13 +18,13 @@ import ( "zotregistry.io/zot/pkg/api/config" extconf "zotregistry.io/zot/pkg/extensions/config" "zotregistry.io/zot/pkg/extensions/monitoring" - . "zotregistry.io/zot/test" + "zotregistry.io/zot/pkg/test" ) func TestExtensionMetrics(t *testing.T) { Convey("Make a new controller with explicitly enabled metrics", t, func() { - port := GetFreePort() - baseURL := GetBaseURL(port) + port := test.GetFreePort() + baseURL := test.GetBaseURL(port) conf := config.New() conf.HTTP.Port = port @@ -46,7 +46,7 @@ func TestExtensionMetrics(t *testing.T) { go startServer(ctlr) defer stopServer(ctlr) - WaitTillServerReady(baseURL) + test.WaitTillServerReady(baseURL) // improve code coverage ctlr.Metrics.SendMetric(baseURL) @@ -60,7 +60,7 @@ func TestExtensionMetrics(t *testing.T) { monitoring.IncDownloadCounter(ctlr.Metrics, "alpine") monitoring.IncUploadCounter(ctlr.Metrics, "alpine") - err = CopyFiles("../../../test/data/zot-test", path.Join(rootDir, "alpine")) + err = test.CopyFiles("../../../test/data/zot-test", path.Join(rootDir, "alpine")) if err != nil { panic(err) } @@ -82,8 +82,8 @@ func TestExtensionMetrics(t *testing.T) { So(respStr, ShouldContainSubstring, "zot_storage_lock_latency_seconds_bucket") }) Convey("Make a new controller with disabled metrics extension", t, func() { - port := GetFreePort() - baseURL := GetBaseURL(port) + port := test.GetFreePort() + baseURL := test.GetBaseURL(port) conf := config.New() conf.HTTP.Port = port @@ -102,7 +102,7 @@ func TestExtensionMetrics(t *testing.T) { go startServer(ctlr) defer stopServer(ctlr) - WaitTillServerReady(baseURL) + test.WaitTillServerReady(baseURL) So(ctlr.Metrics.IsEnabled(), ShouldBeFalse) diff --git a/pkg/extensions/search/common/common_test.go b/pkg/extensions/search/common/common_test.go index 7d9736ec..2dd7643b 100644 --- a/pkg/extensions/search/common/common_test.go +++ b/pkg/extensions/search/common/common_test.go @@ -22,7 +22,7 @@ import ( "zotregistry.io/zot/pkg/extensions/search/common" "zotregistry.io/zot/pkg/log" "zotregistry.io/zot/pkg/storage" - . "zotregistry.io/zot/test" + . "zotregistry.io/zot/pkg/test" ) // nolint:gochecknoglobals diff --git a/pkg/extensions/search/cve/cve_test.go b/pkg/extensions/search/cve/cve_test.go index 185d8a81..74eff7fa 100644 --- a/pkg/extensions/search/cve/cve_test.go +++ b/pkg/extensions/search/cve/cve_test.go @@ -25,7 +25,7 @@ import ( cveinfo "zotregistry.io/zot/pkg/extensions/search/cve" "zotregistry.io/zot/pkg/log" "zotregistry.io/zot/pkg/storage" - . "zotregistry.io/zot/test" + . "zotregistry.io/zot/pkg/test" ) // nolint:gochecknoglobals diff --git a/pkg/extensions/search/digest/digest_test.go b/pkg/extensions/search/digest/digest_test.go index b08a9105..64ef2bc2 100644 --- a/pkg/extensions/search/digest/digest_test.go +++ b/pkg/extensions/search/digest/digest_test.go @@ -21,7 +21,7 @@ import ( digestinfo "zotregistry.io/zot/pkg/extensions/search/digest" "zotregistry.io/zot/pkg/log" "zotregistry.io/zot/pkg/storage" - . "zotregistry.io/zot/test" + . "zotregistry.io/zot/pkg/test" ) // nolint:gochecknoglobals diff --git a/pkg/extensions/sync/sync_internal_test.go b/pkg/extensions/sync/sync_internal_test.go index c4ee1c32..b321d221 100644 --- a/pkg/extensions/sync/sync_internal_test.go +++ b/pkg/extensions/sync/sync_internal_test.go @@ -24,7 +24,7 @@ import ( "zotregistry.io/zot/pkg/extensions/monitoring" "zotregistry.io/zot/pkg/log" "zotregistry.io/zot/pkg/storage" - . "zotregistry.io/zot/test" + . "zotregistry.io/zot/pkg/test" ) const ( diff --git a/pkg/extensions/sync/sync_test.go b/pkg/extensions/sync/sync_test.go index 7eddcad9..8d46c4d9 100644 --- a/pkg/extensions/sync/sync_test.go +++ b/pkg/extensions/sync/sync_test.go @@ -12,6 +12,7 @@ import ( "fmt" "io" "io/ioutil" + "net/http" "os" "os/exec" "path" @@ -24,6 +25,7 @@ import ( godigest "github.com/opencontainers/go-digest" ispec "github.com/opencontainers/image-spec/specs-go/v1" artifactspec "github.com/oras-project/artifacts-spec/specs-go/v1" + perr "github.com/pkg/errors" "github.com/sigstore/cosign/cmd/cosign/cli/generate" "github.com/sigstore/cosign/cmd/cosign/cli/options" "github.com/sigstore/cosign/cmd/cosign/cli/sign" @@ -34,7 +36,7 @@ import ( "zotregistry.io/zot/pkg/api/config" extconf "zotregistry.io/zot/pkg/extensions/config" "zotregistry.io/zot/pkg/extensions/sync" - . "zotregistry.io/zot/test" + "zotregistry.io/zot/pkg/test" ) const ( @@ -49,7 +51,10 @@ const ( testCveImage = "zot-cve-test" ) -var errSync = errors.New("sync error, src oci repo differs from dest one") +var ( + errSync = errors.New("sync error, src oci repo differs from dest one") + errBadStatus = errors.New("bad http status") +) type TagsList struct { Name string @@ -85,7 +90,7 @@ func copyFile(sourceFilePath, destFilePath string) error { } func startUpstreamServer(secure, basicAuth bool) (*api.Controller, string, string, string, *resty.Client) { - srcPort := GetFreePort() + srcPort := test.GetFreePort() srcConfig := config.New() @@ -93,7 +98,7 @@ func startUpstreamServer(secure, basicAuth bool) (*api.Controller, string, strin var srcBaseURL string if secure { - srcBaseURL = GetSecureBaseURL(srcPort) + srcBaseURL = test.GetSecureBaseURL(srcPort) srcConfig.HTTP.TLS = &config.TLSConfig{ Cert: ServerCert, @@ -118,12 +123,12 @@ func startUpstreamServer(secure, basicAuth bool) (*api.Controller, string, strin client.SetCertificates(cert) } else { - srcBaseURL = GetBaseURL(srcPort) + srcBaseURL = test.GetBaseURL(srcPort) } var htpasswdPath string if basicAuth { - htpasswdPath = MakeHtpasswdFile() + htpasswdPath = test.MakeHtpasswdFile() srcConfig.HTTP.Auth = &config.AuthConfig{ HTPasswd: config.AuthHTPasswd{ Path: htpasswdPath, @@ -138,7 +143,7 @@ func startUpstreamServer(secure, basicAuth bool) (*api.Controller, string, strin panic(err) } - err = CopyFiles("../../../test/data", srcDir) + err = test.CopyFiles("../../../test/data", srcDir) if err != nil { panic(err) } @@ -168,7 +173,7 @@ func startUpstreamServer(secure, basicAuth bool) (*api.Controller, string, strin } func startDownstreamServer(secure bool, syncConfig *sync.Config) (*api.Controller, string, string, *resty.Client) { - destPort := GetFreePort() + destPort := test.GetFreePort() destConfig := config.New() @@ -176,7 +181,7 @@ func startDownstreamServer(secure bool, syncConfig *sync.Config) (*api.Controlle var destBaseURL string if secure { - destBaseURL = GetSecureBaseURL(destPort) + destBaseURL = test.GetSecureBaseURL(destPort) destConfig.HTTP.TLS = &config.TLSConfig{ Cert: ServerCert, @@ -201,7 +206,7 @@ func startDownstreamServer(secure bool, syncConfig *sync.Config) (*api.Controlle client.SetCertificates(cert) } else { - destBaseURL = GetBaseURL(destPort) + destBaseURL = test.GetBaseURL(destPort) } destConfig.HTTP.Port = destPort @@ -824,8 +829,8 @@ func TestBasicAuth(t *testing.T) { }) Convey("Verify sync basic auth with wrong file credentials", func() { - destPort := GetFreePort() - destBaseURL := GetBaseURL(destPort) + destPort := test.GetFreePort() + destBaseURL := test.GetBaseURL(destPort) destConfig := config.New() destConfig.HTTP.Port = destPort @@ -1453,10 +1458,10 @@ func TestSubPaths(t *testing.T) { Convey("Verify sync with storage subPaths", t, func() { updateDuration, _ := time.ParseDuration("30m") - srcPort := GetFreePort() + srcPort := test.GetFreePort() srcConfig := config.New() client := resty.New() - srcBaseURL := GetBaseURL(srcPort) + srcBaseURL := test.GetBaseURL(srcPort) srcConfig.HTTP.Port = srcPort @@ -1467,7 +1472,7 @@ func TestSubPaths(t *testing.T) { subpath := "/subpath" - err = CopyFiles("../../../test/data", path.Join(srcDir, subpath)) + err = test.CopyFiles("../../../test/data", path.Join(srcDir, subpath)) if err != nil { panic(err) } @@ -1522,7 +1527,7 @@ func TestSubPaths(t *testing.T) { Registries: []sync.RegistryConfig{syncRegistryConfig}, } - destPort := GetFreePort() + destPort := test.GetFreePort() destConfig := config.New() destDir, err := ioutil.TempDir("", "oci-dest-repo-test") @@ -1547,7 +1552,7 @@ func TestSubPaths(t *testing.T) { }, } - destBaseURL := GetBaseURL(destPort) + destBaseURL := test.GetBaseURL(destPort) destConfig.HTTP.Port = destPort destConfig.Extensions = &extconf.ExtensionConfig{} @@ -2443,7 +2448,7 @@ func pushRepo(url, repoName string) godigest.Digest { panic(err) } - loc := Location(url, resp) + loc := test.Location(url, resp) _, err = resty.R().Get(loc) if err != nil { @@ -2459,11 +2464,41 @@ func pushRepo(url, repoName string) godigest.Digest { panic(err) } + // upload image config blob + resp, err = resty.R(). + Post(fmt.Sprintf("%s/v2/%s/blobs/uploads/", url, repoName)) + if err != nil { + panic(err) + } + + if resp.StatusCode() != http.StatusAccepted { + panic(perr.Wrapf(errBadStatus, "invalid status code: %d", resp.StatusCode())) + } + + loc = test.Location(url, resp) + cblob, cdigest := test.GetRandomImageConfig() + + resp, err = resty.R(). + SetContentLength(true). + SetHeader("Content-Length", fmt.Sprintf("%d", len(cblob))). + SetHeader("Content-Type", "application/octet-stream"). + SetQueryParam("digest", cdigest.String()). + SetBody(cblob). + Put(loc) + if err != nil { + panic(err) + } + + if resp.StatusCode() != http.StatusCreated { + panic(perr.Wrapf(errBadStatus, "invalid status code: %d", resp.StatusCode())) + } + // create a manifest manifest := ispec.Manifest{ Config: ispec.Descriptor{ - Digest: digest, - Size: int64(len(content)), + MediaType: "application/vnd.oci.image.config.v1+json", + Digest: cdigest, + Size: int64(len(cblob)), }, Layers: []ispec.Descriptor{ { diff --git a/pkg/log/log_test.go b/pkg/log/log_test.go index 10c32c68..8eccc28a 100644 --- a/pkg/log/log_test.go +++ b/pkg/log/log_test.go @@ -20,7 +20,7 @@ import ( "gopkg.in/resty.v1" "zotregistry.io/zot/pkg/api" "zotregistry.io/zot/pkg/api/config" - . "zotregistry.io/zot/test" + . "zotregistry.io/zot/pkg/test" ) const ( diff --git a/pkg/storage/storage_fs.go b/pkg/storage/storage_fs.go index f9a2c4c3..5d37a52e 100644 --- a/pkg/storage/storage_fs.go +++ b/pkg/storage/storage_fs.go @@ -466,6 +466,60 @@ func (is *ImageStoreFS) GetImageManifest(repo string, reference string) ([]byte, return buf, digest.String(), mediaType, nil } +func (is *ImageStoreFS) validateOCIManifest(repo string, reference string, manifest *ispec.Manifest) (string, error) { + if manifest.SchemaVersion != SchemaVersion { + is.log.Error().Int("SchemaVersion", manifest.SchemaVersion).Msg("invalid manifest") + + return "", zerr.ErrBadManifest + } + + // validate image config + config := manifest.Config + if config.MediaType != ispec.MediaTypeImageConfig { + return "", zerr.ErrBadManifest + } + + digest := config.Digest + + blobPath := is.BlobPath(repo, digest) + if _, err := os.Stat(blobPath); err != nil { + is.log.Error().Err(err).Str("blobPath", blobPath).Msg("unable to find blob") + + return digest.String(), zerr.ErrBlobNotFound + } + + blobFile, err := os.Open(blobPath) + if err != nil { + is.log.Error().Err(err).Str("blobPath", blobPath).Msg("unable to find blob") + + return digest.String(), zerr.ErrBlobNotFound + } + + defer blobFile.Close() + + dec := json.NewDecoder(blobFile) + + var cspec ispec.Image + if err := dec.Decode(&cspec); err != nil { + return "", zerr.ErrBadManifest + } + + // validate the layers + for _, l := range manifest.Layers { + digest = l.Digest + blobPath = is.BlobPath(repo, digest) + is.log.Info().Str("blobPath", blobPath).Str("reference", reference).Msg("manifest layers") + + if _, err := os.Stat(blobPath); err != nil { + is.log.Error().Err(err).Str("blobPath", blobPath).Msg("unable to find blob") + + return digest.String(), zerr.ErrBlobNotFound + } + } + + return "", nil +} + // PutImageManifest adds an image manifest to the repository. func (is *ImageStoreFS) PutImageManifest(repo string, reference string, mediaType string, body []byte) (string, error) { @@ -475,6 +529,7 @@ func (is *ImageStoreFS) PutImageManifest(repo string, reference string, mediaTyp return "", err } + // validate the manifest if !IsSupportedMediaType(mediaType) { is.log.Debug().Interface("actual", mediaType). Interface("expected", ispec.MediaTypeImageManifest).Msg("bad manifest media type") @@ -489,29 +544,16 @@ func (is *ImageStoreFS) PutImageManifest(repo string, reference string, mediaTyp } if mediaType == ispec.MediaTypeImageManifest { - var m ispec.Manifest - if err := json.Unmarshal(body, &m); err != nil { + var manifest ispec.Manifest + if err := json.Unmarshal(body, &manifest); err != nil { is.log.Error().Err(err).Msg("unable to unmarshal JSON") return "", zerr.ErrBadManifest } - if m.SchemaVersion != SchemaVersion { - is.log.Error().Int("SchemaVersion", m.SchemaVersion).Msg("invalid manifest") - - return "", zerr.ErrBadManifest - } - - for _, l := range m.Layers { - digest := l.Digest - blobPath := is.BlobPath(repo, digest) - is.log.Info().Str("blobPath", blobPath).Str("reference", reference).Msg("manifest layers") - - if _, err := os.Stat(blobPath); err != nil { - is.log.Error().Err(err).Str("blobPath", blobPath).Msg("unable to find blob") - - return digest.String(), zerr.ErrBlobNotFound - } + digest, err := is.validateOCIManifest(repo, reference, &manifest) + if err != nil { + return digest, err } } else if mediaType == artifactspec.MediaTypeArtifactManifest { var m notation.Descriptor diff --git a/pkg/storage/storage_fs_test.go b/pkg/storage/storage_fs_test.go index 3e30de74..deef505d 100644 --- a/pkg/storage/storage_fs_test.go +++ b/pkg/storage/storage_fs_test.go @@ -21,6 +21,7 @@ import ( "zotregistry.io/zot/pkg/extensions/monitoring" "zotregistry.io/zot/pkg/log" "zotregistry.io/zot/pkg/storage" + "zotregistry.io/zot/pkg/test" ) func TestStorageFSAPIs(t *testing.T) { @@ -57,10 +58,20 @@ func TestStorageFSAPIs(t *testing.T) { annotationsMap := make(map[string]string) annotationsMap[ispec.AnnotationRefName] = "1.0" + + cblob, cdigest := test.GetRandomImageConfig() + _, clen, err := imgStore.FullBlobUpload("test", bytes.NewReader(cblob), cdigest.String()) + So(err, ShouldBeNil) + So(clen, ShouldEqual, len(cblob)) + hasBlob, _, err := imgStore.CheckBlob("test", cdigest.String()) + So(err, ShouldBeNil) + So(hasBlob, ShouldEqual, true) + manifest := ispec.Manifest{ Config: ispec.Descriptor{ - Digest: digest, - Size: int64(buflen), + MediaType: "application/vnd.oci.image.config.v1+json", + Digest: cdigest, + Size: int64(len(cblob)), }, Layers: []ispec.Descriptor{ { @@ -188,12 +199,19 @@ func TestDedupeLinks(t *testing.T) { _, _, err = imgStore.GetBlob("dedupe1", digest.String(), "application/vnd.oci.image.layer.v1.tar+gzip") So(err, ShouldBeNil) - manifest := ispec.Manifest{} - manifest.SchemaVersion = 2 - manifest = ispec.Manifest{ + cblob, cdigest := test.GetRandomImageConfig() + _, clen, err := imgStore.FullBlobUpload("dedupe1", bytes.NewReader(cblob), cdigest.String()) + So(err, ShouldBeNil) + So(clen, ShouldEqual, len(cblob)) + hasBlob, _, err := imgStore.CheckBlob("dedupe1", cdigest.String()) + So(err, ShouldBeNil) + So(hasBlob, ShouldEqual, true) + + manifest := ispec.Manifest{ Config: ispec.Descriptor{ - Digest: digest, - Size: int64(buflen), + MediaType: "application/vnd.oci.image.config.v1+json", + Digest: cdigest, + Size: int64(len(cblob)), }, Layers: []ispec.Descriptor{ { @@ -237,12 +255,19 @@ func TestDedupeLinks(t *testing.T) { _, _, err = imgStore.GetBlob("dedupe2", digest.String(), "application/vnd.oci.image.layer.v1.tar+gzip") So(err, ShouldBeNil) - manifest = ispec.Manifest{} - manifest.SchemaVersion = 2 + cblob, cdigest = test.GetRandomImageConfig() + _, clen, err = imgStore.FullBlobUpload("dedupe2", bytes.NewReader(cblob), cdigest.String()) + So(err, ShouldBeNil) + So(clen, ShouldEqual, len(cblob)) + hasBlob, _, err = imgStore.CheckBlob("dedupe2", cdigest.String()) + So(err, ShouldBeNil) + So(hasBlob, ShouldEqual, true) + manifest = ispec.Manifest{ Config: ispec.Descriptor{ - Digest: digest, - Size: int64(buflen), + MediaType: "application/vnd.oci.image.config.v1+json", + Digest: cdigest, + Size: int64(len(cblob)), }, Layers: []ispec.Descriptor{ { diff --git a/pkg/storage/storage_test.go b/pkg/storage/storage_test.go index c4e49fa0..d978a945 100644 --- a/pkg/storage/storage_test.go +++ b/pkg/storage/storage_test.go @@ -28,6 +28,7 @@ import ( "zotregistry.io/zot/pkg/log" "zotregistry.io/zot/pkg/storage" "zotregistry.io/zot/pkg/storage/s3" + "zotregistry.io/zot/pkg/test" ) func cleanupStorage(store driver.StorageDriver, name string) { @@ -267,12 +268,21 @@ func TestStorageAPIs(t *testing.T) { }) Convey("Good image manifest", func() { + cblob, cdigest := test.GetRandomImageConfig() + _, clen, err := imgStore.FullBlobUpload("test", bytes.NewReader(cblob), cdigest.String()) + So(err, ShouldBeNil) + So(clen, ShouldEqual, len(cblob)) + hasBlob, _, err := imgStore.CheckBlob("test", cdigest.String()) + So(err, ShouldBeNil) + So(hasBlob, ShouldEqual, true) + annotationsMap := make(map[string]string) annotationsMap[ispec.AnnotationRefName] = "1.0" manifest := ispec.Manifest{ Config: ispec.Descriptor{ - Digest: digest, - Size: int64(buflen), + MediaType: "application/vnd.oci.image.config.v1+json", + Digest: cdigest, + Size: int64(len(cblob)), }, Layers: []ispec.Descriptor{ { @@ -305,7 +315,7 @@ func TestStorageAPIs(t *testing.T) { _, err = imgStore.PutImageManifest("test", "2.0", ispec.MediaTypeImageManifest, manifestBuf) So(err, ShouldBeNil) - _, err := imgStore.PutImageManifest("test", "3.0", ispec.MediaTypeImageManifest, manifestBuf) + _, err = imgStore.PutImageManifest("test", "3.0", ispec.MediaTypeImageManifest, manifestBuf) So(err, ShouldBeNil) _, err = imgStore.GetImageTags("inexistent") @@ -330,7 +340,7 @@ func TestStorageAPIs(t *testing.T) { So(len(tags), ShouldEqual, 2) // We deleted only one tag, make sure blob should not be removed. - hasBlob, _, err := imgStore.CheckBlob("test", digest.String()) + hasBlob, _, err = imgStore.CheckBlob("test", digest.String()) So(err, ShouldBeNil) So(hasBlob, ShouldEqual, true) @@ -446,10 +456,19 @@ func TestStorageAPIs(t *testing.T) { }) Convey("Good image manifest", func() { + cblob, cdigest := test.GetRandomImageConfig() + _, clen, err := imgStore.FullBlobUpload("test", bytes.NewReader(cblob), cdigest.String()) + So(err, ShouldBeNil) + So(clen, ShouldEqual, len(cblob)) + hasBlob, _, err := imgStore.CheckBlob("test", cdigest.String()) + So(err, ShouldBeNil) + So(hasBlob, ShouldEqual, true) + manifest := ispec.Manifest{ Config: ispec.Descriptor{ - Digest: digest, - Size: int64(buflen), + MediaType: "application/vnd.oci.image.config.v1+json", + Digest: cdigest, + Size: int64(len(cblob)), }, Layers: []ispec.Descriptor{ { @@ -531,12 +550,19 @@ func TestStorageAPIs(t *testing.T) { So(err, ShouldBeNil) So(blob, ShouldEqual, buflen) - manifest := ispec.Manifest{} - manifest.SchemaVersion = 2 - manifest = ispec.Manifest{ + cblob, cdigest := test.GetRandomImageConfig() + _, clen, err := imgStore.FullBlobUpload("replace", bytes.NewReader(cblob), cdigest.String()) + So(err, ShouldBeNil) + So(clen, ShouldEqual, len(cblob)) + hasBlob, _, err := imgStore.CheckBlob("replace", cdigest.String()) + So(err, ShouldBeNil) + So(hasBlob, ShouldEqual, true) + + manifest := ispec.Manifest{ Config: ispec.Descriptor{ - Digest: digest, - Size: int64(buflen), + MediaType: "application/vnd.oci.image.config.v1+json", + Digest: cdigest, + Size: int64(len(cblob)), }, Layers: []ispec.Descriptor{ { @@ -574,10 +600,19 @@ func TestStorageAPIs(t *testing.T) { So(err, ShouldBeNil) So(blob, ShouldEqual, buflen) + cblob, cdigest = test.GetRandomImageConfig() + _, clen, err = imgStore.FullBlobUpload("replace", bytes.NewReader(cblob), cdigest.String()) + So(err, ShouldBeNil) + So(clen, ShouldEqual, len(cblob)) + hasBlob, _, err = imgStore.CheckBlob("replace", cdigest.String()) + So(err, ShouldBeNil) + So(hasBlob, ShouldEqual, true) + manifest = ispec.Manifest{ Config: ispec.Descriptor{ - Digest: digest, - Size: int64(buflen), + MediaType: "application/vnd.oci.image.config.v1+json", + Digest: cdigest, + Size: int64(len(cblob)), }, Layers: []ispec.Descriptor{ { diff --git a/test/common.go b/pkg/test/common.go similarity index 61% rename from test/common.go rename to pkg/test/common.go index 64d341be..f9c3e4c2 100644 --- a/test/common.go +++ b/pkg/test/common.go @@ -1,13 +1,20 @@ package test import ( + "crypto/rand" + "encoding/json" "fmt" "io" "io/ioutil" + "log" + "math/big" + "net/url" "os" "path" "time" + godigest "github.com/opencontainers/go-digest" + imagespec "github.com/opencontainers/image-spec/specs-go/v1" "github.com/phayes/freeport" "gopkg.in/resty.v1" ) @@ -65,7 +72,17 @@ func Location(baseURL string, resp *resty.Response) string { // return the former - this needs to be clarified loc := resp.Header().Get("Location") - return baseURL + loc + uloc, err := url.Parse(loc) + if err != nil { + return "" + } + + path := uloc.Path + if query := uloc.RawQuery; query != "" { + path += "?" + query + } + + return baseURL + path } func CopyFiles(sourceDir string, destDir string) error { @@ -123,3 +140,67 @@ func WaitTillServerReady(url string) { time.Sleep(SleepTime) } } + +// Adapted from https://gist.github.com/dopey/c69559607800d2f2f90b1b1ed4e550fb +func randomString(n int) string { + const letters = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz-" + + ret := make([]byte, n) + + for count := 0; count < n; count++ { + num, err := rand.Int(rand.Reader, big.NewInt(int64(len(letters)))) + if err != nil { + panic(err) + } + + ret[count] = letters[num.Int64()] + } + + return string(ret) +} + +func GetRandomImageConfig() ([]byte, godigest.Digest) { + const maxLen = 16 + + randomAuthor := randomString(maxLen) + + config := imagespec.Image{ + Architecture: "amd64", + OS: "linux", + RootFS: imagespec.RootFS{ + Type: "layers", + DiffIDs: []godigest.Digest{}, + }, + Author: randomAuthor, + } + + configBlobContent, err := json.MarshalIndent(&config, "", "\t") + if err != nil { + log.Fatal(err) + } + + configBlobDigestRaw := godigest.FromBytes(configBlobContent) + + return configBlobContent, configBlobDigestRaw +} + +func GetImageConfig() ([]byte, godigest.Digest) { + config := imagespec.Image{ + Architecture: "amd64", + OS: "linux", + RootFS: imagespec.RootFS{ + Type: "layers", + DiffIDs: []godigest.Digest{}, + }, + Author: "some author", + } + + configBlobContent, err := json.MarshalIndent(&config, "", "\t") + if err != nil { + log.Fatal(err) + } + + configBlobDigestRaw := godigest.FromBytes(configBlobContent) + + return configBlobContent, configBlobDigestRaw +} diff --git a/test/common_test.go b/pkg/test/common_test.go similarity index 82% rename from test/common_test.go rename to pkg/test/common_test.go index cfb800fd..fa55cdf5 100644 --- a/test/common_test.go +++ b/pkg/test/common_test.go @@ -10,12 +10,12 @@ import ( "testing" . "github.com/smartystreets/goconvey/convey" - . "zotregistry.io/zot/test" + "zotregistry.io/zot/pkg/test" ) func TestCopyFiles(t *testing.T) { Convey("sourceDir does not exist", t, func() { - err := CopyFiles("/path/to/some/unexisting/directory", os.TempDir()) + err := test.CopyFiles("/path/to/some/unexisting/directory", os.TempDir()) So(err, ShouldNotBeNil) }) Convey("destDir is a file", t, func() { @@ -24,13 +24,13 @@ func TestCopyFiles(t *testing.T) { panic(err) } - err = CopyFiles("data", dir) + err = test.CopyFiles("../../test/data", dir) if err != nil { panic(err) } defer os.RemoveAll(dir) - err = CopyFiles(dir, "/etc/passwd") + err = test.CopyFiles(dir, "/etc/passwd") So(err, ShouldNotBeNil) }) Convey("sourceDir does not have read permissions", t, func() { @@ -43,7 +43,7 @@ func TestCopyFiles(t *testing.T) { err = os.Chmod(dir, 0o300) So(err, ShouldBeNil) - err = CopyFiles(dir, os.TempDir()) + err = test.CopyFiles(dir, os.TempDir()) So(err, ShouldNotBeNil) }) Convey("sourceDir has a subfolder that does not have read permissions", t, func() { @@ -57,7 +57,7 @@ func TestCopyFiles(t *testing.T) { err = os.Mkdir(path.Join(dir, sdir), 0o300) So(err, ShouldBeNil) - err = CopyFiles(dir, os.TempDir()) + err = test.CopyFiles(dir, os.TempDir()) So(err, ShouldNotBeNil) }) Convey("sourceDir has a file that does not have read permissions", t, func() { @@ -76,7 +76,7 @@ func TestCopyFiles(t *testing.T) { err = os.Chmod(filePath, 0o300) So(err, ShouldBeNil) - err = CopyFiles(dir, os.TempDir()) + err = test.CopyFiles(dir, os.TempDir()) So(err, ShouldNotBeNil) }) }