refactor: move helper functions under common, in usage specific named files (#1540)

Signed-off-by: Alex Stan <alexandrustan96@yahoo.ro>
This commit is contained in:
alexstan12
2023-06-22 14:29:45 +03:00
committed by GitHub
parent 377aff1853
commit ea7dbf9e5c
16 changed files with 498 additions and 446 deletions
-203
View File
@@ -1,26 +1,14 @@
package common
import (
"crypto/tls"
"crypto/x509"
"encoding/json"
"errors"
"fmt"
"io"
"io/fs"
"net/http"
"os"
"path"
"path/filepath"
"strings"
"syscall"
"time"
"unicode/utf8"
"github.com/gorilla/mux"
ispec "github.com/opencontainers/image-spec/specs-go/v1"
"zotregistry.io/zot/pkg/log"
)
const (
@@ -32,10 +20,6 @@ const (
caCertFilename = "ca.crt"
)
func AllowedMethods(methods ...string) []string {
return append(methods, http.MethodOptions)
}
func Contains[T comparable](elems []T, v T) bool {
for _, s := range elems {
if v == s {
@@ -70,162 +54,10 @@ func RemoveFrom(inputSlice []string, item string) []string {
return newSlice
}
func GetTLSConfig(certsPath string, caCertPool *x509.CertPool) (*tls.Config, error) {
clientCert := filepath.Join(certsPath, clientCertFilename)
clientKey := filepath.Join(certsPath, clientKeyFilename)
caCertFile := filepath.Join(certsPath, caCertFilename)
cert, err := tls.LoadX509KeyPair(clientCert, clientKey)
if err != nil {
return nil, err
}
caCert, err := os.ReadFile(caCertFile)
if err != nil {
return nil, err
}
caCertPool.AppendCertsFromPEM(caCert)
return &tls.Config{
Certificates: []tls.Certificate{cert},
RootCAs: caCertPool,
MinVersion: tls.VersionTLS12,
}, nil
}
func loadPerHostCerts(caCertPool *x509.CertPool, host string) *tls.Config {
// Check if the /home/user/.config/containers/certs.d/$IP:$PORT dir exists
home := os.Getenv("HOME")
clientCertsDir := filepath.Join(home, homeCertsDir, host)
if DirExists(clientCertsDir) {
tlsConfig, err := GetTLSConfig(clientCertsDir, caCertPool)
if err == nil {
return tlsConfig
}
}
// Check if the /etc/containers/certs.d/$IP:$PORT dir exists
clientCertsDir = filepath.Join(certsPath, host)
if DirExists(clientCertsDir) {
tlsConfig, err := GetTLSConfig(clientCertsDir, caCertPool)
if err == nil {
return tlsConfig
}
}
return nil
}
func CreateHTTPClient(verifyTLS bool, host string, certDir string) (*http.Client, error) {
htr := http.DefaultTransport.(*http.Transport).Clone() //nolint: forcetypeassert
if !verifyTLS {
htr.TLSClientConfig = &tls.Config{InsecureSkipVerify: true} //nolint: gosec
return &http.Client{
Timeout: httpTimeout,
Transport: htr,
}, nil
}
// Add a copy of the system cert pool
caCertPool, _ := x509.SystemCertPool()
tlsConfig := loadPerHostCerts(caCertPool, host)
if tlsConfig == nil {
tlsConfig = &tls.Config{RootCAs: caCertPool, MinVersion: tls.VersionTLS12}
}
htr.TLSClientConfig = tlsConfig
if certDir != "" {
clientCert := path.Join(certDir, "client.cert")
clientKey := path.Join(certDir, "client.key")
caCertPath := path.Join(certDir, "ca.crt")
caCert, err := os.ReadFile(caCertPath)
if err != nil {
return nil, err
}
caCertPool.AppendCertsFromPEM(caCert)
cert, err := tls.LoadX509KeyPair(clientCert, clientKey)
if err != nil {
return nil, err
}
htr.TLSClientConfig.Certificates = append(htr.TLSClientConfig.Certificates, cert)
}
return &http.Client{
Timeout: httpTimeout,
Transport: htr,
}, nil
}
func TypeOf(v interface{}) string {
return fmt.Sprintf("%T", v)
}
func MakeHTTPGetRequest(httpClient *http.Client, username string, password string, resultPtr interface{},
blobURL string, mediaType string, log log.Logger,
) ([]byte, string, int, error) {
req, err := http.NewRequest(http.MethodGet, blobURL, nil) //nolint
if err != nil {
return nil, "", 0, err
}
if mediaType != "" {
req.Header.Set("Accept", mediaType)
}
if username != "" && password != "" {
req.SetBasicAuth(username, password)
}
resp, err := httpClient.Do(req)
if err != nil {
log.Error().Str("errorType", TypeOf(err)).
Err(err).Str("blobURL", blobURL).Msg("couldn't get blob")
return nil, "", -1, err
}
body, err := io.ReadAll(resp.Body)
if err != nil {
log.Error().Str("errorType", TypeOf(err)).
Err(err).Str("blobURL", blobURL).Msg("couldn't get blob")
return nil, "", resp.StatusCode, err
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
log.Error().Str("status code", fmt.Sprint(resp.StatusCode)).
Err(err).Str("blobURL", blobURL).Msg("couldn't get blob")
return nil, "", resp.StatusCode, errors.New(string(body)) //nolint:goerr113
}
// read blob
if len(body) > 0 {
err = json.Unmarshal(body, &resultPtr)
if err != nil {
log.Error().Str("errorType", TypeOf(err)).Str("blobURL", blobURL).
Err(err).Msg("couldn't unmarshal remote blob")
return body, "", resp.StatusCode, err
}
}
return body, resp.Header.Get("Content-Type"), resp.StatusCode, err
}
func DirExists(d string) bool {
if !utf8.ValidString(d) {
return false
@@ -269,38 +101,3 @@ func MarshalThroughStruct(obj interface{}, throughStruct interface{}) ([]byte, e
return toJSON, nil
}
func GetManifestArtifactType(manifestContent ispec.Manifest) string {
if manifestContent.ArtifactType != "" {
return manifestContent.ArtifactType
}
return manifestContent.Config.MediaType
}
func AddExtensionSecurityHeaders() mux.MiddlewareFunc { //nolint:varnamelen
return func(next http.Handler) http.Handler {
return http.HandlerFunc(func(resp http.ResponseWriter, req *http.Request) {
resp.Header().Set("X-Content-Type-Options", "nosniff")
next.ServeHTTP(resp, req)
})
}
}
func ACHeadersHandler(allowedMethods ...string) mux.MiddlewareFunc {
headerValue := strings.Join(allowedMethods, ",")
return func(next http.Handler) http.Handler {
return http.HandlerFunc(func(resp http.ResponseWriter, req *http.Request) {
resp.Header().Set("Access-Control-Allow-Methods", headerValue)
resp.Header().Set("Access-Control-Allow-Headers", "Authorization,content-type")
if req.Method == http.MethodOptions {
return
}
next.ServeHTTP(resp, req)
})
}
}
-73
View File
@@ -1,19 +1,14 @@
package common_test
import (
"crypto/x509"
"os"
"path"
"testing"
ispec "github.com/opencontainers/image-spec/specs-go/v1"
. "github.com/smartystreets/goconvey/convey"
"zotregistry.io/zot/pkg/api"
"zotregistry.io/zot/pkg/api/config"
"zotregistry.io/zot/pkg/common"
"zotregistry.io/zot/pkg/log"
"zotregistry.io/zot/pkg/test"
)
func TestCommon(t *testing.T) {
@@ -46,21 +41,6 @@ func TestCommon(t *testing.T) {
So(err, ShouldNotBeNil)
})
Convey("test getTLSConfig()", t, func() {
caCertPool, _ := x509.SystemCertPool()
tlsConfig, err := common.GetTLSConfig("wrongPath", caCertPool)
So(tlsConfig, ShouldBeNil)
So(err, ShouldNotBeNil)
tempDir := t.TempDir()
err = test.CopyFiles("../../test/data", tempDir)
So(err, ShouldBeNil)
err = os.Chmod(path.Join(tempDir, "ca.crt"), 0o000)
So(err, ShouldBeNil)
_, err = common.GetTLSConfig(tempDir, caCertPool)
So(err, ShouldNotBeNil)
})
Convey("test dirExists()", t, func() {
exists := common.DirExists("testdir")
So(exists, ShouldBeFalse)
@@ -72,61 +52,8 @@ func TestCommon(t *testing.T) {
So(isDir, ShouldBeFalse)
})
Convey("test CreateHTTPClient() no permissions on certificate", t, func() {
tempDir := t.TempDir()
err := test.CopyFiles("../../test/data", tempDir)
So(err, ShouldBeNil)
err = os.Chmod(path.Join(tempDir, "ca.crt"), 0o000)
So(err, ShouldBeNil)
_, err = common.CreateHTTPClient(true, "localhost", tempDir)
So(err, ShouldNotBeNil)
})
Convey("test CreateHTTPClient() no permissions on key", t, func() {
tempDir := t.TempDir()
err := test.CopyFiles("../../test/data", tempDir)
So(err, ShouldBeNil)
err = os.Chmod(path.Join(tempDir, "client.key"), 0o000)
So(err, ShouldBeNil)
_, err = common.CreateHTTPClient(true, "localhost", tempDir)
So(err, ShouldNotBeNil)
})
Convey("test MakeHTTPGetRequest() no permissions on key", t, func() {
port := test.GetFreePort()
baseURL := test.GetBaseURL(port)
conf := config.New()
conf.HTTP.Port = port
ctlr := api.NewController(conf)
tempDir := t.TempDir()
err := test.CopyFiles("../../test/data", tempDir)
So(err, ShouldBeNil)
ctlr.Config.Storage.RootDirectory = tempDir
cm := test.NewControllerManager(ctlr)
cm.StartServer()
defer cm.StopServer()
test.WaitTillServerReady(baseURL)
var resultPtr interface{}
httpClient, err := common.CreateHTTPClient(true, "localhost", tempDir)
So(err, ShouldBeNil)
_, _, _, err = common.MakeHTTPGetRequest(httpClient, "", "",
resultPtr, baseURL+"/v2/", ispec.MediaTypeImageManifest, log.NewLogger("", ""))
So(err, ShouldBeNil)
})
Convey("Index func", t, func() {
So(common.Index([]string{"a", "b"}, "b"), ShouldEqual, 1)
So(common.Index([]string{"a", "b"}, "c"), ShouldEqual, -1)
})
Convey("Test image dir and digest", t, func() {
repo, digest := common.GetImageDirAndDigest("image")
So(repo, ShouldResemble, "image")
So(digest, ShouldResemble, "")
})
}
+168
View File
@@ -0,0 +1,168 @@
package common
import (
"crypto/tls"
"crypto/x509"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"os"
"path"
"path/filepath"
"zotregistry.io/zot/pkg/log"
)
func GetTLSConfig(certsPath string, caCertPool *x509.CertPool) (*tls.Config, error) {
clientCert := filepath.Join(certsPath, clientCertFilename)
clientKey := filepath.Join(certsPath, clientKeyFilename)
caCertFile := filepath.Join(certsPath, caCertFilename)
cert, err := tls.LoadX509KeyPair(clientCert, clientKey)
if err != nil {
return nil, err
}
caCert, err := os.ReadFile(caCertFile)
if err != nil {
return nil, err
}
caCertPool.AppendCertsFromPEM(caCert)
return &tls.Config{
Certificates: []tls.Certificate{cert},
RootCAs: caCertPool,
MinVersion: tls.VersionTLS12,
}, nil
}
func loadPerHostCerts(caCertPool *x509.CertPool, host string) *tls.Config {
// Check if the /home/user/.config/containers/certs.d/$IP:$PORT dir exists
home := os.Getenv("HOME")
clientCertsDir := filepath.Join(home, homeCertsDir, host)
if DirExists(clientCertsDir) {
tlsConfig, err := GetTLSConfig(clientCertsDir, caCertPool)
if err == nil {
return tlsConfig
}
}
// Check if the /etc/containers/certs.d/$IP:$PORT dir exists
clientCertsDir = filepath.Join(certsPath, host)
if DirExists(clientCertsDir) {
tlsConfig, err := GetTLSConfig(clientCertsDir, caCertPool)
if err == nil {
return tlsConfig
}
}
return nil
}
func CreateHTTPClient(verifyTLS bool, host string, certDir string) (*http.Client, error) {
htr := http.DefaultTransport.(*http.Transport).Clone() //nolint: forcetypeassert
if !verifyTLS {
htr.TLSClientConfig = &tls.Config{InsecureSkipVerify: true} //nolint: gosec
return &http.Client{
Timeout: httpTimeout,
Transport: htr,
}, nil
}
// Add a copy of the system cert pool
caCertPool, _ := x509.SystemCertPool()
tlsConfig := loadPerHostCerts(caCertPool, host)
if tlsConfig == nil {
tlsConfig = &tls.Config{RootCAs: caCertPool, MinVersion: tls.VersionTLS12}
}
htr.TLSClientConfig = tlsConfig
if certDir != "" {
clientCert := path.Join(certDir, "client.cert")
clientKey := path.Join(certDir, "client.key")
caCertPath := path.Join(certDir, "ca.crt")
caCert, err := os.ReadFile(caCertPath)
if err != nil {
return nil, err
}
caCertPool.AppendCertsFromPEM(caCert)
cert, err := tls.LoadX509KeyPair(clientCert, clientKey)
if err != nil {
return nil, err
}
htr.TLSClientConfig.Certificates = append(htr.TLSClientConfig.Certificates, cert)
}
return &http.Client{
Timeout: httpTimeout,
Transport: htr,
}, nil
}
func MakeHTTPGetRequest(httpClient *http.Client, username string, password string, resultPtr interface{},
blobURL string, mediaType string, log log.Logger,
) ([]byte, string, int, error) {
req, err := http.NewRequest(http.MethodGet, blobURL, nil) //nolint
if err != nil {
return nil, "", 0, err
}
if mediaType != "" {
req.Header.Set("Accept", mediaType)
}
if username != "" && password != "" {
req.SetBasicAuth(username, password)
}
resp, err := httpClient.Do(req)
if err != nil {
log.Error().Str("errorType", TypeOf(err)).
Err(err).Str("blobURL", blobURL).Msg("couldn't get blob")
return nil, "", -1, err
}
body, err := io.ReadAll(resp.Body)
if err != nil {
log.Error().Str("errorType", TypeOf(err)).
Err(err).Str("blobURL", blobURL).Msg("couldn't get blob")
return nil, "", resp.StatusCode, err
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
log.Error().Str("status code", fmt.Sprint(resp.StatusCode)).
Err(err).Str("blobURL", blobURL).Msg("couldn't get blob")
return nil, "", resp.StatusCode, errors.New(string(body)) //nolint:goerr113
}
// read blob
if len(body) > 0 {
err = json.Unmarshal(body, &resultPtr)
if err != nil {
log.Error().Str("errorType", TypeOf(err)).Str("blobURL", blobURL).
Err(err).Msg("couldn't unmarshal remote blob")
return body, "", resp.StatusCode, err
}
}
return body, resp.Header.Get("Content-Type"), resp.StatusCode, err
}
+82
View File
@@ -0,0 +1,82 @@
package common_test
import (
"crypto/x509"
"os"
"path"
"testing"
ispec "github.com/opencontainers/image-spec/specs-go/v1"
. "github.com/smartystreets/goconvey/convey"
"zotregistry.io/zot/pkg/api"
"zotregistry.io/zot/pkg/api/config"
"zotregistry.io/zot/pkg/common"
"zotregistry.io/zot/pkg/log"
"zotregistry.io/zot/pkg/test"
)
func TestHTTPClient(t *testing.T) {
Convey("test getTLSConfig()", t, func() {
caCertPool, _ := x509.SystemCertPool()
tlsConfig, err := common.GetTLSConfig("wrongPath", caCertPool)
So(tlsConfig, ShouldBeNil)
So(err, ShouldNotBeNil)
tempDir := t.TempDir()
err = test.CopyFiles("../../test/data", tempDir)
So(err, ShouldBeNil)
err = os.Chmod(path.Join(tempDir, "ca.crt"), 0o000)
So(err, ShouldBeNil)
_, err = common.GetTLSConfig(tempDir, caCertPool)
So(err, ShouldNotBeNil)
})
Convey("test CreateHTTPClient() no permissions on certificate", t, func() {
tempDir := t.TempDir()
err := test.CopyFiles("../../test/data", tempDir)
So(err, ShouldBeNil)
err = os.Chmod(path.Join(tempDir, "ca.crt"), 0o000)
So(err, ShouldBeNil)
_, err = common.CreateHTTPClient(true, "localhost", tempDir)
So(err, ShouldNotBeNil)
})
Convey("test CreateHTTPClient() no permissions on key", t, func() {
tempDir := t.TempDir()
err := test.CopyFiles("../../test/data", tempDir)
So(err, ShouldBeNil)
err = os.Chmod(path.Join(tempDir, "client.key"), 0o000)
So(err, ShouldBeNil)
_, err = common.CreateHTTPClient(true, "localhost", tempDir)
So(err, ShouldNotBeNil)
})
Convey("test MakeHTTPGetRequest() no permissions on key", t, func() {
port := test.GetFreePort()
baseURL := test.GetBaseURL(port)
conf := config.New()
conf.HTTP.Port = port
ctlr := api.NewController(conf)
tempDir := t.TempDir()
err := test.CopyFiles("../../test/data", tempDir)
So(err, ShouldBeNil)
ctlr.Config.Storage.RootDirectory = tempDir
cm := test.NewControllerManager(ctlr)
cm.StartServer()
defer cm.StopServer()
test.WaitTillServerReady(baseURL)
var resultPtr interface{}
httpClient, err := common.CreateHTTPClient(true, "localhost", tempDir)
So(err, ShouldBeNil)
_, _, _, err = common.MakeHTTPGetRequest(httpClient, "", "",
resultPtr, baseURL+"/v2/", ispec.MediaTypeImageManifest, log.NewLogger("", ""))
So(err, ShouldBeNil)
})
}
+68
View File
@@ -0,0 +1,68 @@
package common
import (
"net/http"
"strings"
"time"
"github.com/gorilla/mux"
jsoniter "github.com/json-iterator/go"
"zotregistry.io/zot/pkg/api/constants"
apiErr "zotregistry.io/zot/pkg/api/errors"
)
func AllowedMethods(methods ...string) []string {
return append(methods, http.MethodOptions)
}
func AddExtensionSecurityHeaders() mux.MiddlewareFunc { //nolint:varnamelen
return func(next http.Handler) http.Handler {
return http.HandlerFunc(func(resp http.ResponseWriter, req *http.Request) {
resp.Header().Set("X-Content-Type-Options", "nosniff")
next.ServeHTTP(resp, req)
})
}
}
func ACHeadersHandler(allowedMethods ...string) mux.MiddlewareFunc {
headerValue := strings.Join(allowedMethods, ",")
return func(next http.Handler) http.Handler {
return http.HandlerFunc(func(resp http.ResponseWriter, req *http.Request) {
resp.Header().Set("Access-Control-Allow-Methods", headerValue)
resp.Header().Set("Access-Control-Allow-Headers", "Authorization,content-type")
if req.Method == http.MethodOptions {
return
}
next.ServeHTTP(resp, req)
})
}
}
func AuthzFail(w http.ResponseWriter, realm string, delay int) {
time.Sleep(time.Duration(delay) * time.Second)
w.Header().Set("WWW-Authenticate", realm)
w.Header().Set("Content-Type", "application/json")
WriteJSON(w, http.StatusForbidden, apiErr.NewErrorList(apiErr.NewError(apiErr.DENIED)))
}
func WriteJSON(response http.ResponseWriter, status int, data interface{}) {
json := jsoniter.ConfigCompatibleWithStandardLibrary
body, err := json.Marshal(data)
if err != nil {
panic(err)
}
WriteData(response, status, constants.DefaultMediaType, body)
}
func WriteData(w http.ResponseWriter, status int, mediaType string, data []byte) {
w.Header().Set("Content-Type", mediaType)
w.WriteHeader(status)
_, _ = w.Write(data)
}
@@ -48,6 +48,14 @@ func GetImageDirAndReference(imageName string) (string, string, bool) {
return repo, tag, true
}
func GetManifestArtifactType(manifestContent ispec.Manifest) string {
if manifestContent.ArtifactType != "" {
return manifestContent.ArtifactType
}
return manifestContent.Config.MediaType
}
// GetImageLastUpdated This method will return last updated timestamp.
// The Created timestamp is used, but if it is missing, look at the
// history field and, if provided, return the timestamp of last entry in history.
+17
View File
@@ -0,0 +1,17 @@
package common_test
import (
"testing"
. "github.com/smartystreets/goconvey/convey"
"zotregistry.io/zot/pkg/common"
)
func TestOCI(t *testing.T) {
Convey("Test image dir and digest", t, func() {
repo, digest := common.GetImageDirAndDigest("image")
So(repo, ShouldResemble, "image")
So(digest, ShouldResemble, "")
})
}