diff --git a/pkg/exporter/api/controller_test.go b/pkg/exporter/api/controller_test.go index e97f438b..8b4446fb 100644 --- a/pkg/exporter/api/controller_test.go +++ b/pkg/exporter/api/controller_test.go @@ -94,8 +94,9 @@ func TestNewExporter(t *testing.T) { exporterConfig := api.DefaultConfig() So(exporterConfig, ShouldNotBeNil) - exporterPort := GetFreePort() - serverPort := GetFreePort() + ports := GetFreePorts(2) + exporterPort := ports[0] + serverPort := ports[1] exporterConfig.Exporter.Port = exporterPort exporterConfig.Exporter.Metrics.Path = strings.TrimPrefix(t.TempDir(), "/tmp/") exporterConfig.Server.Port = serverPort diff --git a/pkg/storage/imagestore/imagestore.go b/pkg/storage/imagestore/imagestore.go index ff16558a..da2a6c09 100644 --- a/pkg/storage/imagestore/imagestore.go +++ b/pkg/storage/imagestore/imagestore.go @@ -1936,8 +1936,22 @@ func (is *ImageStore) GetNextDigestWithBlobPaths(repos []string, lastDigests []g return nil } + // Verify path structure follows standard OCI: rootDir/repo/blobs/algorithm/digest + parentDir := path.Clean(path.Dir(fileInfo.Path())) + grandparentDir := path.Clean(path.Dir(parentDir)) + + // Require grandparent directory to be ImageBlobsDir (standard OCI structure) + if path.Base(grandparentDir) != ispec.ImageBlobsDir { + return nil + } + + // Verify parent directory is a valid digest algorithm (e.g., sha256, sha512) + digestAlgorithm := godigest.Algorithm(path.Base(parentDir)) + if !digestAlgorithm.Available() { + return nil + } + digestHash := baseName - digestAlgorithm := godigest.Algorithm(path.Base(path.Dir(fileInfo.Path()))) blobDigest := godigest.NewDigestFromEncoded(digestAlgorithm, digestHash) if err := blobDigest.Validate(); err != nil { //nolint: nilerr diff --git a/pkg/storage/s3/s3_test.go b/pkg/storage/s3/s3_test.go index 62a0d67f..f9dd22d4 100644 --- a/pkg/storage/s3/s3_test.go +++ b/pkg/storage/s3/s3_test.go @@ -2264,6 +2264,11 @@ func TestRebuildDedupeMockStoreDriver(t *testing.T) { validDigest := godigest.FromString("digest") + // Helper function to generate standard OCI blob path + blobPath := func(repo string, digest godigest.Digest) string { + return fmt.Sprintf("%s/%s/%s/%s", repo, ispec.ImageBlobsDir, digest.Algorithm().String(), digest.Encoded()) + } + Convey("Trigger Stat error in getOriginalBlobFromDisk()", t, func() { imgStore := createMockStorage(testDir, tdir, false, &StorageDriverMock{ StatFn: func(ctx context.Context, path string) (driver.FileInfo, error) { @@ -2275,7 +2280,7 @@ func TestRebuildDedupeMockStoreDriver(t *testing.T) { return false }, PathFn: func() string { - return fmt.Sprintf("path/to/%s/%s", validDigest.Algorithm().String(), validDigest.Encoded()) + return blobPath("path/to", validDigest) }, }) }, @@ -2291,7 +2296,7 @@ func TestRebuildDedupeMockStoreDriver(t *testing.T) { Convey("Trigger GetContent error in restoreDedupedBlobs()", t, func() { imgStore := createMockStorage(testDir, tdir, false, &StorageDriverMock{ StatFn: func(ctx context.Context, path string) (driver.FileInfo, error) { - if path == fmt.Sprintf("path/to/%s/%s", validDigest.Algorithm().String(), validDigest.Encoded()) { + if path == blobPath("path/to", validDigest) { return &FileInfoMock{ SizeFn: func() int64 { return int64(0) @@ -2311,7 +2316,7 @@ func TestRebuildDedupeMockStoreDriver(t *testing.T) { return false }, PathFn: func() string { - return fmt.Sprintf("path/to/%s/%s", validDigest.Algorithm().String(), validDigest.Encoded()) + return blobPath("path/to", validDigest) }, }) _ = walkFn(&FileInfoMock{ @@ -2319,7 +2324,7 @@ func TestRebuildDedupeMockStoreDriver(t *testing.T) { return false }, PathFn: func() string { - return fmt.Sprintf("path/to/second/%s/%s", validDigest.Algorithm().String(), validDigest.Encoded()) + return blobPath("path/to/second", validDigest) }, }) @@ -2340,7 +2345,7 @@ func TestRebuildDedupeMockStoreDriver(t *testing.T) { Convey("Trigger GetContent error in restoreDedupedBlobs()", t, func() { imgStore := createMockStorage(testDir, tdir, false, &StorageDriverMock{ StatFn: func(ctx context.Context, path string) (driver.FileInfo, error) { - if path == fmt.Sprintf("path/to/%s/%s", validDigest.Algorithm().String(), validDigest.Encoded()) { + if path == blobPath("path/to", validDigest) { return &FileInfoMock{ SizeFn: func() int64 { return int64(0) @@ -2360,7 +2365,7 @@ func TestRebuildDedupeMockStoreDriver(t *testing.T) { return false }, PathFn: func() string { - return fmt.Sprintf("path/to/%s/%s", validDigest.Algorithm().String(), validDigest.Encoded()) + return blobPath("path/to", validDigest) }, }) _ = walkFn(&FileInfoMock{ @@ -2368,7 +2373,7 @@ func TestRebuildDedupeMockStoreDriver(t *testing.T) { return false }, PathFn: func() string { - return fmt.Sprintf("path/to/second/%s/%s", validDigest.Algorithm().String(), validDigest.Encoded()) + return blobPath("path/to/second", validDigest) }, }) @@ -2389,7 +2394,7 @@ func TestRebuildDedupeMockStoreDriver(t *testing.T) { Convey("Trigger Stat() error in restoreDedupedBlobs()", t, func() { imgStore := createMockStorage(testDir, tdir, false, &StorageDriverMock{ StatFn: func(ctx context.Context, path string) (driver.FileInfo, error) { - if path == fmt.Sprintf("path/to/%s/%s", validDigest.Algorithm().String(), validDigest.Encoded()) { + if path == blobPath("path/to", validDigest) { return &FileInfoMock{ SizeFn: func() int64 { return int64(10) @@ -2409,7 +2414,7 @@ func TestRebuildDedupeMockStoreDriver(t *testing.T) { return false }, PathFn: func() string { - return fmt.Sprintf("path/to/%s/%s", validDigest.Algorithm().String(), validDigest.Encoded()) + return blobPath("path/to", validDigest) }, }) _ = walkFn(&FileInfoMock{ @@ -2417,7 +2422,7 @@ func TestRebuildDedupeMockStoreDriver(t *testing.T) { return false }, PathFn: func() string { - return fmt.Sprintf("path/to/second/%s/%s", validDigest.Algorithm().String(), validDigest.Encoded()) + return blobPath("path/to/second", validDigest) }, }) @@ -2434,7 +2439,7 @@ func TestRebuildDedupeMockStoreDriver(t *testing.T) { Convey("Trigger Stat() error in dedupeBlobs()", func() { imgStore := createMockStorage(testDir, t.TempDir(), true, &StorageDriverMock{ StatFn: func(ctx context.Context, path string) (driver.FileInfo, error) { - if path == fmt.Sprintf("path/to/%s/%s", validDigest.Algorithm().String(), validDigest.Encoded()) { + if path == blobPath("path/to", validDigest) { return &FileInfoMock{ SizeFn: func() int64 { return int64(10) @@ -2454,7 +2459,7 @@ func TestRebuildDedupeMockStoreDriver(t *testing.T) { return false }, PathFn: func() string { - return fmt.Sprintf("path/to/%s/%s", validDigest.Algorithm().String(), validDigest.Encoded()) + return blobPath("path/to", validDigest) }, }) _ = walkFn(&FileInfoMock{ @@ -2462,7 +2467,7 @@ func TestRebuildDedupeMockStoreDriver(t *testing.T) { return false }, PathFn: func() string { - return fmt.Sprintf("path/to/second/%s/%s", validDigest.Algorithm().String(), validDigest.Encoded()) + return blobPath("path/to/second", validDigest) }, }) @@ -2482,7 +2487,7 @@ func TestRebuildDedupeMockStoreDriver(t *testing.T) { tdir := t.TempDir() imgStore := createMockStorage(testDir, tdir, true, &StorageDriverMock{ StatFn: func(ctx context.Context, path string) (driver.FileInfo, error) { - if path == fmt.Sprintf("path/to/%s/%s", validDigest.Algorithm().String(), validDigest.Encoded()) { + if path == blobPath("path/to", validDigest) { return &FileInfoMock{ SizeFn: func() int64 { return int64(0) @@ -2502,7 +2507,7 @@ func TestRebuildDedupeMockStoreDriver(t *testing.T) { return false }, PathFn: func() string { - return fmt.Sprintf("path/to/%s/%s", validDigest.Algorithm().String(), validDigest.Encoded()) + return blobPath("path/to", validDigest) }, }) _ = walkFn(&FileInfoMock{ @@ -2510,7 +2515,7 @@ func TestRebuildDedupeMockStoreDriver(t *testing.T) { return false }, PathFn: func() string { - return fmt.Sprintf("path/to/second/%s/%s", validDigest.Algorithm().String(), validDigest.Encoded()) + return blobPath("path/to/second", validDigest) }, }) @@ -2533,7 +2538,7 @@ func TestRebuildDedupeMockStoreDriver(t *testing.T) { tdir := t.TempDir() imgStore := createMockStorage(testDir, tdir, true, &StorageDriverMock{ StatFn: func(ctx context.Context, path string) (driver.FileInfo, error) { - if path == fmt.Sprintf("path/to/%s/%s", validDigest.Algorithm().String(), validDigest.Encoded()) { + if path == blobPath("path/to", validDigest) { return &FileInfoMock{ SizeFn: func() int64 { return int64(0) @@ -2553,7 +2558,7 @@ func TestRebuildDedupeMockStoreDriver(t *testing.T) { return false }, PathFn: func() string { - return fmt.Sprintf("path/to/%s/%s", validDigest.Algorithm().String(), validDigest.Encoded()) + return blobPath("path/to", validDigest) }, }) _ = walkFn(&FileInfoMock{ @@ -2561,7 +2566,7 @@ func TestRebuildDedupeMockStoreDriver(t *testing.T) { return false }, PathFn: func() string { - return fmt.Sprintf("path/to/second/%s/%s", validDigest.Algorithm().String(), validDigest.Encoded()) + return blobPath("path/to/second", validDigest) }, }) @@ -2581,7 +2586,7 @@ func TestRebuildDedupeMockStoreDriver(t *testing.T) { tdir := t.TempDir() imgStore := createMockStorage(testDir, tdir, true, &StorageDriverMock{ StatFn: func(ctx context.Context, path string) (driver.FileInfo, error) { - if path == "path/to/"+validDigest.Encoded() { + if path == blobPath("path/to", validDigest) { return &FileInfoMock{ SizeFn: func() int64 { return int64(10) @@ -2601,7 +2606,7 @@ func TestRebuildDedupeMockStoreDriver(t *testing.T) { return false }, PathFn: func() string { - return fmt.Sprintf("path/to/%s/%s", validDigest.Algorithm().String(), validDigest.Encoded()) + return blobPath("path/to", validDigest) }, }) _ = walkFn(&FileInfoMock{ @@ -2609,7 +2614,7 @@ func TestRebuildDedupeMockStoreDriver(t *testing.T) { return false }, PathFn: func() string { - return fmt.Sprintf("path/to/second/%s/%s", validDigest.Algorithm().String(), validDigest.Encoded()) + return blobPath("path/to/second", validDigest) }, }) @@ -2636,10 +2641,60 @@ func TestRebuildDedupeMockStoreDriver(t *testing.T) { So(err, ShouldNotBeNil) }) + Convey("Skip files with invalid algorithm directory", t, func() { + tdir := t.TempDir() + imgStore := createMockStorage(testDir, tdir, true, &StorageDriverMock{ + WalkFn: func(ctx context.Context, path string, walkFn driver.WalkFn, options ...func(*driver.WalkOptions)) error { + // File in blobs directory but with invalid algorithm name + _ = walkFn(&FileInfoMock{ + IsDirFn: func() bool { + return false + }, + PathFn: func() string { + return fmt.Sprintf("path/to/%s/invalid-algo/digest-hash", ispec.ImageBlobsDir) + }, + }) + + return nil + }, + }) + + digest, duplicateBlobs, err := imgStore.GetNextDigestWithBlobPaths([]string{"path/to"}, []godigest.Digest{}) + So(err, ShouldBeNil) + // Should return empty digest because invalid algorithm directory is skipped + So(digest.String(), ShouldEqual, "") + So(duplicateBlobs, ShouldBeEmpty) + }) + + Convey("Skip files with invalid digest hash", t, func() { + tdir := t.TempDir() + imgStore := createMockStorage(testDir, tdir, true, &StorageDriverMock{ + WalkFn: func(ctx context.Context, path string, walkFn driver.WalkFn, options ...func(*driver.WalkOptions)) error { + // File with valid algorithm but invalid hash format + _ = walkFn(&FileInfoMock{ + IsDirFn: func() bool { + return false + }, + PathFn: func() string { + return fmt.Sprintf("path/to/%s/sha256/invalid-hash-format", ispec.ImageBlobsDir) + }, + }) + + return nil + }, + }) + + digest, duplicateBlobs, err := imgStore.GetNextDigestWithBlobPaths([]string{"path/to"}, []godigest.Digest{}) + So(err, ShouldBeNil) + // Should return empty digest because invalid hash format is skipped + So(digest.String(), ShouldEqual, "") + So(duplicateBlobs, ShouldBeEmpty) + }) + Convey("Trigger cache errors", t, func() { storageDriverMockIfBranch := &StorageDriverMock{ StatFn: func(ctx context.Context, path string) (driver.FileInfo, error) { - if path == fmt.Sprintf("path/to/%s/%s", validDigest.Algorithm().String(), validDigest.Encoded()) { + if path == blobPath("path/to", validDigest) { return &FileInfoMock{ SizeFn: func() int64 { return int64(0) @@ -2659,7 +2714,7 @@ func TestRebuildDedupeMockStoreDriver(t *testing.T) { return false }, PathFn: func() string { - return fmt.Sprintf("path/to/%s/%s", validDigest.Algorithm().String(), validDigest.Encoded()) + return blobPath("path/to", validDigest) }, }) _ = walkFn(&FileInfoMock{ @@ -2667,7 +2722,7 @@ func TestRebuildDedupeMockStoreDriver(t *testing.T) { return false }, PathFn: func() string { - return fmt.Sprintf("path/to/second/%s/%s", validDigest.Algorithm().String(), validDigest.Encoded()) + return blobPath("path/to/second", validDigest) }, }) @@ -2677,7 +2732,7 @@ func TestRebuildDedupeMockStoreDriver(t *testing.T) { storageDriverMockElseBranch := &StorageDriverMock{ StatFn: func(ctx context.Context, path string) (driver.FileInfo, error) { - if path == "path/to/"+validDigest.Encoded() { + if path == blobPath("path/to", validDigest) { return &FileInfoMock{ SizeFn: func() int64 { return int64(10) @@ -2697,7 +2752,7 @@ func TestRebuildDedupeMockStoreDriver(t *testing.T) { return false }, PathFn: func() string { - return fmt.Sprintf("path/to/%s/%s", validDigest.Algorithm().String(), validDigest.Encoded()) + return blobPath("path/to", validDigest) }, }) _ = walkFn(&FileInfoMock{ @@ -2705,7 +2760,7 @@ func TestRebuildDedupeMockStoreDriver(t *testing.T) { return false }, PathFn: func() string { - return fmt.Sprintf("path/to/second/%s/%s", validDigest.Algorithm().String(), validDigest.Encoded()) + return blobPath("path/to/second", validDigest) }, }) @@ -2738,7 +2793,7 @@ func TestRebuildDedupeMockStoreDriver(t *testing.T) { return false }, PutBlobFn: func(digest godigest.Digest, path string) error { - if path == fmt.Sprintf("path/to/%s/%s", validDigest.Algorithm().String(), validDigest.Encoded()) { + if path == blobPath("path/to", validDigest) { return errCache }