support OCI image index at manifest endpoint (#638)

Signed-off-by: Ramkumar Chinchani <rchincha@cisco.com> Signed-off-by: Ramkumar Chinchani <rchincha@cisco.com>
2026-06-17 12:58:02 +08:00 · 2022-08-20 01:18:48 -07:00
parent b9b233e7fc
commit 5c01c4eab4
6 changed files with 1400 additions and 18 deletions
@@ -538,8 +538,102 @@ func (is *ImageStoreLocal) validateOCIManifest(repo, reference string, manifest
 	return "", nil
 }

+/**
+before an image index manifest is pushed to a repo, its constituent manifests
+are pushed first, so when updating/removing this image index manifest, we also
+need to determine if there are other image index manifests which refer to the
+same constitutent manifests so that they can be garbage-collected correctly
+
+pruneImageManifestsFromIndex is a helper routine to achieve this.
+*/
+func pruneImageManifestsFromIndex(dir string, digest godigest.Digest, // nolint: gocyclo
+	outIndex ispec.Index, otherImgIndexes []ispec.Descriptor, log zerolog.Logger,
+) ([]ispec.Descriptor, error) {
+	indexPath := path.Join(dir, "blobs", digest.Algorithm().String(), digest.Encoded())
+
+	buf, err := ioutil.ReadFile(indexPath)
+	if err != nil {
+		log.Error().Err(err).Str("dir", dir).Msg("failed to read index.json")
+
+		return nil, err
+	}
+
+	var imgIndex ispec.Index
+	if err := json.Unmarshal(buf, &imgIndex); err != nil {
+		log.Error().Err(err).Str("path", indexPath).Msg("invalid JSON")
+
+		return nil, err
+	}
+
+	inUse := map[string]uint{}
+
+	for _, manifest := range imgIndex.Manifests {
+		inUse[manifest.Digest.Encoded()]++
+	}
+
+	for _, otherIndex := range otherImgIndexes {
+		indexPath := path.Join(dir, "blobs", otherIndex.Digest.Algorithm().String(), otherIndex.Digest.Encoded())
+
+		buf, err := ioutil.ReadFile(indexPath)
+		if err != nil {
+			log.Error().Err(err).Str("dir", dir).Msg("failed to read index.json")
+
+			return nil, err
+		}
+
+		var oindex ispec.Index
+		if err := json.Unmarshal(buf, &oindex); err != nil {
+			log.Error().Err(err).Str("path", indexPath).Msg("invalid JSON")
+
+			return nil, err
+		}
+
+		for _, omanifest := range oindex.Manifests {
+			_, ok := inUse[omanifest.Digest.Encoded()]
+			if ok {
+				inUse[omanifest.Digest.Encoded()]++
+			}
+		}
+	}
+
+	prunedManifests := []ispec.Descriptor{}
+
+	// for all manifests in the index, skip those that either have a tag or
+	// are used in other imgIndexes
+	for _, outManifest := range outIndex.Manifests {
+		if outManifest.MediaType != ispec.MediaTypeImageManifest {
+			prunedManifests = append(prunedManifests, outManifest)
+
+			continue
+		}
+
+		_, ok := outManifest.Annotations[ispec.AnnotationRefName]
+		if ok {
+			prunedManifests = append(prunedManifests, outManifest)
+
+			continue
+		}
+
+		count, ok := inUse[outManifest.Digest.Encoded()]
+		if !ok {
+			prunedManifests = append(prunedManifests, outManifest)
+
+			continue
+		}
+
+		if count != 1 {
+			// this manifest is in use in other image indexes
+			prunedManifests = append(prunedManifests, outManifest)
+
+			continue
+		}
+	}
+
+	return prunedManifests, nil
+}
+
 // PutImageManifest adds an image manifest to the repository.
-func (is *ImageStoreLocal) PutImageManifest(repo, reference, mediaType string,
+func (is *ImageStoreLocal) PutImageManifest(repo, reference, mediaType string, // nolint: gocyclo
 	body []byte,
 ) (string, error) {
 	if err := is.InitRepo(repo); err != nil {
@@ -551,7 +645,7 @@ func (is *ImageStoreLocal) PutImageManifest(repo, reference, mediaType string,
 	// validate the manifest
 	if !IsSupportedMediaType(mediaType) {
 		is.log.Debug().Interface("actual", mediaType).
-			Interface("expected", ispec.MediaTypeImageManifest).Msg("bad manifest media type")
+			Msg("bad manifest media type")

 		return "", zerr.ErrBadManifest
 	}
@@ -607,12 +701,13 @@ func (is *ImageStoreLocal) PutImageManifest(repo, reference, mediaType string,
 	// create a new descriptor
 	desc := ispec.Descriptor{
 		MediaType: mediaType, Size: int64(len(body)), Digest: mDigest,
-		Platform: &ispec.Platform{Architecture: "amd64", OS: "linux"},
 	}
 	if !refIsDigest {
 		desc.Annotations = map[string]string{ispec.AnnotationRefName: reference}
 	}

+	var oldDgst godigest.Digest
+
 	for midx, manifest := range index.Manifests {
 		if reference == manifest.Digest.String() {
 			// nothing changed, so don't update
@@ -631,6 +726,7 @@ func (is *ImageStoreLocal) PutImageManifest(repo, reference, mediaType string,

 				break
 			}
+
 			// manifest contents have changed for the same tag,
 			// so update index.json descriptor
 			is.log.Info().
@@ -638,9 +734,22 @@ func (is *ImageStoreLocal) PutImageManifest(repo, reference, mediaType string,
 				Int64("new size", int64(len(body))).
 				Str("old digest", desc.Digest.String()).
 				Str("new digest", mDigest.String()).
+				Str("old mediaType", manifest.MediaType).
+				Str("new mediaType", mediaType).
 				Msg("updating existing tag with new manifest contents")

+			// changing media-type is disallowed!
+			if manifest.MediaType != mediaType {
+				err = zerr.ErrBadManifest
+				is.log.Error().Err(err).
+					Str("old mediaType", manifest.MediaType).
+					Str("new mediaType", mediaType).Msg("cannot change media-type")
+
+				return "", err
+			}
+
 			desc = manifest
+			oldDgst = manifest.Digest
 			desc.Size = int64(len(body))
 			desc.Digest = mDigest

@@ -666,6 +775,30 @@ func (is *ImageStoreLocal) PutImageManifest(repo, reference, mediaType string,
 		return "", err
 	}

+	/* additionally, unmarshal an image index and for all manifests in that
+	index, ensure that they do not have a name or they are not in other
+	manifest indexes else GC can never clean them */
+	if (mediaType == ispec.MediaTypeImageIndex) && (oldDgst != "") {
+		otherImgIndexes := []ispec.Descriptor{}
+
+		for _, manifest := range index.Manifests {
+			if manifest.MediaType == ispec.MediaTypeImageIndex {
+				otherImgIndexes = append(otherImgIndexes, manifest)
+			}
+		}
+
+		otherImgIndexes = append(otherImgIndexes, desc)
+
+		dir := path.Join(is.rootDir, repo)
+
+		prunedManifests, err := pruneImageManifestsFromIndex(dir, oldDgst, index, otherImgIndexes, is.log)
+		if err != nil {
+			return "", err
+		}
+
+		index.Manifests = prunedManifests
+	}
+
 	// now update "index.json"
 	index.Manifests = append(index.Manifests, desc)
 	dir = path.Join(is.rootDir, repo)
@@ -731,7 +864,7 @@ func (is *ImageStoreLocal) DeleteImageManifest(repo, reference string) error {
 	isTag := false

 	// as per spec "reference" can be a digest and a tag
-	digest, err := godigest.Parse(reference)
+	dgst, err := godigest.Parse(reference)
 	if err != nil {
 		is.log.Debug().Str("invalid digest: ", reference).Msg("storage: assuming tag")

@@ -757,38 +890,66 @@ func (is *ImageStoreLocal) DeleteImageManifest(repo, reference string) error {

 	found := false

+	isImageIndex := false
+
 	var manifest ispec.Descriptor

 	// we are deleting, so keep only those manifests that don't match
 	outIndex := index
 	outIndex.Manifests = []ispec.Descriptor{}

+	otherImgIndexes := []ispec.Descriptor{}
+
 	for _, manifest = range index.Manifests {
 		if isTag {
 			tag, ok := manifest.Annotations[ispec.AnnotationRefName]
 			if ok && tag == reference {
 				is.log.Debug().Str("deleting tag", tag).Msg("")

-				digest = manifest.Digest
+				dgst = manifest.Digest

 				found = true

+				if manifest.MediaType == ispec.MediaTypeImageIndex {
+					isImageIndex = true
+				}
+
 				continue
 			}
 		} else if reference == manifest.Digest.String() {
 			is.log.Debug().Str("deleting reference", reference).Msg("")
 			found = true

+			if manifest.MediaType == ispec.MediaTypeImageIndex {
+				isImageIndex = true
+			}
+
 			continue
 		}

 		outIndex.Manifests = append(outIndex.Manifests, manifest)
+
+		if manifest.MediaType == ispec.MediaTypeImageIndex {
+			otherImgIndexes = append(otherImgIndexes, manifest)
+		}
 	}

 	if !found {
 		return zerr.ErrManifestNotFound
 	}

+	/* additionally, unmarshal an image index and for all manifests in that
+	index, ensure that they do not have a name or they are not in other
+	manifest indexes else GC can never clean them */
+	if isImageIndex {
+		prunedManifests, err := pruneImageManifestsFromIndex(dir, dgst, outIndex, otherImgIndexes, is.log)
+		if err != nil {
+			return err
+		}
+
+		outIndex.Manifests = prunedManifests
+	}
+
 	// now update "index.json"
 	dir = path.Join(is.rootDir, repo)
 	file := path.Join(dir, "index.json")
@@ -813,7 +974,7 @@ func (is *ImageStoreLocal) DeleteImageManifest(repo, reference string) error {
 	toDelete := true

 	for _, manifest = range outIndex.Manifests {
-		if digest.String() == manifest.Digest.String() {
+		if dgst.String() == manifest.Digest.String() {
 			toDelete = false

 			break
@@ -821,7 +982,7 @@ func (is *ImageStoreLocal) DeleteImageManifest(repo, reference string) error {
 	}

 	if toDelete {
-		p := path.Join(dir, "blobs", digest.Algorithm().String(), digest.Encoded())
+		p := path.Join(dir, "blobs", dgst.Algorithm().String(), dgst.Encoded())

 		_ = os.Remove(p)
 	}
@@ -1580,7 +1741,8 @@ func (is *ImageStoreLocal) writeFile(filename string, data []byte) error {
 }

 func IsSupportedMediaType(mediaType string) bool {
-	return mediaType == ispec.MediaTypeImageManifest ||
+	return mediaType == ispec.MediaTypeImageIndex ||
+		mediaType == ispec.MediaTypeImageManifest ||
 		mediaType == artifactspec.MediaTypeArtifactManifest
 }