From 819a0ff588019026d92cc21bda764103b4e628c5 Mon Sep 17 00:00:00 2001 From: Arjun Raja Yogidas Date: Thu, 4 Dec 2025 02:58:30 +0000 Subject: [PATCH 1/7] add garbage collection labels to image mainfest Signed-off-by: Arjun Raja Yogidas --- integration/convert_test.go | 22 ++++++++++++++++++++++ soci/soci_convert.go | 12 ++++++++++++ 2 files changed, 34 insertions(+) diff --git a/integration/convert_test.go b/integration/convert_test.go index 06d9c8534..fa48a68bb 100644 --- a/integration/convert_test.go +++ b/integration/convert_test.go @@ -197,7 +197,29 @@ func validateConversion(t *testing.T, sh *shell.Shell, originalDigest, converted if dg, ok := manifest.Annotations[soci.ImageAnnotationSociIndexDigest]; !ok || dg != sociIndexDesc.Digest.String() { t.Errorf("manifest %v does not contain expected soci index digest %v", manifestDesc, sociIndexDesc.Digest) } + + // Get GC labels + manifestLabelOutput := sh.O("ctr", "content", "label", manifestDesc.Digest.String()) + manifestLabels := strings.Split(strings.TrimSpace(string(manifestLabelOutput)), ",") + // verify GC lables exists + if len(manifestLabels) <= 0 { + t.Errorf("manifest does not contain labels, got %d labels", len(manifestLabels)) + } + // verify config label exists + // verify config label exists + hasConfigLabel := false + for _, label := range manifestLabels { + parts := strings.Split(label, "=") + if len(parts) == 2 && parts[0] == "containerd.io/gc.ref.content.config" { + hasConfigLabel = true + break + } + } + if !hasConfigLabel { + t.Errorf("manifest does not contain required config label 'containerd.io/gc.ref.content.config'") + } } + } func TestConvert(t *testing.T) { diff --git a/soci/soci_convert.go b/soci/soci_convert.go index fa7370480..1eaaa9ee9 100644 --- a/soci/soci_convert.go +++ b/soci/soci_convert.go @@ -307,6 +307,18 @@ func (b *IndexBuilder) annotateImages(ctx context.Context, ociIndex *ocispec.Ind } indexWithMetadata.Desc.Annotations[IndexAnnotationImageManifestDigest] = manifestDesc.Digest.String() } + + err = store.LabelGCRefContent(ctx, b.blobStore, *manifestDesc, "config", manifest.Config.Digest.String()) + if err != nil { + return err + } + + for i, layer := range manifest.Layers { + err = store.LabelGCRefContent(ctx, b.blobStore, *manifestDesc, fmt.Sprintf("l.%d", i), layer.Digest.String()) + if err != nil { + return err + } + } } return nil } From 1ef2ce746289f24c2cfd87b830ce9589afabdeb9 Mon Sep 17 00:00:00 2001 From: Arjun Raja Yogidas Date: Thu, 4 Dec 2025 22:13:05 +0000 Subject: [PATCH 2/7] testing do not merge Signed-off-by: Arjun Raja Yogidas --- .github/workflows/build.yml | 44 ++++++++++++++++++------------------- integration/convert_test.go | 17 ++++++++------ 2 files changed, 32 insertions(+), 29 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 7c5cfd1b4..13f992639 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -28,25 +28,25 @@ jobs: setup: uses: ./.github/workflows/setup.yml - test: - needs: setup - runs-on: ${{ fromJSON(needs.setup.outputs.runner-labels)[matrix.os] }} - strategy: - matrix: - os: ${{ fromJSON(needs.setup.outputs.available-runners) }} - timeout-minutes: 15 - steps: - - uses: actions/checkout@v6 - - uses: actions/setup-go@v6 - with: - go-version: ${{ env.GO_VERSION }} - - name: Install zlib static on AL2 ARM instances - if: matrix.os == 'al2-arm' - run: dnf install zlib-static.aarch64 -y - - run: make - - run: make test-with-coverage - - name: Show test coverage - run: make show-test-coverage + # test: + # needs: setup + # runs-on: ${{ fromJSON(needs.setup.outputs.runner-labels)[matrix.os] }} + # strategy: + # matrix: + # os: ${{ fromJSON(needs.setup.outputs.available-runners) }} + # timeout-minutes: 15 + # steps: + # - uses: actions/checkout@v6 + # - uses: actions/setup-go@v6 + # with: + # go-version: ${{ env.GO_VERSION }} + # - name: Install zlib static on AL2 ARM instances + # if: matrix.os == 'al2-arm' + # run: dnf install zlib-static.aarch64 -y + # - run: make + # - run: make test-with-coverage + # - name: Show test coverage + # run: make show-test-coverage integration: needs: setup @@ -72,6 +72,6 @@ jobs: if [[ "${{ matrix.os }}" == "ubuntu-x86" ]]; then SKIP_SYSTEMD_TESTS=1 fi - SKIP_SYSTEMD_TESTS=$SKIP_SYSTEMD_TESTS make integration-with-coverage - - name: Show test coverage - run: make show-integration-coverage + SKIP_SYSTEMD_TESTS=$SKIP_SYSTEMD_TESTS GO_TEST_FLAGS="-count 1 -run TestConvert/convert_and_replace" make integration + # - name: Show test coverage + # run: make show-integration-coverage diff --git a/integration/convert_test.go b/integration/convert_test.go index fa48a68bb..5bbfc15ad 100644 --- a/integration/convert_test.go +++ b/integration/convert_test.go @@ -200,24 +200,27 @@ func validateConversion(t *testing.T, sh *shell.Shell, originalDigest, converted // Get GC labels manifestLabelOutput := sh.O("ctr", "content", "label", manifestDesc.Digest.String()) + fmt.Printf("running ctr content label on manifest digest -> %v", manifestDesc.Digest.String()) manifestLabels := strings.Split(strings.TrimSpace(string(manifestLabelOutput)), ",") + fmt.Printf("length of maifestLabels -> %v", len(manifestLabels)) // verify GC lables exists if len(manifestLabels) <= 0 { t.Errorf("manifest does not contain labels, got %d labels", len(manifestLabels)) } // verify config label exists - // verify config label exists - hasConfigLabel := false + var configInfor ocispec.Image for _, label := range manifestLabels { parts := strings.Split(label, "=") if len(parts) == 2 && parts[0] == "containerd.io/gc.ref.content.config" { - hasConfigLabel = true - break + configBytes := sh.O("ctr", "content", "get", parts[1]) + fmt.Printf("Running ctr content get on -> %v", parts[1]) + if err := json.Unmarshal(configBytes, &configInfor); err != nil { + t.Errorf("failed to decode manifest: %v", err) + continue + } } } - if !hasConfigLabel { - t.Errorf("manifest does not contain required config label 'containerd.io/gc.ref.content.config'") - } + } } From 185518075fc672e3a2598f902939da6de1aacaee Mon Sep 17 00:00:00 2001 From: Arjun Raja Yogidas Date: Fri, 5 Dec 2025 01:36:12 +0000 Subject: [PATCH 3/7] do a reverse test Signed-off-by: Arjun Raja Yogidas --- soci/soci_convert.go | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/soci/soci_convert.go b/soci/soci_convert.go index 1eaaa9ee9..cdfd95ddc 100644 --- a/soci/soci_convert.go +++ b/soci/soci_convert.go @@ -308,17 +308,17 @@ func (b *IndexBuilder) annotateImages(ctx context.Context, ociIndex *ocispec.Ind indexWithMetadata.Desc.Annotations[IndexAnnotationImageManifestDigest] = manifestDesc.Digest.String() } - err = store.LabelGCRefContent(ctx, b.blobStore, *manifestDesc, "config", manifest.Config.Digest.String()) - if err != nil { - return err - } - - for i, layer := range manifest.Layers { - err = store.LabelGCRefContent(ctx, b.blobStore, *manifestDesc, fmt.Sprintf("l.%d", i), layer.Digest.String()) - if err != nil { - return err - } - } + // err = store.LabelGCRefContent(ctx, b.blobStore, *manifestDesc, "config", manifest.Config.Digest.String()) + // if err != nil { + // return err + // } + + // for i, layer := range manifest.Layers { + // err = store.LabelGCRefContent(ctx, b.blobStore, *manifestDesc, fmt.Sprintf("l.%d", i), layer.Digest.String()) + // if err != nil { + // return err + // } + // } } return nil } From b7323fb127676d98e1ff49481b722d5f151ec9e2 Mon Sep 17 00:00:00 2001 From: Arjun Raja Yogidas Date: Fri, 5 Dec 2025 01:56:51 +0000 Subject: [PATCH 4/7] do a reverse test 2 Signed-off-by: Arjun Raja Yogidas --- integration/convert_test.go | 51 ++++++++++++++++++++++--------------- 1 file changed, 31 insertions(+), 20 deletions(-) diff --git a/integration/convert_test.go b/integration/convert_test.go index 5bbfc15ad..4d3b2d8af 100644 --- a/integration/convert_test.go +++ b/integration/convert_test.go @@ -198,33 +198,44 @@ func validateConversion(t *testing.T, sh *shell.Shell, originalDigest, converted t.Errorf("manifest %v does not contain expected soci index digest %v", manifestDesc, sociIndexDesc.Digest) } - // Get GC labels - manifestLabelOutput := sh.O("ctr", "content", "label", manifestDesc.Digest.String()) - fmt.Printf("running ctr content label on manifest digest -> %v", manifestDesc.Digest.String()) - manifestLabels := strings.Split(strings.TrimSpace(string(manifestLabelOutput)), ",") - fmt.Printf("length of maifestLabels -> %v", len(manifestLabels)) - // verify GC lables exists - if len(manifestLabels) <= 0 { - t.Errorf("manifest does not contain labels, got %d labels", len(manifestLabels)) + configDigest, err := getManifestLabelsAndConfigDigest(sh, manifestDesc.Digest.String()) + if err != nil { + t.Errorf("failed to get config digest from manifest: %v", err) + continue } - // verify config label exists - var configInfor ocispec.Image - for _, label := range manifestLabels { - parts := strings.Split(label, "=") - if len(parts) == 2 && parts[0] == "containerd.io/gc.ref.content.config" { - configBytes := sh.O("ctr", "content", "get", parts[1]) - fmt.Printf("Running ctr content get on -> %v", parts[1]) - if err := json.Unmarshal(configBytes, &configInfor); err != nil { - t.Errorf("failed to decode manifest: %v", err) - continue - } - } + configBytes := sh.O("ctr", "content", "get", configDigest) + + var configInfo ocispec.Image + if err := json.Unmarshal(configBytes, &configInfo); err != nil { + fmt.Errorf("failed to decode config: %v", err) } } } +func getManifestLabelsAndConfigDigest(sh *shell.Shell, manifestDigest string) (string, error) { + gcConfigLabelKey := "containerd.io/gc.ref.content.config" + + // Get manifest labels + manifestLabelOutput := sh.O("ctr", "content", "label", manifestDigest) + manifestLabels := strings.Split(strings.TrimSpace(string(manifestLabelOutput)), ",") + + if len(manifestLabels) <= 0 { + return "", fmt.Errorf("manifest does not contain any labels") + } + + // Extract config digest from labels + for _, label := range manifestLabels { + parts := strings.Split(label, "=") + if len(parts) == 2 && parts[0] == gcConfigLabelKey { + return parts[1], nil + } + } + + return "", fmt.Errorf("config label %q not found in manifest labels", gcConfigLabelKey) +} + func TestConvert(t *testing.T) { sh, done := newSnapshotterBaseShell(t) defer done() From cbd4348205a9ec73e72a79ad807000bf2cef9320 Mon Sep 17 00:00:00 2001 From: Arjun Raja Yogidas Date: Fri, 5 Dec 2025 02:06:14 +0000 Subject: [PATCH 5/7] do a reverse test 3 Signed-off-by: Arjun Raja Yogidas --- soci/soci_convert.go | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/soci/soci_convert.go b/soci/soci_convert.go index cdfd95ddc..1eaaa9ee9 100644 --- a/soci/soci_convert.go +++ b/soci/soci_convert.go @@ -308,17 +308,17 @@ func (b *IndexBuilder) annotateImages(ctx context.Context, ociIndex *ocispec.Ind indexWithMetadata.Desc.Annotations[IndexAnnotationImageManifestDigest] = manifestDesc.Digest.String() } - // err = store.LabelGCRefContent(ctx, b.blobStore, *manifestDesc, "config", manifest.Config.Digest.String()) - // if err != nil { - // return err - // } - - // for i, layer := range manifest.Layers { - // err = store.LabelGCRefContent(ctx, b.blobStore, *manifestDesc, fmt.Sprintf("l.%d", i), layer.Digest.String()) - // if err != nil { - // return err - // } - // } + err = store.LabelGCRefContent(ctx, b.blobStore, *manifestDesc, "config", manifest.Config.Digest.String()) + if err != nil { + return err + } + + for i, layer := range manifest.Layers { + err = store.LabelGCRefContent(ctx, b.blobStore, *manifestDesc, fmt.Sprintf("l.%d", i), layer.Digest.String()) + if err != nil { + return err + } + } } return nil } From dfeb7a6dab10bce9c3e5409c35f733823321ccca Mon Sep 17 00:00:00 2001 From: Arjun Raja Yogidas Date: Fri, 5 Dec 2025 21:23:25 +0000 Subject: [PATCH 6/7] verify every layer exists Signed-off-by: Arjun Raja Yogidas --- integration/convert_test.go | 51 ++++++++++++++++++++++++++----------- 1 file changed, 36 insertions(+), 15 deletions(-) diff --git a/integration/convert_test.go b/integration/convert_test.go index 4d3b2d8af..2a6918e9f 100644 --- a/integration/convert_test.go +++ b/integration/convert_test.go @@ -198,42 +198,63 @@ func validateConversion(t *testing.T, sh *shell.Shell, originalDigest, converted t.Errorf("manifest %v does not contain expected soci index digest %v", manifestDesc, sociIndexDesc.Digest) } - configDigest, err := getManifestLabelsAndConfigDigest(sh, manifestDesc.Digest.String()) + // get labels associated with the manifest and extract the digest + configDigest, err := getDigestLabelMapping(sh, manifestDesc.Digest.String()) if err != nil { t.Errorf("failed to get config digest from manifest: %v", err) continue } - configBytes := sh.O("ctr", "content", "get", configDigest) - var configInfo ocispec.Image - if err := json.Unmarshal(configBytes, &configInfo); err != nil { - fmt.Errorf("failed to decode config: %v", err) + _, err = verifyImageLayersContainsLabel(configDigest, manifest) + if err != nil { + t.Errorf("error verifying layers: %v", err) + continue } - } } -func getManifestLabelsAndConfigDigest(sh *shell.Shell, manifestDigest string) (string, error) { - gcConfigLabelKey := "containerd.io/gc.ref.content.config" +func getDigestLabelMapping(sh *shell.Shell, manifestDigest string) (map[string][]string, error) { + digestMap := make(map[string][]string) - // Get manifest labels manifestLabelOutput := sh.O("ctr", "content", "label", manifestDigest) manifestLabels := strings.Split(strings.TrimSpace(string(manifestLabelOutput)), ",") if len(manifestLabels) <= 0 { - return "", fmt.Errorf("manifest does not contain any labels") + return nil, fmt.Errorf("manifest does not contain any labels") } - // Extract config digest from labels for _, label := range manifestLabels { - parts := strings.Split(label, "=") - if len(parts) == 2 && parts[0] == gcConfigLabelKey { - return parts[1], nil + keyVal := strings.Split(label, "=") + if len(keyVal) != 2 { + continue + } else { + digestMap[keyVal[1]] = append(digestMap[keyVal[1]], keyVal[0]) } } + return digestMap, nil +} - return "", fmt.Errorf("config label %q not found in manifest labels", gcConfigLabelKey) +func verifyImageLayersContainsLabel(digestMap map[string][]string, manifest ocispec.Manifest) (bool, error) { + for i := 0; i < len(manifest.Layers); i = i + 1 { + layerDigest := manifest.Layers[i].Digest.String() + if labels, ok := digestMap[layerDigest]; ok { + found := false + for _, label := range labels { + if strings.Contains(label, "containerd.io/gc.ref.content") { + found = true + break + } + } + if !found { + return false, fmt.Errorf("digest %s does not have associated gc label", layerDigest) + } + } else { + // Layer digest not found in digestMap + return false, fmt.Errorf("digest %s not found in digestMap", layerDigest) + } + } + return true, nil } func TestConvert(t *testing.T) { From 5e6cafb069a4c318dd6ad865460733dd10efd152 Mon Sep 17 00:00:00 2001 From: Arjun Raja Yogidas Date: Fri, 5 Dec 2025 21:34:22 +0000 Subject: [PATCH 7/7] verify every layer exists - optimized Signed-off-by: Arjun Raja Yogidas --- integration/convert_test.go | 98 ++++++++++++++++++++----------------- 1 file changed, 53 insertions(+), 45 deletions(-) diff --git a/integration/convert_test.go b/integration/convert_test.go index 2a6918e9f..4778f4487 100644 --- a/integration/convert_test.go +++ b/integration/convert_test.go @@ -101,6 +101,57 @@ func TestConvertWithForceRecreateZtocs(t *testing.T) { func validateConversion(t *testing.T, sh *shell.Shell, originalDigest, convertedDigest string) { t.Helper() + // Helper function to get digest label mapping + getDigestLabelMapping := func(manifestDigest string) (map[string][]string, error) { + digestMap := make(map[string][]string) + + manifestLabelOutput := sh.O("ctr", "content", "label", manifestDigest) + manifestLabels := strings.Split(strings.TrimSpace(string(manifestLabelOutput)), ",") + + if len(manifestLabels) <= 0 { + return nil, fmt.Errorf("manifest does not contain any labels") + } + + for _, label := range manifestLabels { + keyVal := strings.Split(label, "=") + if len(keyVal) != 2 { + continue + } + digestMap[keyVal[1]] = append(digestMap[keyVal[1]], keyVal[0]) + } + return digestMap, nil + } + + // Helper function to verify image layers contain required labels + verifyImageLayersContainsLabel := func(digestMap map[string][]string, manifest ocispec.Manifest) error { + // Helper function to check if a digest has the required label + checkDigestLabels := func(digest string) error { + if labels, ok := digestMap[digest]; ok { + for _, label := range labels { + if strings.Contains(label, "containerd.io/gc.ref.content") { + return nil // Found the required label + } + } + return fmt.Errorf("digest %s does not have associated gc label", digest) + } + return fmt.Errorf("digest %s not found in digestMap", digest) + } + + // Check all layer digests + for _, layer := range manifest.Layers { + if err := checkDigestLabels(layer.Digest.String()); err != nil { + return err + } + } + + // Check config digest + if err := checkDigestLabels(manifest.Config.Digest.String()); err != nil { + return err + } + + return nil + } + if originalDigest == convertedDigest { t.Fatalf("conversion did not change the digest: %s", originalDigest) } @@ -199,13 +250,13 @@ func validateConversion(t *testing.T, sh *shell.Shell, originalDigest, converted } // get labels associated with the manifest and extract the digest - configDigest, err := getDigestLabelMapping(sh, manifestDesc.Digest.String()) + digestLabelMap, err := getDigestLabelMapping(manifestDesc.Digest.String()) if err != nil { t.Errorf("failed to get config digest from manifest: %v", err) continue } - _, err = verifyImageLayersContainsLabel(configDigest, manifest) + err = verifyImageLayersContainsLabel(digestLabelMap, manifest) if err != nil { t.Errorf("error verifying layers: %v", err) continue @@ -214,49 +265,6 @@ func validateConversion(t *testing.T, sh *shell.Shell, originalDigest, converted } -func getDigestLabelMapping(sh *shell.Shell, manifestDigest string) (map[string][]string, error) { - digestMap := make(map[string][]string) - - manifestLabelOutput := sh.O("ctr", "content", "label", manifestDigest) - manifestLabels := strings.Split(strings.TrimSpace(string(manifestLabelOutput)), ",") - - if len(manifestLabels) <= 0 { - return nil, fmt.Errorf("manifest does not contain any labels") - } - - for _, label := range manifestLabels { - keyVal := strings.Split(label, "=") - if len(keyVal) != 2 { - continue - } else { - digestMap[keyVal[1]] = append(digestMap[keyVal[1]], keyVal[0]) - } - } - return digestMap, nil -} - -func verifyImageLayersContainsLabel(digestMap map[string][]string, manifest ocispec.Manifest) (bool, error) { - for i := 0; i < len(manifest.Layers); i = i + 1 { - layerDigest := manifest.Layers[i].Digest.String() - if labels, ok := digestMap[layerDigest]; ok { - found := false - for _, label := range labels { - if strings.Contains(label, "containerd.io/gc.ref.content") { - found = true - break - } - } - if !found { - return false, fmt.Errorf("digest %s does not have associated gc label", layerDigest) - } - } else { - // Layer digest not found in digestMap - return false, fmt.Errorf("digest %s not found in digestMap", layerDigest) - } - } - return true, nil -} - func TestConvert(t *testing.T) { sh, done := newSnapshotterBaseShell(t) defer done()