Conformance test redesign

.github/workflows/build.yml

@@ -4,14 +4,17 @@ on:
   push:
     branches:
       - main
+  pull_request:
+    branches:
+      - main
 
 jobs:
   run:
     runs-on: ubuntu-latest
 
     strategy:
       matrix:
-        go: ['1.21', '1.22']
+        go: ['stable', 'oldstable']
 
     steps:
       - name: checkout source code
@@ -47,5 +50,6 @@ jobs:
         with:
           name: oci-conformance-results-${{ matrix.go }}
           path: |
+            ./results.yaml
             ./report.html
             ./junit.xml

.github/workflows/conformance-action.yml

@@ -4,23 +4,27 @@ on:
   push:
     branches:
       - main
+  pull_request:
+    branches:
+      - main
 
 jobs:
   run:
     runs-on: ubuntu-latest
     steps:
       - name: checkout source code
         uses: actions/checkout@v4
-      - name: Start a test registry (zot)
+      - name: Start a test registry
+        id: setup
         run: |
           set -x
           make registry-ci
+          echo "port=$(docker port oci-conformance-olareg 5000| head -1 | cut -f2 -d:)" >>$GITHUB_OUTPUT
       - name: Run OCI distribution-spec conformance
         env:
-          OCI_ROOT_URL: http://localhost:5000
-          OCI_NAMESPACE: myorg/myrepo
-          OCI_TEST_PULL: 1
-          OCI_TEST_PUSH: 1
-          OCI_TEST_CONTENT_DISCOVERY: 1
-          OCI_TEST_CONTENT_MANAGEMENT: 1
+          OCI_REGISTRY: "localhost:${{ steps.setup.outputs.port }}"
+          OCI_TLS: "disabled"
+          OCI_REPO1: "myorg/myrepo"
+          OCI_REPO2: "myorg/myrepo2"
+          OCI_RESULTS_DIR: "."
         uses: ./

.github/workflows/release.yml

@@ -26,10 +26,6 @@ jobs:
           fi
           VERSION=$(echo "${VERSION}" | sed -r 's#/+#-#g')
           TAGS="${IMAGE}:${VERSION}"
-          if [[ $VERSION == "${{ github.event.repository.default_branch }}" ]]; then
-            GITSHA="$(git rev-parse --short HEAD)"
-            TAGS="${TAGS},${IMAGE}:${GITSHA}"
-          fi
           if [[ $VERSION =~ ^v[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}$ ]]; then
             MINOR=${VERSION%.*}
             MAJOR=${MINOR%.*}

.gitignore

@@ -6,3 +6,4 @@ tags
 go.work
 junit.xml
 report.html
+results.yaml

Makefile

@@ -23,14 +23,13 @@ ifeq "$(strip $(PANDOC))" ''
 	endif
 endif
 
-GOLANGCILINT_CONTAINER ?= ghcr.io/opencontainers/golangci-lint:v1.52.1@sha256:d3d3d56f9706ebe843c1b06686c385877ba65b33f39507cdbeb22f482adce65a
+GOLANGCILINT_CONTAINER ?= docker.io/golangci/golangci-lint:v2.11.3@sha256:e838e8ab68aaefe83e2408691510867ade9329c0e0b895a3fb35eb93d1c2a4ba
 ifeq "$(strip $(GOLANGCILINT))" ''
 	ifneq "$(strip $(DOCKER))" ''
 		GOLANGCILINT = $(DOCKER) run \
 			--rm \
 			-v $(shell pwd)/:/input:ro \
 			-e GOCACHE=/tmp/.cache \
-			-e GO111MODULE=on \
 			-e GOLANGCI_LINT_CACHE=/tmp/.cache \
 			--entrypoint /bin/bash \
 			-u $(shell id -u) \
@@ -83,35 +82,70 @@ install.tools: .install.gitvalidation
 .install.gitvalidation:
 	go install github.com/vbatts/git-validation@latest
 
-conformance: conformance-test conformance-binary
+conformance: conformance-test conformance-cmd
 
 conformance-test:
 	$(GOLANGCILINT) -c 'cd conformance && golangci-lint run -v'
 
 conformance-binary: $(OUTPUT_DIRNAME)/conformance.test
 
-TEST_REGISTRY_CONTAINER ?= ghcr.io/project-zot/zot-minimal-linux-amd64:v2.1.7@sha256:2114797f00696011f38cc94c72f5773c84b1036562df5034d05ea19075179ad1
-registry-ci:
-	docker rm -f oci-conformance && \
+conformance-cmd: $(OUTPUT_DIRNAME)/conformance
+
+registry-ci: registry-ci-olareg
+
+TEST_REGISTRY_IMAGE_OLAREG ?= ghcr.io/olareg/olareg:edge
+registry-ci-olareg:
+	docker rm -f oci-conformance-olareg && \
+		docker run --rm -d \
+			--name=oci-conformance-olareg \
+			-p 5000 \
+			$(TEST_REGISTRY_IMAGE_OLAREG) serve --store-type mem --api-delete --api-blob-delete --api-sparse-image --api-sparse-index && \
+		sleep 2
+
+TEST_REGISTRY_IMAGE_ZOT ?= ghcr.io/project-zot/zot-minimal-linux-amd64:v2.1.7@sha256:2114797f00696011f38cc94c72f5773c84b1036562df5034d05ea19075179ad1
+registry-ci-zot:
+	docker rm -f oci-conformance-zot && \
 		mkdir -p $(OUTPUT_DIRNAME) && \
 		echo '{"distSpecVersion":"1.1.0-dev","storage":{"rootDirectory":"/tmp/zot","gc":false,"dedupe":false},"http":{"address":"0.0.0.0","port":"5000"},"log":{"level":"debug"}}' > $(shell pwd)/$(OUTPUT_DIRNAME)/zot-config.json
-		docker run -d \
+		docker run --rm -d \
 			-v $(shell pwd)/$(OUTPUT_DIRNAME)/zot-config.json:/etc/zot/config.json \
-			--name=oci-conformance \
-			-p 5000:5000 \
-			$(TEST_REGISTRY_CONTAINER) && \
+			--name=oci-conformance-zot \
+			-p 5000 \
+			$(TEST_REGISTRY_IMAGE_ZOT) && \
 		sleep 5
 
-conformance-ci:
-	export OCI_ROOT_URL="http://localhost:5000" && \
-		export OCI_NAMESPACE="myorg/myrepo" && \
-		export OCI_TEST_PULL=1 && \
-		export OCI_TEST_PUSH=1 && \
-		export OCI_TEST_CONTENT_DISCOVERY=1 && \
-		export OCI_TEST_CONTENT_MANAGEMENT=1 && \
-		$(shell pwd)/$(OUTPUT_DIRNAME)/conformance.test
+conformance-ci: conformance-ci-olareg
+
+conformance-ci-olareg: $(OUTPUT_DIRNAME)/conformance
+	export OCI_VERSION="dev" && \
+		export OCI_REGISTRY="localhost:$$(docker port oci-conformance-olareg 5000| head -1 | cut -f2 -d:)" && \
+		export OCI_TLS="disabled" && \
+		export OCI_REPO1="myorg/myrepo" && \
+		export OCI_REPO2="myorg/myrepo2" && \
+		export OCI_RESULTS_DIR="." && \
+		export OCI_DATA_SPARSE=true && \
+		$(shell pwd)/$(OUTPUT_DIRNAME)/conformance
+
+conformance-ci-zot: $(OUTPUT_DIRNAME)/conformance
+	export OCI_REGISTRY="localhost:$$(docker port oci-conformance-zot 5000| head -1 | cut -f2 -d:)" && \
+		export OCI_TLS="disabled" && \
+		export OCI_REPO1="myorg/myrepo" && \
+		export OCI_REPO2="myorg/myrepo2" && \
+		export OCI_RESULTS_DIR="." && \
+		$(shell pwd)/$(OUTPUT_DIRNAME)/conformance
+
+clean-ci:
+	docker rm -f oci-conformance-olareg oci-conformance-zot
+
+$(OUTPUT_DIRNAME)/conformance: conformance/*.go conformance/go.mod
+	cd conformance && \
+		CGO_ENABLED=0 go build -o $(shell pwd)/$(OUTPUT_DIRNAME)/conformance \
+			--ldflags="-X github.com/opencontainers/distribution-spec/conformance.Version=$(CONFORMANCE_VERSION)"
 
-$(OUTPUT_DIRNAME)/conformance.test:
+$(OUTPUT_DIRNAME)/conformance.test: conformance/*.go conformance/go.mod
 	cd conformance && \
 		CGO_ENABLED=0 go test -c -o $(shell pwd)/$(OUTPUT_DIRNAME)/conformance.test \
 			--ldflags="-X github.com/opencontainers/distribution-spec/conformance.Version=$(CONFORMANCE_VERSION)"
+
+clean: clean-ci
+	rm -rf header.html junit.xml report.html results.yaml output conformance/results

action.yml

@@ -14,16 +14,39 @@ runs:
       id: check-conformance
       run: |
         set -x
+        required_env_vars=(
+          "OCI_REGISTRY"
+          "OCI_REPO1"
+        )
+        missing=false
+        for v in ${required_env_vars[@]}; do
+          if [[ "${!v}" == "" ]]; then
+            echo "::error title=Missing variable::Missing required variable ${v}"
+            missing=true
+          fi
+        done
+        if [[ "${missing}" == "false" ]]; then
+          exit 0
+        fi
+        # fallback to previous variables
         required_env_vars=(
           "OCI_ROOT_URL"
           "OCI_NAMESPACE"
         )
+        missing=false
         for v in ${required_env_vars[@]}; do
           if [[ "${!v}" == "" ]]; then
-            echo "Error: the following environment variable is required: ${v}"
-            exit 1
+            echo "::error title=Missing fallback variable::Missing fallback variable ${v}"
+            missing=true
+          else
+            echo "::warning title=Deprecated fallback variable::Variable ${v} has been deprecated"
           fi
         done
+        if [[ "${missing}" == "false" ]]; then
+          exit 0
+        else
+          exit 1
+        fi
 
     - name: Build OCI distribution-spec conformance binary
       shell: bash
@@ -34,14 +57,16 @@ runs:
         # Enter the directory containing the checkout of this action which is surpisingly hard to do (but we did it... #OCI)
         cd "$(dirname $(find $(find ~/work/_actions -name distribution-spec -print -quit) -name Makefile -print -quit))"
 
-        # The .git folder is not present, but the dirname is the requested action ref, so use this as the conformance version
-        conformance_version="$(basename "${PWD}")"
-        echo "conformance-version=${conformance_version}" >> $GITHUB_OUTPUT
+        # The .git folder is not present, but the dirname is the requested action ref, so use this as the commit version
+        commit_version="$(basename "${PWD}")"
 
         # Build the conformance binary
-        CONFORMANCE_VERSION="${conformance_version}" OUTPUT_DIRNAME=bin make conformance-binary
+        CONFORMANCE_VERSION="${commit_version}" OUTPUT_DIRNAME=bin make conformance-cmd
+
+        # The spec version is independent of the conformance git commit
+        echo "oci-spec-version=${OCI_VERSION:-stable}" >> $GITHUB_OUTPUT
 
-        # Add bin to the PATH so we can just run "conformance.test"
+        # Add bin to the PATH so we can just run "conformance"
         echo "${PWD}/bin" >> $GITHUB_PATH
 
     - name: Run OCI distribution-spec conformance binary
@@ -50,7 +75,8 @@ runs:
       run: |
         set -x
         set +e
-        conformance.test
+        OCI_RESULTS_DIR=${OCI_RESULTS_DIR:-.}
+        conformance
         conformance_rc="$?"
         set -e
         if [[ -f report.html ]]; then
@@ -62,9 +88,10 @@ runs:
 
     - name: Upload OCI distribution-spec conformance results as build artifact
       if: always() && steps.run-conformance.outputs.has-report == 'true'
-      uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4.6.0
+      uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
       with:
-        name: oci-distribution-spec-conformance-results-${{ steps.build-conformance.outputs.conformance-version }}
+        name: oci-distribution-spec-conformance-results-${{ steps.build-conformance.outputs.oci-spec-version }}
         path: |
+          results.yaml
           report.html
           junit.xml

conformance/.gitignore

@@ -1,6 +1,5 @@
-vendor/
-junit.xml
-report.html
+conformance
 conformance.test
-tags
-env.sh
+oci-conformance.yaml
+results/**
+vendor/