tests: add a k3s kubernetes integration worker

.github/workflows/build.yml

@@ -94,6 +94,11 @@ jobs:
           - worker: "docker+containerd@27.5"  # same as docker, but with containerd snapshotter
             pkg: ./tests
             mode: experimental
+          - worker: kubernetes
+            pkg: ./tests
+          - worker: kubernetes
+            pkg: ./tests
+            mode: experimental
     steps:
       -
         name: Prepare
@@ -104,7 +109,7 @@ jobs:
           fi
           testFlags="--run=//worker=$(echo "${{ matrix.worker }}" | sed 's/\+/\\+/g')$"
           case "${{ matrix.worker }}" in
-            docker | docker+containerd | docker@* | docker+containerd@* | remote+multinode)
+            docker | docker+containerd | docker@* | docker+containerd@* | remote+multinode | kubernetes)
               echo "TESTFLAGS=${{ env.TESTFLAGS_DOCKER }} $testFlags" >> $GITHUB_ENV
               ;;
             *)

Dockerfile

@@ -14,6 +14,7 @@ ARG REGISTRY_VERSION=3.0.0
 ARG BUILDKIT_VERSION=v0.29.0
 ARG COMPOSE_VERSION=v5.1.0
 ARG UNDOCK_VERSION=0.9.0
+ARG K3S_VERSION=v1.32.13-k3s1
 
 FROM --platform=$BUILDPLATFORM tonistiigi/xx:${XX_VERSION} AS xx
 FROM --platform=$BUILDPLATFORM golang:${GO_VERSION}-alpine${ALPINE_VERSION} AS golatest
@@ -27,6 +28,7 @@ FROM registry:$REGISTRY_VERSION AS registry
 FROM moby/buildkit:$BUILDKIT_VERSION AS buildkit
 FROM docker/compose-bin:$COMPOSE_VERSION AS compose
 FROM crazymax/undock:$UNDOCK_VERSION AS undock
+FROM rancher/k3s:${K3S_VERSION} AS k3s
 
 FROM golatest AS gobase
 COPY --from=xx / /
@@ -128,15 +130,24 @@ FROM gobase AS integration-test-base
 RUN apk add --no-cache \
       bash \
       btrfs-progs \
+      busybox-binsh \
+      cni-plugin-flannel \
+      cni-plugins \
+      conntrack-tools \
+      coreutils \
+      dbus \
       e2fsprogs \
       e2fsprogs-extra \
+      findutils \
       ip6tables \
+      ipset \
       iptables \
       make \
       openssl \
       shadow-uidmap \
       xfsprogs \
       xz
+ENV PATH="/usr/libexec/cni:${PATH}"
 COPY --link --from=gotestsum /out /usr/bin/
 COPY --link --from=registry /bin/registry /usr/bin/
 COPY --link --from=docker-engine / /usr/bin/
@@ -145,13 +156,28 @@ COPY --link --from=docker-engine-alt28 / /opt/docker-alt-28/
 COPY --link --from=docker-engine-alt27 / /opt/docker-alt-27/
 COPY --link --from=docker-cli-alt28 / /opt/docker-alt-28/
 COPY --link --from=docker-cli-alt27 / /opt/docker-alt-27/
+COPY --link --from=k3s /bin/k3s /usr/bin/
+COPY --link --from=k3s /bin/kubectl /usr/bin/
 COPY --link --from=buildkit /usr/bin/buildkitd /usr/bin/
 COPY --link --from=buildkit /usr/bin/buildctl /usr/bin/
 COPY --link --from=compose /docker-compose /usr/bin/compose
 COPY --link --from=undock /usr/local/bin/undock /usr/bin/
 COPY --link --from=binaries /buildx /usr/bin/
 RUN mkdir -p /usr/local/lib/docker/cli-plugins && ln -s /usr/bin/buildx /usr/local/lib/docker/cli-plugins/docker-buildx
 ENV TEST_DOCKER_EXTRA="docker@28.5=/opt/docker-alt-28,docker@27.5=/opt/docker-alt-27"
+COPY --chmod=755 <<-"EOF" /entrypoint.sh
+#!/bin/sh
+set -e
+# cgroup v2: enable nesting
+# https://github.com/moby/moby/blob/v25.0.0/hack/dind#L59-L69
+if [ -f /sys/fs/cgroup/cgroup.controllers ]; then
+  mkdir -p /sys/fs/cgroup/init
+  xargs -rn1 < /sys/fs/cgroup/cgroup.procs > /sys/fs/cgroup/init/cgroup.procs || :
+  sed -e 's/ / +/g' -e 's/^/+/' < /sys/fs/cgroup/cgroup.controllers > /sys/fs/cgroup/cgroup.subtree_control
+fi
+exec "$@"
+EOF
+ENTRYPOINT ["/entrypoint.sh"]
 
 FROM integration-test-base AS integration-test
 COPY . .

tests/helpers/k3s.go

@@ -0,0 +1,165 @@
+package helpers
+
+import (
+	"bytes"
+	"fmt"
+	"net"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/moby/buildkit/util/testutil/integration"
+	"github.com/pkg/errors"
+)
+
+const (
+	k3sBin        = "k3s"
+	kubectlBin    = "kubectl"
+	k3sNodeName   = "integrationk3s"
+	k3sWaitWindow = 3 * time.Minute
+	k3sWaitDelay  = 5 * time.Second
+)
+
+func NewK3sServer(cfg *integration.BackendConfig) (kubeConfig string, cl func() error, err error) {
+	if _, err := exec.LookPath(k3sBin); err != nil {
+		return "", nil, errors.Wrapf(err, "failed to lookup %s binary", k3sBin)
+	}
+	if _, err := exec.LookPath(kubectlBin); err != nil {
+		return "", nil, errors.Wrapf(err, "failed to lookup %s binary", kubectlBin)
+	}
+
+	deferF := &integration.MultiCloser{}
+	cl = deferF.F()
+	defer func() {
+		if err != nil {
+			deferF.F()()
+			cl = nil
+		}
+	}()
+
+	cfgfile, err := os.CreateTemp("", "kubeconfig*.yml")
+	if err != nil {
+		return "", nil, err
+	}
+	if err := cfgfile.Close(); err != nil {
+		return "", nil, err
+	}
+	kubeConfig = cfgfile.Name()
+	deferF.Append(func() error {
+		return os.Remove(cfgfile.Name())
+	})
+
+	k3sDataDir, err := os.MkdirTemp("", "kubedata")
+	if err != nil {
+		return "", nil, err
+	}
+	deferF.Append(func() error {
+		return os.RemoveAll(k3sDataDir)
+	})
+
+	l, err := net.Listen("tcp", "127.0.0.1:0")
+	if err != nil {
+		return "", nil, err
+	}
+	_ = l.Close()
+	lport := strconv.Itoa(l.Addr().(*net.TCPAddr).Port)
+
+	k3sGlobalDir := "/etc/rancher/k3s"
+	if err := os.MkdirAll(k3sGlobalDir, 0o755); err != nil {
+		return "", nil, err
+	}
+
+	registriesFile := filepath.Join(k3sGlobalDir, "registries.yaml")
+	if cfg.Mirror != "" {
+		mirror := cfg.Mirror
+		if port, ok := strings.CutPrefix(mirror, "localhost:"); ok {
+			mirror = "host.k3s.internal:" + port
+		}
+		dt := []byte(fmt.Sprintf(`
+mirrors:
+  "docker.io":
+    endpoint:
+      - "%s"
+
+configs:
+  "%s":
+    insecure_skip_verify: true
+`, mirror, mirror))
+		if err := os.WriteFile(registriesFile, dt, 0o644); err != nil {
+			return "", nil, err
+		}
+		deferF.Append(func() error {
+			if err := os.Remove(registriesFile); errors.Is(err, os.ErrNotExist) {
+				return nil
+			} else {
+				return err
+			}
+		})
+	}
+
+	cmd := exec.Command(
+		k3sBin,
+		"server",
+		"--bind-address", "0.0.0.0",
+		"--https-listen-port", lport,
+		"--data-dir", k3sDataDir,
+		"--write-kubeconfig", cfgfile.Name(),
+		"--write-kubeconfig-mode", "666",
+		"--node-name", k3sNodeName,
+	)
+	stop, err := integration.StartCmd(cmd, cfg.Logs)
+	if err != nil {
+		return "", nil, err
+	}
+
+	if err = waitK3s(cfg, kubeConfig); err != nil {
+		stop()
+		containerdLogs, _ := os.ReadFile(filepath.Join(k3sDataDir, "agent", "containerd", "containerd.log"))
+		containerdConfig, _ := os.ReadFile(filepath.Join(k3sDataDir, "agent", "etc", "containerd", "config.toml"))
+		registries, _ := os.ReadFile(registriesFile)
+		return "", nil, errors.Wrapf(err, "k3s did not start up: %s\ncontainerd.log: %s\ncontainerd.config.toml: %s\nregistries.yaml: %s", formatLogs(cfg.Logs), containerdLogs, containerdConfig, registries)
+	}
+
+	deferF.Append(stop)
+	return kubeConfig, cl, nil
+}
+
+func waitK3s(cfg *integration.BackendConfig, kubeConfig string) error {
+	logbuf := new(bytes.Buffer)
+	defer func() {
+		if logbuf.Len() > 0 {
+			cfg.Logs["waitK3s: "] = logbuf
+		}
+	}()
+
+	deadline := time.Now().Add(k3sWaitWindow)
+	var lastErr error
+	for time.Now().Before(deadline) {
+		cmd := exec.Command(kubectlBin, "--kubeconfig", kubeConfig, "wait", "--timeout=5s", "--for=condition=Ready", "node/"+k3sNodeName)
+		out, err := cmd.CombinedOutput()
+		if err == nil && bytes.Contains(out, []byte("condition met")) {
+			return nil
+		}
+		lastErr = errors.Wrapf(err, "node is not ready: %s %s", cmd.String(), string(out))
+		logbuf.Reset()
+		logbuf.WriteString(lastErr.Error())
+		time.Sleep(k3sWaitDelay)
+	}
+	if lastErr == nil {
+		lastErr = errors.New("node did not become ready")
+	}
+	return lastErr
+}
+
+func formatLogs(m map[string]*bytes.Buffer) string {
+	var ss []string
+	for k, b := range m {
+		if b != nil {
+			ss = append(ss, fmt.Sprintf("%q:%s", k, b.String()))
+		}
+	}
+	return strings.Join(ss, ",")
+}

tests/integration_test.go

@@ -17,6 +17,7 @@ func init() {
 		workers.InitRemoteMultiNodeWorker()
 	} else {
 		workers.InitRemoteWorker()
+		workers.InitKubernetesWorker()
 	}
 }