test kubernetes worker with k3d

.github/workflows/build.yml

@@ -54,6 +54,7 @@ jobs:
         worker:
           - docker-container
           - remote
+          - kubernetes
         pkg:
           - ./tests
         mode:
@@ -105,14 +106,14 @@ jobs:
           fi
           testFlags="--run=//worker=$(echo "${{ matrix.worker }}" | sed 's/\+/\\+/g')$"
           case "${{ matrix.worker }}" in
-            docker | docker+containerd | docker@* | docker+containerd@* | remote+multinode)
+            docker | docker+containerd | docker@* | docker+containerd@* | remote+multinode | kubernetes)
               echo "TESTFLAGS=${{ env.TESTFLAGS_DOCKER }} $testFlags" >> $GITHUB_ENV
               ;;
             *)
               echo "TESTFLAGS=${{ env.TESTFLAGS }} $testFlags" >> $GITHUB_ENV
               ;;
           esac
-          if [[ "${{ matrix.worker }}" == "docker"* || "${{ matrix.worker }}" == "remote+multinode" ]]; then
+          if [[ "${{ matrix.worker }}" == "docker"* || "${{ matrix.worker }}" == "remote+multinode" || "${{ matrix.worker }}" == "kubernetes" ]]; then
             echo "TEST_DOCKERD=1" >> $GITHUB_ENV
           fi
           if [ "${{ matrix.mode }}" = "experimental" ]; then

Dockerfile

@@ -14,6 +14,8 @@ ARG REGISTRY_VERSION=3.0.0
 ARG BUILDKIT_VERSION=v0.29.0
 ARG COMPOSE_VERSION=v5.1.0
 ARG UNDOCK_VERSION=0.9.0
+ARG K3D_VERSION=5.8.3
+ARG K3S_VERSION=v1.32.13-k3s1
 
 FROM --platform=$BUILDPLATFORM tonistiigi/xx:${XX_VERSION} AS xx
 FROM --platform=$BUILDPLATFORM golang:${GO_VERSION}-alpine${ALPINE_VERSION} AS golatest
@@ -27,6 +29,7 @@ FROM registry:$REGISTRY_VERSION AS registry
 FROM moby/buildkit:$BUILDKIT_VERSION AS buildkit
 FROM docker/compose-bin:$COMPOSE_VERSION AS compose
 FROM crazymax/undock:$UNDOCK_VERSION AS undock
+FROM ghcr.io/k3d-io/k3d:${K3D_VERSION} AS k3d
 
 FROM golatest AS gobase
 COPY --from=xx / /
@@ -124,6 +127,8 @@ FROM binaries-$TARGETOS AS binaries
 ARG BUILDKIT_SBOM_SCAN_STAGE=true
 
 FROM gobase AS integration-test-base
+ARG K3D_VERSION
+ARG K3S_VERSION
 # https://github.com/docker/docker/blob/master/project/PACKAGERS.md#runtime-dependencies
 RUN apk add --no-cache \
       bash \
@@ -149,9 +154,13 @@ COPY --link --from=buildkit /usr/bin/buildkitd /usr/bin/
 COPY --link --from=buildkit /usr/bin/buildctl /usr/bin/
 COPY --link --from=compose /docker-compose /usr/bin/compose
 COPY --link --from=undock /usr/local/bin/undock /usr/bin/
+COPY --link --from=k3d /bin/k3d /usr/bin/
 COPY --link --from=binaries /buildx /usr/bin/
 RUN mkdir -p /usr/local/lib/docker/cli-plugins && ln -s /usr/bin/buildx /usr/local/lib/docker/cli-plugins/docker-buildx
 ENV TEST_DOCKER_EXTRA="docker@28.5=/opt/docker-alt-28,docker@27.5=/opt/docker-alt-27"
+ENV TEST_K3S_IMAGE="rancher/k3s:${K3S_VERSION}"
+ENV TEST_K3D_TOOLS_IMAGE="ghcr.io/k3d-io/k3d-tools:${K3D_VERSION}"
+ENV TEST_K3D_LOADBALANCER_IMAGE="ghcr.io/k3d-io/k3d-proxy:${K3D_VERSION}"
 
 FROM integration-test-base AS integration-test
 COPY . .

tests/helpers/k3d.go

@@ -0,0 +1,93 @@
+package helpers
+
+import (
+	"context"
+	"os"
+	"os/exec"
+	"strings"
+	"time"
+
+	"github.com/moby/buildkit/identity"
+	"github.com/moby/buildkit/util/testutil/integration"
+	"github.com/pkg/errors"
+)
+
+const (
+	k3dBin = "k3d"
+)
+
+func NewK3dServer(ctx context.Context, cfg *integration.BackendConfig, dockerAddress string) (clusterName, kubeConfig string, cl func() error, err error) {
+	if _, err := exec.LookPath(k3dBin); err != nil {
+		return "", "", nil, errors.Wrapf(err, "failed to lookup %s binary", k3dBin)
+	}
+
+	deferF := &integration.MultiCloser{}
+	cl = deferF.F()
+
+	defer func() {
+		if err != nil {
+			deferF.F()()
+			cl = nil
+		}
+	}()
+
+	clusterName = "bk-" + identity.NewID()
+
+	createCtx, cancelCreate := context.WithTimeoutCause(ctx, 90*time.Second, errors.New("timed out creating k3d cluster"))
+	defer cancelCreate()
+
+	args := []string{
+		"cluster", "create", clusterName,
+		"--wait",
+		"--k3s-arg=--debug@server:0",
+	}
+	if image := KubernetesK3sImage(); image != "" {
+		args = append(args, "--image="+image)
+	}
+	cmd := exec.CommandContext(createCtx, k3dBin, args...)
+	cmd.Env = k3dEnv(dockerAddress)
+	out, err := cmd.CombinedOutput()
+	if err != nil {
+		diag := KubernetesDiagnostics(ctx, clusterName, dockerAddress)
+		return "", "", nil, errors.Wrapf(err, "failed to create k3d cluster %s: %s\n%s\nouter dockerd logs: %s", clusterName, strings.TrimSpace(string(out)), diag, integration.FormatLogs(cfg.Logs))
+	}
+	deferF.Append(func() error {
+		deleteCtx, cancelDelete := context.WithTimeoutCause(context.WithoutCancel(ctx), 30*time.Second, errors.New("timed out deleting k3d cluster"))
+		defer cancelDelete()
+		cmd := exec.CommandContext(deleteCtx, k3dBin, "cluster", "delete", clusterName)
+		cmd.Env = k3dEnv(dockerAddress)
+		out, err := cmd.CombinedOutput()
+		if err != nil {
+			return errors.Wrapf(err, "failed to delete k3d cluster %s: %s", clusterName, string(out))
+		}
+		return nil
+	})
+
+	kubeconfigCtx, cancelKubeconfig := context.WithTimeoutCause(ctx, 30*time.Second, errors.New("timed out writing k3d kubeconfig"))
+	defer cancelKubeconfig()
+
+	cmd = exec.CommandContext(kubeconfigCtx, k3dBin, "kubeconfig", "write", clusterName)
+	cmd.Env = k3dEnv(dockerAddress)
+	out, err = cmd.CombinedOutput()
+	if err != nil {
+		diag := KubernetesDiagnostics(ctx, clusterName, dockerAddress)
+		return "", "", nil, errors.Wrapf(err, "failed to write kubeconfig for cluster %s: %s\n%s\nouter dockerd logs: %s", clusterName, strings.TrimSpace(string(out)), diag, integration.FormatLogs(cfg.Logs))
+	}
+	kubeConfig = strings.TrimSpace(string(out))
+
+	return
+}
+
+func k3dEnv(dockerAddress string) []string {
+	env := append(
+		os.Environ(),
+		"DOCKER_CONTEXT="+dockerAddress,
+	)
+	if image := KubernetesK3DToolsImage(); image != "" {
+		env = append(env, "K3D_IMAGE_TOOLS="+image)
+	}
+	if image := KubernetesK3DLoadBalancerImage(); image != "" {
+		env = append(env, "K3D_IMAGE_LOADBALANCER="+image)
+	}
+	return env
+}

tests/helpers/kubernetes_diagnostics.go

@@ -0,0 +1,141 @@
+package helpers
+
+import (
+	"bytes"
+	"context"
+	"errors"
+	"fmt"
+	"os"
+	"os/exec"
+	"strings"
+	"time"
+)
+
+const defaultTestBuildkitTag = "buildx-stable-1"
+
+func KubernetesBuildkitImage() string {
+	if v := os.Getenv("TEST_BUILDKIT_IMAGE"); v != "" {
+		return v
+	}
+	tag := os.Getenv("TEST_BUILDKIT_TAG")
+	if tag == "" {
+		tag = defaultTestBuildkitTag
+	}
+	return "moby/buildkit:" + tag
+}
+
+func KubernetesK3sImage() string {
+	return os.Getenv("TEST_K3S_IMAGE")
+}
+
+func KubernetesK3DToolsImage() string {
+	return os.Getenv("TEST_K3D_TOOLS_IMAGE")
+}
+
+func KubernetesK3DLoadBalancerImage() string {
+	return os.Getenv("TEST_K3D_LOADBALANCER_IMAGE")
+}
+
+func KubernetesDiagnostics(ctx context.Context, clusterName, dockerContext string) string {
+	ctx, cancel := context.WithTimeoutCause(context.WithoutCancel(ctx), 20*time.Second, errors.New("timed out collecting kubernetes diagnostics"))
+	defer cancel()
+
+	var buf bytes.Buffer
+	appendK3dDiagnostics(ctx, &buf, clusterName, dockerContext)
+	appendDockerDiagnostics(ctx, &buf, dockerContext)
+	appendK3sServerDiagnostics(ctx, &buf, clusterName, dockerContext)
+	return strings.TrimSpace(buf.String())
+}
+
+func appendK3dDiagnostics(ctx context.Context, buf *bytes.Buffer, clusterName, dockerContext string) {
+	appendCommandOutput(ctx, buf, "k3d cluster list", "k3d", []string{"cluster", "list", clusterName}, []string{"DOCKER_CONTEXT=" + dockerContext})
+	appendCommandOutput(ctx, buf, "k3d node list", "k3d", []string{"node", "list"}, []string{"DOCKER_CONTEXT=" + dockerContext})
+}
+
+func appendDockerDiagnostics(ctx context.Context, buf *bytes.Buffer, dockerContext string) {
+	args := []string{"ps", "-a", "--format", "{{.Names}}\t{{.Image}}\t{{.Status}}"}
+	appendCommandOutput(ctx, buf, "docker ps", "docker", args, []string{"DOCKER_CONTEXT=" + dockerContext})
+}
+
+func appendK3sServerDiagnostics(ctx context.Context, buf *bytes.Buffer, clusterName, dockerContext string) {
+	nodeNames, err := clusterNodeNames(ctx, clusterName, dockerContext)
+	if err != nil {
+		fmt.Fprintf(buf, "cluster node discovery error: %v\n", err)
+		return
+	}
+	if len(nodeNames) == 0 {
+		fmt.Fprintln(buf, "cluster node discovery: no matching k3d containers found")
+		return
+	}
+
+	for _, nodeName := range nodeNames {
+		appendCommandOutput(ctx, buf, "docker inspect "+nodeName, "docker", []string{
+			"inspect",
+			"--format",
+			"Status={{.State.Status}} Health={{if .State.Health}}{{.State.Health.Status}}{{else}}<none>{{end}} Restarting={{.State.Restarting}} ExitCode={{.State.ExitCode}} Error={{.State.Error}} Privileged={{.HostConfig.Privileged}} Cgroupns={{.HostConfig.CgroupnsMode}}",
+			nodeName,
+		}, []string{"DOCKER_CONTEXT=" + dockerContext})
+		appendCommandOutput(ctx, buf, "docker logs "+nodeName, "docker", []string{"logs", "--tail", "80", nodeName}, []string{"DOCKER_CONTEXT=" + dockerContext})
+	}
+
+	for _, nodeName := range nodeNames {
+		if !strings.Contains(nodeName, "-server-") {
+			continue
+		}
+		appendCommandOutput(ctx, buf, "docker exec "+nodeName+" ps", "docker", []string{"exec", nodeName, "sh", "-c", "ps auxww"}, []string{"DOCKER_CONTEXT=" + dockerContext})
+		appendCommandOutput(ctx, buf, "docker exec "+nodeName+" sockets", "docker", []string{"exec", nodeName, "sh", "-c", "ss -lntp || netstat -lnt"}, []string{"DOCKER_CONTEXT=" + dockerContext})
+		appendCommandOutput(ctx, buf, "docker exec "+nodeName+" cgroup", "docker", []string{"exec", nodeName, "sh", "-c", "cat /proc/1/cgroup && echo && mount | grep cgroup"}, []string{"DOCKER_CONTEXT=" + dockerContext})
+		appendCommandOutput(ctx, buf, "docker exec "+nodeName+" env", "docker", []string{"exec", nodeName, "sh", "-c", "env | sort"}, []string{"DOCKER_CONTEXT=" + dockerContext})
+		appendCommandOutput(ctx, buf, "docker exec "+nodeName+" entrypoint", "docker", []string{"exec", nodeName, "sh", "-c", "sed -n '1,200p' /bin/k3d-entrypoint.sh"}, []string{"DOCKER_CONTEXT=" + dockerContext})
+		appendCommandOutput(ctx, buf, "docker exec "+nodeName+" entrypoint logs", "docker", []string{"exec", nodeName, "sh", "-c", "for f in /var/log/k3d-entrypoints_*.log; do if [ -f \"$f\" ]; then echo \"== $f ==\"; tail -n 200 \"$f\"; echo; fi; done"}, []string{"DOCKER_CONTEXT=" + dockerContext})
+		appendCommandOutput(ctx, buf, "docker exec "+nodeName+" k3s files", "docker", []string{"exec", nodeName, "sh", "-c", "find /var/lib/rancher/k3s -maxdepth 3 -type f 2>/dev/null | sort"}, []string{"DOCKER_CONTEXT=" + dockerContext})
+		appendCommandOutput(ctx, buf, "docker exec "+nodeName+" k3s logs", "docker", []string{"exec", nodeName, "sh", "-c", "for f in /var/log/k3s.log /var/lib/rancher/k3s/agent/containerd/containerd.log /var/lib/rancher/k3s/server/logs/*; do if [ -f \"$f\" ]; then echo \"== $f ==\"; tail -n 200 \"$f\"; echo; fi; done"}, []string{"DOCKER_CONTEXT=" + dockerContext})
+		appendCommandOutput(ctx, buf, "docker exec "+nodeName+" kubectl get pods", "docker", []string{"exec", nodeName, "kubectl", "get", "pods", "-A", "-o", "wide"}, []string{"DOCKER_CONTEXT=" + dockerContext})
+		appendCommandOutput(ctx, buf, "docker exec "+nodeName+" kubectl get events", "docker", []string{"exec", nodeName, "kubectl", "get", "events", "-A", "--sort-by=.lastTimestamp"}, []string{"DOCKER_CONTEXT=" + dockerContext})
+		appendCommandOutput(ctx, buf, "docker exec "+nodeName+" kubectl describe pods", "docker", []string{"exec", nodeName, "kubectl", "describe", "pods", "-A"}, []string{"DOCKER_CONTEXT=" + dockerContext})
+		break
+	}
+}
+
+func clusterNodeNames(ctx context.Context, clusterName, dockerContext string) ([]string, error) {
+	out, err := runCommand(ctx, "docker", []string{
+		"ps", "-a",
+		"--filter", "name=k3d-" + clusterName,
+		"--format", "{{.Names}}",
+	}, []string{"DOCKER_CONTEXT=" + dockerContext})
+	if err != nil {
+		return nil, err
+	}
+	var names []string
+	for _, line := range strings.Split(strings.TrimSpace(out), "\n") {
+		line = strings.TrimSpace(line)
+		if line == "" {
+			continue
+		}
+		names = append(names, line)
+	}
+	return names, nil
+}
+
+func appendCommandOutput(ctx context.Context, buf *bytes.Buffer, title, name string, args []string, env []string) {
+	out, err := runCommand(ctx, name, args, env)
+	fmt.Fprintf(buf, "== %s ==\n", title)
+	if err != nil {
+		fmt.Fprintf(buf, "error: %v\n", err)
+	}
+	if strings.TrimSpace(out) == "" {
+		fmt.Fprintln(buf, "<empty>")
+	} else {
+		fmt.Fprintf(buf, "%s\n", strings.TrimSpace(out))
+	}
+	fmt.Fprintln(buf)
+}
+
+func runCommand(ctx context.Context, name string, args []string, env []string) (string, error) {
+	cmd := exec.CommandContext(ctx, name, args...)
+	if len(env) > 0 {
+		cmd.Env = append(os.Environ(), env...)
+	}
+	out, err := cmd.CombinedOutput()
+	return string(out), err
+}

tests/integration_test.go

@@ -15,6 +15,7 @@ func init() {
 		workers.InitDockerWorker()
 		workers.InitDockerContainerWorker()
 		workers.InitRemoteMultiNodeWorker()
+		workers.InitKubernetesWorker()
 	} else {
 		workers.InitRemoteWorker()
 	}