From b6e841ccc8541c5044022900671b36ed7ccaf5e4 Mon Sep 17 00:00:00 2001 From: CrazyMax <1951866+crazy-max@users.noreply.github.com> Date: Tue, 31 Mar 2026 23:25:04 +0200 Subject: [PATCH 1/9] test kubernetes worker with k3d Signed-off-by: CrazyMax <1951866+crazy-max@users.noreply.github.com> --- .github/workflows/build.yml | 5 +- Dockerfile | 3 + tests/helpers/k3d.go | 71 +++++++++++++++++++++ tests/integration_test.go | 1 + tests/workers/kubernetes.go | 123 ++++++++++++++++++++++++++++++++++++ 5 files changed, 201 insertions(+), 2 deletions(-) create mode 100644 tests/helpers/k3d.go create mode 100644 tests/workers/kubernetes.go diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index ddec27555741..1fb051cd5d46 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -54,6 +54,7 @@ jobs: worker: - docker-container - remote + - kubernetes pkg: - ./tests mode: @@ -105,14 +106,14 @@ jobs: fi testFlags="--run=//worker=$(echo "${{ matrix.worker }}" | sed 's/\+/\\+/g')$" case "${{ matrix.worker }}" in - docker | docker+containerd | docker@* | docker+containerd@* | remote+multinode) + docker | docker+containerd | docker@* | docker+containerd@* | remote+multinode | kubernetes) echo "TESTFLAGS=${{ env.TESTFLAGS_DOCKER }} $testFlags" >> $GITHUB_ENV ;; *) echo "TESTFLAGS=${{ env.TESTFLAGS }} $testFlags" >> $GITHUB_ENV ;; esac - if [[ "${{ matrix.worker }}" == "docker"* || "${{ matrix.worker }}" == "remote+multinode" ]]; then + if [[ "${{ matrix.worker }}" == "docker"* || "${{ matrix.worker }}" == "remote+multinode" || "${{ matrix.worker }}" == "kubernetes" ]]; then echo "TEST_DOCKERD=1" >> $GITHUB_ENV fi if [ "${{ matrix.mode }}" = "experimental" ]; then diff --git a/Dockerfile b/Dockerfile index 949cb5911712..a75a6136dfee 100644 --- a/Dockerfile +++ b/Dockerfile @@ -14,6 +14,7 @@ ARG REGISTRY_VERSION=3.0.0 ARG BUILDKIT_VERSION=v0.29.0 ARG COMPOSE_VERSION=v5.1.0 ARG UNDOCK_VERSION=0.9.0 +ARG K3D_VERSION=5.8.3 FROM --platform=$BUILDPLATFORM tonistiigi/xx:${XX_VERSION} AS xx FROM --platform=$BUILDPLATFORM golang:${GO_VERSION}-alpine${ALPINE_VERSION} AS golatest @@ -27,6 +28,7 @@ FROM registry:$REGISTRY_VERSION AS registry FROM moby/buildkit:$BUILDKIT_VERSION AS buildkit FROM docker/compose-bin:$COMPOSE_VERSION AS compose FROM crazymax/undock:$UNDOCK_VERSION AS undock +FROM ghcr.io/k3d-io/k3d:${K3D_VERSION} AS k3d FROM golatest AS gobase COPY --from=xx / / @@ -149,6 +151,7 @@ COPY --link --from=buildkit /usr/bin/buildkitd /usr/bin/ COPY --link --from=buildkit /usr/bin/buildctl /usr/bin/ COPY --link --from=compose /docker-compose /usr/bin/compose COPY --link --from=undock /usr/local/bin/undock /usr/bin/ +COPY --link --from=k3d /bin/k3d /usr/bin/ COPY --link --from=binaries /buildx /usr/bin/ RUN mkdir -p /usr/local/lib/docker/cli-plugins && ln -s /usr/bin/buildx /usr/local/lib/docker/cli-plugins/docker-buildx ENV TEST_DOCKER_EXTRA="docker@28.5=/opt/docker-alt-28,docker@27.5=/opt/docker-alt-27" diff --git a/tests/helpers/k3d.go b/tests/helpers/k3d.go new file mode 100644 index 000000000000..ea67b896add0 --- /dev/null +++ b/tests/helpers/k3d.go @@ -0,0 +1,71 @@ +package helpers + +import ( + "context" + "os" + "os/exec" + "strings" + + "github.com/moby/buildkit/identity" + "github.com/moby/buildkit/util/testutil/integration" + "github.com/pkg/errors" +) + +const ( + k3dBin = "k3d" +) + +func NewK3dServer(ctx context.Context, cfg *integration.BackendConfig, dockerAddress string) (kubeConfig string, cl func() error, err error) { + if _, err := exec.LookPath(k3dBin); err != nil { + return "", nil, errors.Wrapf(err, "failed to lookup %s binary", k3dBin) + } + + deferF := &integration.MultiCloser{} + cl = deferF.F() + + defer func() { + if err != nil { + deferF.F()() + cl = nil + } + }() + + clusterName := "bk-" + identity.NewID() + + cmd := exec.CommandContext(ctx, k3dBin, "cluster", "create", clusterName, + "--wait", + ) + cmd.Env = append( + os.Environ(), + "DOCKER_CONTEXT="+dockerAddress, + ) + out, err := cmd.CombinedOutput() + if err != nil { + return "", nil, errors.Wrapf(err, "failed to create k3d cluster %s: %s", clusterName, string(out)) + } + deferF.Append(func() error { + cmd := exec.Command(k3dBin, "cluster", "delete", clusterName) + cmd.Env = append( + os.Environ(), + "DOCKER_CONTEXT="+dockerAddress, + ) + out, err := cmd.CombinedOutput() + if err != nil { + return errors.Wrapf(err, "failed to delete k3d cluster %s: %s", clusterName, string(out)) + } + return nil + }) + + cmd = exec.CommandContext(ctx, k3dBin, "kubeconfig", "write", clusterName) + cmd.Env = append( + os.Environ(), + "DOCKER_CONTEXT="+dockerAddress, + ) + out, err = cmd.CombinedOutput() + if err != nil { + return "", nil, errors.Wrapf(err, "failed to write kubeconfig for cluster %s: %s", clusterName, string(out)) + } + kubeConfig = strings.TrimSpace(string(out)) + + return +} diff --git a/tests/integration_test.go b/tests/integration_test.go index 574251dc5075..9ce1c8d9c128 100644 --- a/tests/integration_test.go +++ b/tests/integration_test.go @@ -15,6 +15,7 @@ func init() { workers.InitDockerWorker() workers.InitDockerContainerWorker() workers.InitRemoteMultiNodeWorker() + workers.InitKubernetesWorker() } else { workers.InitRemoteWorker() } diff --git a/tests/workers/kubernetes.go b/tests/workers/kubernetes.go new file mode 100644 index 000000000000..3a63cccb0ced --- /dev/null +++ b/tests/workers/kubernetes.go @@ -0,0 +1,123 @@ +package workers + +import ( + "context" + "os" + "os/exec" + "sync" + + "github.com/docker/buildx/tests/helpers" + "github.com/moby/buildkit/identity" + "github.com/moby/buildkit/util/testutil/integration" + "github.com/pkg/errors" +) + +func InitKubernetesWorker() { + integration.Register(&kubernetesWorker{ + id: "kubernetes", + }) +} + +type kubernetesWorker struct { + id string + + unsupported []string + + docker integration.Backend + dockerClose func() error + dockerErr error + dockerOnce sync.Once + + k3dConfig string + k3dClose func() error + k3dErr error + k3dOnce sync.Once +} + +func (w *kubernetesWorker) Name() string { + return w.id +} + +func (w *kubernetesWorker) Rootless() bool { + return false +} + +func (w *kubernetesWorker) NetNSDetached() bool { + return false +} + +func (w *kubernetesWorker) New(ctx context.Context, cfg *integration.BackendConfig) (integration.Backend, func() error, error) { + w.dockerOnce.Do(func() { + w.docker, w.dockerClose, w.dockerErr = dockerWorker{id: w.id}.New(ctx, cfg) + }) + if w.dockerErr != nil { + return w.docker, w.dockerClose, w.dockerErr + } + + w.k3dOnce.Do(func() { + w.k3dConfig, w.k3dClose, w.k3dErr = helpers.NewK3dServer(ctx, cfg, w.docker.DockerAddress()) + }) + if w.k3dErr != nil { + return nil, w.k3dClose, w.k3dErr + } + + name := "integration-kubernetes-" + identity.NewID() + // The generic integration harness injects a host-local registry mirror config. + // That works for host-networked workers, but not for a BuildKit pod where + // localhost points at the pod itself instead of the runner. + cmd := exec.CommandContext(ctx, "buildx", "create", "--bootstrap", "--name="+name, "--driver=kubernetes") + cmd.Env = append( + os.Environ(), + "BUILDX_CONFIG=/tmp/buildx-"+name, + "DOCKER_CONTEXT="+w.docker.DockerAddress(), + "KUBECONFIG="+w.k3dConfig, + ) + if err := cmd.Run(); err != nil { + return nil, nil, errors.Wrapf(err, "failed to create buildx instance %s", name) + } + + cl := func() error { + cmd := exec.CommandContext(context.Background(), "buildx", "rm", "-f", name) + cmd.Env = append( + os.Environ(), + "BUILDX_CONFIG=/tmp/buildx-"+name, + "DOCKER_CONTEXT="+w.docker.DockerAddress(), + "KUBECONFIG="+w.k3dConfig, + ) + return cmd.Run() + } + + return &backend{ + context: w.docker.DockerAddress(), + builder: name, + unsupportedFeatures: w.unsupported, + }, cl, nil +} + +func (w *kubernetesWorker) Close() error { + setErr := func(dst *error, err error) { + if err != nil && *dst == nil { + *dst = err + } + } + + var err error + if c := w.k3dClose; c != nil { + setErr(&err, c()) + } + if c := w.dockerClose; c != nil { + setErr(&err, c()) + } + + // reset the worker to be ready to go again + w.docker = nil + w.dockerClose = nil + w.dockerErr = nil + w.dockerOnce = sync.Once{} + w.k3dConfig = "" + w.k3dClose = nil + w.k3dErr = nil + w.k3dOnce = sync.Once{} + + return err +} From c718b277e750c1a867b0644f468166c72a2c56db Mon Sep 17 00:00:00 2001 From: CrazyMax <1951866+crazy-max@users.noreply.github.com> Date: Wed, 1 Apr 2026 00:19:37 +0200 Subject: [PATCH 2/9] diag Signed-off-by: CrazyMax <1951866+crazy-max@users.noreply.github.com> --- tests/helpers/k3d.go | 10 +-- tests/helpers/kubernetes_diagnostics.go | 114 ++++++++++++++++++++++++ tests/workers/kubernetes.go | 22 +++-- 3 files changed, 134 insertions(+), 12 deletions(-) create mode 100644 tests/helpers/kubernetes_diagnostics.go diff --git a/tests/helpers/k3d.go b/tests/helpers/k3d.go index ea67b896add0..4672d820be25 100644 --- a/tests/helpers/k3d.go +++ b/tests/helpers/k3d.go @@ -15,9 +15,9 @@ const ( k3dBin = "k3d" ) -func NewK3dServer(ctx context.Context, cfg *integration.BackendConfig, dockerAddress string) (kubeConfig string, cl func() error, err error) { +func NewK3dServer(ctx context.Context, cfg *integration.BackendConfig, dockerAddress string) (clusterName, kubeConfig string, cl func() error, err error) { if _, err := exec.LookPath(k3dBin); err != nil { - return "", nil, errors.Wrapf(err, "failed to lookup %s binary", k3dBin) + return "", "", nil, errors.Wrapf(err, "failed to lookup %s binary", k3dBin) } deferF := &integration.MultiCloser{} @@ -30,7 +30,7 @@ func NewK3dServer(ctx context.Context, cfg *integration.BackendConfig, dockerAdd } }() - clusterName := "bk-" + identity.NewID() + clusterName = "bk-" + identity.NewID() cmd := exec.CommandContext(ctx, k3dBin, "cluster", "create", clusterName, "--wait", @@ -41,7 +41,7 @@ func NewK3dServer(ctx context.Context, cfg *integration.BackendConfig, dockerAdd ) out, err := cmd.CombinedOutput() if err != nil { - return "", nil, errors.Wrapf(err, "failed to create k3d cluster %s: %s", clusterName, string(out)) + return "", "", nil, errors.Wrapf(err, "failed to create k3d cluster %s: %s", clusterName, string(out)) } deferF.Append(func() error { cmd := exec.Command(k3dBin, "cluster", "delete", clusterName) @@ -63,7 +63,7 @@ func NewK3dServer(ctx context.Context, cfg *integration.BackendConfig, dockerAdd ) out, err = cmd.CombinedOutput() if err != nil { - return "", nil, errors.Wrapf(err, "failed to write kubeconfig for cluster %s: %s", clusterName, string(out)) + return "", "", nil, errors.Wrapf(err, "failed to write kubeconfig for cluster %s: %s", clusterName, string(out)) } kubeConfig = strings.TrimSpace(string(out)) diff --git a/tests/helpers/kubernetes_diagnostics.go b/tests/helpers/kubernetes_diagnostics.go new file mode 100644 index 000000000000..d296baf352c4 --- /dev/null +++ b/tests/helpers/kubernetes_diagnostics.go @@ -0,0 +1,114 @@ +package helpers + +import ( + "bytes" + "context" + "fmt" + "os" + "os/exec" + "strings" + "time" +) + +const defaultTestBuildkitTag = "buildx-stable-1" + +func KubernetesBuildkitImage() string { + if v := os.Getenv("TEST_BUILDKIT_IMAGE"); v != "" { + return v + } + tag := os.Getenv("TEST_BUILDKIT_TAG") + if tag == "" { + tag = defaultTestBuildkitTag + } + return "moby/buildkit:" + tag +} + +func KubernetesDiagnostics(clusterName, dockerContext string) string { + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) + defer cancel() + + var buf bytes.Buffer + appendK3dDiagnostics(ctx, &buf, clusterName, dockerContext) + appendDockerDiagnostics(ctx, &buf, dockerContext) + appendK3sServerDiagnostics(ctx, &buf, clusterName, dockerContext) + return strings.TrimSpace(buf.String()) +} + +func appendK3dDiagnostics(ctx context.Context, buf *bytes.Buffer, clusterName, dockerContext string) { + appendCommandOutput(ctx, buf, "k3d cluster list", "k3d", []string{"cluster", "list", clusterName}, []string{"DOCKER_CONTEXT=" + dockerContext}) + appendCommandOutput(ctx, buf, "k3d node list", "k3d", []string{"node", "list"}, []string{"DOCKER_CONTEXT=" + dockerContext}) +} + +func appendDockerDiagnostics(ctx context.Context, buf *bytes.Buffer, dockerContext string) { + args := []string{"ps", "-a", "--format", "{{.Names}}\t{{.Image}}\t{{.Status}}"} + appendCommandOutput(ctx, buf, "docker ps", "docker", args, []string{"DOCKER_CONTEXT=" + dockerContext}) +} + +func appendK3sServerDiagnostics(ctx context.Context, buf *bytes.Buffer, clusterName, dockerContext string) { + nodeNames, err := clusterNodeNames(ctx, clusterName, dockerContext) + if err != nil { + fmt.Fprintf(buf, "cluster node discovery error: %v\n", err) + return + } + if len(nodeNames) == 0 { + fmt.Fprintln(buf, "cluster node discovery: no matching k3d containers found") + return + } + + for _, nodeName := range nodeNames { + appendCommandOutput(ctx, buf, "docker logs "+nodeName, "docker", []string{"logs", "--tail", "80", nodeName}, []string{"DOCKER_CONTEXT=" + dockerContext}) + } + + for _, nodeName := range nodeNames { + if !strings.Contains(nodeName, "-server-") { + continue + } + appendCommandOutput(ctx, buf, "docker exec "+nodeName+" k3s kubectl get pods", "docker", []string{"exec", nodeName, "k3s", "kubectl", "get", "pods", "-A", "-o", "wide"}, []string{"DOCKER_CONTEXT=" + dockerContext}) + appendCommandOutput(ctx, buf, "docker exec "+nodeName+" k3s kubectl get events", "docker", []string{"exec", nodeName, "k3s", "kubectl", "get", "events", "-A", "--sort-by=.lastTimestamp"}, []string{"DOCKER_CONTEXT=" + dockerContext}) + appendCommandOutput(ctx, buf, "docker exec "+nodeName+" k3s kubectl describe pods", "docker", []string{"exec", nodeName, "k3s", "kubectl", "describe", "pods", "-A"}, []string{"DOCKER_CONTEXT=" + dockerContext}) + break + } +} + +func clusterNodeNames(ctx context.Context, clusterName, dockerContext string) ([]string, error) { + out, err := runCommand(ctx, "docker", []string{ + "ps", "-a", + "--filter", "name=k3d-" + clusterName, + "--format", "{{.Names}}", + }, []string{"DOCKER_CONTEXT=" + dockerContext}) + if err != nil { + return nil, err + } + var names []string + for _, line := range strings.Split(strings.TrimSpace(out), "\n") { + line = strings.TrimSpace(line) + if line == "" { + continue + } + names = append(names, line) + } + return names, nil +} + +func appendCommandOutput(ctx context.Context, buf *bytes.Buffer, title, name string, args []string, env []string) { + out, err := runCommand(ctx, name, args, env) + fmt.Fprintf(buf, "== %s ==\n", title) + if err != nil { + fmt.Fprintf(buf, "error: %v\n", err) + } + if strings.TrimSpace(out) == "" { + fmt.Fprintln(buf, "") + } else { + fmt.Fprintf(buf, "%s\n", strings.TrimSpace(out)) + } + fmt.Fprintln(buf) +} + +func runCommand(ctx context.Context, name string, args []string, env []string) (string, error) { + cmd := exec.CommandContext(ctx, name, args...) + if len(env) > 0 { + cmd.Env = append(os.Environ(), env...) + } + out, err := cmd.CombinedOutput() + return string(out), err +} diff --git a/tests/workers/kubernetes.go b/tests/workers/kubernetes.go index 3a63cccb0ced..ec9262418906 100644 --- a/tests/workers/kubernetes.go +++ b/tests/workers/kubernetes.go @@ -4,6 +4,7 @@ import ( "context" "os" "os/exec" + "strings" "sync" "github.com/docker/buildx/tests/helpers" @@ -28,6 +29,7 @@ type kubernetesWorker struct { dockerErr error dockerOnce sync.Once + k3dName string k3dConfig string k3dClose func() error k3dErr error @@ -55,25 +57,30 @@ func (w *kubernetesWorker) New(ctx context.Context, cfg *integration.BackendConf } w.k3dOnce.Do(func() { - w.k3dConfig, w.k3dClose, w.k3dErr = helpers.NewK3dServer(ctx, cfg, w.docker.DockerAddress()) + w.k3dName, w.k3dConfig, w.k3dClose, w.k3dErr = helpers.NewK3dServer(ctx, cfg, w.docker.DockerAddress()) }) if w.k3dErr != nil { return nil, w.k3dClose, w.k3dErr } name := "integration-kubernetes-" + identity.NewID() - // The generic integration harness injects a host-local registry mirror config. - // That works for host-networked workers, but not for a BuildKit pod where - // localhost points at the pod itself instead of the runner. - cmd := exec.CommandContext(ctx, "buildx", "create", "--bootstrap", "--name="+name, "--driver=kubernetes") + cmd := exec.CommandContext(ctx, "buildx", "create", + "--bootstrap", + "--name="+name, + "--driver=kubernetes", + "--driver-opt=image="+helpers.KubernetesBuildkitImage(), + "--driver-opt=timeout=60s", + ) cmd.Env = append( os.Environ(), "BUILDX_CONFIG=/tmp/buildx-"+name, "DOCKER_CONTEXT="+w.docker.DockerAddress(), "KUBECONFIG="+w.k3dConfig, ) - if err := cmd.Run(); err != nil { - return nil, nil, errors.Wrapf(err, "failed to create buildx instance %s", name) + out, err := cmd.CombinedOutput() + if err != nil { + diag := helpers.KubernetesDiagnostics(w.k3dName, w.docker.DockerAddress()) + return nil, nil, errors.Wrapf(err, "failed to create buildx instance %s with image %s: %s\n%s", name, helpers.KubernetesBuildkitImage(), strings.TrimSpace(string(out)), diag) } cl := func() error { @@ -114,6 +121,7 @@ func (w *kubernetesWorker) Close() error { w.dockerClose = nil w.dockerErr = nil w.dockerOnce = sync.Once{} + w.k3dName = "" w.k3dConfig = "" w.k3dClose = nil w.k3dErr = nil From 65db9ee4ed0a6952415ecfe7f469a94720f8d20a Mon Sep 17 00:00:00 2001 From: CrazyMax <1951866+crazy-max@users.noreply.github.com> Date: Wed, 1 Apr 2026 08:29:28 +0200 Subject: [PATCH 3/9] k3d timeout Signed-off-by: CrazyMax <1951866+crazy-max@users.noreply.github.com> --- tests/helpers/k3d.go | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/tests/helpers/k3d.go b/tests/helpers/k3d.go index 4672d820be25..44a51f25a391 100644 --- a/tests/helpers/k3d.go +++ b/tests/helpers/k3d.go @@ -5,6 +5,7 @@ import ( "os" "os/exec" "strings" + "time" "github.com/moby/buildkit/identity" "github.com/moby/buildkit/util/testutil/integration" @@ -32,7 +33,10 @@ func NewK3dServer(ctx context.Context, cfg *integration.BackendConfig, dockerAdd clusterName = "bk-" + identity.NewID() - cmd := exec.CommandContext(ctx, k3dBin, "cluster", "create", clusterName, + createCtx, cancelCreate := context.WithTimeout(ctx, 90*time.Second) + defer cancelCreate() + + cmd := exec.CommandContext(createCtx, k3dBin, "cluster", "create", clusterName, "--wait", ) cmd.Env = append( @@ -41,7 +45,8 @@ func NewK3dServer(ctx context.Context, cfg *integration.BackendConfig, dockerAdd ) out, err := cmd.CombinedOutput() if err != nil { - return "", "", nil, errors.Wrapf(err, "failed to create k3d cluster %s: %s", clusterName, string(out)) + diag := KubernetesDiagnostics(clusterName, dockerAddress) + return "", "", nil, errors.Wrapf(err, "failed to create k3d cluster %s: %s\n%s", clusterName, strings.TrimSpace(string(out)), diag) } deferF.Append(func() error { cmd := exec.Command(k3dBin, "cluster", "delete", clusterName) @@ -56,14 +61,18 @@ func NewK3dServer(ctx context.Context, cfg *integration.BackendConfig, dockerAdd return nil }) - cmd = exec.CommandContext(ctx, k3dBin, "kubeconfig", "write", clusterName) + kubeconfigCtx, cancelKubeconfig := context.WithTimeout(ctx, 30*time.Second) + defer cancelKubeconfig() + + cmd = exec.CommandContext(kubeconfigCtx, k3dBin, "kubeconfig", "write", clusterName) cmd.Env = append( os.Environ(), "DOCKER_CONTEXT="+dockerAddress, ) out, err = cmd.CombinedOutput() if err != nil { - return "", "", nil, errors.Wrapf(err, "failed to write kubeconfig for cluster %s: %s", clusterName, string(out)) + diag := KubernetesDiagnostics(clusterName, dockerAddress) + return "", "", nil, errors.Wrapf(err, "failed to write kubeconfig for cluster %s: %s\n%s", clusterName, strings.TrimSpace(string(out)), diag) } kubeConfig = strings.TrimSpace(string(out)) From 1417f6765c17de7659a50bcabfefdd9a267cfd0b Mon Sep 17 00:00:00 2001 From: CrazyMax <1951866+crazy-max@users.noreply.github.com> Date: Wed, 1 Apr 2026 08:51:18 +0200 Subject: [PATCH 4/9] tests: pin k3d images for kubernetes worker Signed-off-by: CrazyMax <1951866+crazy-max@users.noreply.github.com> --- Dockerfile | 4 +++ tests/helpers/k3d.go | 38 ++++++++++++++++--------- tests/helpers/kubernetes_diagnostics.go | 24 ++++++++++++++-- 3 files changed, 49 insertions(+), 17 deletions(-) diff --git a/Dockerfile b/Dockerfile index a75a6136dfee..2401000c5bb0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -15,6 +15,7 @@ ARG BUILDKIT_VERSION=v0.29.0 ARG COMPOSE_VERSION=v5.1.0 ARG UNDOCK_VERSION=0.9.0 ARG K3D_VERSION=5.8.3 +ARG K3S_VERSION=v1.32.2-k3s1 FROM --platform=$BUILDPLATFORM tonistiigi/xx:${XX_VERSION} AS xx FROM --platform=$BUILDPLATFORM golang:${GO_VERSION}-alpine${ALPINE_VERSION} AS golatest @@ -155,6 +156,9 @@ COPY --link --from=k3d /bin/k3d /usr/bin/ COPY --link --from=binaries /buildx /usr/bin/ RUN mkdir -p /usr/local/lib/docker/cli-plugins && ln -s /usr/bin/buildx /usr/local/lib/docker/cli-plugins/docker-buildx ENV TEST_DOCKER_EXTRA="docker@28.5=/opt/docker-alt-28,docker@27.5=/opt/docker-alt-27" +ENV TEST_K3S_IMAGE="rancher/k3s:${K3S_VERSION}" +ENV TEST_K3D_TOOLS_IMAGE="ghcr.io/k3d-io/k3d-tools:${K3D_VERSION}" +ENV TEST_K3D_LOADBALANCER_IMAGE="ghcr.io/k3d-io/k3d-proxy:${K3D_VERSION}" FROM integration-test-base AS integration-test COPY . . diff --git a/tests/helpers/k3d.go b/tests/helpers/k3d.go index 44a51f25a391..92f3be37dba2 100644 --- a/tests/helpers/k3d.go +++ b/tests/helpers/k3d.go @@ -36,13 +36,15 @@ func NewK3dServer(ctx context.Context, cfg *integration.BackendConfig, dockerAdd createCtx, cancelCreate := context.WithTimeout(ctx, 90*time.Second) defer cancelCreate() - cmd := exec.CommandContext(createCtx, k3dBin, "cluster", "create", clusterName, + args := []string{ + "cluster", "create", clusterName, "--wait", - ) - cmd.Env = append( - os.Environ(), - "DOCKER_CONTEXT="+dockerAddress, - ) + } + if image := KubernetesK3sImage(); image != "" { + args = append(args, "--image="+image) + } + cmd := exec.CommandContext(createCtx, k3dBin, args...) + cmd.Env = k3dEnv(dockerAddress) out, err := cmd.CombinedOutput() if err != nil { diag := KubernetesDiagnostics(clusterName, dockerAddress) @@ -50,10 +52,7 @@ func NewK3dServer(ctx context.Context, cfg *integration.BackendConfig, dockerAdd } deferF.Append(func() error { cmd := exec.Command(k3dBin, "cluster", "delete", clusterName) - cmd.Env = append( - os.Environ(), - "DOCKER_CONTEXT="+dockerAddress, - ) + cmd.Env = k3dEnv(dockerAddress) out, err := cmd.CombinedOutput() if err != nil { return errors.Wrapf(err, "failed to delete k3d cluster %s: %s", clusterName, string(out)) @@ -65,10 +64,7 @@ func NewK3dServer(ctx context.Context, cfg *integration.BackendConfig, dockerAdd defer cancelKubeconfig() cmd = exec.CommandContext(kubeconfigCtx, k3dBin, "kubeconfig", "write", clusterName) - cmd.Env = append( - os.Environ(), - "DOCKER_CONTEXT="+dockerAddress, - ) + cmd.Env = k3dEnv(dockerAddress) out, err = cmd.CombinedOutput() if err != nil { diag := KubernetesDiagnostics(clusterName, dockerAddress) @@ -78,3 +74,17 @@ func NewK3dServer(ctx context.Context, cfg *integration.BackendConfig, dockerAdd return } + +func k3dEnv(dockerAddress string) []string { + env := append( + os.Environ(), + "DOCKER_CONTEXT="+dockerAddress, + ) + if image := KubernetesK3DToolsImage(); image != "" { + env = append(env, "K3D_IMAGE_TOOLS="+image) + } + if image := KubernetesK3DLoadBalancerImage(); image != "" { + env = append(env, "K3D_IMAGE_LOADBALANCER="+image) + } + return env +} diff --git a/tests/helpers/kubernetes_diagnostics.go b/tests/helpers/kubernetes_diagnostics.go index d296baf352c4..de124c01f637 100644 --- a/tests/helpers/kubernetes_diagnostics.go +++ b/tests/helpers/kubernetes_diagnostics.go @@ -23,6 +23,18 @@ func KubernetesBuildkitImage() string { return "moby/buildkit:" + tag } +func KubernetesK3sImage() string { + return os.Getenv("TEST_K3S_IMAGE") +} + +func KubernetesK3DToolsImage() string { + return os.Getenv("TEST_K3D_TOOLS_IMAGE") +} + +func KubernetesK3DLoadBalancerImage() string { + return os.Getenv("TEST_K3D_LOADBALANCER_IMAGE") +} + func KubernetesDiagnostics(clusterName, dockerContext string) string { ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) defer cancel() @@ -56,6 +68,12 @@ func appendK3sServerDiagnostics(ctx context.Context, buf *bytes.Buffer, clusterN } for _, nodeName := range nodeNames { + appendCommandOutput(ctx, buf, "docker inspect "+nodeName, "docker", []string{ + "inspect", + "--format", + "Status={{.State.Status}} Health={{if .State.Health}}{{.State.Health.Status}}{{else}}{{end}} Restarting={{.State.Restarting}} ExitCode={{.State.ExitCode}} Error={{.State.Error}}", + nodeName, + }, []string{"DOCKER_CONTEXT=" + dockerContext}) appendCommandOutput(ctx, buf, "docker logs "+nodeName, "docker", []string{"logs", "--tail", "80", nodeName}, []string{"DOCKER_CONTEXT=" + dockerContext}) } @@ -63,9 +81,9 @@ func appendK3sServerDiagnostics(ctx context.Context, buf *bytes.Buffer, clusterN if !strings.Contains(nodeName, "-server-") { continue } - appendCommandOutput(ctx, buf, "docker exec "+nodeName+" k3s kubectl get pods", "docker", []string{"exec", nodeName, "k3s", "kubectl", "get", "pods", "-A", "-o", "wide"}, []string{"DOCKER_CONTEXT=" + dockerContext}) - appendCommandOutput(ctx, buf, "docker exec "+nodeName+" k3s kubectl get events", "docker", []string{"exec", nodeName, "k3s", "kubectl", "get", "events", "-A", "--sort-by=.lastTimestamp"}, []string{"DOCKER_CONTEXT=" + dockerContext}) - appendCommandOutput(ctx, buf, "docker exec "+nodeName+" k3s kubectl describe pods", "docker", []string{"exec", nodeName, "k3s", "kubectl", "describe", "pods", "-A"}, []string{"DOCKER_CONTEXT=" + dockerContext}) + appendCommandOutput(ctx, buf, "docker exec "+nodeName+" kubectl get pods", "docker", []string{"exec", nodeName, "kubectl", "get", "pods", "-A", "-o", "wide"}, []string{"DOCKER_CONTEXT=" + dockerContext}) + appendCommandOutput(ctx, buf, "docker exec "+nodeName+" kubectl get events", "docker", []string{"exec", nodeName, "kubectl", "get", "events", "-A", "--sort-by=.lastTimestamp"}, []string{"DOCKER_CONTEXT=" + dockerContext}) + appendCommandOutput(ctx, buf, "docker exec "+nodeName+" kubectl describe pods", "docker", []string{"exec", nodeName, "kubectl", "describe", "pods", "-A"}, []string{"DOCKER_CONTEXT=" + dockerContext}) break } } From 1d2146af2f84e91763f9611962081ede2866f321 Mon Sep 17 00:00:00 2001 From: CrazyMax <1951866+crazy-max@users.noreply.github.com> Date: Wed, 1 Apr 2026 08:57:06 +0200 Subject: [PATCH 5/9] tests: redeclare k3d args in integration image Signed-off-by: CrazyMax <1951866+crazy-max@users.noreply.github.com> --- Dockerfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Dockerfile b/Dockerfile index 2401000c5bb0..845bb226196a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -127,6 +127,8 @@ FROM binaries-$TARGETOS AS binaries ARG BUILDKIT_SBOM_SCAN_STAGE=true FROM gobase AS integration-test-base +ARG K3D_VERSION +ARG K3S_VERSION # https://github.com/docker/docker/blob/master/project/PACKAGERS.md#runtime-dependencies RUN apk add --no-cache \ bash \ From 31a3b0c7b18ba391599c1dfcf20c96713bc7a088 Mon Sep 17 00:00:00 2001 From: CrazyMax <1951866+crazy-max@users.noreply.github.com> Date: Wed, 1 Apr 2026 09:07:09 +0200 Subject: [PATCH 6/9] tests: expand kubernetes worker diagnostics Signed-off-by: CrazyMax <1951866+crazy-max@users.noreply.github.com> --- tests/helpers/kubernetes_diagnostics.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/helpers/kubernetes_diagnostics.go b/tests/helpers/kubernetes_diagnostics.go index de124c01f637..3c5e2308f0da 100644 --- a/tests/helpers/kubernetes_diagnostics.go +++ b/tests/helpers/kubernetes_diagnostics.go @@ -71,7 +71,7 @@ func appendK3sServerDiagnostics(ctx context.Context, buf *bytes.Buffer, clusterN appendCommandOutput(ctx, buf, "docker inspect "+nodeName, "docker", []string{ "inspect", "--format", - "Status={{.State.Status}} Health={{if .State.Health}}{{.State.Health.Status}}{{else}}{{end}} Restarting={{.State.Restarting}} ExitCode={{.State.ExitCode}} Error={{.State.Error}}", + "Status={{.State.Status}} Health={{if .State.Health}}{{.State.Health.Status}}{{else}}{{end}} Restarting={{.State.Restarting}} ExitCode={{.State.ExitCode}} Error={{.State.Error}} Privileged={{.HostConfig.Privileged}} Cgroupns={{.HostConfig.CgroupnsMode}}", nodeName, }, []string{"DOCKER_CONTEXT=" + dockerContext}) appendCommandOutput(ctx, buf, "docker logs "+nodeName, "docker", []string{"logs", "--tail", "80", nodeName}, []string{"DOCKER_CONTEXT=" + dockerContext}) @@ -81,6 +81,11 @@ func appendK3sServerDiagnostics(ctx context.Context, buf *bytes.Buffer, clusterN if !strings.Contains(nodeName, "-server-") { continue } + appendCommandOutput(ctx, buf, "docker exec "+nodeName+" ps", "docker", []string{"exec", nodeName, "sh", "-c", "ps auxww"}, []string{"DOCKER_CONTEXT=" + dockerContext}) + appendCommandOutput(ctx, buf, "docker exec "+nodeName+" ss", "docker", []string{"exec", nodeName, "sh", "-c", "ss -lntp || netstat -lntp"}, []string{"DOCKER_CONTEXT=" + dockerContext}) + appendCommandOutput(ctx, buf, "docker exec "+nodeName+" cgroup", "docker", []string{"exec", nodeName, "sh", "-c", "cat /proc/1/cgroup && echo && mount | grep cgroup"}, []string{"DOCKER_CONTEXT=" + dockerContext}) + appendCommandOutput(ctx, buf, "docker exec "+nodeName+" k3s files", "docker", []string{"exec", nodeName, "sh", "-c", "find /var/lib/rancher/k3s -maxdepth 3 -type f 2>/dev/null | sort"}, []string{"DOCKER_CONTEXT=" + dockerContext}) + appendCommandOutput(ctx, buf, "docker exec "+nodeName+" k3s logs", "docker", []string{"exec", nodeName, "sh", "-c", "for f in /var/log/k3s.log /var/lib/rancher/k3s/agent/containerd/containerd.log /var/lib/rancher/k3s/server/logs/*; do if [ -f \"$f\" ]; then echo \"== $f ==\"; tail -n 200 \"$f\"; echo; fi; done"}, []string{"DOCKER_CONTEXT=" + dockerContext}) appendCommandOutput(ctx, buf, "docker exec "+nodeName+" kubectl get pods", "docker", []string{"exec", nodeName, "kubectl", "get", "pods", "-A", "-o", "wide"}, []string{"DOCKER_CONTEXT=" + dockerContext}) appendCommandOutput(ctx, buf, "docker exec "+nodeName+" kubectl get events", "docker", []string{"exec", nodeName, "kubectl", "get", "events", "-A", "--sort-by=.lastTimestamp"}, []string{"DOCKER_CONTEXT=" + dockerContext}) appendCommandOutput(ctx, buf, "docker exec "+nodeName+" kubectl describe pods", "docker", []string{"exec", nodeName, "kubectl", "describe", "pods", "-A"}, []string{"DOCKER_CONTEXT=" + dockerContext}) From 0c05598017e0058a47a7055ce3a892407907a49b Mon Sep 17 00:00:00 2001 From: CrazyMax <1951866+crazy-max@users.noreply.github.com> Date: Wed, 1 Apr 2026 09:17:18 +0200 Subject: [PATCH 7/9] tests: include nested dockerd logs in k3d failures Signed-off-by: CrazyMax <1951866+crazy-max@users.noreply.github.com> --- tests/helpers/k3d.go | 4 ++-- tests/helpers/kubernetes_diagnostics.go | 4 +++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/tests/helpers/k3d.go b/tests/helpers/k3d.go index 92f3be37dba2..47fcd13fb247 100644 --- a/tests/helpers/k3d.go +++ b/tests/helpers/k3d.go @@ -48,7 +48,7 @@ func NewK3dServer(ctx context.Context, cfg *integration.BackendConfig, dockerAdd out, err := cmd.CombinedOutput() if err != nil { diag := KubernetesDiagnostics(clusterName, dockerAddress) - return "", "", nil, errors.Wrapf(err, "failed to create k3d cluster %s: %s\n%s", clusterName, strings.TrimSpace(string(out)), diag) + return "", "", nil, errors.Wrapf(err, "failed to create k3d cluster %s: %s\n%s\nouter dockerd logs: %s", clusterName, strings.TrimSpace(string(out)), diag, integration.FormatLogs(cfg.Logs)) } deferF.Append(func() error { cmd := exec.Command(k3dBin, "cluster", "delete", clusterName) @@ -68,7 +68,7 @@ func NewK3dServer(ctx context.Context, cfg *integration.BackendConfig, dockerAdd out, err = cmd.CombinedOutput() if err != nil { diag := KubernetesDiagnostics(clusterName, dockerAddress) - return "", "", nil, errors.Wrapf(err, "failed to write kubeconfig for cluster %s: %s\n%s", clusterName, strings.TrimSpace(string(out)), diag) + return "", "", nil, errors.Wrapf(err, "failed to write kubeconfig for cluster %s: %s\n%s\nouter dockerd logs: %s", clusterName, strings.TrimSpace(string(out)), diag, integration.FormatLogs(cfg.Logs)) } kubeConfig = strings.TrimSpace(string(out)) diff --git a/tests/helpers/kubernetes_diagnostics.go b/tests/helpers/kubernetes_diagnostics.go index 3c5e2308f0da..76c1425acc7c 100644 --- a/tests/helpers/kubernetes_diagnostics.go +++ b/tests/helpers/kubernetes_diagnostics.go @@ -82,8 +82,10 @@ func appendK3sServerDiagnostics(ctx context.Context, buf *bytes.Buffer, clusterN continue } appendCommandOutput(ctx, buf, "docker exec "+nodeName+" ps", "docker", []string{"exec", nodeName, "sh", "-c", "ps auxww"}, []string{"DOCKER_CONTEXT=" + dockerContext}) - appendCommandOutput(ctx, buf, "docker exec "+nodeName+" ss", "docker", []string{"exec", nodeName, "sh", "-c", "ss -lntp || netstat -lntp"}, []string{"DOCKER_CONTEXT=" + dockerContext}) + appendCommandOutput(ctx, buf, "docker exec "+nodeName+" sockets", "docker", []string{"exec", nodeName, "sh", "-c", "ss -lntp || netstat -lnt"}, []string{"DOCKER_CONTEXT=" + dockerContext}) appendCommandOutput(ctx, buf, "docker exec "+nodeName+" cgroup", "docker", []string{"exec", nodeName, "sh", "-c", "cat /proc/1/cgroup && echo && mount | grep cgroup"}, []string{"DOCKER_CONTEXT=" + dockerContext}) + appendCommandOutput(ctx, buf, "docker exec "+nodeName+" env", "docker", []string{"exec", nodeName, "sh", "-c", "env | sort"}, []string{"DOCKER_CONTEXT=" + dockerContext}) + appendCommandOutput(ctx, buf, "docker exec "+nodeName+" entrypoint", "docker", []string{"exec", nodeName, "sh", "-c", "sed -n '1,200p' /bin/k3d-entrypoint.sh"}, []string{"DOCKER_CONTEXT=" + dockerContext}) appendCommandOutput(ctx, buf, "docker exec "+nodeName+" k3s files", "docker", []string{"exec", nodeName, "sh", "-c", "find /var/lib/rancher/k3s -maxdepth 3 -type f 2>/dev/null | sort"}, []string{"DOCKER_CONTEXT=" + dockerContext}) appendCommandOutput(ctx, buf, "docker exec "+nodeName+" k3s logs", "docker", []string{"exec", nodeName, "sh", "-c", "for f in /var/log/k3s.log /var/lib/rancher/k3s/agent/containerd/containerd.log /var/lib/rancher/k3s/server/logs/*; do if [ -f \"$f\" ]; then echo \"== $f ==\"; tail -n 200 \"$f\"; echo; fi; done"}, []string{"DOCKER_CONTEXT=" + dockerContext}) appendCommandOutput(ctx, buf, "docker exec "+nodeName+" kubectl get pods", "docker", []string{"exec", nodeName, "kubectl", "get", "pods", "-A", "-o", "wide"}, []string{"DOCKER_CONTEXT=" + dockerContext}) From 20c7dd079489db6aa89faed5a535a0c3bcf631c0 Mon Sep 17 00:00:00 2001 From: CrazyMax <1951866+crazy-max@users.noreply.github.com> Date: Wed, 1 Apr 2026 09:32:26 +0200 Subject: [PATCH 8/9] tests: capture k3s startup failures Signed-off-by: CrazyMax <1951866+crazy-max@users.noreply.github.com> --- tests/helpers/k3d.go | 1 + tests/helpers/kubernetes_diagnostics.go | 1 + 2 files changed, 2 insertions(+) diff --git a/tests/helpers/k3d.go b/tests/helpers/k3d.go index 47fcd13fb247..fbb56748affe 100644 --- a/tests/helpers/k3d.go +++ b/tests/helpers/k3d.go @@ -39,6 +39,7 @@ func NewK3dServer(ctx context.Context, cfg *integration.BackendConfig, dockerAdd args := []string{ "cluster", "create", clusterName, "--wait", + "--k3s-arg=--debug@server:0", } if image := KubernetesK3sImage(); image != "" { args = append(args, "--image="+image) diff --git a/tests/helpers/kubernetes_diagnostics.go b/tests/helpers/kubernetes_diagnostics.go index 76c1425acc7c..5fa945e62cd7 100644 --- a/tests/helpers/kubernetes_diagnostics.go +++ b/tests/helpers/kubernetes_diagnostics.go @@ -86,6 +86,7 @@ func appendK3sServerDiagnostics(ctx context.Context, buf *bytes.Buffer, clusterN appendCommandOutput(ctx, buf, "docker exec "+nodeName+" cgroup", "docker", []string{"exec", nodeName, "sh", "-c", "cat /proc/1/cgroup && echo && mount | grep cgroup"}, []string{"DOCKER_CONTEXT=" + dockerContext}) appendCommandOutput(ctx, buf, "docker exec "+nodeName+" env", "docker", []string{"exec", nodeName, "sh", "-c", "env | sort"}, []string{"DOCKER_CONTEXT=" + dockerContext}) appendCommandOutput(ctx, buf, "docker exec "+nodeName+" entrypoint", "docker", []string{"exec", nodeName, "sh", "-c", "sed -n '1,200p' /bin/k3d-entrypoint.sh"}, []string{"DOCKER_CONTEXT=" + dockerContext}) + appendCommandOutput(ctx, buf, "docker exec "+nodeName+" entrypoint logs", "docker", []string{"exec", nodeName, "sh", "-c", "for f in /var/log/k3d-entrypoints_*.log; do if [ -f \"$f\" ]; then echo \"== $f ==\"; tail -n 200 \"$f\"; echo; fi; done"}, []string{"DOCKER_CONTEXT=" + dockerContext}) appendCommandOutput(ctx, buf, "docker exec "+nodeName+" k3s files", "docker", []string{"exec", nodeName, "sh", "-c", "find /var/lib/rancher/k3s -maxdepth 3 -type f 2>/dev/null | sort"}, []string{"DOCKER_CONTEXT=" + dockerContext}) appendCommandOutput(ctx, buf, "docker exec "+nodeName+" k3s logs", "docker", []string{"exec", nodeName, "sh", "-c", "for f in /var/log/k3s.log /var/lib/rancher/k3s/agent/containerd/containerd.log /var/lib/rancher/k3s/server/logs/*; do if [ -f \"$f\" ]; then echo \"== $f ==\"; tail -n 200 \"$f\"; echo; fi; done"}, []string{"DOCKER_CONTEXT=" + dockerContext}) appendCommandOutput(ctx, buf, "docker exec "+nodeName+" kubectl get pods", "docker", []string{"exec", nodeName, "kubectl", "get", "pods", "-A", "-o", "wide"}, []string{"DOCKER_CONTEXT=" + dockerContext}) From c2e0980b67cfbc722d1a4052181b5c235768843c Mon Sep 17 00:00:00 2001 From: CrazyMax <1951866+crazy-max@users.noreply.github.com> Date: Wed, 1 Apr 2026 10:19:22 +0200 Subject: [PATCH 9/9] tests: refresh kubernetes k3s pin and timeouts Signed-off-by: CrazyMax <1951866+crazy-max@users.noreply.github.com> --- Dockerfile | 2 +- tests/helpers/k3d.go | 12 +++++++----- tests/helpers/kubernetes_diagnostics.go | 5 +++-- tests/workers/kubernetes.go | 2 +- 4 files changed, 12 insertions(+), 9 deletions(-) diff --git a/Dockerfile b/Dockerfile index 845bb226196a..1767e5f81b99 100644 --- a/Dockerfile +++ b/Dockerfile @@ -15,7 +15,7 @@ ARG BUILDKIT_VERSION=v0.29.0 ARG COMPOSE_VERSION=v5.1.0 ARG UNDOCK_VERSION=0.9.0 ARG K3D_VERSION=5.8.3 -ARG K3S_VERSION=v1.32.2-k3s1 +ARG K3S_VERSION=v1.32.13-k3s1 FROM --platform=$BUILDPLATFORM tonistiigi/xx:${XX_VERSION} AS xx FROM --platform=$BUILDPLATFORM golang:${GO_VERSION}-alpine${ALPINE_VERSION} AS golatest diff --git a/tests/helpers/k3d.go b/tests/helpers/k3d.go index fbb56748affe..6b204da214d0 100644 --- a/tests/helpers/k3d.go +++ b/tests/helpers/k3d.go @@ -33,7 +33,7 @@ func NewK3dServer(ctx context.Context, cfg *integration.BackendConfig, dockerAdd clusterName = "bk-" + identity.NewID() - createCtx, cancelCreate := context.WithTimeout(ctx, 90*time.Second) + createCtx, cancelCreate := context.WithTimeoutCause(ctx, 90*time.Second, errors.New("timed out creating k3d cluster")) defer cancelCreate() args := []string{ @@ -48,11 +48,13 @@ func NewK3dServer(ctx context.Context, cfg *integration.BackendConfig, dockerAdd cmd.Env = k3dEnv(dockerAddress) out, err := cmd.CombinedOutput() if err != nil { - diag := KubernetesDiagnostics(clusterName, dockerAddress) + diag := KubernetesDiagnostics(ctx, clusterName, dockerAddress) return "", "", nil, errors.Wrapf(err, "failed to create k3d cluster %s: %s\n%s\nouter dockerd logs: %s", clusterName, strings.TrimSpace(string(out)), diag, integration.FormatLogs(cfg.Logs)) } deferF.Append(func() error { - cmd := exec.Command(k3dBin, "cluster", "delete", clusterName) + deleteCtx, cancelDelete := context.WithTimeoutCause(context.WithoutCancel(ctx), 30*time.Second, errors.New("timed out deleting k3d cluster")) + defer cancelDelete() + cmd := exec.CommandContext(deleteCtx, k3dBin, "cluster", "delete", clusterName) cmd.Env = k3dEnv(dockerAddress) out, err := cmd.CombinedOutput() if err != nil { @@ -61,14 +63,14 @@ func NewK3dServer(ctx context.Context, cfg *integration.BackendConfig, dockerAdd return nil }) - kubeconfigCtx, cancelKubeconfig := context.WithTimeout(ctx, 30*time.Second) + kubeconfigCtx, cancelKubeconfig := context.WithTimeoutCause(ctx, 30*time.Second, errors.New("timed out writing k3d kubeconfig")) defer cancelKubeconfig() cmd = exec.CommandContext(kubeconfigCtx, k3dBin, "kubeconfig", "write", clusterName) cmd.Env = k3dEnv(dockerAddress) out, err = cmd.CombinedOutput() if err != nil { - diag := KubernetesDiagnostics(clusterName, dockerAddress) + diag := KubernetesDiagnostics(ctx, clusterName, dockerAddress) return "", "", nil, errors.Wrapf(err, "failed to write kubeconfig for cluster %s: %s\n%s\nouter dockerd logs: %s", clusterName, strings.TrimSpace(string(out)), diag, integration.FormatLogs(cfg.Logs)) } kubeConfig = strings.TrimSpace(string(out)) diff --git a/tests/helpers/kubernetes_diagnostics.go b/tests/helpers/kubernetes_diagnostics.go index 5fa945e62cd7..6c2224e45fcb 100644 --- a/tests/helpers/kubernetes_diagnostics.go +++ b/tests/helpers/kubernetes_diagnostics.go @@ -3,6 +3,7 @@ package helpers import ( "bytes" "context" + "errors" "fmt" "os" "os/exec" @@ -35,8 +36,8 @@ func KubernetesK3DLoadBalancerImage() string { return os.Getenv("TEST_K3D_LOADBALANCER_IMAGE") } -func KubernetesDiagnostics(clusterName, dockerContext string) string { - ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) +func KubernetesDiagnostics(ctx context.Context, clusterName, dockerContext string) string { + ctx, cancel := context.WithTimeoutCause(context.WithoutCancel(ctx), 20*time.Second, errors.New("timed out collecting kubernetes diagnostics")) defer cancel() var buf bytes.Buffer diff --git a/tests/workers/kubernetes.go b/tests/workers/kubernetes.go index ec9262418906..ec5d725d3330 100644 --- a/tests/workers/kubernetes.go +++ b/tests/workers/kubernetes.go @@ -79,7 +79,7 @@ func (w *kubernetesWorker) New(ctx context.Context, cfg *integration.BackendConf ) out, err := cmd.CombinedOutput() if err != nil { - diag := helpers.KubernetesDiagnostics(w.k3dName, w.docker.DockerAddress()) + diag := helpers.KubernetesDiagnostics(ctx, w.k3dName, w.docker.DockerAddress()) return nil, nil, errors.Wrapf(err, "failed to create buildx instance %s with image %s: %s\n%s", name, helpers.KubernetesBuildkitImage(), strings.TrimSpace(string(out)), diag) }