diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index ddec27555741..1fb051cd5d46 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -54,6 +54,7 @@ jobs: worker: - docker-container - remote + - kubernetes pkg: - ./tests mode: @@ -105,14 +106,14 @@ jobs: fi testFlags="--run=//worker=$(echo "${{ matrix.worker }}" | sed 's/\+/\\+/g')$" case "${{ matrix.worker }}" in - docker | docker+containerd | docker@* | docker+containerd@* | remote+multinode) + docker | docker+containerd | docker@* | docker+containerd@* | remote+multinode | kubernetes) echo "TESTFLAGS=${{ env.TESTFLAGS_DOCKER }} $testFlags" >> $GITHUB_ENV ;; *) echo "TESTFLAGS=${{ env.TESTFLAGS }} $testFlags" >> $GITHUB_ENV ;; esac - if [[ "${{ matrix.worker }}" == "docker"* || "${{ matrix.worker }}" == "remote+multinode" ]]; then + if [[ "${{ matrix.worker }}" == "docker"* || "${{ matrix.worker }}" == "remote+multinode" || "${{ matrix.worker }}" == "kubernetes" ]]; then echo "TEST_DOCKERD=1" >> $GITHUB_ENV fi if [ "${{ matrix.mode }}" = "experimental" ]; then diff --git a/Dockerfile b/Dockerfile index 949cb5911712..1767e5f81b99 100644 --- a/Dockerfile +++ b/Dockerfile @@ -14,6 +14,8 @@ ARG REGISTRY_VERSION=3.0.0 ARG BUILDKIT_VERSION=v0.29.0 ARG COMPOSE_VERSION=v5.1.0 ARG UNDOCK_VERSION=0.9.0 +ARG K3D_VERSION=5.8.3 +ARG K3S_VERSION=v1.32.13-k3s1 FROM --platform=$BUILDPLATFORM tonistiigi/xx:${XX_VERSION} AS xx FROM --platform=$BUILDPLATFORM golang:${GO_VERSION}-alpine${ALPINE_VERSION} AS golatest @@ -27,6 +29,7 @@ FROM registry:$REGISTRY_VERSION AS registry FROM moby/buildkit:$BUILDKIT_VERSION AS buildkit FROM docker/compose-bin:$COMPOSE_VERSION AS compose FROM crazymax/undock:$UNDOCK_VERSION AS undock +FROM ghcr.io/k3d-io/k3d:${K3D_VERSION} AS k3d FROM golatest AS gobase COPY --from=xx / / @@ -124,6 +127,8 @@ FROM binaries-$TARGETOS AS binaries ARG BUILDKIT_SBOM_SCAN_STAGE=true FROM gobase AS integration-test-base +ARG K3D_VERSION +ARG K3S_VERSION # https://github.com/docker/docker/blob/master/project/PACKAGERS.md#runtime-dependencies RUN apk add --no-cache \ bash \ @@ -149,9 +154,13 @@ COPY --link --from=buildkit /usr/bin/buildkitd /usr/bin/ COPY --link --from=buildkit /usr/bin/buildctl /usr/bin/ COPY --link --from=compose /docker-compose /usr/bin/compose COPY --link --from=undock /usr/local/bin/undock /usr/bin/ +COPY --link --from=k3d /bin/k3d /usr/bin/ COPY --link --from=binaries /buildx /usr/bin/ RUN mkdir -p /usr/local/lib/docker/cli-plugins && ln -s /usr/bin/buildx /usr/local/lib/docker/cli-plugins/docker-buildx ENV TEST_DOCKER_EXTRA="docker@28.5=/opt/docker-alt-28,docker@27.5=/opt/docker-alt-27" +ENV TEST_K3S_IMAGE="rancher/k3s:${K3S_VERSION}" +ENV TEST_K3D_TOOLS_IMAGE="ghcr.io/k3d-io/k3d-tools:${K3D_VERSION}" +ENV TEST_K3D_LOADBALANCER_IMAGE="ghcr.io/k3d-io/k3d-proxy:${K3D_VERSION}" FROM integration-test-base AS integration-test COPY . . diff --git a/tests/helpers/k3d.go b/tests/helpers/k3d.go new file mode 100644 index 000000000000..6b204da214d0 --- /dev/null +++ b/tests/helpers/k3d.go @@ -0,0 +1,93 @@ +package helpers + +import ( + "context" + "os" + "os/exec" + "strings" + "time" + + "github.com/moby/buildkit/identity" + "github.com/moby/buildkit/util/testutil/integration" + "github.com/pkg/errors" +) + +const ( + k3dBin = "k3d" +) + +func NewK3dServer(ctx context.Context, cfg *integration.BackendConfig, dockerAddress string) (clusterName, kubeConfig string, cl func() error, err error) { + if _, err := exec.LookPath(k3dBin); err != nil { + return "", "", nil, errors.Wrapf(err, "failed to lookup %s binary", k3dBin) + } + + deferF := &integration.MultiCloser{} + cl = deferF.F() + + defer func() { + if err != nil { + deferF.F()() + cl = nil + } + }() + + clusterName = "bk-" + identity.NewID() + + createCtx, cancelCreate := context.WithTimeoutCause(ctx, 90*time.Second, errors.New("timed out creating k3d cluster")) + defer cancelCreate() + + args := []string{ + "cluster", "create", clusterName, + "--wait", + "--k3s-arg=--debug@server:0", + } + if image := KubernetesK3sImage(); image != "" { + args = append(args, "--image="+image) + } + cmd := exec.CommandContext(createCtx, k3dBin, args...) + cmd.Env = k3dEnv(dockerAddress) + out, err := cmd.CombinedOutput() + if err != nil { + diag := KubernetesDiagnostics(ctx, clusterName, dockerAddress) + return "", "", nil, errors.Wrapf(err, "failed to create k3d cluster %s: %s\n%s\nouter dockerd logs: %s", clusterName, strings.TrimSpace(string(out)), diag, integration.FormatLogs(cfg.Logs)) + } + deferF.Append(func() error { + deleteCtx, cancelDelete := context.WithTimeoutCause(context.WithoutCancel(ctx), 30*time.Second, errors.New("timed out deleting k3d cluster")) + defer cancelDelete() + cmd := exec.CommandContext(deleteCtx, k3dBin, "cluster", "delete", clusterName) + cmd.Env = k3dEnv(dockerAddress) + out, err := cmd.CombinedOutput() + if err != nil { + return errors.Wrapf(err, "failed to delete k3d cluster %s: %s", clusterName, string(out)) + } + return nil + }) + + kubeconfigCtx, cancelKubeconfig := context.WithTimeoutCause(ctx, 30*time.Second, errors.New("timed out writing k3d kubeconfig")) + defer cancelKubeconfig() + + cmd = exec.CommandContext(kubeconfigCtx, k3dBin, "kubeconfig", "write", clusterName) + cmd.Env = k3dEnv(dockerAddress) + out, err = cmd.CombinedOutput() + if err != nil { + diag := KubernetesDiagnostics(ctx, clusterName, dockerAddress) + return "", "", nil, errors.Wrapf(err, "failed to write kubeconfig for cluster %s: %s\n%s\nouter dockerd logs: %s", clusterName, strings.TrimSpace(string(out)), diag, integration.FormatLogs(cfg.Logs)) + } + kubeConfig = strings.TrimSpace(string(out)) + + return +} + +func k3dEnv(dockerAddress string) []string { + env := append( + os.Environ(), + "DOCKER_CONTEXT="+dockerAddress, + ) + if image := KubernetesK3DToolsImage(); image != "" { + env = append(env, "K3D_IMAGE_TOOLS="+image) + } + if image := KubernetesK3DLoadBalancerImage(); image != "" { + env = append(env, "K3D_IMAGE_LOADBALANCER="+image) + } + return env +} diff --git a/tests/helpers/kubernetes_diagnostics.go b/tests/helpers/kubernetes_diagnostics.go new file mode 100644 index 000000000000..6c2224e45fcb --- /dev/null +++ b/tests/helpers/kubernetes_diagnostics.go @@ -0,0 +1,141 @@ +package helpers + +import ( + "bytes" + "context" + "errors" + "fmt" + "os" + "os/exec" + "strings" + "time" +) + +const defaultTestBuildkitTag = "buildx-stable-1" + +func KubernetesBuildkitImage() string { + if v := os.Getenv("TEST_BUILDKIT_IMAGE"); v != "" { + return v + } + tag := os.Getenv("TEST_BUILDKIT_TAG") + if tag == "" { + tag = defaultTestBuildkitTag + } + return "moby/buildkit:" + tag +} + +func KubernetesK3sImage() string { + return os.Getenv("TEST_K3S_IMAGE") +} + +func KubernetesK3DToolsImage() string { + return os.Getenv("TEST_K3D_TOOLS_IMAGE") +} + +func KubernetesK3DLoadBalancerImage() string { + return os.Getenv("TEST_K3D_LOADBALANCER_IMAGE") +} + +func KubernetesDiagnostics(ctx context.Context, clusterName, dockerContext string) string { + ctx, cancel := context.WithTimeoutCause(context.WithoutCancel(ctx), 20*time.Second, errors.New("timed out collecting kubernetes diagnostics")) + defer cancel() + + var buf bytes.Buffer + appendK3dDiagnostics(ctx, &buf, clusterName, dockerContext) + appendDockerDiagnostics(ctx, &buf, dockerContext) + appendK3sServerDiagnostics(ctx, &buf, clusterName, dockerContext) + return strings.TrimSpace(buf.String()) +} + +func appendK3dDiagnostics(ctx context.Context, buf *bytes.Buffer, clusterName, dockerContext string) { + appendCommandOutput(ctx, buf, "k3d cluster list", "k3d", []string{"cluster", "list", clusterName}, []string{"DOCKER_CONTEXT=" + dockerContext}) + appendCommandOutput(ctx, buf, "k3d node list", "k3d", []string{"node", "list"}, []string{"DOCKER_CONTEXT=" + dockerContext}) +} + +func appendDockerDiagnostics(ctx context.Context, buf *bytes.Buffer, dockerContext string) { + args := []string{"ps", "-a", "--format", "{{.Names}}\t{{.Image}}\t{{.Status}}"} + appendCommandOutput(ctx, buf, "docker ps", "docker", args, []string{"DOCKER_CONTEXT=" + dockerContext}) +} + +func appendK3sServerDiagnostics(ctx context.Context, buf *bytes.Buffer, clusterName, dockerContext string) { + nodeNames, err := clusterNodeNames(ctx, clusterName, dockerContext) + if err != nil { + fmt.Fprintf(buf, "cluster node discovery error: %v\n", err) + return + } + if len(nodeNames) == 0 { + fmt.Fprintln(buf, "cluster node discovery: no matching k3d containers found") + return + } + + for _, nodeName := range nodeNames { + appendCommandOutput(ctx, buf, "docker inspect "+nodeName, "docker", []string{ + "inspect", + "--format", + "Status={{.State.Status}} Health={{if .State.Health}}{{.State.Health.Status}}{{else}}{{end}} Restarting={{.State.Restarting}} ExitCode={{.State.ExitCode}} Error={{.State.Error}} Privileged={{.HostConfig.Privileged}} Cgroupns={{.HostConfig.CgroupnsMode}}", + nodeName, + }, []string{"DOCKER_CONTEXT=" + dockerContext}) + appendCommandOutput(ctx, buf, "docker logs "+nodeName, "docker", []string{"logs", "--tail", "80", nodeName}, []string{"DOCKER_CONTEXT=" + dockerContext}) + } + + for _, nodeName := range nodeNames { + if !strings.Contains(nodeName, "-server-") { + continue + } + appendCommandOutput(ctx, buf, "docker exec "+nodeName+" ps", "docker", []string{"exec", nodeName, "sh", "-c", "ps auxww"}, []string{"DOCKER_CONTEXT=" + dockerContext}) + appendCommandOutput(ctx, buf, "docker exec "+nodeName+" sockets", "docker", []string{"exec", nodeName, "sh", "-c", "ss -lntp || netstat -lnt"}, []string{"DOCKER_CONTEXT=" + dockerContext}) + appendCommandOutput(ctx, buf, "docker exec "+nodeName+" cgroup", "docker", []string{"exec", nodeName, "sh", "-c", "cat /proc/1/cgroup && echo && mount | grep cgroup"}, []string{"DOCKER_CONTEXT=" + dockerContext}) + appendCommandOutput(ctx, buf, "docker exec "+nodeName+" env", "docker", []string{"exec", nodeName, "sh", "-c", "env | sort"}, []string{"DOCKER_CONTEXT=" + dockerContext}) + appendCommandOutput(ctx, buf, "docker exec "+nodeName+" entrypoint", "docker", []string{"exec", nodeName, "sh", "-c", "sed -n '1,200p' /bin/k3d-entrypoint.sh"}, []string{"DOCKER_CONTEXT=" + dockerContext}) + appendCommandOutput(ctx, buf, "docker exec "+nodeName+" entrypoint logs", "docker", []string{"exec", nodeName, "sh", "-c", "for f in /var/log/k3d-entrypoints_*.log; do if [ -f \"$f\" ]; then echo \"== $f ==\"; tail -n 200 \"$f\"; echo; fi; done"}, []string{"DOCKER_CONTEXT=" + dockerContext}) + appendCommandOutput(ctx, buf, "docker exec "+nodeName+" k3s files", "docker", []string{"exec", nodeName, "sh", "-c", "find /var/lib/rancher/k3s -maxdepth 3 -type f 2>/dev/null | sort"}, []string{"DOCKER_CONTEXT=" + dockerContext}) + appendCommandOutput(ctx, buf, "docker exec "+nodeName+" k3s logs", "docker", []string{"exec", nodeName, "sh", "-c", "for f in /var/log/k3s.log /var/lib/rancher/k3s/agent/containerd/containerd.log /var/lib/rancher/k3s/server/logs/*; do if [ -f \"$f\" ]; then echo \"== $f ==\"; tail -n 200 \"$f\"; echo; fi; done"}, []string{"DOCKER_CONTEXT=" + dockerContext}) + appendCommandOutput(ctx, buf, "docker exec "+nodeName+" kubectl get pods", "docker", []string{"exec", nodeName, "kubectl", "get", "pods", "-A", "-o", "wide"}, []string{"DOCKER_CONTEXT=" + dockerContext}) + appendCommandOutput(ctx, buf, "docker exec "+nodeName+" kubectl get events", "docker", []string{"exec", nodeName, "kubectl", "get", "events", "-A", "--sort-by=.lastTimestamp"}, []string{"DOCKER_CONTEXT=" + dockerContext}) + appendCommandOutput(ctx, buf, "docker exec "+nodeName+" kubectl describe pods", "docker", []string{"exec", nodeName, "kubectl", "describe", "pods", "-A"}, []string{"DOCKER_CONTEXT=" + dockerContext}) + break + } +} + +func clusterNodeNames(ctx context.Context, clusterName, dockerContext string) ([]string, error) { + out, err := runCommand(ctx, "docker", []string{ + "ps", "-a", + "--filter", "name=k3d-" + clusterName, + "--format", "{{.Names}}", + }, []string{"DOCKER_CONTEXT=" + dockerContext}) + if err != nil { + return nil, err + } + var names []string + for _, line := range strings.Split(strings.TrimSpace(out), "\n") { + line = strings.TrimSpace(line) + if line == "" { + continue + } + names = append(names, line) + } + return names, nil +} + +func appendCommandOutput(ctx context.Context, buf *bytes.Buffer, title, name string, args []string, env []string) { + out, err := runCommand(ctx, name, args, env) + fmt.Fprintf(buf, "== %s ==\n", title) + if err != nil { + fmt.Fprintf(buf, "error: %v\n", err) + } + if strings.TrimSpace(out) == "" { + fmt.Fprintln(buf, "") + } else { + fmt.Fprintf(buf, "%s\n", strings.TrimSpace(out)) + } + fmt.Fprintln(buf) +} + +func runCommand(ctx context.Context, name string, args []string, env []string) (string, error) { + cmd := exec.CommandContext(ctx, name, args...) + if len(env) > 0 { + cmd.Env = append(os.Environ(), env...) + } + out, err := cmd.CombinedOutput() + return string(out), err +} diff --git a/tests/integration_test.go b/tests/integration_test.go index 574251dc5075..9ce1c8d9c128 100644 --- a/tests/integration_test.go +++ b/tests/integration_test.go @@ -15,6 +15,7 @@ func init() { workers.InitDockerWorker() workers.InitDockerContainerWorker() workers.InitRemoteMultiNodeWorker() + workers.InitKubernetesWorker() } else { workers.InitRemoteWorker() } diff --git a/tests/workers/kubernetes.go b/tests/workers/kubernetes.go new file mode 100644 index 000000000000..ec5d725d3330 --- /dev/null +++ b/tests/workers/kubernetes.go @@ -0,0 +1,131 @@ +package workers + +import ( + "context" + "os" + "os/exec" + "strings" + "sync" + + "github.com/docker/buildx/tests/helpers" + "github.com/moby/buildkit/identity" + "github.com/moby/buildkit/util/testutil/integration" + "github.com/pkg/errors" +) + +func InitKubernetesWorker() { + integration.Register(&kubernetesWorker{ + id: "kubernetes", + }) +} + +type kubernetesWorker struct { + id string + + unsupported []string + + docker integration.Backend + dockerClose func() error + dockerErr error + dockerOnce sync.Once + + k3dName string + k3dConfig string + k3dClose func() error + k3dErr error + k3dOnce sync.Once +} + +func (w *kubernetesWorker) Name() string { + return w.id +} + +func (w *kubernetesWorker) Rootless() bool { + return false +} + +func (w *kubernetesWorker) NetNSDetached() bool { + return false +} + +func (w *kubernetesWorker) New(ctx context.Context, cfg *integration.BackendConfig) (integration.Backend, func() error, error) { + w.dockerOnce.Do(func() { + w.docker, w.dockerClose, w.dockerErr = dockerWorker{id: w.id}.New(ctx, cfg) + }) + if w.dockerErr != nil { + return w.docker, w.dockerClose, w.dockerErr + } + + w.k3dOnce.Do(func() { + w.k3dName, w.k3dConfig, w.k3dClose, w.k3dErr = helpers.NewK3dServer(ctx, cfg, w.docker.DockerAddress()) + }) + if w.k3dErr != nil { + return nil, w.k3dClose, w.k3dErr + } + + name := "integration-kubernetes-" + identity.NewID() + cmd := exec.CommandContext(ctx, "buildx", "create", + "--bootstrap", + "--name="+name, + "--driver=kubernetes", + "--driver-opt=image="+helpers.KubernetesBuildkitImage(), + "--driver-opt=timeout=60s", + ) + cmd.Env = append( + os.Environ(), + "BUILDX_CONFIG=/tmp/buildx-"+name, + "DOCKER_CONTEXT="+w.docker.DockerAddress(), + "KUBECONFIG="+w.k3dConfig, + ) + out, err := cmd.CombinedOutput() + if err != nil { + diag := helpers.KubernetesDiagnostics(ctx, w.k3dName, w.docker.DockerAddress()) + return nil, nil, errors.Wrapf(err, "failed to create buildx instance %s with image %s: %s\n%s", name, helpers.KubernetesBuildkitImage(), strings.TrimSpace(string(out)), diag) + } + + cl := func() error { + cmd := exec.CommandContext(context.Background(), "buildx", "rm", "-f", name) + cmd.Env = append( + os.Environ(), + "BUILDX_CONFIG=/tmp/buildx-"+name, + "DOCKER_CONTEXT="+w.docker.DockerAddress(), + "KUBECONFIG="+w.k3dConfig, + ) + return cmd.Run() + } + + return &backend{ + context: w.docker.DockerAddress(), + builder: name, + unsupportedFeatures: w.unsupported, + }, cl, nil +} + +func (w *kubernetesWorker) Close() error { + setErr := func(dst *error, err error) { + if err != nil && *dst == nil { + *dst = err + } + } + + var err error + if c := w.k3dClose; c != nil { + setErr(&err, c()) + } + if c := w.dockerClose; c != nil { + setErr(&err, c()) + } + + // reset the worker to be ready to go again + w.docker = nil + w.dockerClose = nil + w.dockerErr = nil + w.dockerOnce = sync.Once{} + w.k3dName = "" + w.k3dConfig = "" + w.k3dClose = nil + w.k3dErr = nil + w.k3dOnce = sync.Once{} + + return err +}