Merge pull request linuxkit#3573 from deitch/dockerless

pull and cache images locally without docker

Author: deitch

docs/image-cache.md

@@ -0,0 +1,61 @@
+# Image Caching
+
+linuxkit builds each runtime OS image from a combination of Docker images.
+These images are pulled from a registry and cached locally.
+
+linuxkit does not use the docker image cache to store these images. This is
+for two key reasons.
+
+First, docker does not provide support for different architecture versions. For
+example, if you want to pull down `docker.io/library/alpine:3.11` by manifest,
+with its signature, but get the `arm64` version while you are on an `amd64` device,
+it is not supported.
+
+Second, and more importantly, this requires a running docker daemon. Since the
+very essence of linuxkit is removing daemons and operating systems where unnecessary,
+just laying down bits in a file, removing docker from the image build process
+is valuable. It also simplifies many use cases, like CI, where a docker daemon
+may be unavailable.
+
+## How LinuxKit Caches Images
+
+LinuxKit pulls images down from a registry and stores them in a local cache.
+It stores the root manifest or index of the image, the manifest, and all of the layers
+for the requested architecture. It does not pull down layers, manifest or config
+for all available architectures, only the requested one. If none is requested, it
+defaults to the architecture on which you are running.
+
+By default, LinuxKit caches images in `~/.linuxkit/cache/`. It can be changed
+via a command-line option. The structure of the cache directory matches the
+[OCI spec for image layout](http://github.com/opencontainers/image-spec/blob/master/image-layout.md).
+
+Image names are kept in `index.json` in the [annotation](https://github.com/opencontainers/image-spec/blob/master/annotations.md) `org.opencontainers.image.ref.name`. For example"
+
+```json
+{
+  "schemaVersion": 2,
+  "manifests": [
+    {
+      "mediaType": "application/vnd.docker.distribution.manifest.list.v2+json",
+      "size": 1638,
+      "digest": "sha256:9a839e63dad54c3a6d1834e29692c8492d93f90c59c978c1ed79109ea4fb9a54",
+      "annotations": {
+        "org.opencontainers.image.ref.name": "docker.io/library/alpine:3.11"
+      }
+    }
+  ]
+}
+```
+
+## How LinuxKit Uses the Cache and Registry
+
+For each image that linuxkit needs to read, it does the following. Note that if the `--pull` option
+is provided, it always will pull, independent of what is in the cache.
+
+1. Check in the cache for the image name in the cache `index.json`. If it does not find it, pull it down and store it in cache, using content trust, if enabled.
+1. Read the root hash from `index.json`.
+1. Find the root blob in the `blobs/` directory via the hash and read it.
+1. Proceed to read the manifest, config and layers.
+
+The read process is smart enough to check each blob in the local cache before downloading
+it from a registry.

docs/yaml.md

@@ -11,7 +11,8 @@ are downloaded at build time to create an image. The image is self-contained and
 so it can be tested reliably for continuous delivery.
 
 Components are specified as Docker images which are pulled from a registry during build if they
-are not available locally. The Docker images are optionally verified with Docker Content Trust.
+are not available locally. See [image-cache](./image-cache.md) for more details on local caching.
+The Docker images are optionally verified with Docker Content Trust.
 For private registries or private repositories on a registry credentials provided via
 `docker login` are re-used.
 

src/cmd/linuxkit/build.go

@@ -9,6 +9,7 @@ import (
 	"net/http"
 	"os"
 	"path/filepath"
+	"runtime"
 	"strings"
 
 	"github.com/linuxkit/linuxkit/src/cmd/linuxkit/moby"
@@ -48,9 +49,12 @@ func build(args []string) {
 	buildOutputFile := buildCmd.String("o", "", "File to use for a single output, or '-' for stdout")
 	buildSize := buildCmd.String("size", "1024M", "Size for output image, if supported and fixed size")
 	buildPull := buildCmd.Bool("pull", false, "Always pull images")
+	buildDocker := buildCmd.Bool("docker", false, "Check for images in docker before linuxkit cache")
 	buildDisableTrust := buildCmd.Bool("disable-content-trust", false, "Skip image trust verification specified in trust section of config (default false)")
 	buildDecompressKernel := buildCmd.Bool("decompress-kernel", false, "Decompress the Linux kernel (default false)")
+	buildCacheDir := buildCmd.String("cache", defaultLinuxkitCache(), "Directory for caching and finding cached image")
 	buildCmd.Var(&buildFormats, "format", "Formats to create [ "+strings.Join(outputTypes, " ")+" ]")
+	buildArch := buildCmd.String("arch", runtime.GOARCH, "target architecture for which to build")
 
 	if err := buildCmd.Parse(args); err != nil {
 		log.Fatal("Unable to parse args")
@@ -95,6 +99,8 @@ func build(args []string) {
 		}
 	}
 
+	cacheDir := *buildCacheDir
+
 	if len(buildFormats) == 1 && moby.Streamable(buildFormats[0]) {
 		if *buildOutputFile == "" {
 			*buildOutputFile = filepath.Join(*buildDir, name+"."+buildFormats[0])
@@ -103,7 +109,7 @@ func build(args []string) {
 			*buildDir = ""
 		}
 	} else {
-		err := moby.ValidateFormats(buildFormats)
+		err := moby.ValidateFormats(buildFormats, cacheDir)
 		if err != nil {
 			log.Errorf("Error parsing formats: %v", err)
 			buildCmd.Usage()
@@ -175,6 +181,7 @@ func build(args []string) {
 		if err != nil {
 			log.Fatalf("Invalid config: %v", err)
 		}
+		c.Architecture = *buildArch
 		m, err = moby.AppendConfig(m, c)
 		if err != nil {
 			log.Fatalf("Cannot append config files: %v", err)
@@ -204,7 +211,7 @@ func build(args []string) {
 	if moby.Streamable(buildFormats[0]) {
 		tp = buildFormats[0]
 	}
-	err = moby.Build(m, w, *buildPull, tp, *buildDecompressKernel)
+	err = moby.Build(m, w, *buildPull, tp, *buildDecompressKernel, cacheDir, *buildDocker)
 	if err != nil {
 		log.Fatalf("%v", err)
 	}
@@ -216,7 +223,7 @@ func build(args []string) {
 		}
 
 		log.Infof("Create outputs:")
-		err = moby.Formats(filepath.Join(*buildDir, name), image, buildFormats, size, !*buildDisableTrust)
+		err = moby.Formats(filepath.Join(*buildDir, name), image, buildFormats, size, !*buildDisableTrust, cacheDir)
 		if err != nil {
 			log.Fatalf("Error writing outputs: %v", err)
 		}

src/cmd/linuxkit/cache.go

@@ -0,0 +1,48 @@
+package main
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+
+	"github.com/linuxkit/linuxkit/src/cmd/linuxkit/util"
+	log "github.com/sirupsen/logrus"
+)
+
+func cacheUsage() {
+	invoked := filepath.Base(os.Args[0])
+	fmt.Printf("USAGE: %s cache command [options]\n\n", invoked)
+	fmt.Printf("Supported commands are\n")
+	// Please keep these in alphabetical order
+	fmt.Printf("  clean\n")
+	fmt.Printf("  ls\n")
+	fmt.Printf("\n")
+	fmt.Printf("'options' are the backend specific options.\n")
+	fmt.Printf("See '%s cache [command] --help' for details.\n\n", invoked)
+}
+
+// Process the cache
+func cache(args []string) {
+	if len(args) < 1 {
+		cacheUsage()
+		os.Exit(1)
+	}
+	switch args[0] {
+	// Please keep cases in alphabetical order
+	case "clean":
+		cacheClean(args[1:])
+	case "ls":
+		cacheList(args[1:])
+	case "help", "-h", "-help", "--help":
+		cacheUsage()
+		os.Exit(0)
+	default:
+		log.Errorf("No 'cache' command specified.")
+	}
+}
+
+func defaultLinuxkitCache() string {
+	lktDir := ".linuxkit"
+	home := util.HomeDir()
+	return filepath.Join(home, lktDir, "cache")
+}

src/cmd/linuxkit/cache/find.go

@@ -0,0 +1,50 @@
+package cache
+
+import (
+	"fmt"
+
+	"github.com/google/go-containerregistry/pkg/v1"
+	"github.com/google/go-containerregistry/pkg/v1/layout"
+	"github.com/google/go-containerregistry/pkg/v1/match"
+	"github.com/google/go-containerregistry/pkg/v1/partial"
+)
+
+// matchPlatformsOSArch because match.Platforms rejects it if the provided
+// v1.Platform has a variant of "" but the actual index has a specific one.
+// This becomes an issue with arm64 vs arm64/v8. So this matches only on OS
+// and Architecture.
+func matchPlatformsOSArch(platforms ...v1.Platform) match.Matcher {
+	return func(desc v1.Descriptor) bool {
+		if desc.Platform == nil {
+			return false
+		}
+		for _, platform := range platforms {
+			if desc.Platform.OS == platform.OS && desc.Platform.Architecture == platform.Architecture {
+				return true
+			}
+		}
+		return false
+	}
+}
+
+func findImage(p layout.Path, imageName, architecture string) (v1.Image, error) {
+	root, err := findRootFromLayout(p, imageName)
+	if err != nil {
+		return nil, err
+	}
+	img, err := root.Image()
+	if err == nil {
+		return img, nil
+	}
+	ii, err := root.ImageIndex()
+	if err == nil {
+		// we have the index, get the manifest that represents the manifest for the desired architecture
+		platform := v1.Platform{OS: "linux", Architecture: architecture}
+		images, err := partial.FindImages(ii, matchPlatformsOSArch(platform))
+		if err != nil || len(images) < 1 {
+			return nil, fmt.Errorf("error retrieving image %s for platform %v from cache: %v", imageName, platform, err)
+		}
+		return images[0], nil
+	}
+	return nil, fmt.Errorf("no image found for %s", imageName)
+}

src/cmd/linuxkit/cache/image.go

@@ -0,0 +1,28 @@
+package cache
+
+import (
+	"github.com/google/go-containerregistry/pkg/v1/layout"
+	imagespec "github.com/opencontainers/image-spec/specs-go/v1"
+)
+
+// ListImages list the named images and their root digests from a layout.Path
+func ListImages(p layout.Path) (map[string]string, error) {
+	ii, err := p.ImageIndex()
+	if err != nil {
+		return nil, err
+	}
+	index, err := ii.IndexManifest()
+	if err != nil {
+		return nil, err
+	}
+	names := map[string]string{}
+	for _, i := range index.Manifests {
+		if i.Annotations == nil {
+			continue
+		}
+		if name, ok := i.Annotations[imagespec.AnnotationRefName]; ok {
+			names[name] = i.Digest.String()
+		}
+	}
+	return names, nil
+}

src/cmd/linuxkit/cache/open.go

@@ -0,0 +1,21 @@
+package cache
+
+import (
+	"fmt"
+
+	"github.com/google/go-containerregistry/pkg/v1/empty"
+	"github.com/google/go-containerregistry/pkg/v1/layout"
+)
+
+// Get get or initialize the cache
+func Get(cache string) (layout.Path, error) {
+	// initialize the cache path if needed
+	p, err := layout.FromPath(cache)
+	if err != nil {
+		p, err = layout.Write(cache, empty.Index)
+		if err != nil {
+			return p, fmt.Errorf("could not initialize cache at path %s: %v", cache, err)
+		}
+	}
+	return p, nil
+}

src/cmd/linuxkit/cache/pull.go

@@ -0,0 +1,64 @@
+package cache
+
+import (
+	"errors"
+
+	"github.com/containerd/containerd/reference"
+	"github.com/google/go-containerregistry/pkg/v1"
+	"github.com/google/go-containerregistry/pkg/v1/validate"
+)
+
+// ValidateImage given a reference, validate that it is complete. If not, pull down missing
+// components as necessary.
+func ValidateImage(ref *reference.Spec, cacheDir, architecture string) (ImageSource, error) {
+	var (
+		imageIndex v1.ImageIndex
+		image      v1.Image
+		imageName  = ref.String()
+	)
+	// next try the local cache
+	root, err := FindRoot(cacheDir, imageName)
+	if err == nil {
+		img, err := root.Image()
+		if err == nil {
+			image = img
+		} else {
+			ii, err := root.ImageIndex()
+			if err == nil {
+				imageIndex = ii
+			}
+		}
+	}
+	// three possibilities now:
+	// - we did not find anything locally
+	// - we found an index locally
+	// - we found an image locally
+	switch {
+	case imageIndex == nil && image == nil:
+		// we did not find it yet - either because we were told not to look locally,
+		// or because it was not available - so get it from the remote
+		return ImageSource{}, errors.New("no such image")
+	case imageIndex != nil:
+		// we found a local index, just make sure it is up to date and, if not, download it
+		if err := validate.Index(imageIndex); err == nil {
+			return NewSource(
+				ref,
+				cacheDir,
+				architecture,
+			), nil
+		}
+		return ImageSource{}, errors.New("invalid index")
+	case image != nil:
+		// we found a local image, just make sure it is up to date
+		if err := validate.Image(image); err == nil {
+			return NewSource(
+				ref,
+				cacheDir,
+				architecture,
+			), nil
+		}
+		return ImageSource{}, errors.New("invalid image")
+	}
+	// if we made it to here, we had some strange error
+	return ImageSource{}, errors.New("should not have reached this point, image index and image were both empty and not-empty")
+}