From 08507cf1427353c6c6348c5de88332c63fd8c957 Mon Sep 17 00:00:00 2001 From: Saied Kazemi Date: Thu, 5 Feb 2015 20:32:27 -0800 Subject: [PATCH 01/26] Checkpoint/Restore Support: add exec driver methods Methods for checkpointing and restoring containers were added to the native driver. The LXC driver returns an error message that these methods are not implemented yet. Signed-off-by: Saied Kazemi Conflicts: daemon/execdriver/native/create.go daemon/execdriver/native/driver.go daemon/execdriver/native/init.go --- daemon/execdriver/driver.go | 3 + daemon/execdriver/lxc/driver.go | 8 ++ daemon/execdriver/native/create.go | 19 ++++ daemon/execdriver/native/driver.go | 150 +++++++++++++++++++++++++++++ 4 files changed, 180 insertions(+) diff --git a/daemon/execdriver/driver.go b/daemon/execdriver/driver.go index eca77e921eaac..acb41c5f53376 100644 --- a/daemon/execdriver/driver.go +++ b/daemon/execdriver/driver.go @@ -24,6 +24,7 @@ var ( ) type StartCallback func(*ProcessConfig, int) +type RestoreCallback func(*ProcessConfig, int) // Driver specific information based on // processes registered with the driver @@ -59,6 +60,8 @@ type Driver interface { Kill(c *Command, sig int) error Pause(c *Command) error Unpause(c *Command) error + Checkpoint(c *Command) error + Restore(c *Command, pipes *Pipes, restoreCallback RestoreCallback) (int, error) Name() string // Driver name Info(id string) Info // "temporary" hack (until we move state from core to plugins) GetPidsForContainer(id string) ([]int, error) // Returns a list of pids for the given container. diff --git a/daemon/execdriver/lxc/driver.go b/daemon/execdriver/lxc/driver.go index 692b9f356f3aa..914a7ea5075c2 100644 --- a/daemon/execdriver/lxc/driver.go +++ b/daemon/execdriver/lxc/driver.go @@ -547,6 +547,14 @@ func (d *driver) Unpause(c *execdriver.Command) error { return err } +func (d *driver) Checkpoint(c *execdriver.Command) error { + return fmt.Errorf("Checkpointing lxc containers not supported yet\n") +} + +func (d *driver) Restore(c *execdriver.Command, pipes *execdriver.Pipes, restoreCallback execdriver.RestoreCallback) (int, error) { + return 0, fmt.Errorf("Restoring lxc containers not supported yet\n") +} + func (d *driver) Terminate(c *execdriver.Command) error { return KillLxc(c.ID, 9) } diff --git a/daemon/execdriver/native/create.go b/daemon/execdriver/native/create.go index 1b2d7232d3177..1cf248b0d66fd 100644 --- a/daemon/execdriver/native/create.go +++ b/daemon/execdriver/native/create.go @@ -4,6 +4,7 @@ package native import ( "errors" + "encoding/json" "fmt" "net" "strings" @@ -88,6 +89,24 @@ func generateIfaceName() (string, error) { return "", errors.New("Failed to find name for new interface") } +// Re-create the container type from the image that was saved during checkpoint. +func (d *driver) createRestoreContainer(c *execdriver.Command, imageDir string) (*libcontainer.Config, error) { + // Read the container.json. + f1, err := os.Open(filepath.Join(imageDir, "container.json")) + if err != nil { + return nil, err + } + defer f1.Close() + + var container *libcontainer.Config + err = json.NewDecoder(f1).Decode(&container) + if err != nil { + return nil, err + } + + return container, nil +} + func (d *driver) createNetwork(container *configs.Config, c *execdriver.Command) error { if c.Network == nil { return nil diff --git a/daemon/execdriver/native/driver.go b/daemon/execdriver/native/driver.go index da13c2b165666..8f451bcd7592f 100644 --- a/daemon/execdriver/native/driver.go +++ b/daemon/execdriver/native/driver.go @@ -19,6 +19,7 @@ import ( "github.com/docker/docker/pkg/reexec" sysinfo "github.com/docker/docker/pkg/system" "github.com/docker/docker/pkg/term" + "github.com/docker/docker/utils" "github.com/docker/libcontainer" "github.com/docker/libcontainer/apparmor" "github.com/docker/libcontainer/cgroups/systemd" @@ -276,6 +277,155 @@ func (d *driver) Unpause(c *execdriver.Command) error { return active.Resume() } +// XXX Where is the right place for the following +// const and getCheckpointImageDir() function? +const ( + containersDir = "/var/lib/docker/containers" + criuImgDir = "criu_img" +) + +func getCheckpointImageDir(containerId string) string { + return filepath.Join(containersDir, containerId, criuImgDir) +} + +func (d *driver) Checkpoint(c *execdriver.Command) error { + active := d.activeContainers[c.ID] + if active == nil { + return fmt.Errorf("active container for %s does not exist", c.ID) + } + container := active.container + + // Create an image directory for this container (which + // may already exist from a previous checkpoint). + imageDir := getCheckpointImageDir(c.ID) + err := os.MkdirAll(imageDir, 0700) + if err != nil && !os.IsExist(err) { + return err + } + + // Copy container.json and state.json files to the CRIU + // image directory for later use during restore. Do this + // before checkpointing because after checkpoint the container + // will exit and these files will be removed. + log.CRDbg("saving container.json and state.json before calling CRIU in %s", imageDir) + srcFiles := []string{"container.json", "state.json"} + for _, f := range srcFiles { + srcFile := filepath.Join(d.root, c.ID, f) + dstFile := filepath.Join(imageDir, f) + if _, err := utils.CopyFile(srcFile, dstFile); err != nil { + return err + } + } + + d.Lock() + defer d.Unlock() + err = namespaces.Checkpoint(container, imageDir, c.ProcessConfig.Process.Pid) + if err != nil { + return err + } + + return nil +} + +type restoreOutput struct { + exitCode int + err error +} + +func (d *driver) Restore(c *execdriver.Command, pipes *execdriver.Pipes, restoreCallback execdriver.RestoreCallback) (int, error) { + imageDir := getCheckpointImageDir(c.ID) + container, err := d.createRestoreContainer(c, imageDir) + if err != nil { + return 1, err + } + + var term execdriver.Terminal + + if c.ProcessConfig.Tty { + term, err = NewTtyConsole(&c.ProcessConfig, pipes) + } else { + term, err = execdriver.NewStdConsole(&c.ProcessConfig, pipes) + } + if err != nil { + return -1, err + } + c.ProcessConfig.Terminal = term + + d.Lock() + d.activeContainers[c.ID] = &activeContainer{ + container: container, + cmd: &c.ProcessConfig.Cmd, + } + d.Unlock() + defer d.cleanContainer(c.ID) + + // Since the CRIU binary exits after restoring the container, we + // need to reap its child by setting PR_SET_CHILD_SUBREAPER (36) + // so that it'll be owned by this process (Docker daemon) after restore. + // + // XXX This really belongs to where the Docker daemon starts. + if _, _, syserr := syscall.RawSyscall(syscall.SYS_PRCTL, 36, 1, 0); syserr != 0 { + return -1, fmt.Errorf("Could not set PR_SET_CHILD_SUBREAPER (syserr %d)", syserr) + } + + restoreOutputChan := make(chan restoreOutput, 1) + waitForRestore := make(chan struct{}) + + go func() { + exitCode, err := namespaces.Restore(container, c.ProcessConfig.Stdin, c.ProcessConfig.Stdout, c.ProcessConfig.Stderr, c.ProcessConfig.Console, filepath.Join(d.root, c.ID), imageDir, + func(child *os.File, args []string) *exec.Cmd { + cmd := new(exec.Cmd) + cmd.Path = d.initPath + cmd.Args = append([]string{ + DriverName, + "-restore", + "-pipe", "3", + "--", + }, args...) + cmd.ExtraFiles = []*os.File{child} + return cmd + }, + func(restorePid int) error { + log.CRDbg("restorePid=%d", restorePid) + if restorePid == 0 { + restoreCallback(&c.ProcessConfig, 0) + return nil + } + + // The container.json file should be written *after* the container + // has started because its StdFds cannot be initialized before. + // + // XXX How do we handle error here? + d.writeContainerFile(container, c.ID) + close(waitForRestore) + if restoreCallback != nil { + c.ProcessConfig.Process, err = os.FindProcess(restorePid) + if err != nil { + log.Debugf("cannot find restored process %d", restorePid) + return err + } + c.ContainerPid = c.ProcessConfig.Process.Pid + restoreCallback(&c.ProcessConfig, c.ContainerPid) + } + return nil + }) + restoreOutputChan <- restoreOutput{exitCode, err} + }() + + select { + case restoreOutput := <-restoreOutputChan: + // there was an error + return restoreOutput.exitCode, restoreOutput.err + case <-waitForRestore: + // container restored + break + } + + // Wait for the container to exit. + restoreOutput := <-restoreOutputChan + return restoreOutput.exitCode, restoreOutput.err +} + func (d *driver) Terminate(c *execdriver.Command) error { defer d.cleanContainer(c.ID) container, err := d.factory.Load(c.ID) From 5d19d1aa7e507c35288238846722dde48685071d Mon Sep 17 00:00:00 2001 From: boucher Date: Mon, 25 May 2015 08:32:58 -0700 Subject: [PATCH 02/26] Update checkpoint/restore support to match docker/master Docker-DCO-1.1-Signed-off-by: Ross Boucher (github: boucher) --- daemon/execdriver/driver.go | 4 +- daemon/execdriver/lxc/driver.go | 6 +- daemon/execdriver/native/create.go | 19 --- daemon/execdriver/native/driver.go | 183 ++++++++++------------------- 4 files changed, 69 insertions(+), 143 deletions(-) diff --git a/daemon/execdriver/driver.go b/daemon/execdriver/driver.go index acb41c5f53376..54d3955bb040c 100644 --- a/daemon/execdriver/driver.go +++ b/daemon/execdriver/driver.go @@ -60,8 +60,8 @@ type Driver interface { Kill(c *Command, sig int) error Pause(c *Command) error Unpause(c *Command) error - Checkpoint(c *Command) error - Restore(c *Command, pipes *Pipes, restoreCallback RestoreCallback) (int, error) + Checkpoint(c *Command, opts *libcontainer.CriuOpts) error + Restore(c *Command, pipes *Pipes, restoreCallback RestoreCallback, opts *libcontainer.CriuOpts, forceRestore bool) (ExitStatus, error) Name() string // Driver name Info(id string) Info // "temporary" hack (until we move state from core to plugins) GetPidsForContainer(id string) ([]int, error) // Returns a list of pids for the given container. diff --git a/daemon/execdriver/lxc/driver.go b/daemon/execdriver/lxc/driver.go index 914a7ea5075c2..7ca41abe818c4 100644 --- a/daemon/execdriver/lxc/driver.go +++ b/daemon/execdriver/lxc/driver.go @@ -547,12 +547,12 @@ func (d *driver) Unpause(c *execdriver.Command) error { return err } -func (d *driver) Checkpoint(c *execdriver.Command) error { +func (d *driver) Checkpoint(c *execdriver.Command, opts *libcontainer.CriuOpts) error { return fmt.Errorf("Checkpointing lxc containers not supported yet\n") } -func (d *driver) Restore(c *execdriver.Command, pipes *execdriver.Pipes, restoreCallback execdriver.RestoreCallback) (int, error) { - return 0, fmt.Errorf("Restoring lxc containers not supported yet\n") +func (d *driver) Restore(c *execdriver.Command, pipes *execdriver.Pipes, restoreCallback execdriver.RestoreCallback, opts *libcontainer.CriuOpts, forceRestore bool) (execdriver.ExitStatus, error) { + return execdriver.ExitStatus{ExitCode: 0}, fmt.Errorf("Restoring lxc containers not supported yet\n") } func (d *driver) Terminate(c *execdriver.Command) error { diff --git a/daemon/execdriver/native/create.go b/daemon/execdriver/native/create.go index 1cf248b0d66fd..1b2d7232d3177 100644 --- a/daemon/execdriver/native/create.go +++ b/daemon/execdriver/native/create.go @@ -4,7 +4,6 @@ package native import ( "errors" - "encoding/json" "fmt" "net" "strings" @@ -89,24 +88,6 @@ func generateIfaceName() (string, error) { return "", errors.New("Failed to find name for new interface") } -// Re-create the container type from the image that was saved during checkpoint. -func (d *driver) createRestoreContainer(c *execdriver.Command, imageDir string) (*libcontainer.Config, error) { - // Read the container.json. - f1, err := os.Open(filepath.Join(imageDir, "container.json")) - if err != nil { - return nil, err - } - defer f1.Close() - - var container *libcontainer.Config - err = json.NewDecoder(f1).Decode(&container) - if err != nil { - return nil, err - } - - return container, nil -} - func (d *driver) createNetwork(container *configs.Config, c *execdriver.Command) error { if c.Network == nil { return nil diff --git a/daemon/execdriver/native/driver.go b/daemon/execdriver/native/driver.go index 8f451bcd7592f..3e4658b1f495e 100644 --- a/daemon/execdriver/native/driver.go +++ b/daemon/execdriver/native/driver.go @@ -19,7 +19,6 @@ import ( "github.com/docker/docker/pkg/reexec" sysinfo "github.com/docker/docker/pkg/system" "github.com/docker/docker/pkg/term" - "github.com/docker/docker/utils" "github.com/docker/libcontainer" "github.com/docker/libcontainer/apparmor" "github.com/docker/libcontainer/cgroups/systemd" @@ -277,49 +276,15 @@ func (d *driver) Unpause(c *execdriver.Command) error { return active.Resume() } -// XXX Where is the right place for the following -// const and getCheckpointImageDir() function? -const ( - containersDir = "/var/lib/docker/containers" - criuImgDir = "criu_img" -) - -func getCheckpointImageDir(containerId string) string { - return filepath.Join(containersDir, containerId, criuImgDir) -} - -func (d *driver) Checkpoint(c *execdriver.Command) error { +func (d *driver) Checkpoint(c *execdriver.Command, opts *libcontainer.CriuOpts) error { active := d.activeContainers[c.ID] if active == nil { return fmt.Errorf("active container for %s does not exist", c.ID) } - container := active.container - - // Create an image directory for this container (which - // may already exist from a previous checkpoint). - imageDir := getCheckpointImageDir(c.ID) - err := os.MkdirAll(imageDir, 0700) - if err != nil && !os.IsExist(err) { - return err - } - - // Copy container.json and state.json files to the CRIU - // image directory for later use during restore. Do this - // before checkpointing because after checkpoint the container - // will exit and these files will be removed. - log.CRDbg("saving container.json and state.json before calling CRIU in %s", imageDir) - srcFiles := []string{"container.json", "state.json"} - for _, f := range srcFiles { - srcFile := filepath.Join(d.root, c.ID, f) - dstFile := filepath.Join(imageDir, f) - if _, err := utils.CopyFile(srcFile, dstFile); err != nil { - return err - } - } d.Lock() defer d.Unlock() - err = namespaces.Checkpoint(container, imageDir, c.ProcessConfig.Process.Pid) + err := active.Checkpoint(opts) if err != nil { return err } @@ -327,103 +292,83 @@ func (d *driver) Checkpoint(c *execdriver.Command) error { return nil } -type restoreOutput struct { - exitCode int - err error -} +func (d *driver) Restore(c *execdriver.Command, pipes *execdriver.Pipes, restoreCallback execdriver.RestoreCallback, opts *libcontainer.CriuOpts, forceRestore bool) (execdriver.ExitStatus, error) { + var ( + cont libcontainer.Container + err error + ) -func (d *driver) Restore(c *execdriver.Command, pipes *execdriver.Pipes, restoreCallback execdriver.RestoreCallback) (int, error) { - imageDir := getCheckpointImageDir(c.ID) - container, err := d.createRestoreContainer(c, imageDir) + cont, err = d.factory.Load(c.ID) if err != nil { - return 1, err + if forceRestore { + var config *configs.Config + config, err = d.createContainer(c) + if err != nil { + return execdriver.ExitStatus{ExitCode: -1}, err + } + cont, err = d.factory.Create(c.ID, config) + if err != nil { + return execdriver.ExitStatus{ExitCode: -1}, err + } + } else { + return execdriver.ExitStatus{ExitCode: -1}, err + } } - var term execdriver.Terminal - - if c.ProcessConfig.Tty { - term, err = NewTtyConsole(&c.ProcessConfig, pipes) - } else { - term, err = execdriver.NewStdConsole(&c.ProcessConfig, pipes) + p := &libcontainer.Process{ + Args: append([]string{c.ProcessConfig.Entrypoint}, c.ProcessConfig.Arguments...), + Env: c.ProcessConfig.Env, + Cwd: c.WorkingDir, + User: c.ProcessConfig.User, } - if err != nil { - return -1, err + + config := cont.Config() + if err := setupPipes(&config, &c.ProcessConfig, p, pipes); err != nil { + return execdriver.ExitStatus{ExitCode: -1}, err } - c.ProcessConfig.Terminal = term d.Lock() - d.activeContainers[c.ID] = &activeContainer{ - container: container, - cmd: &c.ProcessConfig.Cmd, - } + d.activeContainers[c.ID] = cont d.Unlock() - defer d.cleanContainer(c.ID) + defer func() { + cont.Destroy() + d.cleanContainer(c.ID) + }() - // Since the CRIU binary exits after restoring the container, we - // need to reap its child by setting PR_SET_CHILD_SUBREAPER (36) - // so that it'll be owned by this process (Docker daemon) after restore. - // - // XXX This really belongs to where the Docker daemon starts. - if _, _, syserr := syscall.RawSyscall(syscall.SYS_PRCTL, 36, 1, 0); syserr != 0 { - return -1, fmt.Errorf("Could not set PR_SET_CHILD_SUBREAPER (syserr %d)", syserr) + if err := cont.Restore(p, opts); err != nil { + return execdriver.ExitStatus{ExitCode: -1}, err } - restoreOutputChan := make(chan restoreOutput, 1) - waitForRestore := make(chan struct{}) - - go func() { - exitCode, err := namespaces.Restore(container, c.ProcessConfig.Stdin, c.ProcessConfig.Stdout, c.ProcessConfig.Stderr, c.ProcessConfig.Console, filepath.Join(d.root, c.ID), imageDir, - func(child *os.File, args []string) *exec.Cmd { - cmd := new(exec.Cmd) - cmd.Path = d.initPath - cmd.Args = append([]string{ - DriverName, - "-restore", - "-pipe", "3", - "--", - }, args...) - cmd.ExtraFiles = []*os.File{child} - return cmd - }, - func(restorePid int) error { - log.CRDbg("restorePid=%d", restorePid) - if restorePid == 0 { - restoreCallback(&c.ProcessConfig, 0) - return nil - } - - // The container.json file should be written *after* the container - // has started because its StdFds cannot be initialized before. - // - // XXX How do we handle error here? - d.writeContainerFile(container, c.ID) - close(waitForRestore) - if restoreCallback != nil { - c.ProcessConfig.Process, err = os.FindProcess(restorePid) - if err != nil { - log.Debugf("cannot find restored process %d", restorePid) - return err - } - c.ContainerPid = c.ProcessConfig.Process.Pid - restoreCallback(&c.ProcessConfig, c.ContainerPid) - } - return nil - }) - restoreOutputChan <- restoreOutput{exitCode, err} - }() + // FIXME: no idea if any of this is needed... + if restoreCallback != nil { + pid, err := p.Pid() + if err != nil { + p.Signal(os.Kill) + p.Wait() + return execdriver.ExitStatus{ExitCode: -1}, err + } + restoreCallback(&c.ProcessConfig, pid) + } - select { - case restoreOutput := <-restoreOutputChan: - // there was an error - return restoreOutput.exitCode, restoreOutput.err - case <-waitForRestore: - // container restored - break + oom := notifyOnOOM(cont) + waitF := p.Wait + if nss := cont.Config().Namespaces; !nss.Contains(configs.NEWPID) { + // we need such hack for tracking processes with inherited fds, + // because cmd.Wait() waiting for all streams to be copied + waitF = waitInPIDHost(p, cont) + } + ps, err := waitF() + if err != nil { + execErr, ok := err.(*exec.ExitError) + if !ok { + return execdriver.ExitStatus{ExitCode: -1}, err + } + ps = execErr.ProcessState } - // Wait for the container to exit. - restoreOutput := <-restoreOutputChan - return restoreOutput.exitCode, restoreOutput.err + cont.Destroy() + _, oomKill := <-oom + return execdriver.ExitStatus{ExitCode: utils.ExitStatus(ps.Sys().(syscall.WaitStatus)), OOMKilled: oomKill}, nil } func (d *driver) Terminate(c *execdriver.Command) error { From de50b10d1535b2c4de1458132dd7a2a9345c243e Mon Sep 17 00:00:00 2001 From: Saied Kazemi Date: Thu, 5 Feb 2015 20:37:07 -0800 Subject: [PATCH 03/26] Checkpoint/Restore Support: add functionality to daemon Support was added to the daemon to use the Checkpoint and Restore methods of the native exec driver for checkpointing and restoring containers. Signed-off-by: Saied Kazemi Conflicts: api/server/server.go daemon/container.go daemon/daemon.go daemon/networkdriver/bridge/driver.go daemon/state.go vendor/src/github.com/docker/libnetwork/ipallocator/allocator.go --- api/server/server.go | 76 +++++++++++++++++++++++++++------------ daemon/checkpoint.go | 55 ++++++++++++++++++++++++++++ daemon/container.go | 66 ++++++++++++++++++++++++++++++++-- daemon/container_linux.go | 47 ++++++++++++++++++++++++ daemon/daemon.go | 31 ++++++++++++++++ daemon/monitor.go | 70 ++++++++++++++++++++++++++++++++++++ daemon/state.go | 23 ++++++++++++ 7 files changed, 344 insertions(+), 24 deletions(-) create mode 100644 daemon/checkpoint.go diff --git a/api/server/server.go b/api/server/server.go index 46210cc7f44a5..c86ce7b7b2fe4 100644 --- a/api/server/server.go +++ b/api/server/server.go @@ -1324,6 +1324,36 @@ func (s *Server) postContainersCopy(version version.Version, w http.ResponseWrit return nil } +func postContainersCheckpoint(eng *engine.Engine, version version.Version, w http.ResponseWriter, r *http.Request, vars map[string]string) error { + if vars == nil { + return fmt.Errorf("Missing parameter") + } + if err := parseForm(r); err != nil { + return err + } + job := eng.Job("checkpoint", vars["name"]) + if err := job.Run(); err != nil { + return err + } + w.WriteHeader(http.StatusNoContent) + return nil +} + +func postContainersRestore(eng *engine.Engine, version version.Version, w http.ResponseWriter, r *http.Request, vars map[string]string) error { + if vars == nil { + return fmt.Errorf("Missing parameter") + } + if err := parseForm(r); err != nil { + return err + } + job := eng.Job("restore", vars["name"]) + if err := job.Run(); err != nil { + return err + } + w.WriteHeader(http.StatusNoContent) + return nil +} + func (s *Server) postContainerExecCreate(version version.Version, w http.ResponseWriter, r *http.Request, vars map[string]string) error { if err := parseForm(r); err != nil { return err @@ -1526,28 +1556,30 @@ func createRouter(s *Server) *mux.Router { "/exec/{id:.*}/json": s.getExecByID, }, "POST": { - "/auth": s.postAuth, - "/commit": s.postCommit, - "/build": s.postBuild, - "/images/create": s.postImagesCreate, - "/images/load": s.postImagesLoad, - "/images/{name:.*}/push": s.postImagesPush, - "/images/{name:.*}/tag": s.postImagesTag, - "/containers/create": s.postContainersCreate, - "/containers/{name:.*}/kill": s.postContainersKill, - "/containers/{name:.*}/pause": s.postContainersPause, - "/containers/{name:.*}/unpause": s.postContainersUnpause, - "/containers/{name:.*}/restart": s.postContainersRestart, - "/containers/{name:.*}/start": s.postContainersStart, - "/containers/{name:.*}/stop": s.postContainersStop, - "/containers/{name:.*}/wait": s.postContainersWait, - "/containers/{name:.*}/resize": s.postContainersResize, - "/containers/{name:.*}/attach": s.postContainersAttach, - "/containers/{name:.*}/copy": s.postContainersCopy, - "/containers/{name:.*}/exec": s.postContainerExecCreate, - "/exec/{name:.*}/start": s.postContainerExecStart, - "/exec/{name:.*}/resize": s.postContainerExecResize, - "/containers/{name:.*}/rename": s.postContainerRename, + "/auth": s.postAuth, + "/commit": s.postCommit, + "/build": s.postBuild, + "/images/create": s.postImagesCreate, + "/images/load": s.postImagesLoad, + "/images/{name:.*}/push": s.postImagesPush, + "/images/{name:.*}/tag": s.postImagesTag, + "/containers/create": s.postContainersCreate, + "/containers/{name:.*}/kill": s.postContainersKill, + "/containers/{name:.*}/pause": s.postContainersPause, + "/containers/{name:.*}/unpause": s.postContainersUnpause, + "/containers/{name:.*}/restart": s.postContainersRestart, + "/containers/{name:.*}/start": s.postContainersStart, + "/containers/{name:.*}/stop": s.postContainersStop, + "/containers/{name:.*}/wait": s.postContainersWait, + "/containers/{name:.*}/resize": s.postContainersResize, + "/containers/{name:.*}/attach": s.postContainersAttach, + "/containers/{name:.*}/copy": s.postContainersCopy, + "/containers/{name:.*}/exec": s.postContainerExecCreate, + "/exec/{name:.*}/start": s.postContainerExecStart, + "/exec/{name:.*}/resize": s.postContainerExecResize, + "/containers/{name:.*}/rename": s.postContainerRename, + "/containers/{name:.*}/checkpoint": s.postContainersCheckpoint, + "/containers/{name:.*}/restore": s.postContainersRestore, }, "DELETE": { "/containers/{name:.*}": s.deleteContainers, diff --git a/daemon/checkpoint.go b/daemon/checkpoint.go new file mode 100644 index 0000000000000..f6057c6a028f9 --- /dev/null +++ b/daemon/checkpoint.go @@ -0,0 +1,55 @@ +package daemon + +import ( + "github.com/docker/docker/engine" +) + +// Checkpoint a running container. +func (daemon *Daemon) ContainerCheckpoint(job *engine.Job) engine.Status { + if len(job.Args) != 1 { + return job.Errorf("Usage: %s CONTAINER\n", job.Name) + } + + name := job.Args[0] + container, err := daemon.Get(name) + if err != nil { + return job.Error(err) + } + if !container.IsRunning() { + return job.Errorf("Container %s not running", name) + } + + if err := container.Checkpoint(); err != nil { + return job.Errorf("Cannot checkpoint container %s: %s", name, err) + } + + container.LogEvent("checkpoint") + return engine.StatusOK +} + +// Restore a checkpointed container. +func (daemon *Daemon) ContainerRestore(job *engine.Job) engine.Status { + if len(job.Args) != 1 { + return job.Errorf("Usage: %s CONTAINER\n", job.Name) + } + + name := job.Args[0] + container, err := daemon.Get(name) + if err != nil { + return job.Error(err) + } + if container.IsRunning() { + return job.Errorf("Container %s already running", name) + } + if !container.State.IsCheckpointed() { + return job.Errorf("Container %s is not checkpointed", name) + } + + if err := container.Restore(); err != nil { + container.LogEvent("die") + return job.Errorf("Cannot restore container %s: %s", name, err) + } + + container.LogEvent("restore") + return engine.StatusOK +} diff --git a/daemon/container.go b/daemon/container.go index 4e44621757b64..8b465553737f3 100644 --- a/daemon/container.go +++ b/daemon/container.go @@ -335,10 +335,15 @@ func (container *Container) isNetworkAllocated() bool { return container.NetworkSettings.IPAddress != "" } + // cleanup releases any network resources allocated to the container along with any rules // around how containers are linked together. It also unmounts the container's root filesystem. func (container *Container) cleanup() { - container.ReleaseNetwork() + if container.IsCheckpointed() { + log.CRDbg("not calling ReleaseNetwork() for checkpointed container %s", container.ID) + } else { + container.ReleaseNetwork() + } disableAllActiveLinks(container) @@ -659,6 +664,41 @@ func (container *Container) Copy(resource string) (io.ReadCloser, error) { return reader, nil } +func (container *Container) Checkpoint() error { + return container.daemon.Checkpoint(container) +} + +func (container *Container) Restore() error { + var err error + + container.Lock() + defer container.Unlock() + + defer func() { + if err != nil { + container.cleanup() + } + }() + + if err = container.initializeNetworking(); err != nil { + return err + } + + linkedEnv, err := container.setupLinkedContainers() + if err != nil { + return err + } + if err = container.setupWorkingDirectory(); err != nil { + return err + } + env := container.createDaemonEnvironment(linkedEnv) + if err = populateCommandRestore(container, env); err != nil { + return err + } + + return container.waitForRestore() +} + // Returns true if the container exposes a certain port func (container *Container) Exposes(p nat.Port) bool { _, exists := container.Config.ExposedPorts[p] @@ -750,6 +790,29 @@ func (container *Container) waitForStart() error { return nil } +// Like waitForStart() but for restoring a container. +// +// XXX Does RestartPolicy apply here? +func (container *Container) waitForRestore() error { + container.monitor = newContainerMonitor(container, container.hostConfig.RestartPolicy) + + // After calling promise.Go() we'll have two goroutines: + // - The current goroutine that will block in the select + // below until restore is done. + // - A new goroutine that will restore the container and + // wait for it to exit. + select { + case <-container.monitor.restoreSignal: + if container.ExitCode != 0 { + return fmt.Errorf("restore process failed") + } + case err := <-promise.Go(container.monitor.Restore): + return err + } + + return nil +} + func (container *Container) GetProcessLabel() string { // even if we have a process label return "" if we are running // in privileged mode @@ -957,7 +1020,6 @@ func attach(streamConfig *StreamConfig, openStdin, stdinOnce, tty bool, stdin io _, err = copyEscapable(cStdin, stdin) } else { _, err = io.Copy(cStdin, stdin) - } if err == io.ErrClosedPipe { err = nil diff --git a/daemon/container_linux.go b/daemon/container_linux.go index b46e6107cdf74..94053ee1335c1 100644 --- a/daemon/container_linux.go +++ b/daemon/container_linux.go @@ -329,6 +329,53 @@ func mergeDevices(defaultDevices, userDevices []*configs.Device) []*configs.Devi return append(devs, userDevices...) } +// Like populateCommand() but for restoring a container. +// +// XXX populateCommand() does a lot more. Not sure if we have +// to do everything it does. +func populateCommandRestore(c *Container, env []string) error { + resources := &execdriver.Resources{ + Memory: c.Config.Memory, + MemorySwap: c.Config.MemorySwap, + CpuShares: c.Config.CpuShares, + Cpuset: c.Config.Cpuset, + } + + processConfig := execdriver.ProcessConfig{ + Privileged: c.hostConfig.Privileged, + Entrypoint: c.Path, + Arguments: c.Args, + Tty: c.Config.Tty, + User: c.Config.User, + } + + processConfig.SysProcAttr = &syscall.SysProcAttr{Setsid: true} + processConfig.Env = env + + c.command = &execdriver.Command{ + ID: c.ID, + Rootfs: c.RootfsPath(), + ReadonlyRootfs: c.hostConfig.ReadonlyRootfs, + InitPath: "/.dockerinit", + WorkingDir: c.Config.WorkingDir, + // Network: en, + // Ipc: ipc, + // Pid: pid, + Resources: resources, + // AllowedDevices: allowedDevices, + // AutoCreatedDevices: autoCreatedDevices, + CapAdd: c.hostConfig.CapAdd, + CapDrop: c.hostConfig.CapDrop, + ProcessConfig: processConfig, + ProcessLabel: c.GetProcessLabel(), + MountLabel: c.GetMountLabel(), + // LxcConfig: lxcConfig, + AppArmorProfile: c.AppArmorProfile, + } + + return nil +} + // GetSize, return real size, virtual size func (container *Container) GetSize() (int64, int64) { var ( diff --git a/daemon/daemon.go b/daemon/daemon.go index cd5f81e574f42..ef75ca283d545 100644 --- a/daemon/daemon.go +++ b/daemon/daemon.go @@ -278,6 +278,18 @@ func (daemon *Daemon) restore() error { logrus.Debugf("Loaded container %v", container.ID) containers[container.ID] = &cr{container: container} + + // If the container was checkpointed, we need to reserve + // the IP address that it was using. + // + // XXX We should also reserve host ports (if any). + if container.IsCheckpointed() { + /*err = bridge.ReserveIP(container.ID, container.NetworkSettings.IPAddress) + if err != nil { + log.Errorf("Failed to reserve IP %s for container %s", + container.ID, container.NetworkSettings.IPAddress) + }*/ + } } else { logrus.Debugf("Cannot load container %s because it was created with another graph driver.", container.ID) } @@ -845,6 +857,25 @@ func (daemon *Daemon) Run(c *Container, pipes *execdriver.Pipes, startCallback e return daemon.execDriver.Run(c.command, pipes, startCallback) } +func (daemon *Daemon) Checkpoint(c *Container) error { + if err := daemon.execDriver.Checkpoint(c.command); err != nil { + return err + } + c.SetCheckpointed() + return nil +} + +func (daemon *Daemon) Restore(c *Container, pipes *execdriver.Pipes, restoreCallback execdriver.RestoreCallback) (int, error) { + // Mount the container's filesystem (daemon/graphdriver/aufs/aufs.go). + _, err := daemon.driver.Get(c.ID, c.GetMountLabel()) + if err != nil { + return 0, err + } + + exitCode, err := daemon.execDriver.Restore(c.command, pipes, restoreCallback) + return exitCode, err +} + func (daemon *Daemon) Kill(c *Container, sig int) error { return daemon.execDriver.Kill(c.command, sig) } diff --git a/daemon/monitor.go b/daemon/monitor.go index dfade8e21847b..241efa32d0eb3 100644 --- a/daemon/monitor.go +++ b/daemon/monitor.go @@ -44,6 +44,9 @@ type containerMonitor struct { // left waiting for nothing to happen during this time stopChan chan struct{} + // like startSignal but for restoring a container + restoreSignal chan struct{} + // timeIncrement is the amount of time to wait between restarts // this is in milliseconds timeIncrement int @@ -61,6 +64,7 @@ func newContainerMonitor(container *Container, policy runconfig.RestartPolicy) * timeIncrement: defaultTimeIncrement, stopChan: make(chan struct{}), startSignal: make(chan struct{}), + restoreSignal: make(chan struct{}), } } @@ -181,6 +185,49 @@ func (m *containerMonitor) Start() error { } } +// Like Start() but for restoring a container. +func (m *containerMonitor) Restore() error { + var ( + err error + // XXX The following line should be changed to + // exitStatus execdriver.ExitStatus to match Start() + exitCode int + afterRestore bool + ) + + defer func() { + if afterRestore { + m.container.Lock() + m.container.setStopped(&execdriver.ExitStatus{exitCode, false}) + defer m.container.Unlock() + } + m.Close() + }() + + if err := m.container.startLoggingToDisk(); err != nil { + m.resetContainer(false) + return err + } + + pipes := execdriver.NewPipes(m.container.stdin, m.container.stdout, m.container.stderr, m.container.Config.OpenStdin) + + m.container.LogEvent("restore") + m.lastStartTime = time.Now() + if exitCode, err = m.container.daemon.Restore(m.container, pipes, m.restoreCallback); err != nil { + log.Errorf("Error restoring container: %s, exitCode=%d", err, exitCode) + m.container.ExitCode = -1 + m.resetContainer(false) + return err + } + afterRestore = true + + m.container.ExitCode = exitCode + m.resetMonitor(err == nil && exitCode == 0) + m.container.LogEvent("die") + m.resetContainer(true) + return err +} + // resetMonitor resets the stateful fields on the containerMonitor based on the // previous runs success or failure. Regardless of success, if the container had // an execution time of more than 10s then reset the timer back to the default @@ -267,6 +314,29 @@ func (m *containerMonitor) callback(processConfig *execdriver.ProcessConfig, pid } } +// Like callback() but for restoring a container. +func (m *containerMonitor) restoreCallback(processConfig *execdriver.ProcessConfig, restorePid int) { + // If restorePid is 0, it means that restore failed. + if restorePid != 0 { + m.container.setRunning(restorePid) + } + + // Unblock the goroutine waiting in waitForRestore(). + select { + case <-m.restoreSignal: + default: + close(m.restoreSignal) + } + + if restorePid != 0 { + // Write config.json and hostconfig.json files + // to /var/lib/docker/containers/. + if err := m.container.ToDisk(); err != nil { + log.Debugf("%s", err) + } + } +} + // resetContainer resets the container's IO and ensures that the command is able to be executed again // by copying the data into a new struct // if lock is true, then container locked during reset diff --git a/daemon/state.go b/daemon/state.go index 0270c88e88774..82eefe75a3a6c 100644 --- a/daemon/state.go +++ b/daemon/state.go @@ -14,6 +14,7 @@ type State struct { Running bool Paused bool Restarting bool + Checkpointed bool OOMKilled bool removalInProgress bool // Not need for this to be persistent on disk. Dead bool @@ -22,7 +23,9 @@ type State struct { Error string // contains last known error when starting the container StartedAt time.Time FinishedAt time.Time + CheckpointedAt time.Time waitChan chan struct{} + } func NewState() *State { @@ -42,6 +45,8 @@ func (s *State) String() string { } return fmt.Sprintf("Up %s", units.HumanDuration(time.Now().UTC().Sub(s.StartedAt))) + } else if s.Checkpointed { + return fmt.Sprintf("Checkpointed %s ago", units.HumanDuration(time.Now().UTC().Sub(s.CheckpointedAt))) } if s.removalInProgress { @@ -166,6 +171,7 @@ func (s *State) setRunning(pid int) { s.Error = "" s.Running = true s.Paused = false + s.Checkpointed = false s.Restarting = false s.ExitCode = 0 s.Pid = pid @@ -262,3 +268,20 @@ func (s *State) SetDead() { s.Dead = true s.Unlock() } + +func (s *State) SetCheckpointed() { + s.Lock() + s.CheckpointedAt = time.Now().UTC() + s.Checkpointed = true + s.Running = false + s.Paused = false + s.Restarting = false + // XXX Not sure if we need to close and recreate waitChan. + // close(s.waitChan) + // s.waitChan = make(chan struct{}) + s.Unlock() +} + +func (s *State) IsCheckpointed() bool { + return s.Checkpointed +} From 55fcafa99fb3436384eef8e9a1b55e4f6c6c984d Mon Sep 17 00:00:00 2001 From: Hui Kang Date: Tue, 19 May 2015 21:08:04 +0000 Subject: [PATCH 04/26] Release the network resource during checkpoint Restore failed if network resource not released during checkpoint, e.g., a container with port open with -p Signed-off-by: Hui Kang Conflicts: daemon/container.go --- daemon/container.go | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/daemon/container.go b/daemon/container.go index 8b465553737f3..ee489d8918228 100644 --- a/daemon/container.go +++ b/daemon/container.go @@ -602,6 +602,19 @@ func validateID(id string) error { return nil } + +func (container *Container) Checkpoint(opts *libcontainer.CriuOpts) error { + if err := container.daemon.Checkpoint(container, opts); err != nil { + return err + } + + if opts.LeaveRunning == false { + container.ReleaseNetwork() + } + return nil +} + + func (container *Container) Copy(resource string) (io.ReadCloser, error) { container.Lock() defer container.Unlock() From 7c452102734e5dad8e851c1cc3c470685228b795 Mon Sep 17 00:00:00 2001 From: boucher Date: Mon, 1 Jun 2015 15:15:02 -0700 Subject: [PATCH 05/26] Update daemon and cli support for checkpoint and restore. Docker-DCO-1.1-Signed-off-by: Ross Boucher (github: boucher) --- api/client/checkpoint.go | 52 ++++++++++++++ api/client/restore.go | 54 ++++++++++++++ api/server/server.go | 24 +++++-- daemon/checkpoint.go | 57 +++++++-------- daemon/container.go | 95 +++++++++++++------------ daemon/container_linux.go | 65 ++++------------- daemon/daemon.go | 12 ++-- daemon/execdriver/driver.go | 5 +- daemon/execdriver/lxc/driver.go | 5 +- daemon/execdriver/native/driver.go | 20 ++++-- daemon/monitor.go | 26 +++---- daemon/state.go | 21 ++++-- docker/flags.go | 4 +- integration-cli/docker_cli_help_test.go | 2 +- runconfig/restore.go | 15 ++++ 15 files changed, 292 insertions(+), 165 deletions(-) create mode 100644 api/client/checkpoint.go create mode 100644 api/client/restore.go create mode 100644 runconfig/restore.go diff --git a/api/client/checkpoint.go b/api/client/checkpoint.go new file mode 100644 index 0000000000000..8c681bcf9716f --- /dev/null +++ b/api/client/checkpoint.go @@ -0,0 +1,52 @@ +package client + +import ( + "fmt" + + flag "github.com/docker/docker/pkg/mflag" + "github.com/docker/docker/runconfig" +) + +func (cli *DockerCli) CmdCheckpoint(args ...string) error { + cmd := cli.Subcmd("checkpoint", []string{"CONTAINER [CONTAINER...]"}, "Checkpoint one or more running containers", true) + cmd.Require(flag.Min, 1) + + var ( + flImgDir = cmd.String([]string{"-image-dir"}, "", "directory for storing checkpoint image files") + flWorkDir = cmd.String([]string{"-work-dir"}, "", "directory for storing log file") + flLeaveRunning = cmd.Bool([]string{"-leave-running"}, false, "leave the container running after checkpoint") + flCheckTcp = cmd.Bool([]string{"-allow-tcp"}, false, "allow checkpointing tcp connections") + flExtUnix = cmd.Bool([]string{"-allow-ext-unix"}, false, "allow checkpointing external unix connections") + flShell = cmd.Bool([]string{"-allow-shell"}, false, "allow checkpointing shell jobs") + ) + + if err := cmd.ParseFlags(args, true); err != nil { + return err + } + + if cmd.NArg() < 1 { + cmd.Usage() + return nil + } + + criuOpts := &runconfig.CriuConfig{ + ImagesDirectory: *flImgDir, + WorkDirectory: *flWorkDir, + LeaveRunning: *flLeaveRunning, + TcpEstablished: *flCheckTcp, + ExternalUnixConnections: *flExtUnix, + ShellJob: *flShell, + } + + var encounteredError error + for _, name := range cmd.Args() { + _, _, err := readBody(cli.call("POST", "/containers/"+name+"/checkpoint", criuOpts, nil)) + if err != nil { + fmt.Fprintf(cli.err, "%s\n", err) + encounteredError = fmt.Errorf("Error: failed to checkpoint one or more containers") + } else { + fmt.Fprintf(cli.out, "%s\n", name) + } + } + return encounteredError +} diff --git a/api/client/restore.go b/api/client/restore.go new file mode 100644 index 0000000000000..0c4085fbbbd84 --- /dev/null +++ b/api/client/restore.go @@ -0,0 +1,54 @@ +package client + +import ( + "fmt" + + flag "github.com/docker/docker/pkg/mflag" + "github.com/docker/docker/runconfig" +) + +func (cli *DockerCli) CmdRestore(args ...string) error { + cmd := cli.Subcmd("restore", []string{"CONTAINER [CONTAINER...]"}, "Restore one or more checkpointed containers", true) + cmd.Require(flag.Min, 1) + + var ( + flImgDir = cmd.String([]string{"-image-dir"}, "", "directory to restore image files from") + flWorkDir = cmd.String([]string{"-work-dir"}, "", "directory for restore log") + flCheckTcp = cmd.Bool([]string{"-allow-tcp"}, false, "allow restoring tcp connections") + flExtUnix = cmd.Bool([]string{"-allow-ext-unix"}, false, "allow restoring external unix connections") + flShell = cmd.Bool([]string{"-allow-shell"}, false, "allow restoring shell jobs") + flForce = cmd.Bool([]string{"-force"}, false, "bypass checks for current container state") + ) + + if err := cmd.ParseFlags(args, true); err != nil { + return err + } + + if cmd.NArg() < 1 { + cmd.Usage() + return nil + } + + restoreOpts := &runconfig.RestoreConfig{ + CriuOpts: runconfig.CriuConfig{ + ImagesDirectory: *flImgDir, + WorkDirectory: *flWorkDir, + TcpEstablished: *flCheckTcp, + ExternalUnixConnections: *flExtUnix, + ShellJob: *flShell, + }, + ForceRestore: *flForce, + } + + var encounteredError error + for _, name := range cmd.Args() { + _, _, err := readBody(cli.call("POST", "/containers/"+name+"/restore", restoreOpts, nil)) + if err != nil { + fmt.Fprintf(cli.err, "%s\n", err) + encounteredError = fmt.Errorf("Error: failed to restore one or more containers") + } else { + fmt.Fprintf(cli.out, "%s\n", name) + } + } + return encounteredError +} diff --git a/api/server/server.go b/api/server/server.go index c86ce7b7b2fe4..3b78dc68b63e1 100644 --- a/api/server/server.go +++ b/api/server/server.go @@ -1324,32 +1324,44 @@ func (s *Server) postContainersCopy(version version.Version, w http.ResponseWrit return nil } -func postContainersCheckpoint(eng *engine.Engine, version version.Version, w http.ResponseWriter, r *http.Request, vars map[string]string) error { +func (s *Server) postContainersCheckpoint(version version.Version, w http.ResponseWriter, r *http.Request, vars map[string]string) error { if vars == nil { return fmt.Errorf("Missing parameter") } if err := parseForm(r); err != nil { return err } - job := eng.Job("checkpoint", vars["name"]) - if err := job.Run(); err != nil { + + criuOpts := &runconfig.CriuConfig{} + if err := json.NewDecoder(r.Body).Decode(criuOpts); err != nil { return err } + + if err := s.daemon.ContainerCheckpoint(vars["name"], criuOpts); err != nil { + return err + } + w.WriteHeader(http.StatusNoContent) return nil } -func postContainersRestore(eng *engine.Engine, version version.Version, w http.ResponseWriter, r *http.Request, vars map[string]string) error { +func (s *Server) postContainersRestore(version version.Version, w http.ResponseWriter, r *http.Request, vars map[string]string) error { if vars == nil { return fmt.Errorf("Missing parameter") } if err := parseForm(r); err != nil { return err } - job := eng.Job("restore", vars["name"]) - if err := job.Run(); err != nil { + + restoreOpts := runconfig.RestoreConfig{} + if err := json.NewDecoder(r.Body).Decode(&restoreOpts); err != nil { return err } + + if err := s.daemon.ContainerRestore(vars["name"], &restoreOpts.CriuOpts, restoreOpts.ForceRestore); err != nil { + return err + } + w.WriteHeader(http.StatusNoContent) return nil } diff --git a/daemon/checkpoint.go b/daemon/checkpoint.go index f6057c6a028f9..a39662cc0f325 100644 --- a/daemon/checkpoint.go +++ b/daemon/checkpoint.go @@ -1,55 +1,56 @@ package daemon import ( - "github.com/docker/docker/engine" + "fmt" + + "github.com/docker/docker/runconfig" ) // Checkpoint a running container. -func (daemon *Daemon) ContainerCheckpoint(job *engine.Job) engine.Status { - if len(job.Args) != 1 { - return job.Errorf("Usage: %s CONTAINER\n", job.Name) - } - - name := job.Args[0] +func (daemon *Daemon) ContainerCheckpoint(name string, opts *runconfig.CriuConfig) error { container, err := daemon.Get(name) if err != nil { - return job.Error(err) + return err } if !container.IsRunning() { - return job.Errorf("Container %s not running", name) + return fmt.Errorf("Container %s not running", name) } - - if err := container.Checkpoint(); err != nil { - return job.Errorf("Cannot checkpoint container %s: %s", name, err) + if err := container.Checkpoint(opts); err != nil { + return fmt.Errorf("Cannot checkpoint container %s: %s", name, err) } container.LogEvent("checkpoint") - return engine.StatusOK + return nil } // Restore a checkpointed container. -func (daemon *Daemon) ContainerRestore(job *engine.Job) engine.Status { - if len(job.Args) != 1 { - return job.Errorf("Usage: %s CONTAINER\n", job.Name) - } - - name := job.Args[0] +func (daemon *Daemon) ContainerRestore(name string, opts *runconfig.CriuConfig, forceRestore bool) error { container, err := daemon.Get(name) if err != nil { - return job.Error(err) - } - if container.IsRunning() { - return job.Errorf("Container %s already running", name) + return err } - if !container.State.IsCheckpointed() { - return job.Errorf("Container %s is not checkpointed", name) + + if !forceRestore { + // TODO: It's possible we only want to bypass the checkpointed check, + // I'm not sure how this will work if the container is already running + if container.IsRunning() { + return fmt.Errorf("Container %s already running", name) + } + + if !container.IsCheckpointed() { + return fmt.Errorf("Container %s is not checkpointed", name) + } + } else { + if !container.HasBeenCheckpointed() && opts.ImagesDirectory == "" { + return fmt.Errorf("You must specify an image directory to restore from %s", name) + } } - if err := container.Restore(); err != nil { + if err = container.Restore(opts, forceRestore); err != nil { container.LogEvent("die") - return job.Errorf("Cannot restore container %s: %s", name, err) + return fmt.Errorf("Cannot restore container %s: %s", name, err) } container.LogEvent("restore") - return engine.StatusOK + return nil } diff --git a/daemon/container.go b/daemon/container.go index ee489d8918228..f1aa62e050024 100644 --- a/daemon/container.go +++ b/daemon/container.go @@ -252,7 +252,7 @@ func (container *Container) Start() (err error) { if err := container.Mount(); err != nil { return err } - if err := container.initializeNetworking(); err != nil { + if err := container.initializeNetworking(false); err != nil { return err } linkedEnv, err := container.setupLinkedContainers() @@ -335,12 +335,11 @@ func (container *Container) isNetworkAllocated() bool { return container.NetworkSettings.IPAddress != "" } - // cleanup releases any network resources allocated to the container along with any rules // around how containers are linked together. It also unmounts the container's root filesystem. func (container *Container) cleanup() { if container.IsCheckpointed() { - log.CRDbg("not calling ReleaseNetwork() for checkpointed container %s", container.ID) + logrus.Debugf("not calling ReleaseNetwork() for checkpointed container %s", container.ID) } else { container.ReleaseNetwork() } @@ -602,8 +601,7 @@ func validateID(id string) error { return nil } - -func (container *Container) Checkpoint(opts *libcontainer.CriuOpts) error { +func (container *Container) Checkpoint(opts *runconfig.CriuConfig) error { if err := container.daemon.Checkpoint(container, opts); err != nil { return err } @@ -614,6 +612,50 @@ func (container *Container) Checkpoint(opts *libcontainer.CriuOpts) error { return nil } +func (container *Container) Restore(opts *runconfig.CriuConfig, forceRestore bool) error { + var err error + container.Lock() + defer container.Unlock() + + defer func() { + if err != nil { + container.setError(err) + // if no one else has set it, make sure we don't leave it at zero + if container.ExitCode == 0 { + container.ExitCode = 128 + } + container.toDisk() + container.cleanup() + } + }() + + if err := container.Mount(); err != nil { + return err + } + if err = container.initializeNetworking(true); err != nil { + return err + } + linkedEnv, err := container.setupLinkedContainers() + if err != nil { + return err + } + if err = container.setupWorkingDirectory(); err != nil { + return err + } + + env := container.createDaemonEnvironment(linkedEnv) + if err = populateCommand(container, env); err != nil { + return err + } + + mounts, err := container.setupMounts() + if err != nil { + return err + } + + container.command.Mounts = mounts + return container.waitForRestore(opts, forceRestore) +} func (container *Container) Copy(resource string) (io.ReadCloser, error) { container.Lock() @@ -677,41 +719,6 @@ func (container *Container) Copy(resource string) (io.ReadCloser, error) { return reader, nil } -func (container *Container) Checkpoint() error { - return container.daemon.Checkpoint(container) -} - -func (container *Container) Restore() error { - var err error - - container.Lock() - defer container.Unlock() - - defer func() { - if err != nil { - container.cleanup() - } - }() - - if err = container.initializeNetworking(); err != nil { - return err - } - - linkedEnv, err := container.setupLinkedContainers() - if err != nil { - return err - } - if err = container.setupWorkingDirectory(); err != nil { - return err - } - env := container.createDaemonEnvironment(linkedEnv) - if err = populateCommandRestore(container, env); err != nil { - return err - } - - return container.waitForRestore() -} - // Returns true if the container exposes a certain port func (container *Container) Exposes(p nat.Port) bool { _, exists := container.Config.ExposedPorts[p] @@ -803,10 +810,7 @@ func (container *Container) waitForStart() error { return nil } -// Like waitForStart() but for restoring a container. -// -// XXX Does RestartPolicy apply here? -func (container *Container) waitForRestore() error { +func (container *Container) waitForRestore(opts *runconfig.CriuConfig, forceRestore bool) error { container.monitor = newContainerMonitor(container, container.hostConfig.RestartPolicy) // After calling promise.Go() we'll have two goroutines: @@ -819,7 +823,7 @@ func (container *Container) waitForRestore() error { if container.ExitCode != 0 { return fmt.Errorf("restore process failed") } - case err := <-promise.Go(container.monitor.Restore): + case err := <-promise.Go(func() error { return container.monitor.Restore(opts, forceRestore) }): return err } @@ -1033,6 +1037,7 @@ func attach(streamConfig *StreamConfig, openStdin, stdinOnce, tty bool, stdin io _, err = copyEscapable(cStdin, stdin) } else { _, err = io.Copy(cStdin, stdin) + } if err == io.ErrClosedPipe { err = nil diff --git a/daemon/container_linux.go b/daemon/container_linux.go index 94053ee1335c1..7fb5e5a8c4eda 100644 --- a/daemon/container_linux.go +++ b/daemon/container_linux.go @@ -329,53 +329,6 @@ func mergeDevices(defaultDevices, userDevices []*configs.Device) []*configs.Devi return append(devs, userDevices...) } -// Like populateCommand() but for restoring a container. -// -// XXX populateCommand() does a lot more. Not sure if we have -// to do everything it does. -func populateCommandRestore(c *Container, env []string) error { - resources := &execdriver.Resources{ - Memory: c.Config.Memory, - MemorySwap: c.Config.MemorySwap, - CpuShares: c.Config.CpuShares, - Cpuset: c.Config.Cpuset, - } - - processConfig := execdriver.ProcessConfig{ - Privileged: c.hostConfig.Privileged, - Entrypoint: c.Path, - Arguments: c.Args, - Tty: c.Config.Tty, - User: c.Config.User, - } - - processConfig.SysProcAttr = &syscall.SysProcAttr{Setsid: true} - processConfig.Env = env - - c.command = &execdriver.Command{ - ID: c.ID, - Rootfs: c.RootfsPath(), - ReadonlyRootfs: c.hostConfig.ReadonlyRootfs, - InitPath: "/.dockerinit", - WorkingDir: c.Config.WorkingDir, - // Network: en, - // Ipc: ipc, - // Pid: pid, - Resources: resources, - // AllowedDevices: allowedDevices, - // AutoCreatedDevices: autoCreatedDevices, - CapAdd: c.hostConfig.CapAdd, - CapDrop: c.hostConfig.CapDrop, - ProcessConfig: processConfig, - ProcessLabel: c.GetProcessLabel(), - MountLabel: c.GetMountLabel(), - // LxcConfig: lxcConfig, - AppArmorProfile: c.AppArmorProfile, - } - - return nil -} - // GetSize, return real size, virtual size func (container *Container) GetSize() (int64, int64) { var ( @@ -710,7 +663,7 @@ func (container *Container) UpdateNetwork() error { return nil } -func (container *Container) buildCreateEndpointOptions() ([]libnetwork.EndpointOption, error) { +func (container *Container) buildCreateEndpointOptions(restoring bool) ([]libnetwork.EndpointOption, error) { var ( portSpecs = make(nat.PortSet) bindings = make(nat.PortMap) @@ -781,10 +734,18 @@ func (container *Container) buildCreateEndpointOptions() ([]libnetwork.EndpointO createOptions = append(createOptions, libnetwork.EndpointOptionGeneric(genericOption)) } + /*if restoring && container.NetworkSettings.IPAddress != "" { + genericOption := options.Generic{ + netlabel.IPAddress: net.ParseIP(container.NetworkSettings.IPAddress), + } + + createOptions = append(createOptions, libnetwork.EndpointOptionGeneric(genericOption)) + }*/ + return createOptions, nil } -func (container *Container) AllocateNetwork() error { +func (container *Container) AllocateNetwork(restoring bool) error { mode := container.hostConfig.NetworkMode if container.Config.NetworkDisabled || mode.IsContainer() { return nil @@ -797,7 +758,7 @@ func (container *Container) AllocateNetwork() error { return fmt.Errorf("error locating network with name %s: %v", string(mode), err) } - createOptions, err := container.buildCreateEndpointOptions() + createOptions, err := container.buildCreateEndpointOptions(restoring) if err != nil { return err } @@ -831,7 +792,7 @@ func (container *Container) AllocateNetwork() error { return nil } -func (container *Container) initializeNetworking() error { +func (container *Container) initializeNetworking(restoring bool) error { var err error // Make sure NetworkMode has an acceptable value before @@ -872,7 +833,7 @@ func (container *Container) initializeNetworking() error { } - if err := container.AllocateNetwork(); err != nil { + if err := container.AllocateNetwork(restoring); err != nil { return err } diff --git a/daemon/daemon.go b/daemon/daemon.go index ef75ca283d545..72b2dacb57983 100644 --- a/daemon/daemon.go +++ b/daemon/daemon.go @@ -857,22 +857,22 @@ func (daemon *Daemon) Run(c *Container, pipes *execdriver.Pipes, startCallback e return daemon.execDriver.Run(c.command, pipes, startCallback) } -func (daemon *Daemon) Checkpoint(c *Container) error { - if err := daemon.execDriver.Checkpoint(c.command); err != nil { +func (daemon *Daemon) Checkpoint(c *Container, opts *runconfig.CriuConfig) error { + if err := daemon.execDriver.Checkpoint(c.command, opts); err != nil { return err } - c.SetCheckpointed() + c.SetCheckpointed(opts.LeaveRunning) return nil } -func (daemon *Daemon) Restore(c *Container, pipes *execdriver.Pipes, restoreCallback execdriver.RestoreCallback) (int, error) { +func (daemon *Daemon) Restore(c *Container, pipes *execdriver.Pipes, restoreCallback execdriver.RestoreCallback, opts *runconfig.CriuConfig, forceRestore bool) (execdriver.ExitStatus, error) { // Mount the container's filesystem (daemon/graphdriver/aufs/aufs.go). _, err := daemon.driver.Get(c.ID, c.GetMountLabel()) if err != nil { - return 0, err + return execdriver.ExitStatus{ExitCode: 0}, err } - exitCode, err := daemon.execDriver.Restore(c.command, pipes, restoreCallback) + exitCode, err := daemon.execDriver.Restore(c.command, pipes, restoreCallback, opts, forceRestore) return exitCode, err } diff --git a/daemon/execdriver/driver.go b/daemon/execdriver/driver.go index 54d3955bb040c..80d775bac2640 100644 --- a/daemon/execdriver/driver.go +++ b/daemon/execdriver/driver.go @@ -8,6 +8,7 @@ import ( // TODO Windows: Factor out ulimit "github.com/docker/docker/pkg/ulimit" + "github.com/docker/docker/runconfig" "github.com/docker/libcontainer" "github.com/docker/libcontainer/configs" ) @@ -60,8 +61,8 @@ type Driver interface { Kill(c *Command, sig int) error Pause(c *Command) error Unpause(c *Command) error - Checkpoint(c *Command, opts *libcontainer.CriuOpts) error - Restore(c *Command, pipes *Pipes, restoreCallback RestoreCallback, opts *libcontainer.CriuOpts, forceRestore bool) (ExitStatus, error) + Checkpoint(c *Command, opts *runconfig.CriuConfig) error + Restore(c *Command, pipes *Pipes, restoreCallback RestoreCallback, opts *runconfig.CriuConfig, forceRestore bool) (ExitStatus, error) Name() string // Driver name Info(id string) Info // "temporary" hack (until we move state from core to plugins) GetPidsForContainer(id string) ([]int, error) // Returns a list of pids for the given container. diff --git a/daemon/execdriver/lxc/driver.go b/daemon/execdriver/lxc/driver.go index 7ca41abe818c4..6116a1e87ce42 100644 --- a/daemon/execdriver/lxc/driver.go +++ b/daemon/execdriver/lxc/driver.go @@ -25,6 +25,7 @@ import ( sysinfo "github.com/docker/docker/pkg/system" "github.com/docker/docker/pkg/term" "github.com/docker/docker/pkg/version" + "github.com/docker/docker/runconfig" "github.com/docker/libcontainer" "github.com/docker/libcontainer/cgroups" "github.com/docker/libcontainer/configs" @@ -547,11 +548,11 @@ func (d *driver) Unpause(c *execdriver.Command) error { return err } -func (d *driver) Checkpoint(c *execdriver.Command, opts *libcontainer.CriuOpts) error { +func (d *driver) Checkpoint(c *execdriver.Command, opts *runconfig.CriuConfig) error { return fmt.Errorf("Checkpointing lxc containers not supported yet\n") } -func (d *driver) Restore(c *execdriver.Command, pipes *execdriver.Pipes, restoreCallback execdriver.RestoreCallback, opts *libcontainer.CriuOpts, forceRestore bool) (execdriver.ExitStatus, error) { +func (d *driver) Restore(c *execdriver.Command, pipes *execdriver.Pipes, restoreCallback execdriver.RestoreCallback, opts *runconfig.CriuConfig, forceRestore bool) (execdriver.ExitStatus, error) { return execdriver.ExitStatus{ExitCode: 0}, fmt.Errorf("Restoring lxc containers not supported yet\n") } diff --git a/daemon/execdriver/native/driver.go b/daemon/execdriver/native/driver.go index 3e4658b1f495e..5d87c85cd5d91 100644 --- a/daemon/execdriver/native/driver.go +++ b/daemon/execdriver/native/driver.go @@ -19,6 +19,7 @@ import ( "github.com/docker/docker/pkg/reexec" sysinfo "github.com/docker/docker/pkg/system" "github.com/docker/docker/pkg/term" + "github.com/docker/docker/runconfig" "github.com/docker/libcontainer" "github.com/docker/libcontainer/apparmor" "github.com/docker/libcontainer/cgroups/systemd" @@ -276,7 +277,18 @@ func (d *driver) Unpause(c *execdriver.Command) error { return active.Resume() } -func (d *driver) Checkpoint(c *execdriver.Command, opts *libcontainer.CriuOpts) error { +func libcontainerCriuOpts(runconfigOpts *runconfig.CriuConfig) *libcontainer.CriuOpts { + return &libcontainer.CriuOpts{ + ImagesDirectory: runconfigOpts.ImagesDirectory, + WorkDirectory: runconfigOpts.WorkDirectory, + LeaveRunning: runconfigOpts.LeaveRunning, + TcpEstablished: runconfigOpts.TcpEstablished, + ExternalUnixConnections: runconfigOpts.ExternalUnixConnections, + ShellJob: runconfigOpts.ShellJob, + } +} + +func (d *driver) Checkpoint(c *execdriver.Command, opts *runconfig.CriuConfig) error { active := d.activeContainers[c.ID] if active == nil { return fmt.Errorf("active container for %s does not exist", c.ID) @@ -284,7 +296,7 @@ func (d *driver) Checkpoint(c *execdriver.Command, opts *libcontainer.CriuOpts) d.Lock() defer d.Unlock() - err := active.Checkpoint(opts) + err := active.Checkpoint(libcontainerCriuOpts(opts)) if err != nil { return err } @@ -292,7 +304,7 @@ func (d *driver) Checkpoint(c *execdriver.Command, opts *libcontainer.CriuOpts) return nil } -func (d *driver) Restore(c *execdriver.Command, pipes *execdriver.Pipes, restoreCallback execdriver.RestoreCallback, opts *libcontainer.CriuOpts, forceRestore bool) (execdriver.ExitStatus, error) { +func (d *driver) Restore(c *execdriver.Command, pipes *execdriver.Pipes, restoreCallback execdriver.RestoreCallback, opts *runconfig.CriuConfig, forceRestore bool) (execdriver.ExitStatus, error) { var ( cont libcontainer.Container err error @@ -335,7 +347,7 @@ func (d *driver) Restore(c *execdriver.Command, pipes *execdriver.Pipes, restore d.cleanContainer(c.ID) }() - if err := cont.Restore(p, opts); err != nil { + if err := cont.Restore(p, libcontainerCriuOpts(opts)); err != nil { return execdriver.ExitStatus{ExitCode: -1}, err } diff --git a/daemon/monitor.go b/daemon/monitor.go index 241efa32d0eb3..5d80d74fc1a88 100644 --- a/daemon/monitor.go +++ b/daemon/monitor.go @@ -186,43 +186,45 @@ func (m *containerMonitor) Start() error { } // Like Start() but for restoring a container. -func (m *containerMonitor) Restore() error { +func (m *containerMonitor) Restore(opts *runconfig.CriuConfig, forceRestore bool) error { var ( err error // XXX The following line should be changed to // exitStatus execdriver.ExitStatus to match Start() - exitCode int + exitCode execdriver.ExitStatus afterRestore bool ) - defer func() { if afterRestore { m.container.Lock() - m.container.setStopped(&execdriver.ExitStatus{exitCode, false}) + m.container.setStopped(&execdriver.ExitStatus{exitCode.ExitCode, false}) defer m.container.Unlock() } m.Close() }() - if err := m.container.startLoggingToDisk(); err != nil { - m.resetContainer(false) - return err + // FIXME: right now if we startLogging again we get double logs after a restore + if m.container.logCopier == nil { + if err := m.container.startLogging(); err != nil { + m.resetContainer(false) + return err + } } pipes := execdriver.NewPipes(m.container.stdin, m.container.stdout, m.container.stderr, m.container.Config.OpenStdin) m.container.LogEvent("restore") m.lastStartTime = time.Now() - if exitCode, err = m.container.daemon.Restore(m.container, pipes, m.restoreCallback); err != nil { - log.Errorf("Error restoring container: %s, exitCode=%d", err, exitCode) + if exitCode, err = m.container.daemon.Restore(m.container, pipes, m.restoreCallback, opts, forceRestore); err != nil { + logrus.Errorf("Error restoring container: %s, exitCode=%d", err, exitCode) m.container.ExitCode = -1 m.resetContainer(false) return err } afterRestore = true - m.container.ExitCode = exitCode - m.resetMonitor(err == nil && exitCode == 0) + m.container.ExitCode = exitCode.ExitCode + m.resetMonitor(err == nil && exitCode.ExitCode == 0) m.container.LogEvent("die") m.resetContainer(true) return err @@ -332,7 +334,7 @@ func (m *containerMonitor) restoreCallback(processConfig *execdriver.ProcessConf // Write config.json and hostconfig.json files // to /var/lib/docker/containers/. if err := m.container.ToDisk(); err != nil { - log.Debugf("%s", err) + logrus.Debugf("%s", err) } } } diff --git a/daemon/state.go b/daemon/state.go index 82eefe75a3a6c..f6ef8977695b0 100644 --- a/daemon/state.go +++ b/daemon/state.go @@ -25,7 +25,6 @@ type State struct { FinishedAt time.Time CheckpointedAt time.Time waitChan chan struct{} - } func NewState() *State { @@ -45,14 +44,16 @@ func (s *State) String() string { } return fmt.Sprintf("Up %s", units.HumanDuration(time.Now().UTC().Sub(s.StartedAt))) - } else if s.Checkpointed { - return fmt.Sprintf("Checkpointed %s ago", units.HumanDuration(time.Now().UTC().Sub(s.CheckpointedAt))) } if s.removalInProgress { return "Removal In Progress" } + if s.Checkpointed { + return fmt.Sprintf("Checkpointed %s ago", units.HumanDuration(time.Now().UTC().Sub(s.CheckpointedAt))) + } + if s.Dead { return "Dead" } @@ -80,6 +81,10 @@ func (s *State) StateString() string { return "running" } + if s.Checkpointed { + return "checkpointed'" + } + if s.Dead { return "dead" } @@ -269,11 +274,11 @@ func (s *State) SetDead() { s.Unlock() } -func (s *State) SetCheckpointed() { +func (s *State) SetCheckpointed(leaveRunning bool) { s.Lock() s.CheckpointedAt = time.Now().UTC() - s.Checkpointed = true - s.Running = false + s.Checkpointed = !leaveRunning + s.Running = leaveRunning s.Paused = false s.Restarting = false // XXX Not sure if we need to close and recreate waitChan. @@ -282,6 +287,10 @@ func (s *State) SetCheckpointed() { s.Unlock() } +func (s *State) HasBeenCheckpointed() bool { + return s.CheckpointedAt != time.Time{} +} + func (s *State) IsCheckpointed() bool { return s.Checkpointed } diff --git a/docker/flags.go b/docker/flags.go index d860ec7e1f703..b193b7bc93e72 100644 --- a/docker/flags.go +++ b/docker/flags.go @@ -31,6 +31,7 @@ var ( dockerCommands = []command{ {"attach", "Attach to a running container"}, {"build", "Build an image from a Dockerfile"}, + {"checkpoint", "Checkpoint one or more running containers"}, {"commit", "Create a new image from a container's changes"}, {"cp", "Copy files/folders from a container's filesystem to the host path"}, {"create", "Create a new container"}, @@ -55,6 +56,7 @@ var ( {"push", "Push an image or a repository to a Docker registry server"}, {"rename", "Rename an existing container"}, {"restart", "Restart a running container"}, + {"restore", "Restore one or more checkpointed containers"}, {"rm", "Remove one or more containers"}, {"rmi", "Remove one or more images"}, {"run", "Run a command in a new container"}, @@ -131,7 +133,7 @@ func init() { sort.Sort(byName(dockerCommands)) for _, cmd := range dockerCommands { - help += fmt.Sprintf(" %-10.10s%s\n", cmd.name, cmd.description) + help += fmt.Sprintf(" %-11.11s%s\n", cmd.name, cmd.description) } help += "\nRun 'docker COMMAND --help' for more information on a command." diff --git a/integration-cli/docker_cli_help_test.go b/integration-cli/docker_cli_help_test.go index ce9f54e69af84..4eb515f02a9a7 100644 --- a/integration-cli/docker_cli_help_test.go +++ b/integration-cli/docker_cli_help_test.go @@ -229,7 +229,7 @@ func (s *DockerSuite) TestHelpTextVerify(c *check.C) { } - expected := 39 + expected := 41 if len(cmds) != expected { c.Fatalf("Wrong # of cmds(%d), it should be: %d\nThe list:\n%q", len(cmds), expected, cmds) diff --git a/runconfig/restore.go b/runconfig/restore.go new file mode 100644 index 0000000000000..22f8b0ab0a096 --- /dev/null +++ b/runconfig/restore.go @@ -0,0 +1,15 @@ +package runconfig + +type CriuConfig struct { + ImagesDirectory string + WorkDirectory string + LeaveRunning bool + TcpEstablished bool + ExternalUnixConnections bool + ShellJob bool +} + +type RestoreConfig struct { + CriuOpts CriuConfig + ForceRestore bool +} From 5638d0540d1dbf2b794f77cf8669f4ddccdffe51 Mon Sep 17 00:00:00 2001 From: boucher Date: Tue, 2 Jun 2015 14:04:14 -0700 Subject: [PATCH 06/26] Add compilation steps for Criu to the Dockerfile Add a basic test for checkpoint/restore to the integration tests Docker-DCO-1.1-Signed-off-by: Ross Boucher (github: boucher) --- Dockerfile | 18 +++++++++ api/types/types.go | 22 ++++++----- daemon/inspect.go | 22 ++++++----- integration-cli/docker_cli_checkpoint_test.go | 37 +++++++++++++++++++ 4 files changed, 79 insertions(+), 20 deletions(-) create mode 100644 integration-cli/docker_cli_checkpoint_test.go diff --git a/Dockerfile b/Dockerfile index bbefb47afbb26..83a43afd3e262 100644 --- a/Dockerfile +++ b/Dockerfile @@ -32,28 +32,39 @@ RUN echo deb http://ppa.launchpad.net/zfs-native/stable/ubuntu trusty main > /et # Packaged dependencies RUN apt-get update && apt-get install -y \ apparmor \ + asciidoc \ aufs-tools \ automake \ bash-completion \ + bsdmainutils \ btrfs-tools \ build-essential \ curl \ dpkg-sig \ git \ iptables \ + libaio-dev \ libapparmor-dev \ libcap-dev \ + libprotobuf-c0-dev \ + libprotobuf-dev \ libsqlite3-dev \ mercurial \ parallel \ + pkg-config \ + protobuf-compiler \ + protobuf-c-compiler \ + python-minimal \ python-mock \ python-pip \ + python-protobuf \ python-websocket \ reprepro \ ruby1.9.1 \ ruby1.9.1-dev \ s3cmd=1.1.0* \ ubuntu-zfs \ + xmlto \ libzfs-dev \ --no-install-recommends @@ -78,6 +89,13 @@ RUN cd /usr/src/lxc \ && make install \ && ldconfig +# Install Criu +RUN mkdir -p /usr/src/criu \ + && curl -sSL https://github.com/xemul/criu/archive/v1.6.tar.gz | tar -v -C /usr/src/criu/ -xz --strip-components=1 +RUN cd /usr/src/criu \ + && make \ + && make install + # Install Go ENV GO_VERSION 1.4.2 RUN curl -sSL https://golang.org/dl/go${GO_VERSION}.src.tar.gz | tar -v -C /usr/local -xz \ diff --git a/api/types/types.go b/api/types/types.go index 86daa126d1490..d93b8c4fa7593 100644 --- a/api/types/types.go +++ b/api/types/types.go @@ -188,16 +188,18 @@ type ExecStartCheck struct { } type ContainerState struct { - Running bool - Paused bool - Restarting bool - OOMKilled bool - Dead bool - Pid int - ExitCode int - Error string - StartedAt time.Time - FinishedAt time.Time + Running bool + Paused bool + Checkpointed bool + Restarting bool + OOMKilled bool + Dead bool + Pid int + ExitCode int + Error string + StartedAt time.Time + FinishedAt time.Time + CheckpointedAt time.Time } // GET "/containers/{name:.*}/json" diff --git a/daemon/inspect.go b/daemon/inspect.go index 73b394ca24c0f..e9d302f1bc9b1 100644 --- a/daemon/inspect.go +++ b/daemon/inspect.go @@ -64,16 +64,18 @@ func (daemon *Daemon) getInspectData(container *Container) (*types.ContainerJSON } containerState := &types.ContainerState{ - Running: container.State.Running, - Paused: container.State.Paused, - Restarting: container.State.Restarting, - OOMKilled: container.State.OOMKilled, - Dead: container.State.Dead, - Pid: container.State.Pid, - ExitCode: container.State.ExitCode, - Error: container.State.Error, - StartedAt: container.State.StartedAt, - FinishedAt: container.State.FinishedAt, + Running: container.State.Running, + Paused: container.State.Paused, + Checkpointed: container.State.Checkpointed, + Restarting: container.State.Restarting, + OOMKilled: container.State.OOMKilled, + Dead: container.State.Dead, + Pid: container.State.Pid, + ExitCode: container.State.ExitCode, + Error: container.State.Error, + StartedAt: container.State.StartedAt, + FinishedAt: container.State.FinishedAt, + CheckpointedAt: container.State.CheckpointedAt, } volumes := make(map[string]string) diff --git a/integration-cli/docker_cli_checkpoint_test.go b/integration-cli/docker_cli_checkpoint_test.go new file mode 100644 index 0000000000000..e19ef524efd43 --- /dev/null +++ b/integration-cli/docker_cli_checkpoint_test.go @@ -0,0 +1,37 @@ +package main + +import ( + "os/exec" + "strings" + + "github.com/go-check/check" +) + +func (s *DockerSuite) TestCheckpointAndRestore(c *check.C) { + defer unpauseAllContainers() + + runCmd := exec.Command(dockerBinary, "run", "-d", "busybox", "top") + out, _, err := runCommandWithOutput(runCmd) + if err != nil { + c.Fatalf("failed to run container: %v, output: %q", err, out) + } + + containerID := strings.TrimSpace(out) + checkpointCmd := exec.Command(dockerBinary, "checkpoint", containerID) + out, _, err = runCommandWithOutput(checkpointCmd) + if err != nil { + c.Fatalf("failed to checkpoint container: %v, output: %q", err, out) + } + + out, err = inspectField(containerID, "State.Checkpointed") + c.Assert(out, check.Equals, "true") + + restoreCmd := exec.Command(dockerBinary, "restore", containerID) + out, _, _, err = runCommandWithStdoutStderr(restoreCmd) + if err != nil { + c.Fatalf("failed to restore container: %v, output: %q", err, out) + } + + out, err = inspectField(containerID, "State.Checkpointed") + c.Assert(out, check.Equals, "false") +} From 931ab3cc4e84e04b08efc0bcf02394d5c104b1f9 Mon Sep 17 00:00:00 2001 From: boucher Date: Tue, 16 Jun 2015 14:41:05 -0700 Subject: [PATCH 07/26] Add optional dependency info to the PACKAGERS file. Docker-DCO-1.1-Signed-off-by: Ross Boucher (github: boucher) --- project/PACKAGERS.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/project/PACKAGERS.md b/project/PACKAGERS.md index fd2156c5855bf..da0d2da9ca0f2 100644 --- a/project/PACKAGERS.md +++ b/project/PACKAGERS.md @@ -304,6 +304,9 @@ by having support for them in the kernel or userspace. A few examples include: least the "auplink" utility from aufs-tools) * BTRFS graph driver (requires BTRFS support enabled in the kernel) * ZFS graph driver (requires userspace zfs-utils and a corresponding kernel module) +* Checkpoint/Restore containers: + - requires criu version 1.5.2 or later (criu.org) + - requires kernel version 3.19 or later if using overlay-fs ## Daemon Init Script From 25627f6e4759733e045ebf5024c2e943481c3324 Mon Sep 17 00:00:00 2001 From: boucher Date: Thu, 18 Jun 2015 15:18:09 -0700 Subject: [PATCH 08/26] Don't destroy/delete the container if it has been checkpointed. Docker-DCO-1.1-Signed-off-by: Ross Boucher (github: boucher) --- daemon/execdriver/native/driver.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/daemon/execdriver/native/driver.go b/daemon/execdriver/native/driver.go index 5d87c85cd5d91..a44db33210e2b 100644 --- a/daemon/execdriver/native/driver.go +++ b/daemon/execdriver/native/driver.go @@ -141,8 +141,11 @@ func (d *driver) Run(c *execdriver.Command, pipes *execdriver.Pipes, startCallba d.activeContainers[c.ID] = cont d.Unlock() defer func() { - cont.Destroy() - d.cleanContainer(c.ID) + status, err := cont.Status() + if err != nil || status != libcontainer.Checkpointed { + cont.Destroy() + d.cleanContainer(c.ID) + } }() if err := cont.Start(p); err != nil { From 637d7aceb401f7cb11522eb459d2e866c9e30d9c Mon Sep 17 00:00:00 2001 From: Hui Kang Date: Fri, 19 Jun 2015 20:57:03 +0000 Subject: [PATCH 09/26] Reuse the endpoint of the checkpointed container when restore Docker-DCO-1.1-Signed-off-by: Hui Kang --- daemon/container.go | 4 ++-- daemon/container_linux.go | 40 ++++++++++++++++++++++++++++----------- 2 files changed, 31 insertions(+), 13 deletions(-) diff --git a/daemon/container.go b/daemon/container.go index f1aa62e050024..f1cac7646ce84 100644 --- a/daemon/container.go +++ b/daemon/container.go @@ -341,7 +341,7 @@ func (container *Container) cleanup() { if container.IsCheckpointed() { logrus.Debugf("not calling ReleaseNetwork() for checkpointed container %s", container.ID) } else { - container.ReleaseNetwork() + container.ReleaseNetwork(false) } disableAllActiveLinks(container) @@ -607,7 +607,7 @@ func (container *Container) Checkpoint(opts *runconfig.CriuConfig) error { } if opts.LeaveRunning == false { - container.ReleaseNetwork() + container.ReleaseNetwork(true) } return nil } diff --git a/daemon/container_linux.go b/daemon/container_linux.go index 7fb5e5a8c4eda..b34971c7bfe6e 100644 --- a/daemon/container_linux.go +++ b/daemon/container_linux.go @@ -758,18 +758,32 @@ func (container *Container) AllocateNetwork(restoring bool) error { return fmt.Errorf("error locating network with name %s: %v", string(mode), err) } - createOptions, err := container.buildCreateEndpointOptions(restoring) - if err != nil { - return err - } + var ep libnetwork.Endpoint + if restoring == true { + // Use existing Endpoint for a checkpointed container + for _, endpoint := range n.Endpoints() { + if endpoint.ID() == container.NetworkSettings.EndpointID { + ep = endpoint + } + } + if ep == nil { + return fmt.Errorf("Fail to find the Endpoint for the checkpointed container") + } + } else { - ep, err := n.CreateEndpoint(container.Name, createOptions...) - if err != nil { - return err - } + createOptions, err := container.buildCreateEndpointOptions(restoring) + if err != nil { + return err + } - if err := container.updateNetworkSettings(n, ep); err != nil { - return err + ep, err = n.CreateEndpoint(container.Name, createOptions...) + if err != nil { + return err + } + + if err := container.updateNetworkSettings(n, ep); err != nil { + return err + } } joinOptions, err := container.buildJoinOptions() @@ -916,7 +930,7 @@ func (container *Container) getNetworkedContainer() (*Container, error) { } } -func (container *Container) ReleaseNetwork() { +func (container *Container) ReleaseNetwork(is_checkpoint bool) { if container.hostConfig.NetworkMode.IsContainer() || container.daemon.config.DisableNetwork { return } @@ -943,6 +957,10 @@ func (container *Container) ReleaseNetwork() { logrus.Errorf("leaving endpoint failed: %v", err) } + if is_checkpoint == true { + return + } + if err := ep.Delete(); err != nil { logrus.Errorf("deleting endpoint failed: %v", err) } From 56cd5a162b604e976b1d87d19a3e51df20471543 Mon Sep 17 00:00:00 2001 From: Saied Kazemi Date: Thu, 5 Feb 2015 20:32:27 -0800 Subject: [PATCH 10/26] Checkpoint/Restore Support: add exec driver methods Methods for checkpointing and restoring containers were added to the native driver. The LXC driver returns an error message that these methods are not implemented yet. Signed-off-by: Saied Kazemi Conflicts: daemon/execdriver/native/create.go daemon/execdriver/native/driver.go daemon/execdriver/native/init.go --- daemon/execdriver/driver.go | 3 + daemon/execdriver/lxc/driver.go | 8 ++ daemon/execdriver/native/create.go | 19 ++++ daemon/execdriver/native/driver.go | 150 +++++++++++++++++++++++++++++ 4 files changed, 180 insertions(+) diff --git a/daemon/execdriver/driver.go b/daemon/execdriver/driver.go index eca77e921eaac..acb41c5f53376 100644 --- a/daemon/execdriver/driver.go +++ b/daemon/execdriver/driver.go @@ -24,6 +24,7 @@ var ( ) type StartCallback func(*ProcessConfig, int) +type RestoreCallback func(*ProcessConfig, int) // Driver specific information based on // processes registered with the driver @@ -59,6 +60,8 @@ type Driver interface { Kill(c *Command, sig int) error Pause(c *Command) error Unpause(c *Command) error + Checkpoint(c *Command) error + Restore(c *Command, pipes *Pipes, restoreCallback RestoreCallback) (int, error) Name() string // Driver name Info(id string) Info // "temporary" hack (until we move state from core to plugins) GetPidsForContainer(id string) ([]int, error) // Returns a list of pids for the given container. diff --git a/daemon/execdriver/lxc/driver.go b/daemon/execdriver/lxc/driver.go index 692b9f356f3aa..914a7ea5075c2 100644 --- a/daemon/execdriver/lxc/driver.go +++ b/daemon/execdriver/lxc/driver.go @@ -547,6 +547,14 @@ func (d *driver) Unpause(c *execdriver.Command) error { return err } +func (d *driver) Checkpoint(c *execdriver.Command) error { + return fmt.Errorf("Checkpointing lxc containers not supported yet\n") +} + +func (d *driver) Restore(c *execdriver.Command, pipes *execdriver.Pipes, restoreCallback execdriver.RestoreCallback) (int, error) { + return 0, fmt.Errorf("Restoring lxc containers not supported yet\n") +} + func (d *driver) Terminate(c *execdriver.Command) error { return KillLxc(c.ID, 9) } diff --git a/daemon/execdriver/native/create.go b/daemon/execdriver/native/create.go index 1b2d7232d3177..1cf248b0d66fd 100644 --- a/daemon/execdriver/native/create.go +++ b/daemon/execdriver/native/create.go @@ -4,6 +4,7 @@ package native import ( "errors" + "encoding/json" "fmt" "net" "strings" @@ -88,6 +89,24 @@ func generateIfaceName() (string, error) { return "", errors.New("Failed to find name for new interface") } +// Re-create the container type from the image that was saved during checkpoint. +func (d *driver) createRestoreContainer(c *execdriver.Command, imageDir string) (*libcontainer.Config, error) { + // Read the container.json. + f1, err := os.Open(filepath.Join(imageDir, "container.json")) + if err != nil { + return nil, err + } + defer f1.Close() + + var container *libcontainer.Config + err = json.NewDecoder(f1).Decode(&container) + if err != nil { + return nil, err + } + + return container, nil +} + func (d *driver) createNetwork(container *configs.Config, c *execdriver.Command) error { if c.Network == nil { return nil diff --git a/daemon/execdriver/native/driver.go b/daemon/execdriver/native/driver.go index da13c2b165666..8f451bcd7592f 100644 --- a/daemon/execdriver/native/driver.go +++ b/daemon/execdriver/native/driver.go @@ -19,6 +19,7 @@ import ( "github.com/docker/docker/pkg/reexec" sysinfo "github.com/docker/docker/pkg/system" "github.com/docker/docker/pkg/term" + "github.com/docker/docker/utils" "github.com/docker/libcontainer" "github.com/docker/libcontainer/apparmor" "github.com/docker/libcontainer/cgroups/systemd" @@ -276,6 +277,155 @@ func (d *driver) Unpause(c *execdriver.Command) error { return active.Resume() } +// XXX Where is the right place for the following +// const and getCheckpointImageDir() function? +const ( + containersDir = "/var/lib/docker/containers" + criuImgDir = "criu_img" +) + +func getCheckpointImageDir(containerId string) string { + return filepath.Join(containersDir, containerId, criuImgDir) +} + +func (d *driver) Checkpoint(c *execdriver.Command) error { + active := d.activeContainers[c.ID] + if active == nil { + return fmt.Errorf("active container for %s does not exist", c.ID) + } + container := active.container + + // Create an image directory for this container (which + // may already exist from a previous checkpoint). + imageDir := getCheckpointImageDir(c.ID) + err := os.MkdirAll(imageDir, 0700) + if err != nil && !os.IsExist(err) { + return err + } + + // Copy container.json and state.json files to the CRIU + // image directory for later use during restore. Do this + // before checkpointing because after checkpoint the container + // will exit and these files will be removed. + log.CRDbg("saving container.json and state.json before calling CRIU in %s", imageDir) + srcFiles := []string{"container.json", "state.json"} + for _, f := range srcFiles { + srcFile := filepath.Join(d.root, c.ID, f) + dstFile := filepath.Join(imageDir, f) + if _, err := utils.CopyFile(srcFile, dstFile); err != nil { + return err + } + } + + d.Lock() + defer d.Unlock() + err = namespaces.Checkpoint(container, imageDir, c.ProcessConfig.Process.Pid) + if err != nil { + return err + } + + return nil +} + +type restoreOutput struct { + exitCode int + err error +} + +func (d *driver) Restore(c *execdriver.Command, pipes *execdriver.Pipes, restoreCallback execdriver.RestoreCallback) (int, error) { + imageDir := getCheckpointImageDir(c.ID) + container, err := d.createRestoreContainer(c, imageDir) + if err != nil { + return 1, err + } + + var term execdriver.Terminal + + if c.ProcessConfig.Tty { + term, err = NewTtyConsole(&c.ProcessConfig, pipes) + } else { + term, err = execdriver.NewStdConsole(&c.ProcessConfig, pipes) + } + if err != nil { + return -1, err + } + c.ProcessConfig.Terminal = term + + d.Lock() + d.activeContainers[c.ID] = &activeContainer{ + container: container, + cmd: &c.ProcessConfig.Cmd, + } + d.Unlock() + defer d.cleanContainer(c.ID) + + // Since the CRIU binary exits after restoring the container, we + // need to reap its child by setting PR_SET_CHILD_SUBREAPER (36) + // so that it'll be owned by this process (Docker daemon) after restore. + // + // XXX This really belongs to where the Docker daemon starts. + if _, _, syserr := syscall.RawSyscall(syscall.SYS_PRCTL, 36, 1, 0); syserr != 0 { + return -1, fmt.Errorf("Could not set PR_SET_CHILD_SUBREAPER (syserr %d)", syserr) + } + + restoreOutputChan := make(chan restoreOutput, 1) + waitForRestore := make(chan struct{}) + + go func() { + exitCode, err := namespaces.Restore(container, c.ProcessConfig.Stdin, c.ProcessConfig.Stdout, c.ProcessConfig.Stderr, c.ProcessConfig.Console, filepath.Join(d.root, c.ID), imageDir, + func(child *os.File, args []string) *exec.Cmd { + cmd := new(exec.Cmd) + cmd.Path = d.initPath + cmd.Args = append([]string{ + DriverName, + "-restore", + "-pipe", "3", + "--", + }, args...) + cmd.ExtraFiles = []*os.File{child} + return cmd + }, + func(restorePid int) error { + log.CRDbg("restorePid=%d", restorePid) + if restorePid == 0 { + restoreCallback(&c.ProcessConfig, 0) + return nil + } + + // The container.json file should be written *after* the container + // has started because its StdFds cannot be initialized before. + // + // XXX How do we handle error here? + d.writeContainerFile(container, c.ID) + close(waitForRestore) + if restoreCallback != nil { + c.ProcessConfig.Process, err = os.FindProcess(restorePid) + if err != nil { + log.Debugf("cannot find restored process %d", restorePid) + return err + } + c.ContainerPid = c.ProcessConfig.Process.Pid + restoreCallback(&c.ProcessConfig, c.ContainerPid) + } + return nil + }) + restoreOutputChan <- restoreOutput{exitCode, err} + }() + + select { + case restoreOutput := <-restoreOutputChan: + // there was an error + return restoreOutput.exitCode, restoreOutput.err + case <-waitForRestore: + // container restored + break + } + + // Wait for the container to exit. + restoreOutput := <-restoreOutputChan + return restoreOutput.exitCode, restoreOutput.err +} + func (d *driver) Terminate(c *execdriver.Command) error { defer d.cleanContainer(c.ID) container, err := d.factory.Load(c.ID) From 456ecbff0b2c2d21dc321f175dfeeb7677921259 Mon Sep 17 00:00:00 2001 From: boucher Date: Mon, 25 May 2015 08:32:58 -0700 Subject: [PATCH 11/26] Update checkpoint/restore support to match docker/master Docker-DCO-1.1-Signed-off-by: Ross Boucher (github: boucher) --- daemon/execdriver/driver.go | 4 +- daemon/execdriver/lxc/driver.go | 6 +- daemon/execdriver/native/create.go | 19 --- daemon/execdriver/native/driver.go | 183 ++++++++++------------------- 4 files changed, 69 insertions(+), 143 deletions(-) diff --git a/daemon/execdriver/driver.go b/daemon/execdriver/driver.go index acb41c5f53376..54d3955bb040c 100644 --- a/daemon/execdriver/driver.go +++ b/daemon/execdriver/driver.go @@ -60,8 +60,8 @@ type Driver interface { Kill(c *Command, sig int) error Pause(c *Command) error Unpause(c *Command) error - Checkpoint(c *Command) error - Restore(c *Command, pipes *Pipes, restoreCallback RestoreCallback) (int, error) + Checkpoint(c *Command, opts *libcontainer.CriuOpts) error + Restore(c *Command, pipes *Pipes, restoreCallback RestoreCallback, opts *libcontainer.CriuOpts, forceRestore bool) (ExitStatus, error) Name() string // Driver name Info(id string) Info // "temporary" hack (until we move state from core to plugins) GetPidsForContainer(id string) ([]int, error) // Returns a list of pids for the given container. diff --git a/daemon/execdriver/lxc/driver.go b/daemon/execdriver/lxc/driver.go index 914a7ea5075c2..7ca41abe818c4 100644 --- a/daemon/execdriver/lxc/driver.go +++ b/daemon/execdriver/lxc/driver.go @@ -547,12 +547,12 @@ func (d *driver) Unpause(c *execdriver.Command) error { return err } -func (d *driver) Checkpoint(c *execdriver.Command) error { +func (d *driver) Checkpoint(c *execdriver.Command, opts *libcontainer.CriuOpts) error { return fmt.Errorf("Checkpointing lxc containers not supported yet\n") } -func (d *driver) Restore(c *execdriver.Command, pipes *execdriver.Pipes, restoreCallback execdriver.RestoreCallback) (int, error) { - return 0, fmt.Errorf("Restoring lxc containers not supported yet\n") +func (d *driver) Restore(c *execdriver.Command, pipes *execdriver.Pipes, restoreCallback execdriver.RestoreCallback, opts *libcontainer.CriuOpts, forceRestore bool) (execdriver.ExitStatus, error) { + return execdriver.ExitStatus{ExitCode: 0}, fmt.Errorf("Restoring lxc containers not supported yet\n") } func (d *driver) Terminate(c *execdriver.Command) error { diff --git a/daemon/execdriver/native/create.go b/daemon/execdriver/native/create.go index 1cf248b0d66fd..1b2d7232d3177 100644 --- a/daemon/execdriver/native/create.go +++ b/daemon/execdriver/native/create.go @@ -4,7 +4,6 @@ package native import ( "errors" - "encoding/json" "fmt" "net" "strings" @@ -89,24 +88,6 @@ func generateIfaceName() (string, error) { return "", errors.New("Failed to find name for new interface") } -// Re-create the container type from the image that was saved during checkpoint. -func (d *driver) createRestoreContainer(c *execdriver.Command, imageDir string) (*libcontainer.Config, error) { - // Read the container.json. - f1, err := os.Open(filepath.Join(imageDir, "container.json")) - if err != nil { - return nil, err - } - defer f1.Close() - - var container *libcontainer.Config - err = json.NewDecoder(f1).Decode(&container) - if err != nil { - return nil, err - } - - return container, nil -} - func (d *driver) createNetwork(container *configs.Config, c *execdriver.Command) error { if c.Network == nil { return nil diff --git a/daemon/execdriver/native/driver.go b/daemon/execdriver/native/driver.go index 8f451bcd7592f..3e4658b1f495e 100644 --- a/daemon/execdriver/native/driver.go +++ b/daemon/execdriver/native/driver.go @@ -19,7 +19,6 @@ import ( "github.com/docker/docker/pkg/reexec" sysinfo "github.com/docker/docker/pkg/system" "github.com/docker/docker/pkg/term" - "github.com/docker/docker/utils" "github.com/docker/libcontainer" "github.com/docker/libcontainer/apparmor" "github.com/docker/libcontainer/cgroups/systemd" @@ -277,49 +276,15 @@ func (d *driver) Unpause(c *execdriver.Command) error { return active.Resume() } -// XXX Where is the right place for the following -// const and getCheckpointImageDir() function? -const ( - containersDir = "/var/lib/docker/containers" - criuImgDir = "criu_img" -) - -func getCheckpointImageDir(containerId string) string { - return filepath.Join(containersDir, containerId, criuImgDir) -} - -func (d *driver) Checkpoint(c *execdriver.Command) error { +func (d *driver) Checkpoint(c *execdriver.Command, opts *libcontainer.CriuOpts) error { active := d.activeContainers[c.ID] if active == nil { return fmt.Errorf("active container for %s does not exist", c.ID) } - container := active.container - - // Create an image directory for this container (which - // may already exist from a previous checkpoint). - imageDir := getCheckpointImageDir(c.ID) - err := os.MkdirAll(imageDir, 0700) - if err != nil && !os.IsExist(err) { - return err - } - - // Copy container.json and state.json files to the CRIU - // image directory for later use during restore. Do this - // before checkpointing because after checkpoint the container - // will exit and these files will be removed. - log.CRDbg("saving container.json and state.json before calling CRIU in %s", imageDir) - srcFiles := []string{"container.json", "state.json"} - for _, f := range srcFiles { - srcFile := filepath.Join(d.root, c.ID, f) - dstFile := filepath.Join(imageDir, f) - if _, err := utils.CopyFile(srcFile, dstFile); err != nil { - return err - } - } d.Lock() defer d.Unlock() - err = namespaces.Checkpoint(container, imageDir, c.ProcessConfig.Process.Pid) + err := active.Checkpoint(opts) if err != nil { return err } @@ -327,103 +292,83 @@ func (d *driver) Checkpoint(c *execdriver.Command) error { return nil } -type restoreOutput struct { - exitCode int - err error -} +func (d *driver) Restore(c *execdriver.Command, pipes *execdriver.Pipes, restoreCallback execdriver.RestoreCallback, opts *libcontainer.CriuOpts, forceRestore bool) (execdriver.ExitStatus, error) { + var ( + cont libcontainer.Container + err error + ) -func (d *driver) Restore(c *execdriver.Command, pipes *execdriver.Pipes, restoreCallback execdriver.RestoreCallback) (int, error) { - imageDir := getCheckpointImageDir(c.ID) - container, err := d.createRestoreContainer(c, imageDir) + cont, err = d.factory.Load(c.ID) if err != nil { - return 1, err + if forceRestore { + var config *configs.Config + config, err = d.createContainer(c) + if err != nil { + return execdriver.ExitStatus{ExitCode: -1}, err + } + cont, err = d.factory.Create(c.ID, config) + if err != nil { + return execdriver.ExitStatus{ExitCode: -1}, err + } + } else { + return execdriver.ExitStatus{ExitCode: -1}, err + } } - var term execdriver.Terminal - - if c.ProcessConfig.Tty { - term, err = NewTtyConsole(&c.ProcessConfig, pipes) - } else { - term, err = execdriver.NewStdConsole(&c.ProcessConfig, pipes) + p := &libcontainer.Process{ + Args: append([]string{c.ProcessConfig.Entrypoint}, c.ProcessConfig.Arguments...), + Env: c.ProcessConfig.Env, + Cwd: c.WorkingDir, + User: c.ProcessConfig.User, } - if err != nil { - return -1, err + + config := cont.Config() + if err := setupPipes(&config, &c.ProcessConfig, p, pipes); err != nil { + return execdriver.ExitStatus{ExitCode: -1}, err } - c.ProcessConfig.Terminal = term d.Lock() - d.activeContainers[c.ID] = &activeContainer{ - container: container, - cmd: &c.ProcessConfig.Cmd, - } + d.activeContainers[c.ID] = cont d.Unlock() - defer d.cleanContainer(c.ID) + defer func() { + cont.Destroy() + d.cleanContainer(c.ID) + }() - // Since the CRIU binary exits after restoring the container, we - // need to reap its child by setting PR_SET_CHILD_SUBREAPER (36) - // so that it'll be owned by this process (Docker daemon) after restore. - // - // XXX This really belongs to where the Docker daemon starts. - if _, _, syserr := syscall.RawSyscall(syscall.SYS_PRCTL, 36, 1, 0); syserr != 0 { - return -1, fmt.Errorf("Could not set PR_SET_CHILD_SUBREAPER (syserr %d)", syserr) + if err := cont.Restore(p, opts); err != nil { + return execdriver.ExitStatus{ExitCode: -1}, err } - restoreOutputChan := make(chan restoreOutput, 1) - waitForRestore := make(chan struct{}) - - go func() { - exitCode, err := namespaces.Restore(container, c.ProcessConfig.Stdin, c.ProcessConfig.Stdout, c.ProcessConfig.Stderr, c.ProcessConfig.Console, filepath.Join(d.root, c.ID), imageDir, - func(child *os.File, args []string) *exec.Cmd { - cmd := new(exec.Cmd) - cmd.Path = d.initPath - cmd.Args = append([]string{ - DriverName, - "-restore", - "-pipe", "3", - "--", - }, args...) - cmd.ExtraFiles = []*os.File{child} - return cmd - }, - func(restorePid int) error { - log.CRDbg("restorePid=%d", restorePid) - if restorePid == 0 { - restoreCallback(&c.ProcessConfig, 0) - return nil - } - - // The container.json file should be written *after* the container - // has started because its StdFds cannot be initialized before. - // - // XXX How do we handle error here? - d.writeContainerFile(container, c.ID) - close(waitForRestore) - if restoreCallback != nil { - c.ProcessConfig.Process, err = os.FindProcess(restorePid) - if err != nil { - log.Debugf("cannot find restored process %d", restorePid) - return err - } - c.ContainerPid = c.ProcessConfig.Process.Pid - restoreCallback(&c.ProcessConfig, c.ContainerPid) - } - return nil - }) - restoreOutputChan <- restoreOutput{exitCode, err} - }() + // FIXME: no idea if any of this is needed... + if restoreCallback != nil { + pid, err := p.Pid() + if err != nil { + p.Signal(os.Kill) + p.Wait() + return execdriver.ExitStatus{ExitCode: -1}, err + } + restoreCallback(&c.ProcessConfig, pid) + } - select { - case restoreOutput := <-restoreOutputChan: - // there was an error - return restoreOutput.exitCode, restoreOutput.err - case <-waitForRestore: - // container restored - break + oom := notifyOnOOM(cont) + waitF := p.Wait + if nss := cont.Config().Namespaces; !nss.Contains(configs.NEWPID) { + // we need such hack for tracking processes with inherited fds, + // because cmd.Wait() waiting for all streams to be copied + waitF = waitInPIDHost(p, cont) + } + ps, err := waitF() + if err != nil { + execErr, ok := err.(*exec.ExitError) + if !ok { + return execdriver.ExitStatus{ExitCode: -1}, err + } + ps = execErr.ProcessState } - // Wait for the container to exit. - restoreOutput := <-restoreOutputChan - return restoreOutput.exitCode, restoreOutput.err + cont.Destroy() + _, oomKill := <-oom + return execdriver.ExitStatus{ExitCode: utils.ExitStatus(ps.Sys().(syscall.WaitStatus)), OOMKilled: oomKill}, nil } func (d *driver) Terminate(c *execdriver.Command) error { From 34bd07f640e4db285e9ab2c25c6a8690f42c8544 Mon Sep 17 00:00:00 2001 From: Saied Kazemi Date: Thu, 5 Feb 2015 20:37:07 -0800 Subject: [PATCH 12/26] Checkpoint/Restore Support: add functionality to daemon Support was added to the daemon to use the Checkpoint and Restore methods of the native exec driver for checkpointing and restoring containers. Signed-off-by: Saied Kazemi Conflicts: api/server/server.go daemon/container.go daemon/daemon.go daemon/networkdriver/bridge/driver.go daemon/state.go vendor/src/github.com/docker/libnetwork/ipallocator/allocator.go --- api/server/server.go | 76 +++++++++++++++++++++++++++------------ daemon/checkpoint.go | 55 ++++++++++++++++++++++++++++ daemon/container.go | 66 ++++++++++++++++++++++++++++++++-- daemon/container_linux.go | 47 ++++++++++++++++++++++++ daemon/daemon.go | 31 ++++++++++++++++ daemon/monitor.go | 70 ++++++++++++++++++++++++++++++++++++ daemon/state.go | 23 ++++++++++++ 7 files changed, 344 insertions(+), 24 deletions(-) create mode 100644 daemon/checkpoint.go diff --git a/api/server/server.go b/api/server/server.go index 6e6631fc41884..cd90ab36962bd 100644 --- a/api/server/server.go +++ b/api/server/server.go @@ -1323,6 +1323,36 @@ func (s *Server) postContainersCopy(version version.Version, w http.ResponseWrit return nil } +func postContainersCheckpoint(eng *engine.Engine, version version.Version, w http.ResponseWriter, r *http.Request, vars map[string]string) error { + if vars == nil { + return fmt.Errorf("Missing parameter") + } + if err := parseForm(r); err != nil { + return err + } + job := eng.Job("checkpoint", vars["name"]) + if err := job.Run(); err != nil { + return err + } + w.WriteHeader(http.StatusNoContent) + return nil +} + +func postContainersRestore(eng *engine.Engine, version version.Version, w http.ResponseWriter, r *http.Request, vars map[string]string) error { + if vars == nil { + return fmt.Errorf("Missing parameter") + } + if err := parseForm(r); err != nil { + return err + } + job := eng.Job("restore", vars["name"]) + if err := job.Run(); err != nil { + return err + } + w.WriteHeader(http.StatusNoContent) + return nil +} + func (s *Server) postContainerExecCreate(version version.Version, w http.ResponseWriter, r *http.Request, vars map[string]string) error { if err := parseForm(r); err != nil { return err @@ -1529,28 +1559,30 @@ func createRouter(s *Server) *mux.Router { "/exec/{id:.*}/json": s.getExecByID, }, "POST": { - "/auth": s.postAuth, - "/commit": s.postCommit, - "/build": s.postBuild, - "/images/create": s.postImagesCreate, - "/images/load": s.postImagesLoad, - "/images/{name:.*}/push": s.postImagesPush, - "/images/{name:.*}/tag": s.postImagesTag, - "/containers/create": s.postContainersCreate, - "/containers/{name:.*}/kill": s.postContainersKill, - "/containers/{name:.*}/pause": s.postContainersPause, - "/containers/{name:.*}/unpause": s.postContainersUnpause, - "/containers/{name:.*}/restart": s.postContainersRestart, - "/containers/{name:.*}/start": s.postContainersStart, - "/containers/{name:.*}/stop": s.postContainersStop, - "/containers/{name:.*}/wait": s.postContainersWait, - "/containers/{name:.*}/resize": s.postContainersResize, - "/containers/{name:.*}/attach": s.postContainersAttach, - "/containers/{name:.*}/copy": s.postContainersCopy, - "/containers/{name:.*}/exec": s.postContainerExecCreate, - "/exec/{name:.*}/start": s.postContainerExecStart, - "/exec/{name:.*}/resize": s.postContainerExecResize, - "/containers/{name:.*}/rename": s.postContainerRename, + "/auth": s.postAuth, + "/commit": s.postCommit, + "/build": s.postBuild, + "/images/create": s.postImagesCreate, + "/images/load": s.postImagesLoad, + "/images/{name:.*}/push": s.postImagesPush, + "/images/{name:.*}/tag": s.postImagesTag, + "/containers/create": s.postContainersCreate, + "/containers/{name:.*}/kill": s.postContainersKill, + "/containers/{name:.*}/pause": s.postContainersPause, + "/containers/{name:.*}/unpause": s.postContainersUnpause, + "/containers/{name:.*}/restart": s.postContainersRestart, + "/containers/{name:.*}/start": s.postContainersStart, + "/containers/{name:.*}/stop": s.postContainersStop, + "/containers/{name:.*}/wait": s.postContainersWait, + "/containers/{name:.*}/resize": s.postContainersResize, + "/containers/{name:.*}/attach": s.postContainersAttach, + "/containers/{name:.*}/copy": s.postContainersCopy, + "/containers/{name:.*}/exec": s.postContainerExecCreate, + "/exec/{name:.*}/start": s.postContainerExecStart, + "/exec/{name:.*}/resize": s.postContainerExecResize, + "/containers/{name:.*}/rename": s.postContainerRename, + "/containers/{name:.*}/checkpoint": s.postContainersCheckpoint, + "/containers/{name:.*}/restore": s.postContainersRestore, }, "DELETE": { "/containers/{name:.*}": s.deleteContainers, diff --git a/daemon/checkpoint.go b/daemon/checkpoint.go new file mode 100644 index 0000000000000..f6057c6a028f9 --- /dev/null +++ b/daemon/checkpoint.go @@ -0,0 +1,55 @@ +package daemon + +import ( + "github.com/docker/docker/engine" +) + +// Checkpoint a running container. +func (daemon *Daemon) ContainerCheckpoint(job *engine.Job) engine.Status { + if len(job.Args) != 1 { + return job.Errorf("Usage: %s CONTAINER\n", job.Name) + } + + name := job.Args[0] + container, err := daemon.Get(name) + if err != nil { + return job.Error(err) + } + if !container.IsRunning() { + return job.Errorf("Container %s not running", name) + } + + if err := container.Checkpoint(); err != nil { + return job.Errorf("Cannot checkpoint container %s: %s", name, err) + } + + container.LogEvent("checkpoint") + return engine.StatusOK +} + +// Restore a checkpointed container. +func (daemon *Daemon) ContainerRestore(job *engine.Job) engine.Status { + if len(job.Args) != 1 { + return job.Errorf("Usage: %s CONTAINER\n", job.Name) + } + + name := job.Args[0] + container, err := daemon.Get(name) + if err != nil { + return job.Error(err) + } + if container.IsRunning() { + return job.Errorf("Container %s already running", name) + } + if !container.State.IsCheckpointed() { + return job.Errorf("Container %s is not checkpointed", name) + } + + if err := container.Restore(); err != nil { + container.LogEvent("die") + return job.Errorf("Cannot restore container %s: %s", name, err) + } + + container.LogEvent("restore") + return engine.StatusOK +} diff --git a/daemon/container.go b/daemon/container.go index 4e44621757b64..8b465553737f3 100644 --- a/daemon/container.go +++ b/daemon/container.go @@ -335,10 +335,15 @@ func (container *Container) isNetworkAllocated() bool { return container.NetworkSettings.IPAddress != "" } + // cleanup releases any network resources allocated to the container along with any rules // around how containers are linked together. It also unmounts the container's root filesystem. func (container *Container) cleanup() { - container.ReleaseNetwork() + if container.IsCheckpointed() { + log.CRDbg("not calling ReleaseNetwork() for checkpointed container %s", container.ID) + } else { + container.ReleaseNetwork() + } disableAllActiveLinks(container) @@ -659,6 +664,41 @@ func (container *Container) Copy(resource string) (io.ReadCloser, error) { return reader, nil } +func (container *Container) Checkpoint() error { + return container.daemon.Checkpoint(container) +} + +func (container *Container) Restore() error { + var err error + + container.Lock() + defer container.Unlock() + + defer func() { + if err != nil { + container.cleanup() + } + }() + + if err = container.initializeNetworking(); err != nil { + return err + } + + linkedEnv, err := container.setupLinkedContainers() + if err != nil { + return err + } + if err = container.setupWorkingDirectory(); err != nil { + return err + } + env := container.createDaemonEnvironment(linkedEnv) + if err = populateCommandRestore(container, env); err != nil { + return err + } + + return container.waitForRestore() +} + // Returns true if the container exposes a certain port func (container *Container) Exposes(p nat.Port) bool { _, exists := container.Config.ExposedPorts[p] @@ -750,6 +790,29 @@ func (container *Container) waitForStart() error { return nil } +// Like waitForStart() but for restoring a container. +// +// XXX Does RestartPolicy apply here? +func (container *Container) waitForRestore() error { + container.monitor = newContainerMonitor(container, container.hostConfig.RestartPolicy) + + // After calling promise.Go() we'll have two goroutines: + // - The current goroutine that will block in the select + // below until restore is done. + // - A new goroutine that will restore the container and + // wait for it to exit. + select { + case <-container.monitor.restoreSignal: + if container.ExitCode != 0 { + return fmt.Errorf("restore process failed") + } + case err := <-promise.Go(container.monitor.Restore): + return err + } + + return nil +} + func (container *Container) GetProcessLabel() string { // even if we have a process label return "" if we are running // in privileged mode @@ -957,7 +1020,6 @@ func attach(streamConfig *StreamConfig, openStdin, stdinOnce, tty bool, stdin io _, err = copyEscapable(cStdin, stdin) } else { _, err = io.Copy(cStdin, stdin) - } if err == io.ErrClosedPipe { err = nil diff --git a/daemon/container_linux.go b/daemon/container_linux.go index 0a5bf38725022..ad1a76f2330cc 100644 --- a/daemon/container_linux.go +++ b/daemon/container_linux.go @@ -329,6 +329,53 @@ func mergeDevices(defaultDevices, userDevices []*configs.Device) []*configs.Devi return append(devs, userDevices...) } +// Like populateCommand() but for restoring a container. +// +// XXX populateCommand() does a lot more. Not sure if we have +// to do everything it does. +func populateCommandRestore(c *Container, env []string) error { + resources := &execdriver.Resources{ + Memory: c.Config.Memory, + MemorySwap: c.Config.MemorySwap, + CpuShares: c.Config.CpuShares, + Cpuset: c.Config.Cpuset, + } + + processConfig := execdriver.ProcessConfig{ + Privileged: c.hostConfig.Privileged, + Entrypoint: c.Path, + Arguments: c.Args, + Tty: c.Config.Tty, + User: c.Config.User, + } + + processConfig.SysProcAttr = &syscall.SysProcAttr{Setsid: true} + processConfig.Env = env + + c.command = &execdriver.Command{ + ID: c.ID, + Rootfs: c.RootfsPath(), + ReadonlyRootfs: c.hostConfig.ReadonlyRootfs, + InitPath: "/.dockerinit", + WorkingDir: c.Config.WorkingDir, + // Network: en, + // Ipc: ipc, + // Pid: pid, + Resources: resources, + // AllowedDevices: allowedDevices, + // AutoCreatedDevices: autoCreatedDevices, + CapAdd: c.hostConfig.CapAdd, + CapDrop: c.hostConfig.CapDrop, + ProcessConfig: processConfig, + ProcessLabel: c.GetProcessLabel(), + MountLabel: c.GetMountLabel(), + // LxcConfig: lxcConfig, + AppArmorProfile: c.AppArmorProfile, + } + + return nil +} + // GetSize, return real size, virtual size func (container *Container) GetSize() (int64, int64) { var ( diff --git a/daemon/daemon.go b/daemon/daemon.go index bfa70a1385ae4..88639d3a56065 100644 --- a/daemon/daemon.go +++ b/daemon/daemon.go @@ -278,6 +278,18 @@ func (daemon *Daemon) restore() error { logrus.Debugf("Loaded container %v", container.ID) containers[container.ID] = &cr{container: container} + + // If the container was checkpointed, we need to reserve + // the IP address that it was using. + // + // XXX We should also reserve host ports (if any). + if container.IsCheckpointed() { + /*err = bridge.ReserveIP(container.ID, container.NetworkSettings.IPAddress) + if err != nil { + log.Errorf("Failed to reserve IP %s for container %s", + container.ID, container.NetworkSettings.IPAddress) + }*/ + } } else { logrus.Debugf("Cannot load container %s because it was created with another graph driver.", container.ID) } @@ -851,6 +863,25 @@ func (daemon *Daemon) Run(c *Container, pipes *execdriver.Pipes, startCallback e return daemon.execDriver.Run(c.command, pipes, startCallback) } +func (daemon *Daemon) Checkpoint(c *Container) error { + if err := daemon.execDriver.Checkpoint(c.command); err != nil { + return err + } + c.SetCheckpointed() + return nil +} + +func (daemon *Daemon) Restore(c *Container, pipes *execdriver.Pipes, restoreCallback execdriver.RestoreCallback) (int, error) { + // Mount the container's filesystem (daemon/graphdriver/aufs/aufs.go). + _, err := daemon.driver.Get(c.ID, c.GetMountLabel()) + if err != nil { + return 0, err + } + + exitCode, err := daemon.execDriver.Restore(c.command, pipes, restoreCallback) + return exitCode, err +} + func (daemon *Daemon) Kill(c *Container, sig int) error { return daemon.execDriver.Kill(c.command, sig) } diff --git a/daemon/monitor.go b/daemon/monitor.go index dfade8e21847b..241efa32d0eb3 100644 --- a/daemon/monitor.go +++ b/daemon/monitor.go @@ -44,6 +44,9 @@ type containerMonitor struct { // left waiting for nothing to happen during this time stopChan chan struct{} + // like startSignal but for restoring a container + restoreSignal chan struct{} + // timeIncrement is the amount of time to wait between restarts // this is in milliseconds timeIncrement int @@ -61,6 +64,7 @@ func newContainerMonitor(container *Container, policy runconfig.RestartPolicy) * timeIncrement: defaultTimeIncrement, stopChan: make(chan struct{}), startSignal: make(chan struct{}), + restoreSignal: make(chan struct{}), } } @@ -181,6 +185,49 @@ func (m *containerMonitor) Start() error { } } +// Like Start() but for restoring a container. +func (m *containerMonitor) Restore() error { + var ( + err error + // XXX The following line should be changed to + // exitStatus execdriver.ExitStatus to match Start() + exitCode int + afterRestore bool + ) + + defer func() { + if afterRestore { + m.container.Lock() + m.container.setStopped(&execdriver.ExitStatus{exitCode, false}) + defer m.container.Unlock() + } + m.Close() + }() + + if err := m.container.startLoggingToDisk(); err != nil { + m.resetContainer(false) + return err + } + + pipes := execdriver.NewPipes(m.container.stdin, m.container.stdout, m.container.stderr, m.container.Config.OpenStdin) + + m.container.LogEvent("restore") + m.lastStartTime = time.Now() + if exitCode, err = m.container.daemon.Restore(m.container, pipes, m.restoreCallback); err != nil { + log.Errorf("Error restoring container: %s, exitCode=%d", err, exitCode) + m.container.ExitCode = -1 + m.resetContainer(false) + return err + } + afterRestore = true + + m.container.ExitCode = exitCode + m.resetMonitor(err == nil && exitCode == 0) + m.container.LogEvent("die") + m.resetContainer(true) + return err +} + // resetMonitor resets the stateful fields on the containerMonitor based on the // previous runs success or failure. Regardless of success, if the container had // an execution time of more than 10s then reset the timer back to the default @@ -267,6 +314,29 @@ func (m *containerMonitor) callback(processConfig *execdriver.ProcessConfig, pid } } +// Like callback() but for restoring a container. +func (m *containerMonitor) restoreCallback(processConfig *execdriver.ProcessConfig, restorePid int) { + // If restorePid is 0, it means that restore failed. + if restorePid != 0 { + m.container.setRunning(restorePid) + } + + // Unblock the goroutine waiting in waitForRestore(). + select { + case <-m.restoreSignal: + default: + close(m.restoreSignal) + } + + if restorePid != 0 { + // Write config.json and hostconfig.json files + // to /var/lib/docker/containers/. + if err := m.container.ToDisk(); err != nil { + log.Debugf("%s", err) + } + } +} + // resetContainer resets the container's IO and ensures that the command is able to be executed again // by copying the data into a new struct // if lock is true, then container locked during reset diff --git a/daemon/state.go b/daemon/state.go index 0270c88e88774..82eefe75a3a6c 100644 --- a/daemon/state.go +++ b/daemon/state.go @@ -14,6 +14,7 @@ type State struct { Running bool Paused bool Restarting bool + Checkpointed bool OOMKilled bool removalInProgress bool // Not need for this to be persistent on disk. Dead bool @@ -22,7 +23,9 @@ type State struct { Error string // contains last known error when starting the container StartedAt time.Time FinishedAt time.Time + CheckpointedAt time.Time waitChan chan struct{} + } func NewState() *State { @@ -42,6 +45,8 @@ func (s *State) String() string { } return fmt.Sprintf("Up %s", units.HumanDuration(time.Now().UTC().Sub(s.StartedAt))) + } else if s.Checkpointed { + return fmt.Sprintf("Checkpointed %s ago", units.HumanDuration(time.Now().UTC().Sub(s.CheckpointedAt))) } if s.removalInProgress { @@ -166,6 +171,7 @@ func (s *State) setRunning(pid int) { s.Error = "" s.Running = true s.Paused = false + s.Checkpointed = false s.Restarting = false s.ExitCode = 0 s.Pid = pid @@ -262,3 +268,20 @@ func (s *State) SetDead() { s.Dead = true s.Unlock() } + +func (s *State) SetCheckpointed() { + s.Lock() + s.CheckpointedAt = time.Now().UTC() + s.Checkpointed = true + s.Running = false + s.Paused = false + s.Restarting = false + // XXX Not sure if we need to close and recreate waitChan. + // close(s.waitChan) + // s.waitChan = make(chan struct{}) + s.Unlock() +} + +func (s *State) IsCheckpointed() bool { + return s.Checkpointed +} From ca085a806bb61ccf24f36844aeec71773a91cec1 Mon Sep 17 00:00:00 2001 From: Hui Kang Date: Tue, 19 May 2015 21:08:04 +0000 Subject: [PATCH 13/26] Release the network resource during checkpoint Restore failed if network resource not released during checkpoint, e.g., a container with port open with -p Signed-off-by: Hui Kang Conflicts: daemon/container.go --- daemon/container.go | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/daemon/container.go b/daemon/container.go index 8b465553737f3..ee489d8918228 100644 --- a/daemon/container.go +++ b/daemon/container.go @@ -602,6 +602,19 @@ func validateID(id string) error { return nil } + +func (container *Container) Checkpoint(opts *libcontainer.CriuOpts) error { + if err := container.daemon.Checkpoint(container, opts); err != nil { + return err + } + + if opts.LeaveRunning == false { + container.ReleaseNetwork() + } + return nil +} + + func (container *Container) Copy(resource string) (io.ReadCloser, error) { container.Lock() defer container.Unlock() From d06dedf1d08e046843f76497187a5bbbee5dff13 Mon Sep 17 00:00:00 2001 From: boucher Date: Mon, 1 Jun 2015 15:15:02 -0700 Subject: [PATCH 14/26] Update daemon and cli support for checkpoint and restore. Docker-DCO-1.1-Signed-off-by: Ross Boucher (github: boucher) --- api/client/checkpoint.go | 52 ++++++++++++++ api/client/restore.go | 54 ++++++++++++++ api/server/server.go | 24 +++++-- daemon/checkpoint.go | 57 +++++++-------- daemon/container.go | 95 +++++++++++++------------ daemon/container_linux.go | 71 +++++------------- daemon/container_windows.go | 2 +- daemon/daemon.go | 12 ++-- daemon/execdriver/driver.go | 5 +- daemon/execdriver/lxc/driver.go | 5 +- daemon/execdriver/native/driver.go | 20 ++++-- daemon/execdriver/windows/windows.go | 9 +++ daemon/monitor.go | 26 +++---- daemon/state.go | 21 ++++-- docker/flags.go | 4 +- integration-cli/docker_cli_help_test.go | 2 +- runconfig/restore.go | 15 ++++ 17 files changed, 305 insertions(+), 169 deletions(-) create mode 100644 api/client/checkpoint.go create mode 100644 api/client/restore.go create mode 100644 runconfig/restore.go diff --git a/api/client/checkpoint.go b/api/client/checkpoint.go new file mode 100644 index 0000000000000..8c681bcf9716f --- /dev/null +++ b/api/client/checkpoint.go @@ -0,0 +1,52 @@ +package client + +import ( + "fmt" + + flag "github.com/docker/docker/pkg/mflag" + "github.com/docker/docker/runconfig" +) + +func (cli *DockerCli) CmdCheckpoint(args ...string) error { + cmd := cli.Subcmd("checkpoint", []string{"CONTAINER [CONTAINER...]"}, "Checkpoint one or more running containers", true) + cmd.Require(flag.Min, 1) + + var ( + flImgDir = cmd.String([]string{"-image-dir"}, "", "directory for storing checkpoint image files") + flWorkDir = cmd.String([]string{"-work-dir"}, "", "directory for storing log file") + flLeaveRunning = cmd.Bool([]string{"-leave-running"}, false, "leave the container running after checkpoint") + flCheckTcp = cmd.Bool([]string{"-allow-tcp"}, false, "allow checkpointing tcp connections") + flExtUnix = cmd.Bool([]string{"-allow-ext-unix"}, false, "allow checkpointing external unix connections") + flShell = cmd.Bool([]string{"-allow-shell"}, false, "allow checkpointing shell jobs") + ) + + if err := cmd.ParseFlags(args, true); err != nil { + return err + } + + if cmd.NArg() < 1 { + cmd.Usage() + return nil + } + + criuOpts := &runconfig.CriuConfig{ + ImagesDirectory: *flImgDir, + WorkDirectory: *flWorkDir, + LeaveRunning: *flLeaveRunning, + TcpEstablished: *flCheckTcp, + ExternalUnixConnections: *flExtUnix, + ShellJob: *flShell, + } + + var encounteredError error + for _, name := range cmd.Args() { + _, _, err := readBody(cli.call("POST", "/containers/"+name+"/checkpoint", criuOpts, nil)) + if err != nil { + fmt.Fprintf(cli.err, "%s\n", err) + encounteredError = fmt.Errorf("Error: failed to checkpoint one or more containers") + } else { + fmt.Fprintf(cli.out, "%s\n", name) + } + } + return encounteredError +} diff --git a/api/client/restore.go b/api/client/restore.go new file mode 100644 index 0000000000000..0c4085fbbbd84 --- /dev/null +++ b/api/client/restore.go @@ -0,0 +1,54 @@ +package client + +import ( + "fmt" + + flag "github.com/docker/docker/pkg/mflag" + "github.com/docker/docker/runconfig" +) + +func (cli *DockerCli) CmdRestore(args ...string) error { + cmd := cli.Subcmd("restore", []string{"CONTAINER [CONTAINER...]"}, "Restore one or more checkpointed containers", true) + cmd.Require(flag.Min, 1) + + var ( + flImgDir = cmd.String([]string{"-image-dir"}, "", "directory to restore image files from") + flWorkDir = cmd.String([]string{"-work-dir"}, "", "directory for restore log") + flCheckTcp = cmd.Bool([]string{"-allow-tcp"}, false, "allow restoring tcp connections") + flExtUnix = cmd.Bool([]string{"-allow-ext-unix"}, false, "allow restoring external unix connections") + flShell = cmd.Bool([]string{"-allow-shell"}, false, "allow restoring shell jobs") + flForce = cmd.Bool([]string{"-force"}, false, "bypass checks for current container state") + ) + + if err := cmd.ParseFlags(args, true); err != nil { + return err + } + + if cmd.NArg() < 1 { + cmd.Usage() + return nil + } + + restoreOpts := &runconfig.RestoreConfig{ + CriuOpts: runconfig.CriuConfig{ + ImagesDirectory: *flImgDir, + WorkDirectory: *flWorkDir, + TcpEstablished: *flCheckTcp, + ExternalUnixConnections: *flExtUnix, + ShellJob: *flShell, + }, + ForceRestore: *flForce, + } + + var encounteredError error + for _, name := range cmd.Args() { + _, _, err := readBody(cli.call("POST", "/containers/"+name+"/restore", restoreOpts, nil)) + if err != nil { + fmt.Fprintf(cli.err, "%s\n", err) + encounteredError = fmt.Errorf("Error: failed to restore one or more containers") + } else { + fmt.Fprintf(cli.out, "%s\n", name) + } + } + return encounteredError +} diff --git a/api/server/server.go b/api/server/server.go index cd90ab36962bd..591005322c718 100644 --- a/api/server/server.go +++ b/api/server/server.go @@ -1323,32 +1323,44 @@ func (s *Server) postContainersCopy(version version.Version, w http.ResponseWrit return nil } -func postContainersCheckpoint(eng *engine.Engine, version version.Version, w http.ResponseWriter, r *http.Request, vars map[string]string) error { +func (s *Server) postContainersCheckpoint(version version.Version, w http.ResponseWriter, r *http.Request, vars map[string]string) error { if vars == nil { return fmt.Errorf("Missing parameter") } if err := parseForm(r); err != nil { return err } - job := eng.Job("checkpoint", vars["name"]) - if err := job.Run(); err != nil { + + criuOpts := &runconfig.CriuConfig{} + if err := json.NewDecoder(r.Body).Decode(criuOpts); err != nil { return err } + + if err := s.daemon.ContainerCheckpoint(vars["name"], criuOpts); err != nil { + return err + } + w.WriteHeader(http.StatusNoContent) return nil } -func postContainersRestore(eng *engine.Engine, version version.Version, w http.ResponseWriter, r *http.Request, vars map[string]string) error { +func (s *Server) postContainersRestore(version version.Version, w http.ResponseWriter, r *http.Request, vars map[string]string) error { if vars == nil { return fmt.Errorf("Missing parameter") } if err := parseForm(r); err != nil { return err } - job := eng.Job("restore", vars["name"]) - if err := job.Run(); err != nil { + + restoreOpts := runconfig.RestoreConfig{} + if err := json.NewDecoder(r.Body).Decode(&restoreOpts); err != nil { return err } + + if err := s.daemon.ContainerRestore(vars["name"], &restoreOpts.CriuOpts, restoreOpts.ForceRestore); err != nil { + return err + } + w.WriteHeader(http.StatusNoContent) return nil } diff --git a/daemon/checkpoint.go b/daemon/checkpoint.go index f6057c6a028f9..a39662cc0f325 100644 --- a/daemon/checkpoint.go +++ b/daemon/checkpoint.go @@ -1,55 +1,56 @@ package daemon import ( - "github.com/docker/docker/engine" + "fmt" + + "github.com/docker/docker/runconfig" ) // Checkpoint a running container. -func (daemon *Daemon) ContainerCheckpoint(job *engine.Job) engine.Status { - if len(job.Args) != 1 { - return job.Errorf("Usage: %s CONTAINER\n", job.Name) - } - - name := job.Args[0] +func (daemon *Daemon) ContainerCheckpoint(name string, opts *runconfig.CriuConfig) error { container, err := daemon.Get(name) if err != nil { - return job.Error(err) + return err } if !container.IsRunning() { - return job.Errorf("Container %s not running", name) + return fmt.Errorf("Container %s not running", name) } - - if err := container.Checkpoint(); err != nil { - return job.Errorf("Cannot checkpoint container %s: %s", name, err) + if err := container.Checkpoint(opts); err != nil { + return fmt.Errorf("Cannot checkpoint container %s: %s", name, err) } container.LogEvent("checkpoint") - return engine.StatusOK + return nil } // Restore a checkpointed container. -func (daemon *Daemon) ContainerRestore(job *engine.Job) engine.Status { - if len(job.Args) != 1 { - return job.Errorf("Usage: %s CONTAINER\n", job.Name) - } - - name := job.Args[0] +func (daemon *Daemon) ContainerRestore(name string, opts *runconfig.CriuConfig, forceRestore bool) error { container, err := daemon.Get(name) if err != nil { - return job.Error(err) - } - if container.IsRunning() { - return job.Errorf("Container %s already running", name) + return err } - if !container.State.IsCheckpointed() { - return job.Errorf("Container %s is not checkpointed", name) + + if !forceRestore { + // TODO: It's possible we only want to bypass the checkpointed check, + // I'm not sure how this will work if the container is already running + if container.IsRunning() { + return fmt.Errorf("Container %s already running", name) + } + + if !container.IsCheckpointed() { + return fmt.Errorf("Container %s is not checkpointed", name) + } + } else { + if !container.HasBeenCheckpointed() && opts.ImagesDirectory == "" { + return fmt.Errorf("You must specify an image directory to restore from %s", name) + } } - if err := container.Restore(); err != nil { + if err = container.Restore(opts, forceRestore); err != nil { container.LogEvent("die") - return job.Errorf("Cannot restore container %s: %s", name, err) + return fmt.Errorf("Cannot restore container %s: %s", name, err) } container.LogEvent("restore") - return engine.StatusOK + return nil } diff --git a/daemon/container.go b/daemon/container.go index ee489d8918228..f1aa62e050024 100644 --- a/daemon/container.go +++ b/daemon/container.go @@ -252,7 +252,7 @@ func (container *Container) Start() (err error) { if err := container.Mount(); err != nil { return err } - if err := container.initializeNetworking(); err != nil { + if err := container.initializeNetworking(false); err != nil { return err } linkedEnv, err := container.setupLinkedContainers() @@ -335,12 +335,11 @@ func (container *Container) isNetworkAllocated() bool { return container.NetworkSettings.IPAddress != "" } - // cleanup releases any network resources allocated to the container along with any rules // around how containers are linked together. It also unmounts the container's root filesystem. func (container *Container) cleanup() { if container.IsCheckpointed() { - log.CRDbg("not calling ReleaseNetwork() for checkpointed container %s", container.ID) + logrus.Debugf("not calling ReleaseNetwork() for checkpointed container %s", container.ID) } else { container.ReleaseNetwork() } @@ -602,8 +601,7 @@ func validateID(id string) error { return nil } - -func (container *Container) Checkpoint(opts *libcontainer.CriuOpts) error { +func (container *Container) Checkpoint(opts *runconfig.CriuConfig) error { if err := container.daemon.Checkpoint(container, opts); err != nil { return err } @@ -614,6 +612,50 @@ func (container *Container) Checkpoint(opts *libcontainer.CriuOpts) error { return nil } +func (container *Container) Restore(opts *runconfig.CriuConfig, forceRestore bool) error { + var err error + container.Lock() + defer container.Unlock() + + defer func() { + if err != nil { + container.setError(err) + // if no one else has set it, make sure we don't leave it at zero + if container.ExitCode == 0 { + container.ExitCode = 128 + } + container.toDisk() + container.cleanup() + } + }() + + if err := container.Mount(); err != nil { + return err + } + if err = container.initializeNetworking(true); err != nil { + return err + } + linkedEnv, err := container.setupLinkedContainers() + if err != nil { + return err + } + if err = container.setupWorkingDirectory(); err != nil { + return err + } + + env := container.createDaemonEnvironment(linkedEnv) + if err = populateCommand(container, env); err != nil { + return err + } + + mounts, err := container.setupMounts() + if err != nil { + return err + } + + container.command.Mounts = mounts + return container.waitForRestore(opts, forceRestore) +} func (container *Container) Copy(resource string) (io.ReadCloser, error) { container.Lock() @@ -677,41 +719,6 @@ func (container *Container) Copy(resource string) (io.ReadCloser, error) { return reader, nil } -func (container *Container) Checkpoint() error { - return container.daemon.Checkpoint(container) -} - -func (container *Container) Restore() error { - var err error - - container.Lock() - defer container.Unlock() - - defer func() { - if err != nil { - container.cleanup() - } - }() - - if err = container.initializeNetworking(); err != nil { - return err - } - - linkedEnv, err := container.setupLinkedContainers() - if err != nil { - return err - } - if err = container.setupWorkingDirectory(); err != nil { - return err - } - env := container.createDaemonEnvironment(linkedEnv) - if err = populateCommandRestore(container, env); err != nil { - return err - } - - return container.waitForRestore() -} - // Returns true if the container exposes a certain port func (container *Container) Exposes(p nat.Port) bool { _, exists := container.Config.ExposedPorts[p] @@ -803,10 +810,7 @@ func (container *Container) waitForStart() error { return nil } -// Like waitForStart() but for restoring a container. -// -// XXX Does RestartPolicy apply here? -func (container *Container) waitForRestore() error { +func (container *Container) waitForRestore(opts *runconfig.CriuConfig, forceRestore bool) error { container.monitor = newContainerMonitor(container, container.hostConfig.RestartPolicy) // After calling promise.Go() we'll have two goroutines: @@ -819,7 +823,7 @@ func (container *Container) waitForRestore() error { if container.ExitCode != 0 { return fmt.Errorf("restore process failed") } - case err := <-promise.Go(container.monitor.Restore): + case err := <-promise.Go(func() error { return container.monitor.Restore(opts, forceRestore) }): return err } @@ -1033,6 +1037,7 @@ func attach(streamConfig *StreamConfig, openStdin, stdinOnce, tty bool, stdin io _, err = copyEscapable(cStdin, stdin) } else { _, err = io.Copy(cStdin, stdin) + } if err == io.ErrClosedPipe { err = nil diff --git a/daemon/container_linux.go b/daemon/container_linux.go index ad1a76f2330cc..6b9e2e4d7f4cd 100644 --- a/daemon/container_linux.go +++ b/daemon/container_linux.go @@ -329,53 +329,6 @@ func mergeDevices(defaultDevices, userDevices []*configs.Device) []*configs.Devi return append(devs, userDevices...) } -// Like populateCommand() but for restoring a container. -// -// XXX populateCommand() does a lot more. Not sure if we have -// to do everything it does. -func populateCommandRestore(c *Container, env []string) error { - resources := &execdriver.Resources{ - Memory: c.Config.Memory, - MemorySwap: c.Config.MemorySwap, - CpuShares: c.Config.CpuShares, - Cpuset: c.Config.Cpuset, - } - - processConfig := execdriver.ProcessConfig{ - Privileged: c.hostConfig.Privileged, - Entrypoint: c.Path, - Arguments: c.Args, - Tty: c.Config.Tty, - User: c.Config.User, - } - - processConfig.SysProcAttr = &syscall.SysProcAttr{Setsid: true} - processConfig.Env = env - - c.command = &execdriver.Command{ - ID: c.ID, - Rootfs: c.RootfsPath(), - ReadonlyRootfs: c.hostConfig.ReadonlyRootfs, - InitPath: "/.dockerinit", - WorkingDir: c.Config.WorkingDir, - // Network: en, - // Ipc: ipc, - // Pid: pid, - Resources: resources, - // AllowedDevices: allowedDevices, - // AutoCreatedDevices: autoCreatedDevices, - CapAdd: c.hostConfig.CapAdd, - CapDrop: c.hostConfig.CapDrop, - ProcessConfig: processConfig, - ProcessLabel: c.GetProcessLabel(), - MountLabel: c.GetMountLabel(), - // LxcConfig: lxcConfig, - AppArmorProfile: c.AppArmorProfile, - } - - return nil -} - // GetSize, return real size, virtual size func (container *Container) GetSize() (int64, int64) { var ( @@ -710,7 +663,7 @@ func (container *Container) UpdateNetwork() error { return nil } -func (container *Container) buildCreateEndpointOptions() ([]libnetwork.EndpointOption, error) { +func (container *Container) buildCreateEndpointOptions(restoring bool) ([]libnetwork.EndpointOption, error) { var ( portSpecs = make(nat.PortSet) bindings = make(nat.PortMap) @@ -781,6 +734,14 @@ func (container *Container) buildCreateEndpointOptions() ([]libnetwork.EndpointO createOptions = append(createOptions, libnetwork.EndpointOptionGeneric(genericOption)) } + /*if restoring && container.NetworkSettings.IPAddress != "" { + genericOption := options.Generic{ + netlabel.IPAddress: net.ParseIP(container.NetworkSettings.IPAddress), + } + + createOptions = append(createOptions, libnetwork.EndpointOptionGeneric(genericOption)) + }*/ + return createOptions, nil } @@ -829,7 +790,7 @@ func (container *Container) secondaryNetworkRequired(primaryNetworkType string) return false } -func (container *Container) AllocateNetwork() error { +func (container *Container) AllocateNetwork(isRestoring bool) error { mode := container.hostConfig.NetworkMode controller := container.daemon.netController if container.Config.NetworkDisabled || mode.IsContainer() { @@ -860,19 +821,19 @@ func (container *Container) AllocateNetwork() error { if container.secondaryNetworkRequired(networkDriver) { // Configure Bridge as secondary network for port binding purposes - if err := container.configureNetwork("bridge", service, "bridge", false); err != nil { + if err := container.configureNetwork("bridge", service, "bridge", false, isRestoring); err != nil { return err } } - if err := container.configureNetwork(networkName, service, networkDriver, mode.IsDefault()); err != nil { + if err := container.configureNetwork(networkName, service, networkDriver, mode.IsDefault(), isRestoring); err != nil { return err } return container.WriteHostConfig() } -func (container *Container) configureNetwork(networkName, service, networkDriver string, canCreateNetwork bool) error { +func (container *Container) configureNetwork(networkName, service, networkDriver string, canCreateNetwork bool, isRestoring bool) error { controller := container.daemon.netController n, err := controller.NetworkByName(networkName) if err != nil { @@ -891,7 +852,7 @@ func (container *Container) configureNetwork(networkName, service, networkDriver return err } - createOptions, err := container.buildCreateEndpointOptions() + createOptions, err := container.buildCreateEndpointOptions(isRestoring) if err != nil { return err } @@ -922,7 +883,7 @@ func (container *Container) configureNetwork(networkName, service, networkDriver return nil } -func (container *Container) initializeNetworking() error { +func (container *Container) initializeNetworking(restoring bool) error { var err error // Make sure NetworkMode has an acceptable value before @@ -962,7 +923,7 @@ func (container *Container) initializeNetworking() error { } - if err := container.AllocateNetwork(); err != nil { + if err := container.AllocateNetwork(restoring); err != nil { return err } diff --git a/daemon/container_windows.go b/daemon/container_windows.go index 8bbbf65bfbefb..252c6b8bd221c 100644 --- a/daemon/container_windows.go +++ b/daemon/container_windows.go @@ -51,7 +51,7 @@ func (container *Container) createDaemonEnvironment(linkedEnv []string) []string return nil } -func (container *Container) initializeNetworking() error { +func (container *Container) initializeNetworking(restoring bool) error { return nil } diff --git a/daemon/daemon.go b/daemon/daemon.go index 88639d3a56065..de28f60cf95bc 100644 --- a/daemon/daemon.go +++ b/daemon/daemon.go @@ -863,22 +863,22 @@ func (daemon *Daemon) Run(c *Container, pipes *execdriver.Pipes, startCallback e return daemon.execDriver.Run(c.command, pipes, startCallback) } -func (daemon *Daemon) Checkpoint(c *Container) error { - if err := daemon.execDriver.Checkpoint(c.command); err != nil { +func (daemon *Daemon) Checkpoint(c *Container, opts *runconfig.CriuConfig) error { + if err := daemon.execDriver.Checkpoint(c.command, opts); err != nil { return err } - c.SetCheckpointed() + c.SetCheckpointed(opts.LeaveRunning) return nil } -func (daemon *Daemon) Restore(c *Container, pipes *execdriver.Pipes, restoreCallback execdriver.RestoreCallback) (int, error) { +func (daemon *Daemon) Restore(c *Container, pipes *execdriver.Pipes, restoreCallback execdriver.RestoreCallback, opts *runconfig.CriuConfig, forceRestore bool) (execdriver.ExitStatus, error) { // Mount the container's filesystem (daemon/graphdriver/aufs/aufs.go). _, err := daemon.driver.Get(c.ID, c.GetMountLabel()) if err != nil { - return 0, err + return execdriver.ExitStatus{ExitCode: 0}, err } - exitCode, err := daemon.execDriver.Restore(c.command, pipes, restoreCallback) + exitCode, err := daemon.execDriver.Restore(c.command, pipes, restoreCallback, opts, forceRestore) return exitCode, err } diff --git a/daemon/execdriver/driver.go b/daemon/execdriver/driver.go index 54d3955bb040c..80d775bac2640 100644 --- a/daemon/execdriver/driver.go +++ b/daemon/execdriver/driver.go @@ -8,6 +8,7 @@ import ( // TODO Windows: Factor out ulimit "github.com/docker/docker/pkg/ulimit" + "github.com/docker/docker/runconfig" "github.com/docker/libcontainer" "github.com/docker/libcontainer/configs" ) @@ -60,8 +61,8 @@ type Driver interface { Kill(c *Command, sig int) error Pause(c *Command) error Unpause(c *Command) error - Checkpoint(c *Command, opts *libcontainer.CriuOpts) error - Restore(c *Command, pipes *Pipes, restoreCallback RestoreCallback, opts *libcontainer.CriuOpts, forceRestore bool) (ExitStatus, error) + Checkpoint(c *Command, opts *runconfig.CriuConfig) error + Restore(c *Command, pipes *Pipes, restoreCallback RestoreCallback, opts *runconfig.CriuConfig, forceRestore bool) (ExitStatus, error) Name() string // Driver name Info(id string) Info // "temporary" hack (until we move state from core to plugins) GetPidsForContainer(id string) ([]int, error) // Returns a list of pids for the given container. diff --git a/daemon/execdriver/lxc/driver.go b/daemon/execdriver/lxc/driver.go index 7ca41abe818c4..6116a1e87ce42 100644 --- a/daemon/execdriver/lxc/driver.go +++ b/daemon/execdriver/lxc/driver.go @@ -25,6 +25,7 @@ import ( sysinfo "github.com/docker/docker/pkg/system" "github.com/docker/docker/pkg/term" "github.com/docker/docker/pkg/version" + "github.com/docker/docker/runconfig" "github.com/docker/libcontainer" "github.com/docker/libcontainer/cgroups" "github.com/docker/libcontainer/configs" @@ -547,11 +548,11 @@ func (d *driver) Unpause(c *execdriver.Command) error { return err } -func (d *driver) Checkpoint(c *execdriver.Command, opts *libcontainer.CriuOpts) error { +func (d *driver) Checkpoint(c *execdriver.Command, opts *runconfig.CriuConfig) error { return fmt.Errorf("Checkpointing lxc containers not supported yet\n") } -func (d *driver) Restore(c *execdriver.Command, pipes *execdriver.Pipes, restoreCallback execdriver.RestoreCallback, opts *libcontainer.CriuOpts, forceRestore bool) (execdriver.ExitStatus, error) { +func (d *driver) Restore(c *execdriver.Command, pipes *execdriver.Pipes, restoreCallback execdriver.RestoreCallback, opts *runconfig.CriuConfig, forceRestore bool) (execdriver.ExitStatus, error) { return execdriver.ExitStatus{ExitCode: 0}, fmt.Errorf("Restoring lxc containers not supported yet\n") } diff --git a/daemon/execdriver/native/driver.go b/daemon/execdriver/native/driver.go index 3e4658b1f495e..5d87c85cd5d91 100644 --- a/daemon/execdriver/native/driver.go +++ b/daemon/execdriver/native/driver.go @@ -19,6 +19,7 @@ import ( "github.com/docker/docker/pkg/reexec" sysinfo "github.com/docker/docker/pkg/system" "github.com/docker/docker/pkg/term" + "github.com/docker/docker/runconfig" "github.com/docker/libcontainer" "github.com/docker/libcontainer/apparmor" "github.com/docker/libcontainer/cgroups/systemd" @@ -276,7 +277,18 @@ func (d *driver) Unpause(c *execdriver.Command) error { return active.Resume() } -func (d *driver) Checkpoint(c *execdriver.Command, opts *libcontainer.CriuOpts) error { +func libcontainerCriuOpts(runconfigOpts *runconfig.CriuConfig) *libcontainer.CriuOpts { + return &libcontainer.CriuOpts{ + ImagesDirectory: runconfigOpts.ImagesDirectory, + WorkDirectory: runconfigOpts.WorkDirectory, + LeaveRunning: runconfigOpts.LeaveRunning, + TcpEstablished: runconfigOpts.TcpEstablished, + ExternalUnixConnections: runconfigOpts.ExternalUnixConnections, + ShellJob: runconfigOpts.ShellJob, + } +} + +func (d *driver) Checkpoint(c *execdriver.Command, opts *runconfig.CriuConfig) error { active := d.activeContainers[c.ID] if active == nil { return fmt.Errorf("active container for %s does not exist", c.ID) @@ -284,7 +296,7 @@ func (d *driver) Checkpoint(c *execdriver.Command, opts *libcontainer.CriuOpts) d.Lock() defer d.Unlock() - err := active.Checkpoint(opts) + err := active.Checkpoint(libcontainerCriuOpts(opts)) if err != nil { return err } @@ -292,7 +304,7 @@ func (d *driver) Checkpoint(c *execdriver.Command, opts *libcontainer.CriuOpts) return nil } -func (d *driver) Restore(c *execdriver.Command, pipes *execdriver.Pipes, restoreCallback execdriver.RestoreCallback, opts *libcontainer.CriuOpts, forceRestore bool) (execdriver.ExitStatus, error) { +func (d *driver) Restore(c *execdriver.Command, pipes *execdriver.Pipes, restoreCallback execdriver.RestoreCallback, opts *runconfig.CriuConfig, forceRestore bool) (execdriver.ExitStatus, error) { var ( cont libcontainer.Container err error @@ -335,7 +347,7 @@ func (d *driver) Restore(c *execdriver.Command, pipes *execdriver.Pipes, restore d.cleanContainer(c.ID) }() - if err := cont.Restore(p, opts); err != nil { + if err := cont.Restore(p, libcontainerCriuOpts(opts)); err != nil { return execdriver.ExitStatus{ExitCode: -1}, err } diff --git a/daemon/execdriver/windows/windows.go b/daemon/execdriver/windows/windows.go index 9837270235d5c..d49f76a95cf93 100644 --- a/daemon/execdriver/windows/windows.go +++ b/daemon/execdriver/windows/windows.go @@ -13,6 +13,7 @@ import ( "fmt" "github.com/docker/docker/daemon/execdriver" + "github.com/docker/docker/runconfig" ) const ( @@ -95,3 +96,11 @@ func (d *driver) Stats(id string) (*execdriver.ResourceStats, error) { func (d *driver) Exec(c *execdriver.Command, processConfig *execdriver.ProcessConfig, pipes *execdriver.Pipes, startCallback execdriver.StartCallback) (int, error) { return 0, nil } + +func (d *driver) Checkpoint(c *execdriver.Command, opts *runconfig.CriuConfig) error { + return fmt.Errorf("Windows: Containers cannot be checkpointed") +} + +func (d *driver) Restore(c *execdriver.Command, pipes *execdriver.Pipes, restoreCallback execdriver.RestoreCallback, opts *runconfig.CriuConfig, forceRestore bool) (execdriver.ExitStatus, error) { + return execdriver.ExitStatus{ExitCode: 0}, fmt.Errorf("Windows: Containers cannot be restored") +} diff --git a/daemon/monitor.go b/daemon/monitor.go index 241efa32d0eb3..5d80d74fc1a88 100644 --- a/daemon/monitor.go +++ b/daemon/monitor.go @@ -186,43 +186,45 @@ func (m *containerMonitor) Start() error { } // Like Start() but for restoring a container. -func (m *containerMonitor) Restore() error { +func (m *containerMonitor) Restore(opts *runconfig.CriuConfig, forceRestore bool) error { var ( err error // XXX The following line should be changed to // exitStatus execdriver.ExitStatus to match Start() - exitCode int + exitCode execdriver.ExitStatus afterRestore bool ) - defer func() { if afterRestore { m.container.Lock() - m.container.setStopped(&execdriver.ExitStatus{exitCode, false}) + m.container.setStopped(&execdriver.ExitStatus{exitCode.ExitCode, false}) defer m.container.Unlock() } m.Close() }() - if err := m.container.startLoggingToDisk(); err != nil { - m.resetContainer(false) - return err + // FIXME: right now if we startLogging again we get double logs after a restore + if m.container.logCopier == nil { + if err := m.container.startLogging(); err != nil { + m.resetContainer(false) + return err + } } pipes := execdriver.NewPipes(m.container.stdin, m.container.stdout, m.container.stderr, m.container.Config.OpenStdin) m.container.LogEvent("restore") m.lastStartTime = time.Now() - if exitCode, err = m.container.daemon.Restore(m.container, pipes, m.restoreCallback); err != nil { - log.Errorf("Error restoring container: %s, exitCode=%d", err, exitCode) + if exitCode, err = m.container.daemon.Restore(m.container, pipes, m.restoreCallback, opts, forceRestore); err != nil { + logrus.Errorf("Error restoring container: %s, exitCode=%d", err, exitCode) m.container.ExitCode = -1 m.resetContainer(false) return err } afterRestore = true - m.container.ExitCode = exitCode - m.resetMonitor(err == nil && exitCode == 0) + m.container.ExitCode = exitCode.ExitCode + m.resetMonitor(err == nil && exitCode.ExitCode == 0) m.container.LogEvent("die") m.resetContainer(true) return err @@ -332,7 +334,7 @@ func (m *containerMonitor) restoreCallback(processConfig *execdriver.ProcessConf // Write config.json and hostconfig.json files // to /var/lib/docker/containers/. if err := m.container.ToDisk(); err != nil { - log.Debugf("%s", err) + logrus.Debugf("%s", err) } } } diff --git a/daemon/state.go b/daemon/state.go index 82eefe75a3a6c..f6ef8977695b0 100644 --- a/daemon/state.go +++ b/daemon/state.go @@ -25,7 +25,6 @@ type State struct { FinishedAt time.Time CheckpointedAt time.Time waitChan chan struct{} - } func NewState() *State { @@ -45,14 +44,16 @@ func (s *State) String() string { } return fmt.Sprintf("Up %s", units.HumanDuration(time.Now().UTC().Sub(s.StartedAt))) - } else if s.Checkpointed { - return fmt.Sprintf("Checkpointed %s ago", units.HumanDuration(time.Now().UTC().Sub(s.CheckpointedAt))) } if s.removalInProgress { return "Removal In Progress" } + if s.Checkpointed { + return fmt.Sprintf("Checkpointed %s ago", units.HumanDuration(time.Now().UTC().Sub(s.CheckpointedAt))) + } + if s.Dead { return "Dead" } @@ -80,6 +81,10 @@ func (s *State) StateString() string { return "running" } + if s.Checkpointed { + return "checkpointed'" + } + if s.Dead { return "dead" } @@ -269,11 +274,11 @@ func (s *State) SetDead() { s.Unlock() } -func (s *State) SetCheckpointed() { +func (s *State) SetCheckpointed(leaveRunning bool) { s.Lock() s.CheckpointedAt = time.Now().UTC() - s.Checkpointed = true - s.Running = false + s.Checkpointed = !leaveRunning + s.Running = leaveRunning s.Paused = false s.Restarting = false // XXX Not sure if we need to close and recreate waitChan. @@ -282,6 +287,10 @@ func (s *State) SetCheckpointed() { s.Unlock() } +func (s *State) HasBeenCheckpointed() bool { + return s.CheckpointedAt != time.Time{} +} + func (s *State) IsCheckpointed() bool { return s.Checkpointed } diff --git a/docker/flags.go b/docker/flags.go index d860ec7e1f703..b193b7bc93e72 100644 --- a/docker/flags.go +++ b/docker/flags.go @@ -31,6 +31,7 @@ var ( dockerCommands = []command{ {"attach", "Attach to a running container"}, {"build", "Build an image from a Dockerfile"}, + {"checkpoint", "Checkpoint one or more running containers"}, {"commit", "Create a new image from a container's changes"}, {"cp", "Copy files/folders from a container's filesystem to the host path"}, {"create", "Create a new container"}, @@ -55,6 +56,7 @@ var ( {"push", "Push an image or a repository to a Docker registry server"}, {"rename", "Rename an existing container"}, {"restart", "Restart a running container"}, + {"restore", "Restore one or more checkpointed containers"}, {"rm", "Remove one or more containers"}, {"rmi", "Remove one or more images"}, {"run", "Run a command in a new container"}, @@ -131,7 +133,7 @@ func init() { sort.Sort(byName(dockerCommands)) for _, cmd := range dockerCommands { - help += fmt.Sprintf(" %-10.10s%s\n", cmd.name, cmd.description) + help += fmt.Sprintf(" %-11.11s%s\n", cmd.name, cmd.description) } help += "\nRun 'docker COMMAND --help' for more information on a command." diff --git a/integration-cli/docker_cli_help_test.go b/integration-cli/docker_cli_help_test.go index ce9f54e69af84..4eb515f02a9a7 100644 --- a/integration-cli/docker_cli_help_test.go +++ b/integration-cli/docker_cli_help_test.go @@ -229,7 +229,7 @@ func (s *DockerSuite) TestHelpTextVerify(c *check.C) { } - expected := 39 + expected := 41 if len(cmds) != expected { c.Fatalf("Wrong # of cmds(%d), it should be: %d\nThe list:\n%q", len(cmds), expected, cmds) diff --git a/runconfig/restore.go b/runconfig/restore.go new file mode 100644 index 0000000000000..22f8b0ab0a096 --- /dev/null +++ b/runconfig/restore.go @@ -0,0 +1,15 @@ +package runconfig + +type CriuConfig struct { + ImagesDirectory string + WorkDirectory string + LeaveRunning bool + TcpEstablished bool + ExternalUnixConnections bool + ShellJob bool +} + +type RestoreConfig struct { + CriuOpts CriuConfig + ForceRestore bool +} From 87179b42d0c221c2eb2f246427c5f9a220759bcc Mon Sep 17 00:00:00 2001 From: boucher Date: Tue, 2 Jun 2015 14:04:14 -0700 Subject: [PATCH 15/26] Add compilation steps for Criu to the Dockerfile Add a basic test for checkpoint/restore to the integration tests Docker-DCO-1.1-Signed-off-by: Ross Boucher (github: boucher) --- Dockerfile | 18 +++++++++ api/types/types.go | 22 ++++++----- daemon/inspect.go | 22 ++++++----- integration-cli/docker_cli_checkpoint_test.go | 37 +++++++++++++++++++ 4 files changed, 79 insertions(+), 20 deletions(-) create mode 100644 integration-cli/docker_cli_checkpoint_test.go diff --git a/Dockerfile b/Dockerfile index eddeefc67d84f..532b8cd54797e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -32,28 +32,39 @@ RUN echo deb http://ppa.launchpad.net/zfs-native/stable/ubuntu trusty main > /et # Packaged dependencies RUN apt-get update && apt-get install -y \ apparmor \ + asciidoc \ aufs-tools \ automake \ bash-completion \ + bsdmainutils \ btrfs-tools \ build-essential \ curl \ dpkg-sig \ git \ iptables \ + libaio-dev \ libapparmor-dev \ libcap-dev \ + libprotobuf-c0-dev \ + libprotobuf-dev \ libsqlite3-dev \ mercurial \ parallel \ + pkg-config \ + protobuf-compiler \ + protobuf-c-compiler \ + python-minimal \ python-mock \ python-pip \ + python-protobuf \ python-websocket \ reprepro \ ruby1.9.1 \ ruby1.9.1-dev \ s3cmd=1.1.0* \ ubuntu-zfs \ + xmlto \ libzfs-dev \ --no-install-recommends @@ -78,6 +89,13 @@ RUN cd /usr/src/lxc \ && make install \ && ldconfig +# Install Criu +RUN mkdir -p /usr/src/criu \ + && curl -sSL https://github.com/xemul/criu/archive/v1.6.tar.gz | tar -v -C /usr/src/criu/ -xz --strip-components=1 +RUN cd /usr/src/criu \ + && make \ + && make install + # Install Go ENV GO_VERSION 1.4.2 RUN curl -sSL https://golang.org/dl/go${GO_VERSION}.src.tar.gz | tar -v -C /usr/local -xz \ diff --git a/api/types/types.go b/api/types/types.go index fdcc2284394e1..e9a913542dfc3 100644 --- a/api/types/types.go +++ b/api/types/types.go @@ -193,16 +193,18 @@ type ExecStartCheck struct { } type ContainerState struct { - Running bool - Paused bool - Restarting bool - OOMKilled bool - Dead bool - Pid int - ExitCode int - Error string - StartedAt time.Time - FinishedAt time.Time + Running bool + Paused bool + Checkpointed bool + Restarting bool + OOMKilled bool + Dead bool + Pid int + ExitCode int + Error string + StartedAt time.Time + FinishedAt time.Time + CheckpointedAt time.Time } // GET "/containers/{name:.*}/json" diff --git a/daemon/inspect.go b/daemon/inspect.go index 73b394ca24c0f..e9d302f1bc9b1 100644 --- a/daemon/inspect.go +++ b/daemon/inspect.go @@ -64,16 +64,18 @@ func (daemon *Daemon) getInspectData(container *Container) (*types.ContainerJSON } containerState := &types.ContainerState{ - Running: container.State.Running, - Paused: container.State.Paused, - Restarting: container.State.Restarting, - OOMKilled: container.State.OOMKilled, - Dead: container.State.Dead, - Pid: container.State.Pid, - ExitCode: container.State.ExitCode, - Error: container.State.Error, - StartedAt: container.State.StartedAt, - FinishedAt: container.State.FinishedAt, + Running: container.State.Running, + Paused: container.State.Paused, + Checkpointed: container.State.Checkpointed, + Restarting: container.State.Restarting, + OOMKilled: container.State.OOMKilled, + Dead: container.State.Dead, + Pid: container.State.Pid, + ExitCode: container.State.ExitCode, + Error: container.State.Error, + StartedAt: container.State.StartedAt, + FinishedAt: container.State.FinishedAt, + CheckpointedAt: container.State.CheckpointedAt, } volumes := make(map[string]string) diff --git a/integration-cli/docker_cli_checkpoint_test.go b/integration-cli/docker_cli_checkpoint_test.go new file mode 100644 index 0000000000000..e19ef524efd43 --- /dev/null +++ b/integration-cli/docker_cli_checkpoint_test.go @@ -0,0 +1,37 @@ +package main + +import ( + "os/exec" + "strings" + + "github.com/go-check/check" +) + +func (s *DockerSuite) TestCheckpointAndRestore(c *check.C) { + defer unpauseAllContainers() + + runCmd := exec.Command(dockerBinary, "run", "-d", "busybox", "top") + out, _, err := runCommandWithOutput(runCmd) + if err != nil { + c.Fatalf("failed to run container: %v, output: %q", err, out) + } + + containerID := strings.TrimSpace(out) + checkpointCmd := exec.Command(dockerBinary, "checkpoint", containerID) + out, _, err = runCommandWithOutput(checkpointCmd) + if err != nil { + c.Fatalf("failed to checkpoint container: %v, output: %q", err, out) + } + + out, err = inspectField(containerID, "State.Checkpointed") + c.Assert(out, check.Equals, "true") + + restoreCmd := exec.Command(dockerBinary, "restore", containerID) + out, _, _, err = runCommandWithStdoutStderr(restoreCmd) + if err != nil { + c.Fatalf("failed to restore container: %v, output: %q", err, out) + } + + out, err = inspectField(containerID, "State.Checkpointed") + c.Assert(out, check.Equals, "false") +} From b8cebfc0f4d8a59a15c029dedce4c8e61c2b6001 Mon Sep 17 00:00:00 2001 From: boucher Date: Tue, 16 Jun 2015 14:41:05 -0700 Subject: [PATCH 16/26] Add optional dependency info to the PACKAGERS file. Docker-DCO-1.1-Signed-off-by: Ross Boucher (github: boucher) --- project/PACKAGERS.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/project/PACKAGERS.md b/project/PACKAGERS.md index fd2156c5855bf..da0d2da9ca0f2 100644 --- a/project/PACKAGERS.md +++ b/project/PACKAGERS.md @@ -304,6 +304,9 @@ by having support for them in the kernel or userspace. A few examples include: least the "auplink" utility from aufs-tools) * BTRFS graph driver (requires BTRFS support enabled in the kernel) * ZFS graph driver (requires userspace zfs-utils and a corresponding kernel module) +* Checkpoint/Restore containers: + - requires criu version 1.5.2 or later (criu.org) + - requires kernel version 3.19 or later if using overlay-fs ## Daemon Init Script From c1a59ac46daa0a06af1c6a54c65cdea90a3bedbc Mon Sep 17 00:00:00 2001 From: boucher Date: Thu, 18 Jun 2015 15:18:09 -0700 Subject: [PATCH 17/26] Don't destroy/delete the container if it has been checkpointed. Docker-DCO-1.1-Signed-off-by: Ross Boucher (github: boucher) --- daemon/execdriver/native/driver.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/daemon/execdriver/native/driver.go b/daemon/execdriver/native/driver.go index 5d87c85cd5d91..a44db33210e2b 100644 --- a/daemon/execdriver/native/driver.go +++ b/daemon/execdriver/native/driver.go @@ -141,8 +141,11 @@ func (d *driver) Run(c *execdriver.Command, pipes *execdriver.Pipes, startCallba d.activeContainers[c.ID] = cont d.Unlock() defer func() { - cont.Destroy() - d.cleanContainer(c.ID) + status, err := cont.Status() + if err != nil || status != libcontainer.Checkpointed { + cont.Destroy() + d.cleanContainer(c.ID) + } }() if err := cont.Start(p); err != nil { From b654a746ef18fa6003f2f8265556d295ef5ef29f Mon Sep 17 00:00:00 2001 From: Saied Kazemi Date: Thu, 5 Feb 2015 20:32:27 -0800 Subject: [PATCH 18/26] Checkpoint/Restore Support: add exec driver methods Methods for checkpointing and restoring containers were added to the native driver. The LXC driver returns an error message that these methods are not implemented yet. Signed-off-by: Saied Kazemi Conflicts: daemon/execdriver/native/create.go daemon/execdriver/native/driver.go daemon/execdriver/native/init.go --- daemon/execdriver/driver.go | 3 + daemon/execdriver/lxc/driver.go | 8 ++ daemon/execdriver/native/create.go | 19 ++++ daemon/execdriver/native/driver.go | 150 +++++++++++++++++++++++++++++ 4 files changed, 180 insertions(+) diff --git a/daemon/execdriver/driver.go b/daemon/execdriver/driver.go index eca77e921eaac..acb41c5f53376 100644 --- a/daemon/execdriver/driver.go +++ b/daemon/execdriver/driver.go @@ -24,6 +24,7 @@ var ( ) type StartCallback func(*ProcessConfig, int) +type RestoreCallback func(*ProcessConfig, int) // Driver specific information based on // processes registered with the driver @@ -59,6 +60,8 @@ type Driver interface { Kill(c *Command, sig int) error Pause(c *Command) error Unpause(c *Command) error + Checkpoint(c *Command) error + Restore(c *Command, pipes *Pipes, restoreCallback RestoreCallback) (int, error) Name() string // Driver name Info(id string) Info // "temporary" hack (until we move state from core to plugins) GetPidsForContainer(id string) ([]int, error) // Returns a list of pids for the given container. diff --git a/daemon/execdriver/lxc/driver.go b/daemon/execdriver/lxc/driver.go index 692b9f356f3aa..914a7ea5075c2 100644 --- a/daemon/execdriver/lxc/driver.go +++ b/daemon/execdriver/lxc/driver.go @@ -547,6 +547,14 @@ func (d *driver) Unpause(c *execdriver.Command) error { return err } +func (d *driver) Checkpoint(c *execdriver.Command) error { + return fmt.Errorf("Checkpointing lxc containers not supported yet\n") +} + +func (d *driver) Restore(c *execdriver.Command, pipes *execdriver.Pipes, restoreCallback execdriver.RestoreCallback) (int, error) { + return 0, fmt.Errorf("Restoring lxc containers not supported yet\n") +} + func (d *driver) Terminate(c *execdriver.Command) error { return KillLxc(c.ID, 9) } diff --git a/daemon/execdriver/native/create.go b/daemon/execdriver/native/create.go index 1b2d7232d3177..1cf248b0d66fd 100644 --- a/daemon/execdriver/native/create.go +++ b/daemon/execdriver/native/create.go @@ -4,6 +4,7 @@ package native import ( "errors" + "encoding/json" "fmt" "net" "strings" @@ -88,6 +89,24 @@ func generateIfaceName() (string, error) { return "", errors.New("Failed to find name for new interface") } +// Re-create the container type from the image that was saved during checkpoint. +func (d *driver) createRestoreContainer(c *execdriver.Command, imageDir string) (*libcontainer.Config, error) { + // Read the container.json. + f1, err := os.Open(filepath.Join(imageDir, "container.json")) + if err != nil { + return nil, err + } + defer f1.Close() + + var container *libcontainer.Config + err = json.NewDecoder(f1).Decode(&container) + if err != nil { + return nil, err + } + + return container, nil +} + func (d *driver) createNetwork(container *configs.Config, c *execdriver.Command) error { if c.Network == nil { return nil diff --git a/daemon/execdriver/native/driver.go b/daemon/execdriver/native/driver.go index 2017a9c65ca7e..43c7bdc18afb3 100644 --- a/daemon/execdriver/native/driver.go +++ b/daemon/execdriver/native/driver.go @@ -20,6 +20,7 @@ import ( "github.com/docker/docker/pkg/reexec" sysinfo "github.com/docker/docker/pkg/system" "github.com/docker/docker/pkg/term" + "github.com/docker/docker/utils" "github.com/docker/libcontainer" "github.com/docker/libcontainer/apparmor" "github.com/docker/libcontainer/cgroups/systemd" @@ -277,6 +278,155 @@ func (d *driver) Unpause(c *execdriver.Command) error { return active.Resume() } +// XXX Where is the right place for the following +// const and getCheckpointImageDir() function? +const ( + containersDir = "/var/lib/docker/containers" + criuImgDir = "criu_img" +) + +func getCheckpointImageDir(containerId string) string { + return filepath.Join(containersDir, containerId, criuImgDir) +} + +func (d *driver) Checkpoint(c *execdriver.Command) error { + active := d.activeContainers[c.ID] + if active == nil { + return fmt.Errorf("active container for %s does not exist", c.ID) + } + container := active.container + + // Create an image directory for this container (which + // may already exist from a previous checkpoint). + imageDir := getCheckpointImageDir(c.ID) + err := os.MkdirAll(imageDir, 0700) + if err != nil && !os.IsExist(err) { + return err + } + + // Copy container.json and state.json files to the CRIU + // image directory for later use during restore. Do this + // before checkpointing because after checkpoint the container + // will exit and these files will be removed. + log.CRDbg("saving container.json and state.json before calling CRIU in %s", imageDir) + srcFiles := []string{"container.json", "state.json"} + for _, f := range srcFiles { + srcFile := filepath.Join(d.root, c.ID, f) + dstFile := filepath.Join(imageDir, f) + if _, err := utils.CopyFile(srcFile, dstFile); err != nil { + return err + } + } + + d.Lock() + defer d.Unlock() + err = namespaces.Checkpoint(container, imageDir, c.ProcessConfig.Process.Pid) + if err != nil { + return err + } + + return nil +} + +type restoreOutput struct { + exitCode int + err error +} + +func (d *driver) Restore(c *execdriver.Command, pipes *execdriver.Pipes, restoreCallback execdriver.RestoreCallback) (int, error) { + imageDir := getCheckpointImageDir(c.ID) + container, err := d.createRestoreContainer(c, imageDir) + if err != nil { + return 1, err + } + + var term execdriver.Terminal + + if c.ProcessConfig.Tty { + term, err = NewTtyConsole(&c.ProcessConfig, pipes) + } else { + term, err = execdriver.NewStdConsole(&c.ProcessConfig, pipes) + } + if err != nil { + return -1, err + } + c.ProcessConfig.Terminal = term + + d.Lock() + d.activeContainers[c.ID] = &activeContainer{ + container: container, + cmd: &c.ProcessConfig.Cmd, + } + d.Unlock() + defer d.cleanContainer(c.ID) + + // Since the CRIU binary exits after restoring the container, we + // need to reap its child by setting PR_SET_CHILD_SUBREAPER (36) + // so that it'll be owned by this process (Docker daemon) after restore. + // + // XXX This really belongs to where the Docker daemon starts. + if _, _, syserr := syscall.RawSyscall(syscall.SYS_PRCTL, 36, 1, 0); syserr != 0 { + return -1, fmt.Errorf("Could not set PR_SET_CHILD_SUBREAPER (syserr %d)", syserr) + } + + restoreOutputChan := make(chan restoreOutput, 1) + waitForRestore := make(chan struct{}) + + go func() { + exitCode, err := namespaces.Restore(container, c.ProcessConfig.Stdin, c.ProcessConfig.Stdout, c.ProcessConfig.Stderr, c.ProcessConfig.Console, filepath.Join(d.root, c.ID), imageDir, + func(child *os.File, args []string) *exec.Cmd { + cmd := new(exec.Cmd) + cmd.Path = d.initPath + cmd.Args = append([]string{ + DriverName, + "-restore", + "-pipe", "3", + "--", + }, args...) + cmd.ExtraFiles = []*os.File{child} + return cmd + }, + func(restorePid int) error { + log.CRDbg("restorePid=%d", restorePid) + if restorePid == 0 { + restoreCallback(&c.ProcessConfig, 0) + return nil + } + + // The container.json file should be written *after* the container + // has started because its StdFds cannot be initialized before. + // + // XXX How do we handle error here? + d.writeContainerFile(container, c.ID) + close(waitForRestore) + if restoreCallback != nil { + c.ProcessConfig.Process, err = os.FindProcess(restorePid) + if err != nil { + log.Debugf("cannot find restored process %d", restorePid) + return err + } + c.ContainerPid = c.ProcessConfig.Process.Pid + restoreCallback(&c.ProcessConfig, c.ContainerPid) + } + return nil + }) + restoreOutputChan <- restoreOutput{exitCode, err} + }() + + select { + case restoreOutput := <-restoreOutputChan: + // there was an error + return restoreOutput.exitCode, restoreOutput.err + case <-waitForRestore: + // container restored + break + } + + // Wait for the container to exit. + restoreOutput := <-restoreOutputChan + return restoreOutput.exitCode, restoreOutput.err +} + func (d *driver) Terminate(c *execdriver.Command) error { defer d.cleanContainer(c.ID) container, err := d.factory.Load(c.ID) From 90f59172b68f515f1306119db97fcfbdffc05614 Mon Sep 17 00:00:00 2001 From: boucher Date: Mon, 25 May 2015 08:32:58 -0700 Subject: [PATCH 19/26] Update checkpoint/restore support to match docker/master Docker-DCO-1.1-Signed-off-by: Ross Boucher (github: boucher) --- daemon/execdriver/driver.go | 4 +- daemon/execdriver/lxc/driver.go | 6 +- daemon/execdriver/native/create.go | 19 --- daemon/execdriver/native/driver.go | 183 ++++++++++------------------- 4 files changed, 69 insertions(+), 143 deletions(-) diff --git a/daemon/execdriver/driver.go b/daemon/execdriver/driver.go index acb41c5f53376..54d3955bb040c 100644 --- a/daemon/execdriver/driver.go +++ b/daemon/execdriver/driver.go @@ -60,8 +60,8 @@ type Driver interface { Kill(c *Command, sig int) error Pause(c *Command) error Unpause(c *Command) error - Checkpoint(c *Command) error - Restore(c *Command, pipes *Pipes, restoreCallback RestoreCallback) (int, error) + Checkpoint(c *Command, opts *libcontainer.CriuOpts) error + Restore(c *Command, pipes *Pipes, restoreCallback RestoreCallback, opts *libcontainer.CriuOpts, forceRestore bool) (ExitStatus, error) Name() string // Driver name Info(id string) Info // "temporary" hack (until we move state from core to plugins) GetPidsForContainer(id string) ([]int, error) // Returns a list of pids for the given container. diff --git a/daemon/execdriver/lxc/driver.go b/daemon/execdriver/lxc/driver.go index 914a7ea5075c2..7ca41abe818c4 100644 --- a/daemon/execdriver/lxc/driver.go +++ b/daemon/execdriver/lxc/driver.go @@ -547,12 +547,12 @@ func (d *driver) Unpause(c *execdriver.Command) error { return err } -func (d *driver) Checkpoint(c *execdriver.Command) error { +func (d *driver) Checkpoint(c *execdriver.Command, opts *libcontainer.CriuOpts) error { return fmt.Errorf("Checkpointing lxc containers not supported yet\n") } -func (d *driver) Restore(c *execdriver.Command, pipes *execdriver.Pipes, restoreCallback execdriver.RestoreCallback) (int, error) { - return 0, fmt.Errorf("Restoring lxc containers not supported yet\n") +func (d *driver) Restore(c *execdriver.Command, pipes *execdriver.Pipes, restoreCallback execdriver.RestoreCallback, opts *libcontainer.CriuOpts, forceRestore bool) (execdriver.ExitStatus, error) { + return execdriver.ExitStatus{ExitCode: 0}, fmt.Errorf("Restoring lxc containers not supported yet\n") } func (d *driver) Terminate(c *execdriver.Command) error { diff --git a/daemon/execdriver/native/create.go b/daemon/execdriver/native/create.go index 1cf248b0d66fd..1b2d7232d3177 100644 --- a/daemon/execdriver/native/create.go +++ b/daemon/execdriver/native/create.go @@ -4,7 +4,6 @@ package native import ( "errors" - "encoding/json" "fmt" "net" "strings" @@ -89,24 +88,6 @@ func generateIfaceName() (string, error) { return "", errors.New("Failed to find name for new interface") } -// Re-create the container type from the image that was saved during checkpoint. -func (d *driver) createRestoreContainer(c *execdriver.Command, imageDir string) (*libcontainer.Config, error) { - // Read the container.json. - f1, err := os.Open(filepath.Join(imageDir, "container.json")) - if err != nil { - return nil, err - } - defer f1.Close() - - var container *libcontainer.Config - err = json.NewDecoder(f1).Decode(&container) - if err != nil { - return nil, err - } - - return container, nil -} - func (d *driver) createNetwork(container *configs.Config, c *execdriver.Command) error { if c.Network == nil { return nil diff --git a/daemon/execdriver/native/driver.go b/daemon/execdriver/native/driver.go index 43c7bdc18afb3..7e1bdf3a6ad4b 100644 --- a/daemon/execdriver/native/driver.go +++ b/daemon/execdriver/native/driver.go @@ -20,7 +20,6 @@ import ( "github.com/docker/docker/pkg/reexec" sysinfo "github.com/docker/docker/pkg/system" "github.com/docker/docker/pkg/term" - "github.com/docker/docker/utils" "github.com/docker/libcontainer" "github.com/docker/libcontainer/apparmor" "github.com/docker/libcontainer/cgroups/systemd" @@ -278,49 +277,15 @@ func (d *driver) Unpause(c *execdriver.Command) error { return active.Resume() } -// XXX Where is the right place for the following -// const and getCheckpointImageDir() function? -const ( - containersDir = "/var/lib/docker/containers" - criuImgDir = "criu_img" -) - -func getCheckpointImageDir(containerId string) string { - return filepath.Join(containersDir, containerId, criuImgDir) -} - -func (d *driver) Checkpoint(c *execdriver.Command) error { +func (d *driver) Checkpoint(c *execdriver.Command, opts *libcontainer.CriuOpts) error { active := d.activeContainers[c.ID] if active == nil { return fmt.Errorf("active container for %s does not exist", c.ID) } - container := active.container - - // Create an image directory for this container (which - // may already exist from a previous checkpoint). - imageDir := getCheckpointImageDir(c.ID) - err := os.MkdirAll(imageDir, 0700) - if err != nil && !os.IsExist(err) { - return err - } - - // Copy container.json and state.json files to the CRIU - // image directory for later use during restore. Do this - // before checkpointing because after checkpoint the container - // will exit and these files will be removed. - log.CRDbg("saving container.json and state.json before calling CRIU in %s", imageDir) - srcFiles := []string{"container.json", "state.json"} - for _, f := range srcFiles { - srcFile := filepath.Join(d.root, c.ID, f) - dstFile := filepath.Join(imageDir, f) - if _, err := utils.CopyFile(srcFile, dstFile); err != nil { - return err - } - } d.Lock() defer d.Unlock() - err = namespaces.Checkpoint(container, imageDir, c.ProcessConfig.Process.Pid) + err := active.Checkpoint(opts) if err != nil { return err } @@ -328,103 +293,83 @@ func (d *driver) Checkpoint(c *execdriver.Command) error { return nil } -type restoreOutput struct { - exitCode int - err error -} +func (d *driver) Restore(c *execdriver.Command, pipes *execdriver.Pipes, restoreCallback execdriver.RestoreCallback, opts *libcontainer.CriuOpts, forceRestore bool) (execdriver.ExitStatus, error) { + var ( + cont libcontainer.Container + err error + ) -func (d *driver) Restore(c *execdriver.Command, pipes *execdriver.Pipes, restoreCallback execdriver.RestoreCallback) (int, error) { - imageDir := getCheckpointImageDir(c.ID) - container, err := d.createRestoreContainer(c, imageDir) + cont, err = d.factory.Load(c.ID) if err != nil { - return 1, err + if forceRestore { + var config *configs.Config + config, err = d.createContainer(c) + if err != nil { + return execdriver.ExitStatus{ExitCode: -1}, err + } + cont, err = d.factory.Create(c.ID, config) + if err != nil { + return execdriver.ExitStatus{ExitCode: -1}, err + } + } else { + return execdriver.ExitStatus{ExitCode: -1}, err + } } - var term execdriver.Terminal - - if c.ProcessConfig.Tty { - term, err = NewTtyConsole(&c.ProcessConfig, pipes) - } else { - term, err = execdriver.NewStdConsole(&c.ProcessConfig, pipes) + p := &libcontainer.Process{ + Args: append([]string{c.ProcessConfig.Entrypoint}, c.ProcessConfig.Arguments...), + Env: c.ProcessConfig.Env, + Cwd: c.WorkingDir, + User: c.ProcessConfig.User, } - if err != nil { - return -1, err + + config := cont.Config() + if err := setupPipes(&config, &c.ProcessConfig, p, pipes); err != nil { + return execdriver.ExitStatus{ExitCode: -1}, err } - c.ProcessConfig.Terminal = term d.Lock() - d.activeContainers[c.ID] = &activeContainer{ - container: container, - cmd: &c.ProcessConfig.Cmd, - } + d.activeContainers[c.ID] = cont d.Unlock() - defer d.cleanContainer(c.ID) + defer func() { + cont.Destroy() + d.cleanContainer(c.ID) + }() - // Since the CRIU binary exits after restoring the container, we - // need to reap its child by setting PR_SET_CHILD_SUBREAPER (36) - // so that it'll be owned by this process (Docker daemon) after restore. - // - // XXX This really belongs to where the Docker daemon starts. - if _, _, syserr := syscall.RawSyscall(syscall.SYS_PRCTL, 36, 1, 0); syserr != 0 { - return -1, fmt.Errorf("Could not set PR_SET_CHILD_SUBREAPER (syserr %d)", syserr) + if err := cont.Restore(p, opts); err != nil { + return execdriver.ExitStatus{ExitCode: -1}, err } - restoreOutputChan := make(chan restoreOutput, 1) - waitForRestore := make(chan struct{}) - - go func() { - exitCode, err := namespaces.Restore(container, c.ProcessConfig.Stdin, c.ProcessConfig.Stdout, c.ProcessConfig.Stderr, c.ProcessConfig.Console, filepath.Join(d.root, c.ID), imageDir, - func(child *os.File, args []string) *exec.Cmd { - cmd := new(exec.Cmd) - cmd.Path = d.initPath - cmd.Args = append([]string{ - DriverName, - "-restore", - "-pipe", "3", - "--", - }, args...) - cmd.ExtraFiles = []*os.File{child} - return cmd - }, - func(restorePid int) error { - log.CRDbg("restorePid=%d", restorePid) - if restorePid == 0 { - restoreCallback(&c.ProcessConfig, 0) - return nil - } - - // The container.json file should be written *after* the container - // has started because its StdFds cannot be initialized before. - // - // XXX How do we handle error here? - d.writeContainerFile(container, c.ID) - close(waitForRestore) - if restoreCallback != nil { - c.ProcessConfig.Process, err = os.FindProcess(restorePid) - if err != nil { - log.Debugf("cannot find restored process %d", restorePid) - return err - } - c.ContainerPid = c.ProcessConfig.Process.Pid - restoreCallback(&c.ProcessConfig, c.ContainerPid) - } - return nil - }) - restoreOutputChan <- restoreOutput{exitCode, err} - }() + // FIXME: no idea if any of this is needed... + if restoreCallback != nil { + pid, err := p.Pid() + if err != nil { + p.Signal(os.Kill) + p.Wait() + return execdriver.ExitStatus{ExitCode: -1}, err + } + restoreCallback(&c.ProcessConfig, pid) + } - select { - case restoreOutput := <-restoreOutputChan: - // there was an error - return restoreOutput.exitCode, restoreOutput.err - case <-waitForRestore: - // container restored - break + oom := notifyOnOOM(cont) + waitF := p.Wait + if nss := cont.Config().Namespaces; !nss.Contains(configs.NEWPID) { + // we need such hack for tracking processes with inherited fds, + // because cmd.Wait() waiting for all streams to be copied + waitF = waitInPIDHost(p, cont) + } + ps, err := waitF() + if err != nil { + execErr, ok := err.(*exec.ExitError) + if !ok { + return execdriver.ExitStatus{ExitCode: -1}, err + } + ps = execErr.ProcessState } - // Wait for the container to exit. - restoreOutput := <-restoreOutputChan - return restoreOutput.exitCode, restoreOutput.err + cont.Destroy() + _, oomKill := <-oom + return execdriver.ExitStatus{ExitCode: utils.ExitStatus(ps.Sys().(syscall.WaitStatus)), OOMKilled: oomKill}, nil } func (d *driver) Terminate(c *execdriver.Command) error { From a3b970f11aaf2dc8fb4891d890c2497bc4888446 Mon Sep 17 00:00:00 2001 From: Saied Kazemi Date: Thu, 5 Feb 2015 20:37:07 -0800 Subject: [PATCH 20/26] Checkpoint/Restore Support: add functionality to daemon Support was added to the daemon to use the Checkpoint and Restore methods of the native exec driver for checkpointing and restoring containers. Signed-off-by: Saied Kazemi Conflicts: api/server/server.go daemon/container.go daemon/daemon.go daemon/networkdriver/bridge/driver.go daemon/state.go vendor/src/github.com/docker/libnetwork/ipallocator/allocator.go --- api/server/server.go | 76 +++++++++++++++++++++++++++------------ daemon/checkpoint.go | 55 ++++++++++++++++++++++++++++ daemon/container.go | 66 ++++++++++++++++++++++++++++++++-- daemon/container_linux.go | 47 ++++++++++++++++++++++++ daemon/daemon.go | 31 ++++++++++++++++ daemon/monitor.go | 70 ++++++++++++++++++++++++++++++++++++ daemon/state.go | 23 ++++++++++++ 7 files changed, 344 insertions(+), 24 deletions(-) create mode 100644 daemon/checkpoint.go diff --git a/api/server/server.go b/api/server/server.go index 0c165d024e198..2d9132f75be01 100644 --- a/api/server/server.go +++ b/api/server/server.go @@ -1348,6 +1348,36 @@ func (s *Server) postContainersCopy(version version.Version, w http.ResponseWrit return nil } +func postContainersCheckpoint(eng *engine.Engine, version version.Version, w http.ResponseWriter, r *http.Request, vars map[string]string) error { + if vars == nil { + return fmt.Errorf("Missing parameter") + } + if err := parseForm(r); err != nil { + return err + } + job := eng.Job("checkpoint", vars["name"]) + if err := job.Run(); err != nil { + return err + } + w.WriteHeader(http.StatusNoContent) + return nil +} + +func postContainersRestore(eng *engine.Engine, version version.Version, w http.ResponseWriter, r *http.Request, vars map[string]string) error { + if vars == nil { + return fmt.Errorf("Missing parameter") + } + if err := parseForm(r); err != nil { + return err + } + job := eng.Job("restore", vars["name"]) + if err := job.Run(); err != nil { + return err + } + w.WriteHeader(http.StatusNoContent) + return nil +} + func (s *Server) postContainerExecCreate(version version.Version, w http.ResponseWriter, r *http.Request, vars map[string]string) error { if err := parseForm(r); err != nil { return err @@ -1559,28 +1589,30 @@ func createRouter(s *Server) *mux.Router { "/exec/{id:.*}/json": s.getExecByID, }, "POST": { - "/auth": s.postAuth, - "/commit": s.postCommit, - "/build": s.postBuild, - "/images/create": s.postImagesCreate, - "/images/load": s.postImagesLoad, - "/images/{name:.*}/push": s.postImagesPush, - "/images/{name:.*}/tag": s.postImagesTag, - "/containers/create": s.postContainersCreate, - "/containers/{name:.*}/kill": s.postContainersKill, - "/containers/{name:.*}/pause": s.postContainersPause, - "/containers/{name:.*}/unpause": s.postContainersUnpause, - "/containers/{name:.*}/restart": s.postContainersRestart, - "/containers/{name:.*}/start": s.postContainersStart, - "/containers/{name:.*}/stop": s.postContainersStop, - "/containers/{name:.*}/wait": s.postContainersWait, - "/containers/{name:.*}/resize": s.postContainersResize, - "/containers/{name:.*}/attach": s.postContainersAttach, - "/containers/{name:.*}/copy": s.postContainersCopy, - "/containers/{name:.*}/exec": s.postContainerExecCreate, - "/exec/{name:.*}/start": s.postContainerExecStart, - "/exec/{name:.*}/resize": s.postContainerExecResize, - "/containers/{name:.*}/rename": s.postContainerRename, + "/auth": s.postAuth, + "/commit": s.postCommit, + "/build": s.postBuild, + "/images/create": s.postImagesCreate, + "/images/load": s.postImagesLoad, + "/images/{name:.*}/push": s.postImagesPush, + "/images/{name:.*}/tag": s.postImagesTag, + "/containers/create": s.postContainersCreate, + "/containers/{name:.*}/kill": s.postContainersKill, + "/containers/{name:.*}/pause": s.postContainersPause, + "/containers/{name:.*}/unpause": s.postContainersUnpause, + "/containers/{name:.*}/restart": s.postContainersRestart, + "/containers/{name:.*}/start": s.postContainersStart, + "/containers/{name:.*}/stop": s.postContainersStop, + "/containers/{name:.*}/wait": s.postContainersWait, + "/containers/{name:.*}/resize": s.postContainersResize, + "/containers/{name:.*}/attach": s.postContainersAttach, + "/containers/{name:.*}/copy": s.postContainersCopy, + "/containers/{name:.*}/exec": s.postContainerExecCreate, + "/exec/{name:.*}/start": s.postContainerExecStart, + "/exec/{name:.*}/resize": s.postContainerExecResize, + "/containers/{name:.*}/rename": s.postContainerRename, + "/containers/{name:.*}/checkpoint": s.postContainersCheckpoint, + "/containers/{name:.*}/restore": s.postContainersRestore, }, "DELETE": { "/containers/{name:.*}": s.deleteContainers, diff --git a/daemon/checkpoint.go b/daemon/checkpoint.go new file mode 100644 index 0000000000000..f6057c6a028f9 --- /dev/null +++ b/daemon/checkpoint.go @@ -0,0 +1,55 @@ +package daemon + +import ( + "github.com/docker/docker/engine" +) + +// Checkpoint a running container. +func (daemon *Daemon) ContainerCheckpoint(job *engine.Job) engine.Status { + if len(job.Args) != 1 { + return job.Errorf("Usage: %s CONTAINER\n", job.Name) + } + + name := job.Args[0] + container, err := daemon.Get(name) + if err != nil { + return job.Error(err) + } + if !container.IsRunning() { + return job.Errorf("Container %s not running", name) + } + + if err := container.Checkpoint(); err != nil { + return job.Errorf("Cannot checkpoint container %s: %s", name, err) + } + + container.LogEvent("checkpoint") + return engine.StatusOK +} + +// Restore a checkpointed container. +func (daemon *Daemon) ContainerRestore(job *engine.Job) engine.Status { + if len(job.Args) != 1 { + return job.Errorf("Usage: %s CONTAINER\n", job.Name) + } + + name := job.Args[0] + container, err := daemon.Get(name) + if err != nil { + return job.Error(err) + } + if container.IsRunning() { + return job.Errorf("Container %s already running", name) + } + if !container.State.IsCheckpointed() { + return job.Errorf("Container %s is not checkpointed", name) + } + + if err := container.Restore(); err != nil { + container.LogEvent("die") + return job.Errorf("Cannot restore container %s: %s", name, err) + } + + container.LogEvent("restore") + return engine.StatusOK +} diff --git a/daemon/container.go b/daemon/container.go index a06033c890df6..27b34cfa36a46 100644 --- a/daemon/container.go +++ b/daemon/container.go @@ -341,10 +341,15 @@ func (container *Container) isNetworkAllocated() bool { return container.NetworkSettings.IPAddress != "" } + // cleanup releases any network resources allocated to the container along with any rules // around how containers are linked together. It also unmounts the container's root filesystem. func (container *Container) cleanup() { - container.ReleaseNetwork() + if container.IsCheckpointed() { + log.CRDbg("not calling ReleaseNetwork() for checkpointed container %s", container.ID) + } else { + container.ReleaseNetwork() + } disableAllActiveLinks(container) @@ -674,6 +679,41 @@ func (container *Container) Copy(resource string) (io.ReadCloser, error) { return reader, nil } +func (container *Container) Checkpoint() error { + return container.daemon.Checkpoint(container) +} + +func (container *Container) Restore() error { + var err error + + container.Lock() + defer container.Unlock() + + defer func() { + if err != nil { + container.cleanup() + } + }() + + if err = container.initializeNetworking(); err != nil { + return err + } + + linkedEnv, err := container.setupLinkedContainers() + if err != nil { + return err + } + if err = container.setupWorkingDirectory(); err != nil { + return err + } + env := container.createDaemonEnvironment(linkedEnv) + if err = populateCommandRestore(container, env); err != nil { + return err + } + + return container.waitForRestore() +} + // Returns true if the container exposes a certain port func (container *Container) Exposes(p nat.Port) bool { _, exists := container.Config.ExposedPorts[p] @@ -765,6 +805,29 @@ func (container *Container) waitForStart() error { return nil } +// Like waitForStart() but for restoring a container. +// +// XXX Does RestartPolicy apply here? +func (container *Container) waitForRestore() error { + container.monitor = newContainerMonitor(container, container.hostConfig.RestartPolicy) + + // After calling promise.Go() we'll have two goroutines: + // - The current goroutine that will block in the select + // below until restore is done. + // - A new goroutine that will restore the container and + // wait for it to exit. + select { + case <-container.monitor.restoreSignal: + if container.ExitCode != 0 { + return fmt.Errorf("restore process failed") + } + case err := <-promise.Go(container.monitor.Restore): + return err + } + + return nil +} + func (container *Container) GetProcessLabel() string { // even if we have a process label return "" if we are running // in privileged mode @@ -972,7 +1035,6 @@ func attach(streamConfig *StreamConfig, openStdin, stdinOnce, tty bool, stdin io _, err = copyEscapable(cStdin, stdin) } else { _, err = io.Copy(cStdin, stdin) - } if err == io.ErrClosedPipe { err = nil diff --git a/daemon/container_linux.go b/daemon/container_linux.go index 8c5b15be0a382..57dad06e73d7b 100644 --- a/daemon/container_linux.go +++ b/daemon/container_linux.go @@ -329,6 +329,53 @@ func mergeDevices(defaultDevices, userDevices []*configs.Device) []*configs.Devi return append(devs, userDevices...) } +// Like populateCommand() but for restoring a container. +// +// XXX populateCommand() does a lot more. Not sure if we have +// to do everything it does. +func populateCommandRestore(c *Container, env []string) error { + resources := &execdriver.Resources{ + Memory: c.Config.Memory, + MemorySwap: c.Config.MemorySwap, + CpuShares: c.Config.CpuShares, + Cpuset: c.Config.Cpuset, + } + + processConfig := execdriver.ProcessConfig{ + Privileged: c.hostConfig.Privileged, + Entrypoint: c.Path, + Arguments: c.Args, + Tty: c.Config.Tty, + User: c.Config.User, + } + + processConfig.SysProcAttr = &syscall.SysProcAttr{Setsid: true} + processConfig.Env = env + + c.command = &execdriver.Command{ + ID: c.ID, + Rootfs: c.RootfsPath(), + ReadonlyRootfs: c.hostConfig.ReadonlyRootfs, + InitPath: "/.dockerinit", + WorkingDir: c.Config.WorkingDir, + // Network: en, + // Ipc: ipc, + // Pid: pid, + Resources: resources, + // AllowedDevices: allowedDevices, + // AutoCreatedDevices: autoCreatedDevices, + CapAdd: c.hostConfig.CapAdd, + CapDrop: c.hostConfig.CapDrop, + ProcessConfig: processConfig, + ProcessLabel: c.GetProcessLabel(), + MountLabel: c.GetMountLabel(), + // LxcConfig: lxcConfig, + AppArmorProfile: c.AppArmorProfile, + } + + return nil +} + // GetSize, return real size, virtual size func (container *Container) GetSize() (int64, int64) { var ( diff --git a/daemon/daemon.go b/daemon/daemon.go index 549a1c188ead3..652a46b518ed6 100644 --- a/daemon/daemon.go +++ b/daemon/daemon.go @@ -277,6 +277,18 @@ func (daemon *Daemon) restore() error { logrus.Debugf("Loaded container %v", container.ID) containers[container.ID] = &cr{container: container} + + // If the container was checkpointed, we need to reserve + // the IP address that it was using. + // + // XXX We should also reserve host ports (if any). + if container.IsCheckpointed() { + /*err = bridge.ReserveIP(container.ID, container.NetworkSettings.IPAddress) + if err != nil { + log.Errorf("Failed to reserve IP %s for container %s", + container.ID, container.NetworkSettings.IPAddress) + }*/ + } } else { logrus.Debugf("Cannot load container %s because it was created with another graph driver.", container.ID) } @@ -812,6 +824,25 @@ func (daemon *Daemon) Run(c *Container, pipes *execdriver.Pipes, startCallback e return daemon.execDriver.Run(c.command, pipes, startCallback) } +func (daemon *Daemon) Checkpoint(c *Container) error { + if err := daemon.execDriver.Checkpoint(c.command); err != nil { + return err + } + c.SetCheckpointed() + return nil +} + +func (daemon *Daemon) Restore(c *Container, pipes *execdriver.Pipes, restoreCallback execdriver.RestoreCallback) (int, error) { + // Mount the container's filesystem (daemon/graphdriver/aufs/aufs.go). + _, err := daemon.driver.Get(c.ID, c.GetMountLabel()) + if err != nil { + return 0, err + } + + exitCode, err := daemon.execDriver.Restore(c.command, pipes, restoreCallback) + return exitCode, err +} + func (daemon *Daemon) Kill(c *Container, sig int) error { return daemon.execDriver.Kill(c.command, sig) } diff --git a/daemon/monitor.go b/daemon/monitor.go index ff173c8f0324e..a1310cfa9efa8 100644 --- a/daemon/monitor.go +++ b/daemon/monitor.go @@ -44,6 +44,9 @@ type containerMonitor struct { // left waiting for nothing to happen during this time stopChan chan struct{} + // like startSignal but for restoring a container + restoreSignal chan struct{} + // timeIncrement is the amount of time to wait between restarts // this is in milliseconds timeIncrement int @@ -61,6 +64,7 @@ func newContainerMonitor(container *Container, policy runconfig.RestartPolicy) * timeIncrement: defaultTimeIncrement, stopChan: make(chan struct{}), startSignal: make(chan struct{}), + restoreSignal: make(chan struct{}), } } @@ -181,6 +185,49 @@ func (m *containerMonitor) Start() error { } } +// Like Start() but for restoring a container. +func (m *containerMonitor) Restore() error { + var ( + err error + // XXX The following line should be changed to + // exitStatus execdriver.ExitStatus to match Start() + exitCode int + afterRestore bool + ) + + defer func() { + if afterRestore { + m.container.Lock() + m.container.setStopped(&execdriver.ExitStatus{exitCode, false}) + defer m.container.Unlock() + } + m.Close() + }() + + if err := m.container.startLoggingToDisk(); err != nil { + m.resetContainer(false) + return err + } + + pipes := execdriver.NewPipes(m.container.stdin, m.container.stdout, m.container.stderr, m.container.Config.OpenStdin) + + m.container.LogEvent("restore") + m.lastStartTime = time.Now() + if exitCode, err = m.container.daemon.Restore(m.container, pipes, m.restoreCallback); err != nil { + log.Errorf("Error restoring container: %s, exitCode=%d", err, exitCode) + m.container.ExitCode = -1 + m.resetContainer(false) + return err + } + afterRestore = true + + m.container.ExitCode = exitCode + m.resetMonitor(err == nil && exitCode == 0) + m.container.LogEvent("die") + m.resetContainer(true) + return err +} + // resetMonitor resets the stateful fields on the containerMonitor based on the // previous runs success or failure. Regardless of success, if the container had // an execution time of more than 10s then reset the timer back to the default @@ -267,6 +314,29 @@ func (m *containerMonitor) callback(processConfig *execdriver.ProcessConfig, pid } } +// Like callback() but for restoring a container. +func (m *containerMonitor) restoreCallback(processConfig *execdriver.ProcessConfig, restorePid int) { + // If restorePid is 0, it means that restore failed. + if restorePid != 0 { + m.container.setRunning(restorePid) + } + + // Unblock the goroutine waiting in waitForRestore(). + select { + case <-m.restoreSignal: + default: + close(m.restoreSignal) + } + + if restorePid != 0 { + // Write config.json and hostconfig.json files + // to /var/lib/docker/containers/. + if err := m.container.ToDisk(); err != nil { + log.Debugf("%s", err) + } + } +} + // resetContainer resets the container's IO and ensures that the command is able to be executed again // by copying the data into a new struct // if lock is true, then container locked during reset diff --git a/daemon/state.go b/daemon/state.go index 0270c88e88774..82eefe75a3a6c 100644 --- a/daemon/state.go +++ b/daemon/state.go @@ -14,6 +14,7 @@ type State struct { Running bool Paused bool Restarting bool + Checkpointed bool OOMKilled bool removalInProgress bool // Not need for this to be persistent on disk. Dead bool @@ -22,7 +23,9 @@ type State struct { Error string // contains last known error when starting the container StartedAt time.Time FinishedAt time.Time + CheckpointedAt time.Time waitChan chan struct{} + } func NewState() *State { @@ -42,6 +45,8 @@ func (s *State) String() string { } return fmt.Sprintf("Up %s", units.HumanDuration(time.Now().UTC().Sub(s.StartedAt))) + } else if s.Checkpointed { + return fmt.Sprintf("Checkpointed %s ago", units.HumanDuration(time.Now().UTC().Sub(s.CheckpointedAt))) } if s.removalInProgress { @@ -166,6 +171,7 @@ func (s *State) setRunning(pid int) { s.Error = "" s.Running = true s.Paused = false + s.Checkpointed = false s.Restarting = false s.ExitCode = 0 s.Pid = pid @@ -262,3 +268,20 @@ func (s *State) SetDead() { s.Dead = true s.Unlock() } + +func (s *State) SetCheckpointed() { + s.Lock() + s.CheckpointedAt = time.Now().UTC() + s.Checkpointed = true + s.Running = false + s.Paused = false + s.Restarting = false + // XXX Not sure if we need to close and recreate waitChan. + // close(s.waitChan) + // s.waitChan = make(chan struct{}) + s.Unlock() +} + +func (s *State) IsCheckpointed() bool { + return s.Checkpointed +} From b3c2a8ed29770b99f25e6159adee645eeeada5e7 Mon Sep 17 00:00:00 2001 From: Hui Kang Date: Tue, 19 May 2015 21:08:04 +0000 Subject: [PATCH 21/26] Release the network resource during checkpoint Restore failed if network resource not released during checkpoint, e.g., a container with port open with -p Signed-off-by: Hui Kang Conflicts: daemon/container.go --- daemon/container.go | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/daemon/container.go b/daemon/container.go index 27b34cfa36a46..0b4c4a2d632f4 100644 --- a/daemon/container.go +++ b/daemon/container.go @@ -608,6 +608,19 @@ func validateID(id string) error { return nil } + +func (container *Container) Checkpoint(opts *libcontainer.CriuOpts) error { + if err := container.daemon.Checkpoint(container, opts); err != nil { + return err + } + + if opts.LeaveRunning == false { + container.ReleaseNetwork() + } + return nil +} + + func (container *Container) Copy(resource string) (io.ReadCloser, error) { container.Lock() defer container.Unlock() From 10003bf85a5979b8a9f03c4511df284e761860a6 Mon Sep 17 00:00:00 2001 From: boucher Date: Mon, 1 Jun 2015 15:15:02 -0700 Subject: [PATCH 22/26] Update daemon and cli support for checkpoint and restore. Docker-DCO-1.1-Signed-off-by: Ross Boucher (github: boucher) --- api/client/checkpoint.go | 52 ++++++++++ api/client/restore.go | 54 +++++++++++ api/server/server.go | 24 +++-- daemon/checkpoint.go | 57 +++++------ daemon/container.go | 95 ++++++++++--------- daemon/container_linux.go | 71 ++++---------- daemon/container_windows.go | 2 +- daemon/daemon.go | 12 +-- daemon/execdriver/driver.go | 5 +- daemon/execdriver/lxc/driver.go | 5 +- daemon/execdriver/native/driver.go | 20 +++- daemon/execdriver/windows/windows.go | 9 ++ daemon/monitor.go | 26 ++--- daemon/state.go | 21 ++-- docker/flags.go | 4 +- integration-cli/docker_cli_help_test.go | 2 +- runconfig/restore.go | 15 +++ .../docker/libcontainer/container_linux.go | 12 +-- 18 files changed, 311 insertions(+), 175 deletions(-) create mode 100644 api/client/checkpoint.go create mode 100644 api/client/restore.go create mode 100644 runconfig/restore.go diff --git a/api/client/checkpoint.go b/api/client/checkpoint.go new file mode 100644 index 0000000000000..8c681bcf9716f --- /dev/null +++ b/api/client/checkpoint.go @@ -0,0 +1,52 @@ +package client + +import ( + "fmt" + + flag "github.com/docker/docker/pkg/mflag" + "github.com/docker/docker/runconfig" +) + +func (cli *DockerCli) CmdCheckpoint(args ...string) error { + cmd := cli.Subcmd("checkpoint", []string{"CONTAINER [CONTAINER...]"}, "Checkpoint one or more running containers", true) + cmd.Require(flag.Min, 1) + + var ( + flImgDir = cmd.String([]string{"-image-dir"}, "", "directory for storing checkpoint image files") + flWorkDir = cmd.String([]string{"-work-dir"}, "", "directory for storing log file") + flLeaveRunning = cmd.Bool([]string{"-leave-running"}, false, "leave the container running after checkpoint") + flCheckTcp = cmd.Bool([]string{"-allow-tcp"}, false, "allow checkpointing tcp connections") + flExtUnix = cmd.Bool([]string{"-allow-ext-unix"}, false, "allow checkpointing external unix connections") + flShell = cmd.Bool([]string{"-allow-shell"}, false, "allow checkpointing shell jobs") + ) + + if err := cmd.ParseFlags(args, true); err != nil { + return err + } + + if cmd.NArg() < 1 { + cmd.Usage() + return nil + } + + criuOpts := &runconfig.CriuConfig{ + ImagesDirectory: *flImgDir, + WorkDirectory: *flWorkDir, + LeaveRunning: *flLeaveRunning, + TcpEstablished: *flCheckTcp, + ExternalUnixConnections: *flExtUnix, + ShellJob: *flShell, + } + + var encounteredError error + for _, name := range cmd.Args() { + _, _, err := readBody(cli.call("POST", "/containers/"+name+"/checkpoint", criuOpts, nil)) + if err != nil { + fmt.Fprintf(cli.err, "%s\n", err) + encounteredError = fmt.Errorf("Error: failed to checkpoint one or more containers") + } else { + fmt.Fprintf(cli.out, "%s\n", name) + } + } + return encounteredError +} diff --git a/api/client/restore.go b/api/client/restore.go new file mode 100644 index 0000000000000..0c4085fbbbd84 --- /dev/null +++ b/api/client/restore.go @@ -0,0 +1,54 @@ +package client + +import ( + "fmt" + + flag "github.com/docker/docker/pkg/mflag" + "github.com/docker/docker/runconfig" +) + +func (cli *DockerCli) CmdRestore(args ...string) error { + cmd := cli.Subcmd("restore", []string{"CONTAINER [CONTAINER...]"}, "Restore one or more checkpointed containers", true) + cmd.Require(flag.Min, 1) + + var ( + flImgDir = cmd.String([]string{"-image-dir"}, "", "directory to restore image files from") + flWorkDir = cmd.String([]string{"-work-dir"}, "", "directory for restore log") + flCheckTcp = cmd.Bool([]string{"-allow-tcp"}, false, "allow restoring tcp connections") + flExtUnix = cmd.Bool([]string{"-allow-ext-unix"}, false, "allow restoring external unix connections") + flShell = cmd.Bool([]string{"-allow-shell"}, false, "allow restoring shell jobs") + flForce = cmd.Bool([]string{"-force"}, false, "bypass checks for current container state") + ) + + if err := cmd.ParseFlags(args, true); err != nil { + return err + } + + if cmd.NArg() < 1 { + cmd.Usage() + return nil + } + + restoreOpts := &runconfig.RestoreConfig{ + CriuOpts: runconfig.CriuConfig{ + ImagesDirectory: *flImgDir, + WorkDirectory: *flWorkDir, + TcpEstablished: *flCheckTcp, + ExternalUnixConnections: *flExtUnix, + ShellJob: *flShell, + }, + ForceRestore: *flForce, + } + + var encounteredError error + for _, name := range cmd.Args() { + _, _, err := readBody(cli.call("POST", "/containers/"+name+"/restore", restoreOpts, nil)) + if err != nil { + fmt.Fprintf(cli.err, "%s\n", err) + encounteredError = fmt.Errorf("Error: failed to restore one or more containers") + } else { + fmt.Fprintf(cli.out, "%s\n", name) + } + } + return encounteredError +} diff --git a/api/server/server.go b/api/server/server.go index 2d9132f75be01..b8c44b6531491 100644 --- a/api/server/server.go +++ b/api/server/server.go @@ -1348,32 +1348,44 @@ func (s *Server) postContainersCopy(version version.Version, w http.ResponseWrit return nil } -func postContainersCheckpoint(eng *engine.Engine, version version.Version, w http.ResponseWriter, r *http.Request, vars map[string]string) error { +func (s *Server) postContainersCheckpoint(version version.Version, w http.ResponseWriter, r *http.Request, vars map[string]string) error { if vars == nil { return fmt.Errorf("Missing parameter") } if err := parseForm(r); err != nil { return err } - job := eng.Job("checkpoint", vars["name"]) - if err := job.Run(); err != nil { + + criuOpts := &runconfig.CriuConfig{} + if err := json.NewDecoder(r.Body).Decode(criuOpts); err != nil { return err } + + if err := s.daemon.ContainerCheckpoint(vars["name"], criuOpts); err != nil { + return err + } + w.WriteHeader(http.StatusNoContent) return nil } -func postContainersRestore(eng *engine.Engine, version version.Version, w http.ResponseWriter, r *http.Request, vars map[string]string) error { +func (s *Server) postContainersRestore(version version.Version, w http.ResponseWriter, r *http.Request, vars map[string]string) error { if vars == nil { return fmt.Errorf("Missing parameter") } if err := parseForm(r); err != nil { return err } - job := eng.Job("restore", vars["name"]) - if err := job.Run(); err != nil { + + restoreOpts := runconfig.RestoreConfig{} + if err := json.NewDecoder(r.Body).Decode(&restoreOpts); err != nil { return err } + + if err := s.daemon.ContainerRestore(vars["name"], &restoreOpts.CriuOpts, restoreOpts.ForceRestore); err != nil { + return err + } + w.WriteHeader(http.StatusNoContent) return nil } diff --git a/daemon/checkpoint.go b/daemon/checkpoint.go index f6057c6a028f9..a39662cc0f325 100644 --- a/daemon/checkpoint.go +++ b/daemon/checkpoint.go @@ -1,55 +1,56 @@ package daemon import ( - "github.com/docker/docker/engine" + "fmt" + + "github.com/docker/docker/runconfig" ) // Checkpoint a running container. -func (daemon *Daemon) ContainerCheckpoint(job *engine.Job) engine.Status { - if len(job.Args) != 1 { - return job.Errorf("Usage: %s CONTAINER\n", job.Name) - } - - name := job.Args[0] +func (daemon *Daemon) ContainerCheckpoint(name string, opts *runconfig.CriuConfig) error { container, err := daemon.Get(name) if err != nil { - return job.Error(err) + return err } if !container.IsRunning() { - return job.Errorf("Container %s not running", name) + return fmt.Errorf("Container %s not running", name) } - - if err := container.Checkpoint(); err != nil { - return job.Errorf("Cannot checkpoint container %s: %s", name, err) + if err := container.Checkpoint(opts); err != nil { + return fmt.Errorf("Cannot checkpoint container %s: %s", name, err) } container.LogEvent("checkpoint") - return engine.StatusOK + return nil } // Restore a checkpointed container. -func (daemon *Daemon) ContainerRestore(job *engine.Job) engine.Status { - if len(job.Args) != 1 { - return job.Errorf("Usage: %s CONTAINER\n", job.Name) - } - - name := job.Args[0] +func (daemon *Daemon) ContainerRestore(name string, opts *runconfig.CriuConfig, forceRestore bool) error { container, err := daemon.Get(name) if err != nil { - return job.Error(err) - } - if container.IsRunning() { - return job.Errorf("Container %s already running", name) + return err } - if !container.State.IsCheckpointed() { - return job.Errorf("Container %s is not checkpointed", name) + + if !forceRestore { + // TODO: It's possible we only want to bypass the checkpointed check, + // I'm not sure how this will work if the container is already running + if container.IsRunning() { + return fmt.Errorf("Container %s already running", name) + } + + if !container.IsCheckpointed() { + return fmt.Errorf("Container %s is not checkpointed", name) + } + } else { + if !container.HasBeenCheckpointed() && opts.ImagesDirectory == "" { + return fmt.Errorf("You must specify an image directory to restore from %s", name) + } } - if err := container.Restore(); err != nil { + if err = container.Restore(opts, forceRestore); err != nil { container.LogEvent("die") - return job.Errorf("Cannot restore container %s: %s", name, err) + return fmt.Errorf("Cannot restore container %s: %s", name, err) } container.LogEvent("restore") - return engine.StatusOK + return nil } diff --git a/daemon/container.go b/daemon/container.go index 0b4c4a2d632f4..efaeaa71867a4 100644 --- a/daemon/container.go +++ b/daemon/container.go @@ -258,7 +258,7 @@ func (container *Container) Start() (err error) { if err := container.Mount(); err != nil { return err } - if err := container.initializeNetworking(); err != nil { + if err := container.initializeNetworking(false); err != nil { return err } linkedEnv, err := container.setupLinkedContainers() @@ -341,12 +341,11 @@ func (container *Container) isNetworkAllocated() bool { return container.NetworkSettings.IPAddress != "" } - // cleanup releases any network resources allocated to the container along with any rules // around how containers are linked together. It also unmounts the container's root filesystem. func (container *Container) cleanup() { if container.IsCheckpointed() { - log.CRDbg("not calling ReleaseNetwork() for checkpointed container %s", container.ID) + logrus.Debugf("not calling ReleaseNetwork() for checkpointed container %s", container.ID) } else { container.ReleaseNetwork() } @@ -608,8 +607,7 @@ func validateID(id string) error { return nil } - -func (container *Container) Checkpoint(opts *libcontainer.CriuOpts) error { +func (container *Container) Checkpoint(opts *runconfig.CriuConfig) error { if err := container.daemon.Checkpoint(container, opts); err != nil { return err } @@ -620,6 +618,50 @@ func (container *Container) Checkpoint(opts *libcontainer.CriuOpts) error { return nil } +func (container *Container) Restore(opts *runconfig.CriuConfig, forceRestore bool) error { + var err error + container.Lock() + defer container.Unlock() + + defer func() { + if err != nil { + container.setError(err) + // if no one else has set it, make sure we don't leave it at zero + if container.ExitCode == 0 { + container.ExitCode = 128 + } + container.toDisk() + container.cleanup() + } + }() + + if err := container.Mount(); err != nil { + return err + } + if err = container.initializeNetworking(true); err != nil { + return err + } + linkedEnv, err := container.setupLinkedContainers() + if err != nil { + return err + } + if err = container.setupWorkingDirectory(); err != nil { + return err + } + + env := container.createDaemonEnvironment(linkedEnv) + if err = populateCommand(container, env); err != nil { + return err + } + + mounts, err := container.setupMounts() + if err != nil { + return err + } + + container.command.Mounts = mounts + return container.waitForRestore(opts, forceRestore) +} func (container *Container) Copy(resource string) (io.ReadCloser, error) { container.Lock() @@ -692,41 +734,6 @@ func (container *Container) Copy(resource string) (io.ReadCloser, error) { return reader, nil } -func (container *Container) Checkpoint() error { - return container.daemon.Checkpoint(container) -} - -func (container *Container) Restore() error { - var err error - - container.Lock() - defer container.Unlock() - - defer func() { - if err != nil { - container.cleanup() - } - }() - - if err = container.initializeNetworking(); err != nil { - return err - } - - linkedEnv, err := container.setupLinkedContainers() - if err != nil { - return err - } - if err = container.setupWorkingDirectory(); err != nil { - return err - } - env := container.createDaemonEnvironment(linkedEnv) - if err = populateCommandRestore(container, env); err != nil { - return err - } - - return container.waitForRestore() -} - // Returns true if the container exposes a certain port func (container *Container) Exposes(p nat.Port) bool { _, exists := container.Config.ExposedPorts[p] @@ -818,10 +825,7 @@ func (container *Container) waitForStart() error { return nil } -// Like waitForStart() but for restoring a container. -// -// XXX Does RestartPolicy apply here? -func (container *Container) waitForRestore() error { +func (container *Container) waitForRestore(opts *runconfig.CriuConfig, forceRestore bool) error { container.monitor = newContainerMonitor(container, container.hostConfig.RestartPolicy) // After calling promise.Go() we'll have two goroutines: @@ -834,7 +838,7 @@ func (container *Container) waitForRestore() error { if container.ExitCode != 0 { return fmt.Errorf("restore process failed") } - case err := <-promise.Go(container.monitor.Restore): + case err := <-promise.Go(func() error { return container.monitor.Restore(opts, forceRestore) }): return err } @@ -1048,6 +1052,7 @@ func attach(streamConfig *StreamConfig, openStdin, stdinOnce, tty bool, stdin io _, err = copyEscapable(cStdin, stdin) } else { _, err = io.Copy(cStdin, stdin) + } if err == io.ErrClosedPipe { err = nil diff --git a/daemon/container_linux.go b/daemon/container_linux.go index 57dad06e73d7b..7e51782cdc8f9 100644 --- a/daemon/container_linux.go +++ b/daemon/container_linux.go @@ -329,53 +329,6 @@ func mergeDevices(defaultDevices, userDevices []*configs.Device) []*configs.Devi return append(devs, userDevices...) } -// Like populateCommand() but for restoring a container. -// -// XXX populateCommand() does a lot more. Not sure if we have -// to do everything it does. -func populateCommandRestore(c *Container, env []string) error { - resources := &execdriver.Resources{ - Memory: c.Config.Memory, - MemorySwap: c.Config.MemorySwap, - CpuShares: c.Config.CpuShares, - Cpuset: c.Config.Cpuset, - } - - processConfig := execdriver.ProcessConfig{ - Privileged: c.hostConfig.Privileged, - Entrypoint: c.Path, - Arguments: c.Args, - Tty: c.Config.Tty, - User: c.Config.User, - } - - processConfig.SysProcAttr = &syscall.SysProcAttr{Setsid: true} - processConfig.Env = env - - c.command = &execdriver.Command{ - ID: c.ID, - Rootfs: c.RootfsPath(), - ReadonlyRootfs: c.hostConfig.ReadonlyRootfs, - InitPath: "/.dockerinit", - WorkingDir: c.Config.WorkingDir, - // Network: en, - // Ipc: ipc, - // Pid: pid, - Resources: resources, - // AllowedDevices: allowedDevices, - // AutoCreatedDevices: autoCreatedDevices, - CapAdd: c.hostConfig.CapAdd, - CapDrop: c.hostConfig.CapDrop, - ProcessConfig: processConfig, - ProcessLabel: c.GetProcessLabel(), - MountLabel: c.GetMountLabel(), - // LxcConfig: lxcConfig, - AppArmorProfile: c.AppArmorProfile, - } - - return nil -} - // GetSize, return real size, virtual size func (container *Container) GetSize() (int64, int64) { var ( @@ -710,7 +663,7 @@ func (container *Container) UpdateNetwork() error { return nil } -func (container *Container) buildCreateEndpointOptions() ([]libnetwork.EndpointOption, error) { +func (container *Container) buildCreateEndpointOptions(restoring bool) ([]libnetwork.EndpointOption, error) { var ( portSpecs = make(nat.PortSet) bindings = make(nat.PortMap) @@ -781,6 +734,14 @@ func (container *Container) buildCreateEndpointOptions() ([]libnetwork.EndpointO createOptions = append(createOptions, libnetwork.EndpointOptionGeneric(genericOption)) } + /*if restoring && container.NetworkSettings.IPAddress != "" { + genericOption := options.Generic{ + netlabel.IPAddress: net.ParseIP(container.NetworkSettings.IPAddress), + } + + createOptions = append(createOptions, libnetwork.EndpointOptionGeneric(genericOption)) + }*/ + return createOptions, nil } @@ -834,7 +795,7 @@ func (container *Container) secondaryNetworkRequired(primaryNetworkType string) return false } -func (container *Container) AllocateNetwork() error { +func (container *Container) AllocateNetwork(isRestoring bool) error { mode := container.hostConfig.NetworkMode controller := container.daemon.netController if container.Config.NetworkDisabled || mode.IsContainer() { @@ -870,19 +831,19 @@ func (container *Container) AllocateNetwork() error { if container.secondaryNetworkRequired(networkDriver) { // Configure Bridge as secondary network for port binding purposes - if err := container.configureNetwork("bridge", service, "bridge", false); err != nil { + if err := container.configureNetwork("bridge", service, "bridge", false, isRestoring); err != nil { return err } } - if err := container.configureNetwork(networkName, service, networkDriver, mode.IsDefault()); err != nil { + if err := container.configureNetwork(networkName, service, networkDriver, mode.IsDefault(), isRestoring); err != nil { return err } return container.WriteHostConfig() } -func (container *Container) configureNetwork(networkName, service, networkDriver string, canCreateNetwork bool) error { +func (container *Container) configureNetwork(networkName, service, networkDriver string, canCreateNetwork bool, isRestoring bool) error { controller := container.daemon.netController n, err := controller.NetworkByName(networkName) if err != nil { @@ -901,7 +862,7 @@ func (container *Container) configureNetwork(networkName, service, networkDriver return err } - createOptions, err := container.buildCreateEndpointOptions() + createOptions, err := container.buildCreateEndpointOptions(isRestoring) if err != nil { return err } @@ -932,7 +893,7 @@ func (container *Container) configureNetwork(networkName, service, networkDriver return nil } -func (container *Container) initializeNetworking() error { +func (container *Container) initializeNetworking(restoring bool) error { var err error // Make sure NetworkMode has an acceptable value before @@ -968,7 +929,7 @@ func (container *Container) initializeNetworking() error { } - if err := container.AllocateNetwork(); err != nil { + if err := container.AllocateNetwork(restoring); err != nil { return err } diff --git a/daemon/container_windows.go b/daemon/container_windows.go index 8b8ef3e0401c0..c7288f335dd68 100644 --- a/daemon/container_windows.go +++ b/daemon/container_windows.go @@ -53,7 +53,7 @@ func (container *Container) createDaemonEnvironment(linkedEnv []string) []string return container.Config.Env } -func (container *Container) initializeNetworking() error { +func (container *Container) initializeNetworking(restoring bool) error { return nil } diff --git a/daemon/daemon.go b/daemon/daemon.go index 652a46b518ed6..a0b05024a3adf 100644 --- a/daemon/daemon.go +++ b/daemon/daemon.go @@ -824,22 +824,22 @@ func (daemon *Daemon) Run(c *Container, pipes *execdriver.Pipes, startCallback e return daemon.execDriver.Run(c.command, pipes, startCallback) } -func (daemon *Daemon) Checkpoint(c *Container) error { - if err := daemon.execDriver.Checkpoint(c.command); err != nil { +func (daemon *Daemon) Checkpoint(c *Container, opts *runconfig.CriuConfig) error { + if err := daemon.execDriver.Checkpoint(c.command, opts); err != nil { return err } - c.SetCheckpointed() + c.SetCheckpointed(opts.LeaveRunning) return nil } -func (daemon *Daemon) Restore(c *Container, pipes *execdriver.Pipes, restoreCallback execdriver.RestoreCallback) (int, error) { +func (daemon *Daemon) Restore(c *Container, pipes *execdriver.Pipes, restoreCallback execdriver.RestoreCallback, opts *runconfig.CriuConfig, forceRestore bool) (execdriver.ExitStatus, error) { // Mount the container's filesystem (daemon/graphdriver/aufs/aufs.go). _, err := daemon.driver.Get(c.ID, c.GetMountLabel()) if err != nil { - return 0, err + return execdriver.ExitStatus{ExitCode: 0}, err } - exitCode, err := daemon.execDriver.Restore(c.command, pipes, restoreCallback) + exitCode, err := daemon.execDriver.Restore(c.command, pipes, restoreCallback, opts, forceRestore) return exitCode, err } diff --git a/daemon/execdriver/driver.go b/daemon/execdriver/driver.go index 54d3955bb040c..80d775bac2640 100644 --- a/daemon/execdriver/driver.go +++ b/daemon/execdriver/driver.go @@ -8,6 +8,7 @@ import ( // TODO Windows: Factor out ulimit "github.com/docker/docker/pkg/ulimit" + "github.com/docker/docker/runconfig" "github.com/docker/libcontainer" "github.com/docker/libcontainer/configs" ) @@ -60,8 +61,8 @@ type Driver interface { Kill(c *Command, sig int) error Pause(c *Command) error Unpause(c *Command) error - Checkpoint(c *Command, opts *libcontainer.CriuOpts) error - Restore(c *Command, pipes *Pipes, restoreCallback RestoreCallback, opts *libcontainer.CriuOpts, forceRestore bool) (ExitStatus, error) + Checkpoint(c *Command, opts *runconfig.CriuConfig) error + Restore(c *Command, pipes *Pipes, restoreCallback RestoreCallback, opts *runconfig.CriuConfig, forceRestore bool) (ExitStatus, error) Name() string // Driver name Info(id string) Info // "temporary" hack (until we move state from core to plugins) GetPidsForContainer(id string) ([]int, error) // Returns a list of pids for the given container. diff --git a/daemon/execdriver/lxc/driver.go b/daemon/execdriver/lxc/driver.go index 7ca41abe818c4..6116a1e87ce42 100644 --- a/daemon/execdriver/lxc/driver.go +++ b/daemon/execdriver/lxc/driver.go @@ -25,6 +25,7 @@ import ( sysinfo "github.com/docker/docker/pkg/system" "github.com/docker/docker/pkg/term" "github.com/docker/docker/pkg/version" + "github.com/docker/docker/runconfig" "github.com/docker/libcontainer" "github.com/docker/libcontainer/cgroups" "github.com/docker/libcontainer/configs" @@ -547,11 +548,11 @@ func (d *driver) Unpause(c *execdriver.Command) error { return err } -func (d *driver) Checkpoint(c *execdriver.Command, opts *libcontainer.CriuOpts) error { +func (d *driver) Checkpoint(c *execdriver.Command, opts *runconfig.CriuConfig) error { return fmt.Errorf("Checkpointing lxc containers not supported yet\n") } -func (d *driver) Restore(c *execdriver.Command, pipes *execdriver.Pipes, restoreCallback execdriver.RestoreCallback, opts *libcontainer.CriuOpts, forceRestore bool) (execdriver.ExitStatus, error) { +func (d *driver) Restore(c *execdriver.Command, pipes *execdriver.Pipes, restoreCallback execdriver.RestoreCallback, opts *runconfig.CriuConfig, forceRestore bool) (execdriver.ExitStatus, error) { return execdriver.ExitStatus{ExitCode: 0}, fmt.Errorf("Restoring lxc containers not supported yet\n") } diff --git a/daemon/execdriver/native/driver.go b/daemon/execdriver/native/driver.go index 7e1bdf3a6ad4b..04353131efe9e 100644 --- a/daemon/execdriver/native/driver.go +++ b/daemon/execdriver/native/driver.go @@ -20,6 +20,7 @@ import ( "github.com/docker/docker/pkg/reexec" sysinfo "github.com/docker/docker/pkg/system" "github.com/docker/docker/pkg/term" + "github.com/docker/docker/runconfig" "github.com/docker/libcontainer" "github.com/docker/libcontainer/apparmor" "github.com/docker/libcontainer/cgroups/systemd" @@ -277,7 +278,18 @@ func (d *driver) Unpause(c *execdriver.Command) error { return active.Resume() } -func (d *driver) Checkpoint(c *execdriver.Command, opts *libcontainer.CriuOpts) error { +func libcontainerCriuOpts(runconfigOpts *runconfig.CriuConfig) *libcontainer.CriuOpts { + return &libcontainer.CriuOpts{ + ImagesDirectory: runconfigOpts.ImagesDirectory, + WorkDirectory: runconfigOpts.WorkDirectory, + LeaveRunning: runconfigOpts.LeaveRunning, + TcpEstablished: runconfigOpts.TcpEstablished, + ExternalUnixConnections: runconfigOpts.ExternalUnixConnections, + ShellJob: runconfigOpts.ShellJob, + } +} + +func (d *driver) Checkpoint(c *execdriver.Command, opts *runconfig.CriuConfig) error { active := d.activeContainers[c.ID] if active == nil { return fmt.Errorf("active container for %s does not exist", c.ID) @@ -285,7 +297,7 @@ func (d *driver) Checkpoint(c *execdriver.Command, opts *libcontainer.CriuOpts) d.Lock() defer d.Unlock() - err := active.Checkpoint(opts) + err := active.Checkpoint(libcontainerCriuOpts(opts)) if err != nil { return err } @@ -293,7 +305,7 @@ func (d *driver) Checkpoint(c *execdriver.Command, opts *libcontainer.CriuOpts) return nil } -func (d *driver) Restore(c *execdriver.Command, pipes *execdriver.Pipes, restoreCallback execdriver.RestoreCallback, opts *libcontainer.CriuOpts, forceRestore bool) (execdriver.ExitStatus, error) { +func (d *driver) Restore(c *execdriver.Command, pipes *execdriver.Pipes, restoreCallback execdriver.RestoreCallback, opts *runconfig.CriuConfig, forceRestore bool) (execdriver.ExitStatus, error) { var ( cont libcontainer.Container err error @@ -336,7 +348,7 @@ func (d *driver) Restore(c *execdriver.Command, pipes *execdriver.Pipes, restore d.cleanContainer(c.ID) }() - if err := cont.Restore(p, opts); err != nil { + if err := cont.Restore(p, libcontainerCriuOpts(opts)); err != nil { return execdriver.ExitStatus{ExitCode: -1}, err } diff --git a/daemon/execdriver/windows/windows.go b/daemon/execdriver/windows/windows.go index 9837270235d5c..d49f76a95cf93 100644 --- a/daemon/execdriver/windows/windows.go +++ b/daemon/execdriver/windows/windows.go @@ -13,6 +13,7 @@ import ( "fmt" "github.com/docker/docker/daemon/execdriver" + "github.com/docker/docker/runconfig" ) const ( @@ -95,3 +96,11 @@ func (d *driver) Stats(id string) (*execdriver.ResourceStats, error) { func (d *driver) Exec(c *execdriver.Command, processConfig *execdriver.ProcessConfig, pipes *execdriver.Pipes, startCallback execdriver.StartCallback) (int, error) { return 0, nil } + +func (d *driver) Checkpoint(c *execdriver.Command, opts *runconfig.CriuConfig) error { + return fmt.Errorf("Windows: Containers cannot be checkpointed") +} + +func (d *driver) Restore(c *execdriver.Command, pipes *execdriver.Pipes, restoreCallback execdriver.RestoreCallback, opts *runconfig.CriuConfig, forceRestore bool) (execdriver.ExitStatus, error) { + return execdriver.ExitStatus{ExitCode: 0}, fmt.Errorf("Windows: Containers cannot be restored") +} diff --git a/daemon/monitor.go b/daemon/monitor.go index a1310cfa9efa8..c4abcfe68af3b 100644 --- a/daemon/monitor.go +++ b/daemon/monitor.go @@ -186,43 +186,45 @@ func (m *containerMonitor) Start() error { } // Like Start() but for restoring a container. -func (m *containerMonitor) Restore() error { +func (m *containerMonitor) Restore(opts *runconfig.CriuConfig, forceRestore bool) error { var ( err error // XXX The following line should be changed to // exitStatus execdriver.ExitStatus to match Start() - exitCode int + exitCode execdriver.ExitStatus afterRestore bool ) - defer func() { if afterRestore { m.container.Lock() - m.container.setStopped(&execdriver.ExitStatus{exitCode, false}) + m.container.setStopped(&execdriver.ExitStatus{exitCode.ExitCode, false}) defer m.container.Unlock() } m.Close() }() - if err := m.container.startLoggingToDisk(); err != nil { - m.resetContainer(false) - return err + // FIXME: right now if we startLogging again we get double logs after a restore + if m.container.logCopier == nil { + if err := m.container.startLogging(); err != nil { + m.resetContainer(false) + return err + } } pipes := execdriver.NewPipes(m.container.stdin, m.container.stdout, m.container.stderr, m.container.Config.OpenStdin) m.container.LogEvent("restore") m.lastStartTime = time.Now() - if exitCode, err = m.container.daemon.Restore(m.container, pipes, m.restoreCallback); err != nil { - log.Errorf("Error restoring container: %s, exitCode=%d", err, exitCode) + if exitCode, err = m.container.daemon.Restore(m.container, pipes, m.restoreCallback, opts, forceRestore); err != nil { + logrus.Errorf("Error restoring container: %s, exitCode=%d", err, exitCode) m.container.ExitCode = -1 m.resetContainer(false) return err } afterRestore = true - m.container.ExitCode = exitCode - m.resetMonitor(err == nil && exitCode == 0) + m.container.ExitCode = exitCode.ExitCode + m.resetMonitor(err == nil && exitCode.ExitCode == 0) m.container.LogEvent("die") m.resetContainer(true) return err @@ -332,7 +334,7 @@ func (m *containerMonitor) restoreCallback(processConfig *execdriver.ProcessConf // Write config.json and hostconfig.json files // to /var/lib/docker/containers/. if err := m.container.ToDisk(); err != nil { - log.Debugf("%s", err) + logrus.Debugf("%s", err) } } } diff --git a/daemon/state.go b/daemon/state.go index 82eefe75a3a6c..f6ef8977695b0 100644 --- a/daemon/state.go +++ b/daemon/state.go @@ -25,7 +25,6 @@ type State struct { FinishedAt time.Time CheckpointedAt time.Time waitChan chan struct{} - } func NewState() *State { @@ -45,14 +44,16 @@ func (s *State) String() string { } return fmt.Sprintf("Up %s", units.HumanDuration(time.Now().UTC().Sub(s.StartedAt))) - } else if s.Checkpointed { - return fmt.Sprintf("Checkpointed %s ago", units.HumanDuration(time.Now().UTC().Sub(s.CheckpointedAt))) } if s.removalInProgress { return "Removal In Progress" } + if s.Checkpointed { + return fmt.Sprintf("Checkpointed %s ago", units.HumanDuration(time.Now().UTC().Sub(s.CheckpointedAt))) + } + if s.Dead { return "Dead" } @@ -80,6 +81,10 @@ func (s *State) StateString() string { return "running" } + if s.Checkpointed { + return "checkpointed'" + } + if s.Dead { return "dead" } @@ -269,11 +274,11 @@ func (s *State) SetDead() { s.Unlock() } -func (s *State) SetCheckpointed() { +func (s *State) SetCheckpointed(leaveRunning bool) { s.Lock() s.CheckpointedAt = time.Now().UTC() - s.Checkpointed = true - s.Running = false + s.Checkpointed = !leaveRunning + s.Running = leaveRunning s.Paused = false s.Restarting = false // XXX Not sure if we need to close and recreate waitChan. @@ -282,6 +287,10 @@ func (s *State) SetCheckpointed() { s.Unlock() } +func (s *State) HasBeenCheckpointed() bool { + return s.CheckpointedAt != time.Time{} +} + func (s *State) IsCheckpointed() bool { return s.Checkpointed } diff --git a/docker/flags.go b/docker/flags.go index 969469c176214..1f34079a6eb53 100644 --- a/docker/flags.go +++ b/docker/flags.go @@ -31,6 +31,7 @@ var ( dockerCommands = []command{ {"attach", "Attach to a running container"}, {"build", "Build an image from a Dockerfile"}, + {"checkpoint", "Checkpoint one or more running containers"}, {"commit", "Create a new image from a container's changes"}, {"cp", "Copy files/folders from a container to a HOSTDIR or to STDOUT"}, {"create", "Create a new container"}, @@ -55,6 +56,7 @@ var ( {"push", "Push an image or a repository to a registry"}, {"rename", "Rename a container"}, {"restart", "Restart a running container"}, + {"restore", "Restore one or more checkpointed containers"}, {"rm", "Remove one or more containers"}, {"rmi", "Remove one or more images"}, {"run", "Run a command in a new container"}, @@ -131,7 +133,7 @@ func init() { sort.Sort(byName(dockerCommands)) for _, cmd := range dockerCommands { - help += fmt.Sprintf(" %-10.10s%s\n", cmd.name, cmd.description) + help += fmt.Sprintf(" %-11.11s%s\n", cmd.name, cmd.description) } help += "\nRun 'docker COMMAND --help' for more information on a command." diff --git a/integration-cli/docker_cli_help_test.go b/integration-cli/docker_cli_help_test.go index ce9f54e69af84..4eb515f02a9a7 100644 --- a/integration-cli/docker_cli_help_test.go +++ b/integration-cli/docker_cli_help_test.go @@ -229,7 +229,7 @@ func (s *DockerSuite) TestHelpTextVerify(c *check.C) { } - expected := 39 + expected := 41 if len(cmds) != expected { c.Fatalf("Wrong # of cmds(%d), it should be: %d\nThe list:\n%q", len(cmds), expected, cmds) diff --git a/runconfig/restore.go b/runconfig/restore.go new file mode 100644 index 0000000000000..22f8b0ab0a096 --- /dev/null +++ b/runconfig/restore.go @@ -0,0 +1,15 @@ +package runconfig + +type CriuConfig struct { + ImagesDirectory string + WorkDirectory string + LeaveRunning bool + TcpEstablished bool + ExternalUnixConnections bool + ShellJob bool +} + +type RestoreConfig struct { + CriuOpts CriuConfig + ForceRestore bool +} diff --git a/vendor/src/github.com/docker/libcontainer/container_linux.go b/vendor/src/github.com/docker/libcontainer/container_linux.go index 215f35d38d8c6..f944bfded779b 100644 --- a/vendor/src/github.com/docker/libcontainer/container_linux.go +++ b/vendor/src/github.com/docker/libcontainer/container_linux.go @@ -386,7 +386,7 @@ func (c *linuxContainer) Checkpoint(criuOpts *CriuOpts) error { return err } - err = c.criuSwrk(nil, &req, criuOpts) + err = c.criuSwrk(nil, &req, criuOpts.LeaveRunning) if err != nil { return err } @@ -513,14 +513,14 @@ func (c *linuxContainer) Restore(process *Process, criuOpts *CriuOpts) error { } } - err = c.criuSwrk(process, &req, criuOpts) + err = c.criuSwrk(process, &req, false) if err != nil { return err } return nil } -func (c *linuxContainer) criuSwrk(process *Process, req *criurpc.CriuReq, opts *CriuOpts) error { +func (c *linuxContainer) criuSwrk(process *Process, req *criurpc.CriuReq, leaveRunning bool) error { fds, err := syscall.Socketpair(syscall.AF_LOCAL, syscall.SOCK_SEQPACKET|syscall.SOCK_CLOEXEC, 0) if err != nil { return err @@ -595,7 +595,7 @@ func (c *linuxContainer) criuSwrk(process *Process, req *criurpc.CriuReq, opts * t := resp.GetType() switch { case t == criurpc.CriuReqType_NOTIFY: - if err := c.criuNotifications(resp, process, opts, extFds); err != nil { + if err := c.criuNotifications(resp, process, leaveRunning, extFds); err != nil { return err } t = criurpc.CriuReqType_NOTIFY @@ -662,7 +662,7 @@ func unlockNetwork(config *configs.Config) error { return nil } -func (c *linuxContainer) criuNotifications(resp *criurpc.CriuResp, process *Process, opts *CriuOpts, fds []string) error { +func (c *linuxContainer) criuNotifications(resp *criurpc.CriuResp, process *Process, leaveRunning bool, fds []string) error { notify := resp.GetNotify() if notify == nil { return fmt.Errorf("invalid response: %s", resp.String()) @@ -670,7 +670,7 @@ func (c *linuxContainer) criuNotifications(resp *criurpc.CriuResp, process *Proc switch { case notify.GetScript() == "post-dump": - if !opts.LeaveRunning { + if !leaveRunning { f, err := os.Create(filepath.Join(c.root, "checkpoint")) if err != nil { return err From b7c8e382f634c1b02843f40d7108b0205543dac4 Mon Sep 17 00:00:00 2001 From: boucher Date: Tue, 2 Jun 2015 14:04:14 -0700 Subject: [PATCH 23/26] Add compilation steps for Criu to the Dockerfile Add a basic test for checkpoint/restore to the integration tests Docker-DCO-1.1-Signed-off-by: Ross Boucher (github: boucher) --- Dockerfile | 18 +++++++++ api/types/types.go | 22 ++++++----- daemon/inspect.go | 22 ++++++----- integration-cli/docker_cli_checkpoint_test.go | 37 +++++++++++++++++++ 4 files changed, 79 insertions(+), 20 deletions(-) create mode 100644 integration-cli/docker_cli_checkpoint_test.go diff --git a/Dockerfile b/Dockerfile index eddeefc67d84f..532b8cd54797e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -32,28 +32,39 @@ RUN echo deb http://ppa.launchpad.net/zfs-native/stable/ubuntu trusty main > /et # Packaged dependencies RUN apt-get update && apt-get install -y \ apparmor \ + asciidoc \ aufs-tools \ automake \ bash-completion \ + bsdmainutils \ btrfs-tools \ build-essential \ curl \ dpkg-sig \ git \ iptables \ + libaio-dev \ libapparmor-dev \ libcap-dev \ + libprotobuf-c0-dev \ + libprotobuf-dev \ libsqlite3-dev \ mercurial \ parallel \ + pkg-config \ + protobuf-compiler \ + protobuf-c-compiler \ + python-minimal \ python-mock \ python-pip \ + python-protobuf \ python-websocket \ reprepro \ ruby1.9.1 \ ruby1.9.1-dev \ s3cmd=1.1.0* \ ubuntu-zfs \ + xmlto \ libzfs-dev \ --no-install-recommends @@ -78,6 +89,13 @@ RUN cd /usr/src/lxc \ && make install \ && ldconfig +# Install Criu +RUN mkdir -p /usr/src/criu \ + && curl -sSL https://github.com/xemul/criu/archive/v1.6.tar.gz | tar -v -C /usr/src/criu/ -xz --strip-components=1 +RUN cd /usr/src/criu \ + && make \ + && make install + # Install Go ENV GO_VERSION 1.4.2 RUN curl -sSL https://golang.org/dl/go${GO_VERSION}.src.tar.gz | tar -v -C /usr/local -xz \ diff --git a/api/types/types.go b/api/types/types.go index a27755f69f589..4151d4ee39d28 100644 --- a/api/types/types.go +++ b/api/types/types.go @@ -194,16 +194,18 @@ type ExecStartCheck struct { } type ContainerState struct { - Running bool - Paused bool - Restarting bool - OOMKilled bool - Dead bool - Pid int - ExitCode int - Error string - StartedAt time.Time - FinishedAt time.Time + Running bool + Paused bool + Checkpointed bool + Restarting bool + OOMKilled bool + Dead bool + Pid int + ExitCode int + Error string + StartedAt time.Time + FinishedAt time.Time + CheckpointedAt time.Time } // GET "/containers/{name:.*}/json" diff --git a/daemon/inspect.go b/daemon/inspect.go index 73b394ca24c0f..e9d302f1bc9b1 100644 --- a/daemon/inspect.go +++ b/daemon/inspect.go @@ -64,16 +64,18 @@ func (daemon *Daemon) getInspectData(container *Container) (*types.ContainerJSON } containerState := &types.ContainerState{ - Running: container.State.Running, - Paused: container.State.Paused, - Restarting: container.State.Restarting, - OOMKilled: container.State.OOMKilled, - Dead: container.State.Dead, - Pid: container.State.Pid, - ExitCode: container.State.ExitCode, - Error: container.State.Error, - StartedAt: container.State.StartedAt, - FinishedAt: container.State.FinishedAt, + Running: container.State.Running, + Paused: container.State.Paused, + Checkpointed: container.State.Checkpointed, + Restarting: container.State.Restarting, + OOMKilled: container.State.OOMKilled, + Dead: container.State.Dead, + Pid: container.State.Pid, + ExitCode: container.State.ExitCode, + Error: container.State.Error, + StartedAt: container.State.StartedAt, + FinishedAt: container.State.FinishedAt, + CheckpointedAt: container.State.CheckpointedAt, } volumes := make(map[string]string) diff --git a/integration-cli/docker_cli_checkpoint_test.go b/integration-cli/docker_cli_checkpoint_test.go new file mode 100644 index 0000000000000..e19ef524efd43 --- /dev/null +++ b/integration-cli/docker_cli_checkpoint_test.go @@ -0,0 +1,37 @@ +package main + +import ( + "os/exec" + "strings" + + "github.com/go-check/check" +) + +func (s *DockerSuite) TestCheckpointAndRestore(c *check.C) { + defer unpauseAllContainers() + + runCmd := exec.Command(dockerBinary, "run", "-d", "busybox", "top") + out, _, err := runCommandWithOutput(runCmd) + if err != nil { + c.Fatalf("failed to run container: %v, output: %q", err, out) + } + + containerID := strings.TrimSpace(out) + checkpointCmd := exec.Command(dockerBinary, "checkpoint", containerID) + out, _, err = runCommandWithOutput(checkpointCmd) + if err != nil { + c.Fatalf("failed to checkpoint container: %v, output: %q", err, out) + } + + out, err = inspectField(containerID, "State.Checkpointed") + c.Assert(out, check.Equals, "true") + + restoreCmd := exec.Command(dockerBinary, "restore", containerID) + out, _, _, err = runCommandWithStdoutStderr(restoreCmd) + if err != nil { + c.Fatalf("failed to restore container: %v, output: %q", err, out) + } + + out, err = inspectField(containerID, "State.Checkpointed") + c.Assert(out, check.Equals, "false") +} From 28d3e844add3432878d72e65f0d3630ae28e1153 Mon Sep 17 00:00:00 2001 From: boucher Date: Tue, 16 Jun 2015 14:41:05 -0700 Subject: [PATCH 24/26] Add optional dependency info to the PACKAGERS file. Docker-DCO-1.1-Signed-off-by: Ross Boucher (github: boucher) --- project/PACKAGERS.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/project/PACKAGERS.md b/project/PACKAGERS.md index fd2156c5855bf..da0d2da9ca0f2 100644 --- a/project/PACKAGERS.md +++ b/project/PACKAGERS.md @@ -304,6 +304,9 @@ by having support for them in the kernel or userspace. A few examples include: least the "auplink" utility from aufs-tools) * BTRFS graph driver (requires BTRFS support enabled in the kernel) * ZFS graph driver (requires userspace zfs-utils and a corresponding kernel module) +* Checkpoint/Restore containers: + - requires criu version 1.5.2 or later (criu.org) + - requires kernel version 3.19 or later if using overlay-fs ## Daemon Init Script From 590ae6ba2ea83d59c60ec17f317fce32eb0692a1 Mon Sep 17 00:00:00 2001 From: boucher Date: Thu, 18 Jun 2015 15:18:09 -0700 Subject: [PATCH 25/26] Don't destroy/delete the container if it has been checkpointed. Docker-DCO-1.1-Signed-off-by: Ross Boucher (github: boucher) --- daemon/execdriver/native/driver.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/daemon/execdriver/native/driver.go b/daemon/execdriver/native/driver.go index 04353131efe9e..4176b14b13454 100644 --- a/daemon/execdriver/native/driver.go +++ b/daemon/execdriver/native/driver.go @@ -142,8 +142,11 @@ func (d *driver) Run(c *execdriver.Command, pipes *execdriver.Pipes, startCallba d.activeContainers[c.ID] = cont d.Unlock() defer func() { - cont.Destroy() - d.cleanContainer(c.ID) + status, err := cont.Status() + if err != nil || status != libcontainer.Checkpointed { + cont.Destroy() + d.cleanContainer(c.ID) + } }() if err := cont.Start(p); err != nil { From ff844f86797d426729c80513bb5753495142a0cd Mon Sep 17 00:00:00 2001 From: boucher Date: Thu, 9 Jul 2015 09:40:43 -0700 Subject: [PATCH 26/26] Move checkpoint methods into a separate container_checkpoint file. Docker-DCO-1.1-Signed-off-by: Ross Boucher (github: boucher) --- daemon/container.go | 76 ------------------------------ daemon/container_checkpoint.go | 84 ++++++++++++++++++++++++++++++++++ 2 files changed, 84 insertions(+), 76 deletions(-) create mode 100644 daemon/container_checkpoint.go diff --git a/daemon/container.go b/daemon/container.go index efaeaa71867a4..5593e6987be65 100644 --- a/daemon/container.go +++ b/daemon/container.go @@ -607,62 +607,6 @@ func validateID(id string) error { return nil } -func (container *Container) Checkpoint(opts *runconfig.CriuConfig) error { - if err := container.daemon.Checkpoint(container, opts); err != nil { - return err - } - - if opts.LeaveRunning == false { - container.ReleaseNetwork() - } - return nil -} - -func (container *Container) Restore(opts *runconfig.CriuConfig, forceRestore bool) error { - var err error - container.Lock() - defer container.Unlock() - - defer func() { - if err != nil { - container.setError(err) - // if no one else has set it, make sure we don't leave it at zero - if container.ExitCode == 0 { - container.ExitCode = 128 - } - container.toDisk() - container.cleanup() - } - }() - - if err := container.Mount(); err != nil { - return err - } - if err = container.initializeNetworking(true); err != nil { - return err - } - linkedEnv, err := container.setupLinkedContainers() - if err != nil { - return err - } - if err = container.setupWorkingDirectory(); err != nil { - return err - } - - env := container.createDaemonEnvironment(linkedEnv) - if err = populateCommand(container, env); err != nil { - return err - } - - mounts, err := container.setupMounts() - if err != nil { - return err - } - - container.command.Mounts = mounts - return container.waitForRestore(opts, forceRestore) -} - func (container *Container) Copy(resource string) (io.ReadCloser, error) { container.Lock() defer container.Unlock() @@ -825,26 +769,6 @@ func (container *Container) waitForStart() error { return nil } -func (container *Container) waitForRestore(opts *runconfig.CriuConfig, forceRestore bool) error { - container.monitor = newContainerMonitor(container, container.hostConfig.RestartPolicy) - - // After calling promise.Go() we'll have two goroutines: - // - The current goroutine that will block in the select - // below until restore is done. - // - A new goroutine that will restore the container and - // wait for it to exit. - select { - case <-container.monitor.restoreSignal: - if container.ExitCode != 0 { - return fmt.Errorf("restore process failed") - } - case err := <-promise.Go(func() error { return container.monitor.Restore(opts, forceRestore) }): - return err - } - - return nil -} - func (container *Container) GetProcessLabel() string { // even if we have a process label return "" if we are running // in privileged mode diff --git a/daemon/container_checkpoint.go b/daemon/container_checkpoint.go new file mode 100644 index 0000000000000..468816e448dc1 --- /dev/null +++ b/daemon/container_checkpoint.go @@ -0,0 +1,84 @@ +package daemon + +import ( + "fmt" + + "github.com/docker/docker/pkg/promise" + "github.com/docker/docker/runconfig" +) + +func (container *Container) Checkpoint(opts *runconfig.CriuConfig) error { + if err := container.daemon.Checkpoint(container, opts); err != nil { + return err + } + + if opts.LeaveRunning == false { + container.ReleaseNetwork() + } + return nil +} + +func (container *Container) Restore(opts *runconfig.CriuConfig, forceRestore bool) error { + var err error + container.Lock() + defer container.Unlock() + + defer func() { + if err != nil { + container.setError(err) + // if no one else has set it, make sure we don't leave it at zero + if container.ExitCode == 0 { + container.ExitCode = 128 + } + container.toDisk() + container.cleanup() + } + }() + + if err := container.Mount(); err != nil { + return err + } + if err = container.initializeNetworking(true); err != nil { + return err + } + linkedEnv, err := container.setupLinkedContainers() + if err != nil { + return err + } + if err = container.setupWorkingDirectory(); err != nil { + return err + } + + env := container.createDaemonEnvironment(linkedEnv) + if err = populateCommand(container, env); err != nil { + return err + } + + mounts, err := container.setupMounts() + if err != nil { + return err + } + + container.command.Mounts = mounts + return container.waitForRestore(opts, forceRestore) +} + +func (container *Container) waitForRestore(opts *runconfig.CriuConfig, forceRestore bool) error { + container.monitor = newContainerMonitor(container, container.hostConfig.RestartPolicy) + + // After calling promise.Go() we'll have two goroutines: + // - The current goroutine that will block in the select + // below until restore is done. + // - A new goroutine that will restore the container and + // wait for it to exit. + select { + case <-container.monitor.restoreSignal: + if container.ExitCode != 0 { + return fmt.Errorf("restore process failed") + } + case err := <-promise.Go(func() error { return container.monitor.Restore(opts, forceRestore) }): + return err + } + + return nil +}