The following pull request was submitted through Github. It can be accessed and reviewed at: https://github.com/lxc/lxd/pull/2662
This e-mail was sent by the LXC bot, direct replies will not reach the author unless they happen to be subscribed to this list. === Description (from pull-request) ===
From cf02ecaead9b1ce288ec3a8c7861c51258e96020 Mon Sep 17 00:00:00 2001 From: Tycho Andersen <tycho.ander...@canonical.com> Date: Mon, 28 Nov 2016 09:04:23 -0700 Subject: [PATCH 1/3] Change ContainerStart to take the name and path to start Instead of taking the container and computing the name and path, let's just take the name and path themselves. We'll use this in the next patch to start the storage for a container that we haven't created in memory or in the database yet, when we're reading its slurp file. Signed-off-by: Tycho Andersen <tycho.ander...@canonical.com> --- lxd/container_lxc.go | 4 ++-- lxd/storage.go | 16 ++++++++-------- lxd/storage_btrfs.go | 4 ++-- lxd/storage_dir.go | 4 ++-- lxd/storage_lvm.go | 20 ++++++++++---------- lxd/storage_test.go | 4 ++-- lxd/storage_zfs.go | 6 +++--- 7 files changed, 29 insertions(+), 29 deletions(-) diff --git a/lxd/container_lxc.go b/lxd/container_lxc.go index 82a91d3..8419cb6 100644 --- a/lxd/container_lxc.go +++ b/lxd/container_lxc.go @@ -4663,7 +4663,7 @@ func (c *containerLXC) StorageStart() error { return c.storage.ContainerSnapshotStart(c) } - return c.storage.ContainerStart(c) + return c.storage.ContainerStart(c.Name(), c.Path()) } func (c *containerLXC) StorageStop() error { @@ -4671,7 +4671,7 @@ func (c *containerLXC) StorageStop() error { return c.storage.ContainerSnapshotStop(c) } - return c.storage.ContainerStop(c) + return c.storage.ContainerStop(c.Name(), c.Path()) } // Mount handling diff --git a/lxd/storage.go b/lxd/storage.go index a41ca1b..2ae706d 100644 --- a/lxd/storage.go +++ b/lxd/storage.go @@ -149,8 +149,8 @@ type storage interface { ContainerCanRestore(container container, sourceContainer container) error ContainerDelete(container container) error ContainerCopy(container container, sourceContainer container) error - ContainerStart(container container) error - ContainerStop(container container) error + ContainerStart(name string, path string) error + ContainerStop(name string, path string) error ContainerRename(container container, newName string) error ContainerRestore(container container, sourceContainer container) error ContainerSetQuota(container container, size int64) error @@ -431,14 +431,14 @@ func (lw *storageLogWrapper) ContainerCopy( return lw.w.ContainerCopy(container, sourceContainer) } -func (lw *storageLogWrapper) ContainerStart(container container) error { - lw.log.Debug("ContainerStart", log.Ctx{"container": container.Name()}) - return lw.w.ContainerStart(container) +func (lw *storageLogWrapper) ContainerStart(name string, path string) error { + lw.log.Debug("ContainerStart", log.Ctx{"container": name}) + return lw.w.ContainerStart(name, path) } -func (lw *storageLogWrapper) ContainerStop(container container) error { - lw.log.Debug("ContainerStop", log.Ctx{"container": container.Name()}) - return lw.w.ContainerStop(container) +func (lw *storageLogWrapper) ContainerStop(name string, path string) error { + lw.log.Debug("ContainerStop", log.Ctx{"container": name}) + return lw.w.ContainerStop(name, path) } func (lw *storageLogWrapper) ContainerRename( diff --git a/lxd/storage_btrfs.go b/lxd/storage_btrfs.go index 4a8d63f..cf70b21 100644 --- a/lxd/storage_btrfs.go +++ b/lxd/storage_btrfs.go @@ -172,11 +172,11 @@ func (s *storageBtrfs) ContainerCopy(container container, sourceContainer contai return container.TemplateApply("copy") } -func (s *storageBtrfs) ContainerStart(container container) error { +func (s *storageBtrfs) ContainerStart(name string, path string) error { return nil } -func (s *storageBtrfs) ContainerStop(container container) error { +func (s *storageBtrfs) ContainerStop(name string, path string) error { return nil } diff --git a/lxd/storage_dir.go b/lxd/storage_dir.go index 18f4985..5adcd45 100644 --- a/lxd/storage_dir.go +++ b/lxd/storage_dir.go @@ -123,11 +123,11 @@ func (s *storageDir) ContainerCopy( return container.TemplateApply("copy") } -func (s *storageDir) ContainerStart(container container) error { +func (s *storageDir) ContainerStart(name string, path string) error { return nil } -func (s *storageDir) ContainerStop(container container) error { +func (s *storageDir) ContainerStop(name string, path string) error { return nil } diff --git a/lxd/storage_lvm.go b/lxd/storage_lvm.go index 3e9404d..a3bc02f 100644 --- a/lxd/storage_lvm.go +++ b/lxd/storage_lvm.go @@ -404,7 +404,7 @@ func (s *storageLvm) ContainerCopy(container container, sourceContainer containe return err } - if err := s.ContainerStart(container); err != nil { + if err := s.ContainerStart(container.Name(), container.Path()); err != nil { s.log.Error("Error starting/mounting container", log.Ctx{"err": err, "container": container.Name()}) s.ContainerDelete(container) return err @@ -419,36 +419,36 @@ func (s *storageLvm) ContainerCopy(container container, sourceContainer containe return fmt.Errorf("rsync failed: %s", string(output)) } - if err := s.ContainerStop(container); err != nil { + if err := s.ContainerStop(container.Name(), container.Path()); err != nil { return err } } return container.TemplateApply("copy") } -func (s *storageLvm) ContainerStart(container container) error { - lvName := containerNameToLVName(container.Name()) +func (s *storageLvm) ContainerStart(name string, path string) error { + lvName := containerNameToLVName(name) lvpath := fmt.Sprintf("/dev/%s/%s", s.vgName, lvName) fstype := daemonConfig["storage.lvm_fstype"].Get() mountOptions := daemonConfig["storage.lvm_mount_options"].Get() - err := tryMount(lvpath, container.Path(), fstype, 0, mountOptions) + err := tryMount(lvpath, path, fstype, 0, mountOptions) if err != nil { return fmt.Errorf( "Error mounting snapshot LV path='%s': %v", - container.Path(), + path, err) } return nil } -func (s *storageLvm) ContainerStop(container container) error { - err := tryUnmount(container.Path(), 0) +func (s *storageLvm) ContainerStop(name string, path string) error { + err := tryUnmount(path, 0) if err != nil { return fmt.Errorf( "failed to unmount container path '%s'.\nError: %v", - container.Path(), + path, err) } @@ -690,7 +690,7 @@ func (s *storageLvm) ContainerSnapshotStart(container container) error { } func (s *storageLvm) ContainerSnapshotStop(container container) error { - err := s.ContainerStop(container) + err := s.ContainerStop(container.Name(), container.Path()) if err != nil { return err } diff --git a/lxd/storage_test.go b/lxd/storage_test.go index bd723d4..63006c9 100644 --- a/lxd/storage_test.go +++ b/lxd/storage_test.go @@ -61,11 +61,11 @@ func (s *storageMock) ContainerCopy( return nil } -func (s *storageMock) ContainerStart(container container) error { +func (s *storageMock) ContainerStart(name string, path string) error { return nil } -func (s *storageMock) ContainerStop(container container) error { +func (s *storageMock) ContainerStop(name string, path string) error { return nil } diff --git a/lxd/storage_zfs.go b/lxd/storage_zfs.go index 333ac12..6e12f31 100644 --- a/lxd/storage_zfs.go +++ b/lxd/storage_zfs.go @@ -92,8 +92,8 @@ func (s *storageZfs) Init(config map[string]interface{}) (storage, error) { } // Things we don't need to care about -func (s *storageZfs) ContainerStart(container container) error { - fs := fmt.Sprintf("containers/%s", container.Name()) +func (s *storageZfs) ContainerStart(name string, path string) error { + fs := fmt.Sprintf("containers/%s", name) // Just in case the container filesystem got unmounted if !shared.IsMountPoint(shared.VarPath(fs)) { @@ -103,7 +103,7 @@ func (s *storageZfs) ContainerStart(container container) error { return nil } -func (s *storageZfs) ContainerStop(container container) error { +func (s *storageZfs) ContainerStop(name string, path string) error { return nil } From 017b304ead613f74a85c1fb2564d591b5d661f96 Mon Sep 17 00:00:00 2001 From: Tycho Andersen <tycho.ander...@canonical.com> Date: Mon, 28 Nov 2016 18:40:45 +0000 Subject: [PATCH 2/3] rework EEXISTS detection on create In particular: let's use the database and not stat the filesystem, so that if we are doing a `lxd import`, it doesn't fail. Signed-off-by: Tycho Andersen <tycho.ander...@canonical.com> --- lxd/container.go | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/lxd/container.go b/lxd/container.go index 95e9c12..3d00e46 100644 --- a/lxd/container.go +++ b/lxd/container.go @@ -634,22 +634,22 @@ func containerCreateInternal(d *Daemon, args containerArgs) (container, error) { } } - path := containerPath(args.Name, args.Ctype == cTypeSnapshot) - if shared.PathExists(path) { - if shared.IsSnapshot(args.Name) { - return nil, fmt.Errorf("Snapshot '%s' already exists", args.Name) + // Create the container entry + id, err := dbContainerCreate(d.db, args) + if err != nil { + if err == DbErrAlreadyDefined { + thing := "Container" + if shared.IsSnapshot(args.Name) { + thing = "Snapshot" + } + return nil, fmt.Errorf("%s '%s' already exists", thing, args.Name) } - return nil, fmt.Errorf("The container already exists") + return nil, err } // Wipe any existing log for this container name os.RemoveAll(shared.LogPath(args.Name)) - // Create the container entry - id, err := dbContainerCreate(d.db, args) - if err != nil { - return nil, err - } args.Id = id // Read the timestamp from the database From 189641641f1effe9d53449d3e0a14f08b91bff69 Mon Sep 17 00:00:00 2001 From: Tycho Andersen <tycho.ander...@canonical.com> Date: Tue, 8 Nov 2016 16:56:40 +0200 Subject: [PATCH 3/3] implement `lxd import` The basic idea here is to maintain a file in the container's directory that indicates what configuration and snapshots it has, so that if someone just backs up the filesystem the container is on (e.g. the zfs pool), they can re-import their containers with a simple `lxd import` command. Closes #2286 Signed-off-by: Tycho Andersen <tycho.ander...@canonical.com> --- lxd/api_internal.go | 102 +++++++++++++++++++++++++++++++++++++++++++++++ lxd/container.go | 12 +++++- lxd/container_lxc.go | 75 ++++++++++++++++++++++++++++++++++ lxd/main.go | 28 +++++++++++++ shared/log.go | 2 +- test/suites/migration.sh | 9 +++++ 6 files changed, 226 insertions(+), 2 deletions(-) diff --git a/lxd/api_internal.go b/lxd/api_internal.go index 51c11e3..11ca5aa 100644 --- a/lxd/api_internal.go +++ b/lxd/api_internal.go @@ -2,10 +2,13 @@ package main import ( "fmt" + "io/ioutil" "net/http" "strconv" + "strings" "github.com/gorilla/mux" + "gopkg.in/yaml.v2" "github.com/lxc/lxd/shared" @@ -17,6 +20,7 @@ var apiInternal = []Command{ internalShutdownCmd, internalContainerOnStartCmd, internalContainerOnStopCmd, + internalContainersCmd, } func internalReady(d *Daemon, r *http.Request) Response { @@ -95,3 +99,101 @@ var internalShutdownCmd = Command{name: "shutdown", put: internalShutdown} var internalReadyCmd = Command{name: "ready", put: internalReady, get: internalWaitReady} var internalContainerOnStartCmd = Command{name: "containers/{id}/onstart", get: internalContainerOnStart} var internalContainerOnStopCmd = Command{name: "containers/{id}/onstop", get: internalContainerOnStop} + +func slurpSlurpFile(path string) (*slurpFile, error) { + data, err := ioutil.ReadFile(path) + if err != nil { + return nil, err + } + + sf := slurpFile{} + + if err := yaml.Unmarshal(data, &sf); err != nil { + return nil, err + } + + return &sf, nil +} + +func internalImport(d *Daemon, r *http.Request) Response { + name := r.FormValue("target") + if name == "" { + return BadRequest(fmt.Errorf("target is required")) + } + + path := containerPath(name, false) + err := d.Storage.ContainerStart(name, path) + if err != nil { + return SmartError(err) + } + + defer d.Storage.ContainerStop(name, path) + + sf, err := slurpSlurpFile(shared.VarPath("containers", name, "backup.yaml")) + if err != nil { + return SmartError(err) + } + + baseImage := sf.Container.Config["volatile.base_image"] + for k := range sf.Container.Config { + if strings.HasPrefix(k, "volatile") { + delete(sf.Container.Config, k) + } + } + + arch, err := shared.ArchitectureId(sf.Container.Architecture) + if err != nil { + return SmartError(err) + } + _, err = containerCreateInternal(d, containerArgs{ + Architecture: arch, + BaseImage: baseImage, + Config: sf.Container.Config, + CreationDate: sf.Container.CreationDate, + LastUsedDate: sf.Container.LastUsedDate, + Ctype: cTypeRegular, + Devices: sf.Container.Devices, + Ephemeral: sf.Container.Ephemeral, + Name: sf.Container.Name, + Profiles: sf.Container.Profiles, + Stateful: sf.Container.Stateful, + }) + if err != nil { + return SmartError(err) + } + + for _, snap := range sf.Snapshots { + baseImage := snap.Config["volatile.base_image"] + for k := range snap.Config { + if strings.HasPrefix(k, "volatile") { + delete(snap.Config, k) + } + } + + arch, err := shared.ArchitectureId(snap.Architecture) + if err != nil { + return SmartError(err) + } + + _, err = containerCreateInternal(d, containerArgs{ + Architecture: arch, + BaseImage: baseImage, + Config: snap.Config, + CreationDate: snap.CreationDate, + LastUsedDate: snap.LastUsedDate, + Ctype: cTypeSnapshot, + Devices: snap.Devices, + Ephemeral: snap.Ephemeral, + Name: snap.Name, + Profiles: snap.Profiles, + Stateful: snap.Stateful, + }) + if err != nil { + return SmartError(err) + } + } + + return EmptySyncResponse +} + +var internalContainersCmd = Command{name: "containers", post: internalImport} diff --git a/lxd/container.go b/lxd/container.go index 3d00e46..8c8170f 100644 --- a/lxd/container.go +++ b/lxd/container.go @@ -660,7 +660,12 @@ func containerCreateInternal(d *Daemon, args containerArgs) (container, error) { args.CreationDate = dbArgs.CreationDate args.LastUsedDate = dbArgs.LastUsedDate - return containerLXCCreate(d, args) + c, err := containerLXCCreate(d, args) + if err != nil { + return nil, err + } + + return c, nil } func containerConfigureInternal(c container) error { @@ -683,6 +688,11 @@ func containerConfigureInternal(c container) error { break } + err := writeSlurpFile(c) + if err != nil { + return err + } + return nil } diff --git a/lxd/container_lxc.go b/lxd/container_lxc.go index 8419cb6..9602965 100644 --- a/lxd/container_lxc.go +++ b/lxd/container_lxc.go @@ -2511,6 +2511,14 @@ func (c *containerLXC) Restore(sourceContainer container) error { return err } + // The old slurp file may be out of date (e.g. it doesn't have all the + // current snapshots of the container listed); let's write a new one to + // be safe. + err = writeSlurpFile(c) + if err != nil { + return err + } + // If the container wasn't running but was stateful, should we restore // it as running? if shared.PathExists(c.StatePath()) { @@ -2738,6 +2746,65 @@ func (c *containerLXC) ConfigKeySet(key string, value string) error { return c.Update(args, false) } +type slurpFile struct { + Container *shared.ContainerInfo `yaml:"container"` + Snapshots []*shared.SnapshotInfo `yaml:"snapshots"` +} + +func writeSlurpFile(c container) error { + /* we only write slurp files out for actual containers */ + if c.IsSnapshot() { + return nil + } + + ci, _, err := c.Render() + if err != nil { + return err + } + + snapshots, err := c.Snapshots() + if err != nil { + return err + } + + var sis []*shared.SnapshotInfo + + for _, s := range snapshots { + si, _, err := s.Render() + if err != nil { + return err + } + + sis = append(sis, si.(*shared.SnapshotInfo)) + } + + data, err := yaml.Marshal(&slurpFile{ + Container: ci.(*shared.ContainerInfo), + Snapshots: sis, + }) + if err != nil { + return err + } + + f, err := os.Create(shared.VarPath("containers", c.Name(), "backup.yaml")) + if err != nil { + return err + } + defer f.Close() + + err = f.Chmod(0400) + if err != nil { + return err + } + + err = shared.WriteAll(f, data) + if err != nil { + return err + } + + return nil +} + func (c *containerLXC) Update(args containerArgs, userRequested bool) error { // Set sane defaults for unset keys if args.Architecture == 0 { @@ -3486,6 +3553,14 @@ func (c *containerLXC) Update(args containerArgs, userRequested bool) error { return err } + /* we can call Update in some cases when the directory doesn't exist + * yet before container creation; this is okay, because at the end of + * container creation we write the slurp file, so let's not worry about + * ENOENT. */ + if err := writeSlurpFile(c); err != nil && !os.IsNotExist(err) { + return err + } + // Update network leases needsUpdate := false for _, m := range updateDevices { diff --git a/lxd/main.go b/lxd/main.go index cba11bb..02204b5 100644 --- a/lxd/main.go +++ b/lxd/main.go @@ -88,6 +88,8 @@ func run() error { fmt.Printf(" Perform a clean shutdown of LXD and all running containers\n") fmt.Printf(" waitready [--timeout=15]\n") fmt.Printf(" Wait until LXD is ready to handle requests\n") + fmt.Printf(" import <container name>\n") + fmt.Printf(" Import a pre-existing container from storage\n") fmt.Printf("\n\nCommon options:\n") fmt.Printf(" --debug\n") @@ -223,6 +225,8 @@ func run() error { return cmdShutdown() case "waitready": return cmdWaitReady() + case "import": + return cmdImport(os.Args[1:]) // Internal commands case "forkgetnet": @@ -1218,3 +1222,27 @@ func cmdMigrateDumpSuccess(args []string) error { return c.WaitForSuccess(args[1]) } + +func cmdImport(args []string) error { + name := args[1] + + c, err := lxd.NewClient(&lxd.DefaultConfig, "local") + if err != nil { + return err + } + + url := fmt.Sprintf("%s/internal/containers?target=%s", c.BaseURL, name) + + req, err := http.NewRequest("POST", url, nil) + if err != nil { + return err + } + + raw, err := c.Http.Do(req) + _, err = lxd.HoistResponse(raw, lxd.Sync) + if err != nil { + return err + } + + return nil +} diff --git a/shared/log.go b/shared/log.go index 792ccf6..f696feb 100644 --- a/shared/log.go +++ b/shared/log.go @@ -93,5 +93,5 @@ func LogCritf(format string, args ...interface{}) { func PrintStack() { buf := make([]byte, 1<<16) runtime.Stack(buf, true) - LogDebugf("%s", buf) + LogErrorf("%s", buf) } diff --git a/test/suites/migration.sh b/test/suites/migration.sh index 675049f..a3bf48f 100644 --- a/test/suites/migration.sh +++ b/test/suites/migration.sh @@ -21,6 +21,15 @@ test_migration() { lxc_remote move l1:nonlive l2: lxc_remote config show l2:nonlive/snap0 | grep user.tester | grep foo + # test `lxd import` + if [ "${LXD_BACKEND}" = "zfs" ]; then + lxc_remote init testimage backup + lxc_remote snapshot backup + sqlite3 "${LXD_DIR}/lxd.db" "DELETE FROM containers WHERE name='backup'" + lxd import backup + lxc_remote info backup | grep snap0 + fi + # FIXME: make this backend agnostic if [ "${LXD_BACKEND}" != "lvm" ]; then [ -d "${LXD2_DIR}/containers/nonlive/rootfs" ]
_______________________________________________ lxc-devel mailing list lxc-devel@lists.linuxcontainers.org http://lists.linuxcontainers.org/listinfo/lxc-devel